1package aggregator
  2
  3import (
  4	"context"
  5	"fmt"
  6	"io"
  7	"net/http"
  8	"os"
  9	"strings"
 10	"sync"
 11	"time"
 12
 13	"llm_aggregator/internal/progress"
 14
 15	"github.com/PuerkitoBio/goquery"
 16	"github.com/mmcdole/gofeed"
 17	"golang.org/x/sync/errgroup"
 18)
 19
 20// FeedAggregator aggregates articles from multiple RSS feeds.
 21type FeedAggregator struct {
 22	maxArticlesPerFeed int
 23	maxDaysOld         int
 24	maxContentLength   int
 25	client             *http.Client
 26	userAgent          string
 27	progressCtx        *progress.Context
 28}
 29
 30// NewFeedAggregatorWithProgress creates a new FeedAggregator with progress context.
 31func NewFeedAggregatorWithProgress(maxArticlesPerFeed, maxDaysOld, maxContentLength int, progressCtx *progress.Context) *FeedAggregator {
 32	return &FeedAggregator{
 33		maxArticlesPerFeed: maxArticlesPerFeed,
 34		maxDaysOld:         maxDaysOld,
 35		maxContentLength:   maxContentLength,
 36		client: &http.Client{
 37			Timeout: 30 * time.Second,
 38		},
 39		userAgent:   "LLM-Aggregator/0.1.0 (+https://codeberg.org/maxwelljensen/llm-aggregator)",
 40		progressCtx: progressCtx,
 41	}
 42}
 43
 44// ParseFeedsFromFile parses RSS feeds from a file containing one URL per line.
 45// Feeds are fetched concurrently for improved performance.
 46func (fa *FeedAggregator) ParseFeedsFromFile(filePath string) ([]*Article, error) {
 47	file, err := os.Open(filePath)
 48	if err != nil {
 49		return nil, fmt.Errorf("failed to open feeds file: %w", err)
 50	}
 51	defer file.Close()
 52
 53	content, err := io.ReadAll(file)
 54	if err != nil {
 55		return nil, fmt.Errorf("failed to read feeds file: %w", err)
 56	}
 57
 58	lines := strings.Split(string(content), "\n")
 59	feedURLs := []string{}
 60	for _, line := range lines {
 61		line = strings.TrimSpace(line)
 62		if line != "" && !strings.HasPrefix(line, "#") {
 63			feedURLs = append(feedURLs, line)
 64		}
 65	}
 66
 67	if fa.progressCtx != nil {
 68		fa.progressCtx.Logf("Found %d feed URLs in %s", len(feedURLs), filePath)
 69	}
 70
 71	// Use mutex to protect shared state
 72	var mu sync.Mutex
 73	var allArticles []*Article
 74	var feedErrors []string
 75
 76	// Limit concurrency to avoid overwhelming servers
 77	// NOTE: maxConcurrency is hardcoded to 10. Changing this requires a code
 78	// modification.
 79	const maxConcurrency = 10
 80	sem := make(chan struct{}, maxConcurrency)
 81
 82	g, _ := errgroup.WithContext(context.Background())
 83	for i, feedURL := range feedURLs {
 84		currentIndex := i
 85		sem <- struct{}{} // Acquire semaphore
 86		g.Go(func() error {
 87			defer func() { <-sem }() // Release semaphore
 88			if fa.progressCtx != nil {
 89				fa.progressCtx.Logf("Processing feed %d/%d: %s", currentIndex+1, len(feedURLs), feedURL)
 90			}
 91			feedArticles, err := fa.ParseFeed(feedURL)
 92			if err != nil {
 93				if fa.progressCtx != nil {
 94					fa.progressCtx.Warningf("Failed to parse feed %s: %v", feedURL, err)
 95				}
 96				mu.Lock()
 97				feedErrors = append(feedErrors, fmt.Sprintf("%s: %v", feedURL, err))
 98				mu.Unlock()
 99				return nil // Don't return error to continue processing other feeds
100			}
101			mu.Lock()
102			allArticles = append(allArticles, feedArticles...)
103			mu.Unlock()
104			return nil
105		})
106	}
107
108	if err := g.Wait(); err != nil {
109		return nil, fmt.Errorf("failed to process feeds: %w", err)
110	}
111
112	if len(feedErrors) > 0 {
113		if fa.progressCtx != nil {
114			fa.progressCtx.Warningf("Encountered %d feed errors: %v", len(feedErrors), feedErrors)
115		}
116	}
117
118	return allArticles, nil
119}
120
121// ParseFeed parses a single RSS feed and extracts articles.
122func (fa *FeedAggregator) ParseFeed(feedURL string) ([]*Article, error) {
123	articles := []*Article{}
124
125	fp := gofeed.NewParser()
126	feed, err := fp.ParseURL(feedURL)
127	if err != nil {
128		return nil, fmt.Errorf("failed to parse feed URL %s: %w", feedURL, err)
129	}
130
131	feedTitle := feed.Title
132	if feedTitle == "" {
133		feedTitle = feedURL
134	}
135
136	if fa.progressCtx != nil {
137		fa.progressCtx.Logf("Parsing feed: %s (%d entries)", feedTitle, len(feed.Items))
138	}
139
140	var cutoffTime time.Time
141	if fa.maxDaysOld > 0 {
142		cutoffTime = time.Now().Add(-time.Duration(fa.maxDaysOld) * 24 * time.Hour)
143	}
144
145	maxItems := min(fa.maxArticlesPerFeed, len(feed.Items))
146
147	for i, item := range feed.Items[:maxItems] {
148		article, err := fa.extractArticle(item, feedTitle, cutoffTime)
149		if err != nil {
150			if fa.progressCtx != nil {
151				fa.progressCtx.Warningf("Failed to extract article %d from %s: %v", i, feedURL, err)
152			}
153			continue
154		}
155		if article != nil {
156			articles = append(articles, article)
157		}
158	}
159
160	return articles, nil
161}
162
163func (fa *FeedAggregator) extractArticle(item *gofeed.Item, feedTitle string, cutoffTime time.Time) (*Article, error) {
164	// Extract metadata
165	title := item.Title
166	if title == "" {
167		title = "Untitled"
168	}
169
170	link := item.Link
171	if link == "" {
172		return nil, nil
173	}
174
175	// Parse publication date
176	published := fa.parsePublishedDate(item)
177
178	// Check if article is too old
179	if !cutoffTime.IsZero() && !published.IsZero() && published.Before(cutoffTime) {
180		if fa.progressCtx != nil {
181			fa.progressCtx.Debugf("Skipping old article: %s (%s)", title, published.Format("2006-01-02"))
182		}
183		return nil, nil
184	}
185
186	// Extract author
187	author := ""
188	if item.Author != nil {
189		author = item.Author.Name
190		if author == "" {
191			author = item.Author.Email
192		}
193	}
194	if author == "" && item.DublinCoreExt != nil && len(item.DublinCoreExt.Creator) > 0 {
195		author = item.DublinCoreExt.Creator[0]
196	}
197
198	// Get content
199	content := fa.extractContent(item, link)
200
201	// Truncate content if too long
202	if len(content) > fa.maxContentLength {
203		content = content[:fa.maxContentLength] + "... [truncated]"
204	}
205
206	return &Article{
207		Title:      title,
208		Link:       link,
209		Content:    content,
210		Published:  published,
211		Author:     author,
212		SourceFeed: feedTitle,
213	}, nil
214}
215
216func (fa *FeedAggregator) parsePublishedDate(item *gofeed.Item) time.Time {
217	if item.PublishedParsed != nil {
218		return *item.PublishedParsed
219	}
220	if item.UpdatedParsed != nil {
221		return *item.UpdatedParsed
222	}
223	// Try to parse from string
224	if item.Published != "" {
225		if t, err := time.Parse(time.RFC1123, item.Published); err == nil {
226			return t
227		}
228		if t, err := time.Parse(time.RFC1123Z, item.Published); err == nil {
229			return t
230		}
231		if t, err := time.Parse(time.RFC3339, item.Published); err == nil {
232			return t
233		}
234	}
235	return time.Time{}
236}
237
238func (fa *FeedAggregator) extractContent(item *gofeed.Item, link string) string {
239	// Priority: Content -> Description -> Fetch webpage
240	// NOTE: Webpage scraping is only attempted when content is empty or <100
241	// chars. This balances content quality against latency (webpage fetch adds
242	// ~1-2s per article).
243	content := ""
244
245	// Try to get content from item
246	if item.Content != "" {
247		content = item.Content
248	} else if item.Description != "" {
249		content = item.Description
250	}
251
252	// If still no content or very short, fetch webpage
253	if content == "" || len(content) < 100 {
254		fetchedContent, err := fa.fetchWebpageContent(link)
255		if err == nil && fetchedContent != "" {
256			content = fetchedContent
257		}
258	}
259
260	return content
261}
262
263func (fa *FeedAggregator) fetchWebpageContent(url string) (string, error) {
264	req, err := http.NewRequest("GET", url, nil)
265	if err != nil {
266		return "", err
267	}
268	req.Header.Set("User-Agent", fa.userAgent)
269
270	resp, err := fa.client.Do(req)
271	if err != nil {
272		return "", err
273	}
274	defer resp.Body.Close()
275
276	if resp.StatusCode != http.StatusOK {
277		return "", fmt.Errorf("HTTP status %d", resp.StatusCode)
278	}
279
280	doc, err := goquery.NewDocumentFromReader(resp.Body)
281	if err != nil {
282		return "", err
283	}
284
285	// Remove script and style elements
286	doc.Find("script, style, nav, footer, header").Remove()
287
288	// Try to find article content
289	articleSelectors := []string{
290		"article",
291		"main",
292		".article-content",
293		".post-content",
294		".entry-content",
295		"#content",
296		".content",
297	}
298
299	for _, selector := range articleSelectors {
300		articleElement := doc.Find(selector).First()
301		if articleElement.Length() > 0 {
302			text := articleElement.Text()
303			text = strings.Join(strings.Fields(text), " ") // Normalise whitespace
304			if len(text) > 200 {
305				return text, nil
306			}
307		}
308	}
309
310	// Fallback: get all paragraphs
311	paragraphs := doc.Find("p")
312	if paragraphs.Length() > 0 {
313		var texts []string
314		paragraphs.Each(func(i int, s *goquery.Selection) {
315			texts = append(texts, strings.TrimSpace(s.Text()))
316		})
317		text := strings.Join(texts, " ")
318		if len(text) > 200 {
319			return text, nil
320		}
321	}
322
323	// Last resort: get all text
324	text := doc.Text()
325	text = strings.Join(strings.Fields(text), " ")
326	if len(text) > 200 {
327		return text, nil
328	}
329
330	return "", nil
331}