1package aggregator
  2
  3import (
  4	"context"
  5	"fmt"
  6	"io"
  7	"net/http"
  8	"os"
  9	"strings"
 10	"sync"
 11	"time"
 12
 13	"llm_aggregator/internal/progress"
 14
 15	"github.com/PuerkitoBio/goquery"
 16	"github.com/mmcdole/gofeed"
 17	"golang.org/x/sync/errgroup"
 18)
 19
 20// FeedAggregator aggregates articles from multiple RSS feeds.
 21type FeedAggregator struct {
 22	maxArticlesPerFeed int
 23	maxDaysOld         int
 24	maxContentLength   int
 25	client             *http.Client
 26	userAgent          string
 27	progressCtx        *progress.Context
 28}
 29
 30// NewFeedAggregator creates a new FeedAggregator with the specified options.
 31func NewFeedAggregator(maxArticlesPerFeed, maxDaysOld, maxContentLength int) *FeedAggregator {
 32	return NewFeedAggregatorWithProgress(maxArticlesPerFeed, maxDaysOld, maxContentLength, nil)
 33}
 34
 35// NewFeedAggregatorWithProgress creates a new FeedAggregator with progress context.
 36func NewFeedAggregatorWithProgress(maxArticlesPerFeed, maxDaysOld, maxContentLength int, progressCtx *progress.Context) *FeedAggregator {
 37	return &FeedAggregator{
 38		maxArticlesPerFeed: maxArticlesPerFeed,
 39		maxDaysOld:         maxDaysOld,
 40		maxContentLength:   maxContentLength,
 41		client: &http.Client{
 42			Timeout: 30 * time.Second,
 43		},
 44		userAgent:   "LLM-Aggregator/0.1.0 (+https://codeberg.org/maxwelljensen/llm-aggregator)",
 45		progressCtx: progressCtx,
 46	}
 47}
 48
 49// ParseFeedsFromFile parses RSS feeds from a file containing one URL per line.
 50// Feeds are fetched concurrently for improved performance.
 51func (fa *FeedAggregator) ParseFeedsFromFile(filePath string) ([]*Article, error) {
 52	file, err := os.Open(filePath)
 53	if err != nil {
 54		return nil, fmt.Errorf("failed to open feeds file: %w", err)
 55	}
 56	defer file.Close()
 57
 58	content, err := io.ReadAll(file)
 59	if err != nil {
 60		return nil, fmt.Errorf("failed to read feeds file: %w", err)
 61	}
 62
 63	lines := strings.Split(string(content), "\n")
 64	feedURLs := []string{}
 65	for _, line := range lines {
 66		line = strings.TrimSpace(line)
 67		if line != "" && !strings.HasPrefix(line, "#") {
 68			feedURLs = append(feedURLs, line)
 69		}
 70	}
 71
 72	if fa.progressCtx != nil {
 73		fa.progressCtx.Logf("Found %d feed URLs in %s", len(feedURLs), filePath)
 74	}
 75
 76	// Use mutex to protect shared state
 77	var mu sync.Mutex
 78	var allArticles []*Article
 79	var feedErrors []string
 80
 81	// Limit concurrency to avoid overwhelming servers
 82	const maxConcurrency = 10
 83	sem := make(chan struct{}, maxConcurrency)
 84
 85	g, _ := errgroup.WithContext(context.Background())
 86	for i, feedURL := range feedURLs {
 87		currentIndex := i
 88		sem <- struct{}{} // Acquire semaphore
 89		g.Go(func() error {
 90			defer func() { <-sem }() // Release semaphore
 91			if fa.progressCtx != nil {
 92				fa.progressCtx.Logf("Processing feed %d/%d: %s", currentIndex+1, len(feedURLs), feedURL)
 93			}
 94			feedArticles, err := fa.ParseFeed(feedURL)
 95			if err != nil {
 96				if fa.progressCtx != nil {
 97					fa.progressCtx.Warningf("Failed to parse feed %s: %v", feedURL, err)
 98				}
 99				mu.Lock()
100				feedErrors = append(feedErrors, fmt.Sprintf("%s: %v", feedURL, err))
101				mu.Unlock()
102				return nil // Don't return error to continue processing other feeds
103			}
104			mu.Lock()
105			allArticles = append(allArticles, feedArticles...)
106			mu.Unlock()
107			return nil
108		})
109	}
110
111	if err := g.Wait(); err != nil {
112		return nil, fmt.Errorf("failed to process feeds: %w", err)
113	}
114
115	if len(feedErrors) > 0 {
116		if fa.progressCtx != nil {
117			fa.progressCtx.Warningf("Encountered %d feed errors: %v", len(feedErrors), feedErrors)
118		}
119	}
120
121	return allArticles, nil
122}
123
124// ParseFeed parses a single RSS feed and extracts articles.
125func (fa *FeedAggregator) ParseFeed(feedURL string) ([]*Article, error) {
126	articles := []*Article{}
127
128	fp := gofeed.NewParser()
129	feed, err := fp.ParseURL(feedURL)
130	if err != nil {
131		return nil, fmt.Errorf("failed to parse feed URL %s: %w", feedURL, err)
132	}
133
134	feedTitle := feed.Title
135	if feedTitle == "" {
136		feedTitle = feedURL
137	}
138
139	if fa.progressCtx != nil {
140		fa.progressCtx.Logf("Parsing feed: %s (%d entries)", feedTitle, len(feed.Items))
141	}
142
143	var cutoffTime time.Time
144	if fa.maxDaysOld > 0 {
145		cutoffTime = time.Now().Add(-time.Duration(fa.maxDaysOld) * 24 * time.Hour)
146	}
147
148	maxItems := min(fa.maxArticlesPerFeed, len(feed.Items))
149
150	for i, item := range feed.Items[:maxItems] {
151		article, err := fa.extractArticle(item, feedTitle, cutoffTime)
152		if err != nil {
153			if fa.progressCtx != nil {
154				fa.progressCtx.Warningf("Failed to extract article %d from %s: %v", i, feedURL, err)
155			}
156			continue
157		}
158		if article != nil {
159			articles = append(articles, article)
160		}
161	}
162
163	return articles, nil
164}
165
166func (fa *FeedAggregator) extractArticle(item *gofeed.Item, feedTitle string, cutoffTime time.Time) (*Article, error) {
167	// Extract metadata
168	title := item.Title
169	if title == "" {
170		title = "Untitled"
171	}
172
173	link := item.Link
174	if link == "" {
175		return nil, nil
176	}
177
178	// Parse publication date
179	published := fa.parsePublishedDate(item)
180
181	// Check if article is too old
182	if !cutoffTime.IsZero() && !published.IsZero() && published.Before(cutoffTime) {
183		if fa.progressCtx != nil {
184			fa.progressCtx.Debugf("Skipping old article: %s (%s)", title, published.Format("2006-01-02"))
185		}
186		return nil, nil
187	}
188
189	// Extract author
190	author := ""
191	if item.Author != nil {
192		author = item.Author.Name
193		if author == "" {
194			author = item.Author.Email
195		}
196	}
197	if author == "" && item.DublinCoreExt != nil && len(item.DublinCoreExt.Creator) > 0 {
198		author = item.DublinCoreExt.Creator[0]
199	}
200
201	// Get content
202	content := fa.extractContent(item, link)
203
204	// Truncate content if too long
205	if len(content) > fa.maxContentLength {
206		content = content[:fa.maxContentLength] + "... [truncated]"
207	}
208
209	return &Article{
210		Title:      title,
211		Link:       link,
212		Content:    content,
213		Published:  published,
214		Author:     author,
215		SourceFeed: feedTitle,
216	}, nil
217}
218
219func (fa *FeedAggregator) parsePublishedDate(item *gofeed.Item) time.Time {
220	if item.PublishedParsed != nil {
221		return *item.PublishedParsed
222	}
223	if item.UpdatedParsed != nil {
224		return *item.UpdatedParsed
225	}
226	// Try to parse from string
227	if item.Published != "" {
228		if t, err := time.Parse(time.RFC1123, item.Published); err == nil {
229			return t
230		}
231		if t, err := time.Parse(time.RFC1123Z, item.Published); err == nil {
232			return t
233		}
234		if t, err := time.Parse(time.RFC3339, item.Published); err == nil {
235			return t
236		}
237	}
238	return time.Time{}
239}
240
241func (fa *FeedAggregator) extractContent(item *gofeed.Item, link string) string {
242	// Priority: Content -> Description -> Fetch webpage
243	content := ""
244
245	// Try to get content from item
246	if item.Content != "" {
247		content = item.Content
248	} else if item.Description != "" {
249		content = item.Description
250	}
251
252	// If still no content or very short, fetch webpage
253	if content == "" || len(content) < 100 {
254		fetchedContent, err := fa.fetchWebpageContent(link)
255		if err == nil && fetchedContent != "" {
256			content = fetchedContent
257		}
258	}
259
260	return content
261}
262
263func (fa *FeedAggregator) fetchWebpageContent(url string) (string, error) {
264	req, err := http.NewRequest("GET", url, nil)
265	if err != nil {
266		return "", err
267	}
268	req.Header.Set("User-Agent", fa.userAgent)
269
270	resp, err := fa.client.Do(req)
271	if err != nil {
272		return "", err
273	}
274	defer resp.Body.Close()
275
276	if resp.StatusCode != http.StatusOK {
277		return "", fmt.Errorf("HTTP status %d", resp.StatusCode)
278	}
279
280	doc, err := goquery.NewDocumentFromReader(resp.Body)
281	if err != nil {
282		return "", err
283	}
284
285	// Remove script and style elements
286	doc.Find("script, style, nav, footer, header").Remove()
287
288	// Try to find article content
289	articleSelectors := []string{
290		"article",
291		"main",
292		".article-content",
293		".post-content",
294		".entry-content",
295		"#content",
296		".content",
297	}
298
299	for _, selector := range articleSelectors {
300		articleElement := doc.Find(selector).First()
301		if articleElement.Length() > 0 {
302			text := articleElement.Text()
303			text = strings.Join(strings.Fields(text), " ") // Normalise whitespace
304			if len(text) > 200 {
305				return text, nil
306			}
307		}
308	}
309
310	// Fallback: get all paragraphs
311	paragraphs := doc.Find("p")
312	if paragraphs.Length() > 0 {
313		var texts []string
314		paragraphs.Each(func(i int, s *goquery.Selection) {
315			texts = append(texts, strings.TrimSpace(s.Text()))
316		})
317		text := strings.Join(texts, " ")
318		if len(text) > 200 {
319			return text, nil
320		}
321	}
322
323	// Last resort: get all text
324	text := doc.Text()
325	text = strings.Join(strings.Fields(text), " ")
326	if len(text) > 200 {
327		return text, nil
328	}
329
330	return "", nil
331}