1package aggregator
  2
  3import (
  4	"context"
  5	"fmt"
  6	"io"
  7	"net/http"
  8	"os"
  9	"strings"
 10	"sync"
 11	"time"
 12
 13	"llm_aggregator/internal/progress"
 14
 15	"github.com/PuerkitoBio/goquery"
 16	"github.com/mmcdole/gofeed"
 17	"golang.org/x/sync/errgroup"
 18)
 19
 20// FeedAggregator aggregates articles from multiple RSS feeds.
 21type FeedAggregator struct {
 22	maxArticlesPerFeed int
 23	maxDaysOld         int
 24	maxContentLength   int
 25	client             *http.Client
 26	userAgent          string
 27	progressCtx        *progress.Context
 28}
 29
 30// NewFeedAggregator creates a new FeedAggregator with the specified options.
 31func NewFeedAggregator(maxArticlesPerFeed, maxDaysOld, maxContentLength int) *FeedAggregator {
 32	return NewFeedAggregatorWithProgress(maxArticlesPerFeed, maxDaysOld, maxContentLength, nil)
 33}
 34
 35// NewFeedAggregatorWithProgress creates a new FeedAggregator with progress context.
 36func NewFeedAggregatorWithProgress(maxArticlesPerFeed, maxDaysOld, maxContentLength int, progressCtx *progress.Context) *FeedAggregator {
 37	return &FeedAggregator{
 38		maxArticlesPerFeed: maxArticlesPerFeed,
 39		maxDaysOld:         maxDaysOld,
 40		maxContentLength:   maxContentLength,
 41		client: &http.Client{
 42			Timeout: 30 * time.Second,
 43		},
 44		userAgent:   "LLM-Aggregator/0.1.0 (+https://codeberg.org/maxwelljensen/llm-aggregator)",
 45		progressCtx: progressCtx,
 46	}
 47}
 48
 49// ParseFeedsFromFile parses RSS feeds from a file containing one URL per line.
 50// Feeds are fetched concurrently for improved performance.
 51func (fa *FeedAggregator) ParseFeedsFromFile(filePath string) ([]*Article, error) {
 52	file, err := os.Open(filePath)
 53	if err != nil {
 54		return nil, fmt.Errorf("failed to open feeds file: %w", err)
 55	}
 56	defer file.Close()
 57
 58	content, err := io.ReadAll(file)
 59	if err != nil {
 60		return nil, fmt.Errorf("failed to read feeds file: %w", err)
 61	}
 62
 63	lines := strings.Split(string(content), "\n")
 64	feedURLs := []string{}
 65	for _, line := range lines {
 66		line = strings.TrimSpace(line)
 67		if line != "" && !strings.HasPrefix(line, "#") {
 68			feedURLs = append(feedURLs, line)
 69		}
 70	}
 71
 72	fa.progressCtx.Logf("Found %d feed URLs in %s", len(feedURLs), filePath)
 73
 74	// Use mutex to protect shared state
 75	var mu sync.Mutex
 76	var allArticles []*Article
 77	var feedErrors []string
 78
 79	// Limit concurrency to avoid overwhelming servers
 80	const maxConcurrency = 10
 81	sem := make(chan struct{}, maxConcurrency)
 82
 83	g, _ := errgroup.WithContext(context.Background())
 84	for i, feedURL := range feedURLs {
 85		currentIndex := i
 86		sem <- struct{}{} // Acquire semaphore
 87		g.Go(func() error {
 88			defer func() { <-sem }() // Release semaphore
 89			fa.progressCtx.Logf("Processing feed %d/%d: %s", currentIndex+1, len(feedURLs), feedURL)
 90			feedArticles, err := fa.ParseFeed(feedURL)
 91			if err != nil {
 92				fa.progressCtx.Warningf("Failed to parse feed %s: %v", feedURL, err)
 93				mu.Lock()
 94				feedErrors = append(feedErrors, fmt.Sprintf("%s: %v", feedURL, err))
 95				mu.Unlock()
 96				return nil // Don't return error to continue processing other feeds
 97			}
 98			mu.Lock()
 99			allArticles = append(allArticles, feedArticles...)
100			mu.Unlock()
101			return nil
102		})
103	}
104
105	if err := g.Wait(); err != nil {
106		return nil, fmt.Errorf("failed to process feeds: %w", err)
107	}
108
109	if len(feedErrors) > 0 {
110		fa.progressCtx.Warningf("Encountered %d feed errors: %v", len(feedErrors), feedErrors)
111	}
112
113	return allArticles, nil
114}
115
116// ParseFeed parses a single RSS feed and extracts articles.
117func (fa *FeedAggregator) ParseFeed(feedURL string) ([]*Article, error) {
118	articles := []*Article{}
119
120	fp := gofeed.NewParser()
121	feed, err := fp.ParseURL(feedURL)
122	if err != nil {
123		return nil, fmt.Errorf("failed to parse feed URL %s: %w", feedURL, err)
124	}
125
126	feedTitle := feed.Title
127	if feedTitle == "" {
128		feedTitle = feedURL
129	}
130
131	fa.progressCtx.Logf("Parsing feed: %s (%d entries)", feedTitle, len(feed.Items))
132
133	var cutoffTime time.Time
134	if fa.maxDaysOld > 0 {
135		cutoffTime = time.Now().Add(-time.Duration(fa.maxDaysOld) * 24 * time.Hour)
136	}
137
138	maxItems := min(fa.maxArticlesPerFeed, len(feed.Items))
139
140	for i, item := range feed.Items[:maxItems] {
141		article, err := fa.extractArticle(item, feedTitle, cutoffTime)
142		if err != nil {
143			fa.progressCtx.Warningf("Failed to extract article %d from %s: %v", i, feedURL, err)
144			continue
145		}
146		if article != nil {
147			articles = append(articles, article)
148		}
149	}
150
151	return articles, nil
152}
153
154func (fa *FeedAggregator) extractArticle(item *gofeed.Item, feedTitle string, cutoffTime time.Time) (*Article, error) {
155	// Extract metadata
156	title := item.Title
157	if title == "" {
158		title = "Untitled"
159	}
160
161	link := item.Link
162	if link == "" {
163		return nil, nil
164	}
165
166	// Parse publication date
167	published := fa.parsePublishedDate(item)
168
169	// Check if article is too old
170	if !cutoffTime.IsZero() && !published.IsZero() && published.Before(cutoffTime) {
171		fa.progressCtx.Debugf("Skipping old article: %s (%s)", title, published.Format("2006-01-02"))
172		return nil, nil
173	}
174
175	// Extract author
176	author := ""
177	if item.Author != nil {
178		author = item.Author.Name
179		if author == "" {
180			author = item.Author.Email
181		}
182	}
183	if author == "" && item.DublinCoreExt != nil && len(item.DublinCoreExt.Creator) > 0 {
184		author = item.DublinCoreExt.Creator[0]
185	}
186
187	// Get content
188	content := fa.extractContent(item, link)
189
190	// Truncate content if too long
191	if len(content) > fa.maxContentLength {
192		content = content[:fa.maxContentLength] + "... [truncated]"
193	}
194
195	return &Article{
196		Title:      title,
197		Link:       link,
198		Content:    content,
199		Published:  published,
200		Author:     author,
201		SourceFeed: feedTitle,
202	}, nil
203}
204
205func (fa *FeedAggregator) parsePublishedDate(item *gofeed.Item) time.Time {
206	if item.PublishedParsed != nil {
207		return *item.PublishedParsed
208	}
209	if item.UpdatedParsed != nil {
210		return *item.UpdatedParsed
211	}
212	// Try to parse from string
213	if item.Published != "" {
214		if t, err := time.Parse(time.RFC1123, item.Published); err == nil {
215			return t
216		}
217		if t, err := time.Parse(time.RFC1123Z, item.Published); err == nil {
218			return t
219		}
220		if t, err := time.Parse(time.RFC3339, item.Published); err == nil {
221			return t
222		}
223	}
224	return time.Time{}
225}
226
227func (fa *FeedAggregator) extractContent(item *gofeed.Item, link string) string {
228	// Priority: Content -> Description -> Fetch webpage
229	content := ""
230
231	// Try to get content from item
232	if item.Content != "" {
233		content = item.Content
234	} else if item.Description != "" {
235		content = item.Description
236	}
237
238	// If still no content or very short, fetch webpage
239	if content == "" || len(content) < 100 {
240		fetchedContent, err := fa.fetchWebpageContent(link)
241		if err == nil && fetchedContent != "" {
242			content = fetchedContent
243		}
244	}
245
246	return content
247}
248
249func (fa *FeedAggregator) fetchWebpageContent(url string) (string, error) {
250	req, err := http.NewRequest("GET", url, nil)
251	if err != nil {
252		return "", err
253	}
254	req.Header.Set("User-Agent", fa.userAgent)
255
256	resp, err := fa.client.Do(req)
257	if err != nil {
258		return "", err
259	}
260	defer resp.Body.Close()
261
262	if resp.StatusCode != http.StatusOK {
263		return "", fmt.Errorf("HTTP status %d", resp.StatusCode)
264	}
265
266	doc, err := goquery.NewDocumentFromReader(resp.Body)
267	if err != nil {
268		return "", err
269	}
270
271	// Remove script and style elements
272	doc.Find("script, style, nav, footer, header").Remove()
273
274	// Try to find article content
275	articleSelectors := []string{
276		"article",
277		"main",
278		".article-content",
279		".post-content",
280		".entry-content",
281		"#content",
282		".content",
283	}
284
285	for _, selector := range articleSelectors {
286		articleElement := doc.Find(selector).First()
287		if articleElement.Length() > 0 {
288			text := articleElement.Text()
289			text = strings.Join(strings.Fields(text), " ") // Normalise whitespace
290			if len(text) > 200 {
291				return text, nil
292			}
293		}
294	}
295
296	// Fallback: get all paragraphs
297	paragraphs := doc.Find("p")
298	if paragraphs.Length() > 0 {
299		var texts []string
300		paragraphs.Each(func(i int, s *goquery.Selection) {
301			texts = append(texts, strings.TrimSpace(s.Text()))
302		})
303		text := strings.Join(texts, " ")
304		if len(text) > 200 {
305			return text, nil
306		}
307	}
308
309	// Last resort: get all text
310	text := doc.Text()
311	text = strings.Join(strings.Fields(text), " ")
312	if len(text) > 200 {
313		return text, nil
314	}
315
316	return "", nil
317}