1package aggregator
  2
  3import (
  4	"fmt"
  5	"io"
  6	"net/http"
  7	"os"
  8	"strings"
  9	"time"
 10
 11	"llm_aggregator/internal/progress"
 12
 13	"github.com/PuerkitoBio/goquery"
 14	"github.com/mmcdole/gofeed"
 15)
 16
 17// FeedAggregator aggregates articles from multiple RSS feeds.
 18type FeedAggregator struct {
 19	maxArticlesPerFeed int
 20	maxDaysOld         int
 21	maxContentLength   int
 22	client             *http.Client
 23	userAgent          string
 24	progressCtx        *progress.Context
 25}
 26
 27// NewFeedAggregator creates a new FeedAggregator with the specified options.
 28func NewFeedAggregator(maxArticlesPerFeed, maxDaysOld, maxContentLength int) *FeedAggregator {
 29	return NewFeedAggregatorWithProgress(maxArticlesPerFeed, maxDaysOld, maxContentLength, nil)
 30}
 31
 32// NewFeedAggregatorWithProgress creates a new FeedAggregator with progress context.
 33func NewFeedAggregatorWithProgress(maxArticlesPerFeed, maxDaysOld, maxContentLength int, progressCtx *progress.Context) *FeedAggregator {
 34	return &FeedAggregator{
 35		maxArticlesPerFeed: maxArticlesPerFeed,
 36		maxDaysOld:         maxDaysOld,
 37		maxContentLength:   maxContentLength,
 38		client: &http.Client{
 39			Timeout: 30 * time.Second,
 40		},
 41		userAgent:   "LLM-Aggregator/0.1.0 (+https://codeberg.org/maxwelljensen/llm-aggregator)",
 42		progressCtx: progressCtx,
 43	}
 44}
 45
 46// ParseFeedsFromFile parses RSS feeds from a file containing one URL per line.
 47func (fa *FeedAggregator) ParseFeedsFromFile(filePath string) ([]*Article, error) {
 48	articles := []*Article{}
 49
 50	file, err := os.Open(filePath)
 51	if err != nil {
 52		return nil, fmt.Errorf("failed to open feeds file: %w", err)
 53	}
 54	defer file.Close()
 55
 56	content, err := io.ReadAll(file)
 57	if err != nil {
 58		return nil, fmt.Errorf("failed to read feeds file: %w", err)
 59	}
 60
 61	lines := strings.Split(string(content), "\n")
 62	feedURLs := []string{}
 63	for _, line := range lines {
 64		line = strings.TrimSpace(line)
 65		if line != "" && !strings.HasPrefix(line, "#") {
 66			feedURLs = append(feedURLs, line)
 67		}
 68	}
 69
 70	fa.progressCtx.Logf("Found %d feed URLs in %s", len(feedURLs), filePath)
 71
 72	for i, feedURL := range feedURLs {
 73		fa.progressCtx.Logf("Processing feed %d/%d: %s", i+1, len(feedURLs), feedURL)
 74		feedArticles, err := fa.ParseFeed(feedURL)
 75		if err != nil {
 76			fa.progressCtx.Warningf("Failed to parse feed %s: %v", feedURL, err)
 77			continue
 78		}
 79		articles = append(articles, feedArticles...)
 80	}
 81
 82	return articles, nil
 83}
 84
 85// ParseFeed parses a single RSS feed and extracts articles.
 86func (fa *FeedAggregator) ParseFeed(feedURL string) ([]*Article, error) {
 87	articles := []*Article{}
 88
 89	fp := gofeed.NewParser()
 90	feed, err := fp.ParseURL(feedURL)
 91	if err != nil {
 92		return nil, fmt.Errorf("failed to parse feed URL %s: %w", feedURL, err)
 93	}
 94
 95	feedTitle := feed.Title
 96	if feedTitle == "" {
 97		feedTitle = feedURL
 98	}
 99
100	fa.progressCtx.Logf("Parsing feed: %s (%d entries)", feedTitle, len(feed.Items))
101
102	var cutoffTime time.Time
103	if fa.maxDaysOld > 0 {
104		cutoffTime = time.Now().Add(-time.Duration(fa.maxDaysOld) * 24 * time.Hour)
105	}
106
107	maxItems := min(fa.maxArticlesPerFeed, len(feed.Items))
108
109	for i, item := range feed.Items[:maxItems] {
110		article, err := fa.extractArticle(item, feedTitle, cutoffTime)
111		if err != nil {
112			fa.progressCtx.Warningf("Failed to extract article %d from %s: %v", i, feedURL, err)
113			continue
114		}
115		if article != nil {
116			articles = append(articles, article)
117		}
118	}
119
120	return articles, nil
121}
122
123func (fa *FeedAggregator) extractArticle(item *gofeed.Item, feedTitle string, cutoffTime time.Time) (*Article, error) {
124	// Extract metadata
125	title := item.Title
126	if title == "" {
127		title = "Untitled"
128	}
129
130	link := item.Link
131	if link == "" {
132		return nil, nil
133	}
134
135	// Parse publication date
136	published := fa.parsePublishedDate(item)
137
138	// Check if article is too old
139	if !cutoffTime.IsZero() && !published.IsZero() && published.Before(cutoffTime) {
140		fa.progressCtx.Debugf("Skipping old article: %s (%s)", title, published.Format("2006-01-02"))
141		return nil, nil
142	}
143
144	// Extract author
145	author := ""
146	if item.Author != nil {
147		author = item.Author.Name
148		if author == "" {
149			author = item.Author.Email
150		}
151	}
152	if author == "" && item.DublinCoreExt != nil && len(item.DublinCoreExt.Creator) > 0 {
153		author = item.DublinCoreExt.Creator[0]
154	}
155
156	// Get content
157	content := fa.extractContent(item, link)
158
159	// Truncate content if too long
160	if len(content) > fa.maxContentLength {
161		content = content[:fa.maxContentLength] + "... [truncated]"
162	}
163
164	return &Article{
165		Title:      title,
166		Link:       link,
167		Content:    content,
168		Published:  published,
169		Author:     author,
170		SourceFeed: feedTitle,
171	}, nil
172}
173
174func (fa *FeedAggregator) parsePublishedDate(item *gofeed.Item) time.Time {
175	if item.PublishedParsed != nil {
176		return *item.PublishedParsed
177	}
178	if item.UpdatedParsed != nil {
179		return *item.UpdatedParsed
180	}
181	// Try to parse from string
182	if item.Published != "" {
183		if t, err := time.Parse(time.RFC1123, item.Published); err == nil {
184			return t
185		}
186		if t, err := time.Parse(time.RFC1123Z, item.Published); err == nil {
187			return t
188		}
189		if t, err := time.Parse(time.RFC3339, item.Published); err == nil {
190			return t
191		}
192	}
193	return time.Time{}
194}
195
196func (fa *FeedAggregator) extractContent(item *gofeed.Item, link string) string {
197	// Priority: Content -> Description -> Fetch webpage
198	content := ""
199
200	// Try to get content from item
201	if item.Content != "" {
202		content = item.Content
203	} else if item.Description != "" {
204		content = item.Description
205	}
206
207	// If still no content or very short, fetch webpage
208	if content == "" || len(content) < 100 {
209		fetchedContent, err := fa.fetchWebpageContent(link)
210		if err == nil && fetchedContent != "" {
211			content = fetchedContent
212		}
213	}
214
215	return content
216}
217
218func (fa *FeedAggregator) fetchWebpageContent(url string) (string, error) {
219	req, err := http.NewRequest("GET", url, nil)
220	if err != nil {
221		return "", err
222	}
223	req.Header.Set("User-Agent", fa.userAgent)
224
225	resp, err := fa.client.Do(req)
226	if err != nil {
227		return "", err
228	}
229	defer resp.Body.Close()
230
231	if resp.StatusCode != http.StatusOK {
232		return "", fmt.Errorf("HTTP status %d", resp.StatusCode)
233	}
234
235	doc, err := goquery.NewDocumentFromReader(resp.Body)
236	if err != nil {
237		return "", err
238	}
239
240	// Remove script and style elements
241	doc.Find("script, style, nav, footer, header").Remove()
242
243	// Try to find article content
244	articleSelectors := []string{
245		"article",
246		"main",
247		".article-content",
248		".post-content",
249		".entry-content",
250		"#content",
251		".content",
252	}
253
254	for _, selector := range articleSelectors {
255		articleElement := doc.Find(selector).First()
256		if articleElement.Length() > 0 {
257			text := articleElement.Text()
258			text = strings.Join(strings.Fields(text), " ") // Normalise whitespace
259			if len(text) > 200 {
260				return text, nil
261			}
262		}
263	}
264
265	// Fallback: get all paragraphs
266	paragraphs := doc.Find("p")
267	if paragraphs.Length() > 0 {
268		var texts []string
269		paragraphs.Each(func(i int, s *goquery.Selection) {
270			texts = append(texts, strings.TrimSpace(s.Text()))
271		})
272		text := strings.Join(texts, " ")
273		if len(text) > 200 {
274			return text, nil
275		}
276	}
277
278	// Last resort: get all text
279	text := doc.Text()
280	text = strings.Join(strings.Fields(text), " ")
281	if len(text) > 200 {
282		return text, nil
283	}
284
285	return "", nil
286}
287