1package aggregator
  2
  3import (
  4	"fmt"
  5	"io"
  6	"net/http"
  7	"os"
  8	"strings"
  9	"time"
 10
 11	"llm_aggregator/internal/progress"
 12
 13	"github.com/PuerkitoBio/goquery"
 14	"github.com/mmcdole/gofeed"
 15)
 16
 17// FeedAggregator aggregates articles from multiple RSS feeds.
 18type FeedAggregator struct {
 19	maxArticlesPerFeed int
 20	maxDaysOld         int
 21	maxContentLength   int
 22	client             *http.Client
 23	userAgent          string
 24	progressCtx        *progress.Context
 25}
 26
 27// NewFeedAggregator creates a new FeedAggregator with the specified options.
 28func NewFeedAggregator(maxArticlesPerFeed, maxDaysOld, maxContentLength int) *FeedAggregator {
 29	return NewFeedAggregatorWithProgress(maxArticlesPerFeed, maxDaysOld, maxContentLength, nil)
 30}
 31
 32// NewFeedAggregatorWithProgress creates a new FeedAggregator with progress context.
 33func NewFeedAggregatorWithProgress(maxArticlesPerFeed, maxDaysOld, maxContentLength int, progressCtx *progress.Context) *FeedAggregator {
 34	return &FeedAggregator{
 35		maxArticlesPerFeed: maxArticlesPerFeed,
 36		maxDaysOld:         maxDaysOld,
 37		maxContentLength:   maxContentLength,
 38		client: &http.Client{
 39			Timeout: 30 * time.Second,
 40		},
 41		userAgent:   "LLM-Aggregator/0.1.0 (+https://codeberg.org/maxwelljensen/llm-aggregator)",
 42		progressCtx: progressCtx,
 43	}
 44}
 45
 46// ParseFeedsFromFile parses RSS feeds from a file containing one URL per line.
 47func (fa *FeedAggregator) ParseFeedsFromFile(filePath string) ([]*Article, error) {
 48	articles := []*Article{}
 49
 50	file, err := os.Open(filePath)
 51	if err != nil {
 52		return nil, fmt.Errorf("failed to open feeds file: %w", err)
 53	}
 54	defer file.Close()
 55
 56	content, err := io.ReadAll(file)
 57	if err != nil {
 58		return nil, fmt.Errorf("failed to read feeds file: %w", err)
 59	}
 60
 61	lines := strings.Split(string(content), "\n")
 62	feedURLs := []string{}
 63	for _, line := range lines {
 64		line = strings.TrimSpace(line)
 65		if line != "" && !strings.HasPrefix(line, "#") {
 66			feedURLs = append(feedURLs, line)
 67		}
 68	}
 69
 70	if fa.progressCtx != nil {
 71		fa.progressCtx.Logf("Found %d feed URLs in %s", len(feedURLs), filePath)
 72	} else {
 73		fmt.Printf("Found %d feed URLs in %s\n", len(feedURLs), filePath)
 74	}
 75
 76	for i, feedURL := range feedURLs {
 77		if fa.progressCtx != nil {
 78			fa.progressCtx.Logf("Processing feed %d/%d: %s", i+1, len(feedURLs), feedURL)
 79		} else {
 80			fmt.Printf("Processing feed %d/%d: %s\n", i+1, len(feedURLs), feedURL)
 81		}
 82		feedArticles, err := fa.ParseFeed(feedURL)
 83		if err != nil {
 84			if fa.progressCtx != nil {
 85				fa.progressCtx.Warningf("Failed to parse feed %s: %v", feedURL, err)
 86			} else {
 87				fmt.Printf("Warning: Failed to parse feed %s: %v\n", feedURL, err)
 88			}
 89			continue
 90		}
 91		articles = append(articles, feedArticles...)
 92	}
 93
 94	return articles, nil
 95}
 96
 97// ParseFeed parses a single RSS feed and extracts articles.
 98func (fa *FeedAggregator) ParseFeed(feedURL string) ([]*Article, error) {
 99	articles := []*Article{}
100
101	fp := gofeed.NewParser()
102	feed, err := fp.ParseURL(feedURL)
103	if err != nil {
104		return nil, fmt.Errorf("failed to parse feed URL %s: %w", feedURL, err)
105	}
106
107	feedTitle := feed.Title
108	if feedTitle == "" {
109		feedTitle = feedURL
110	}
111
112	if fa.progressCtx != nil {
113		fa.progressCtx.Logf("Parsing feed: %s (%d entries)", feedTitle, len(feed.Items))
114	} else {
115		fmt.Printf("Parsing feed: %s (%d entries)\n", feedTitle, len(feed.Items))
116	}
117
118	var cutoffTime time.Time
119	if fa.maxDaysOld > 0 {
120		cutoffTime = time.Now().Add(-time.Duration(fa.maxDaysOld) * 24 * time.Hour)
121	}
122
123	maxItems := min(fa.maxArticlesPerFeed, len(feed.Items))
124
125	for i, item := range feed.Items[:maxItems] {
126		article, err := fa.extractArticle(item, feedTitle, cutoffTime)
127		if err != nil {
128			if fa.progressCtx != nil {
129				fa.progressCtx.Warningf("Failed to extract article %d from %s: %v", i, feedURL, err)
130			} else {
131				fmt.Printf("Warning: Failed to extract article %d from %s: %v\n", i, feedURL, err)
132			}
133			continue
134		}
135		if article != nil {
136			articles = append(articles, article)
137		}
138	}
139
140	return articles, nil
141}
142
143func (fa *FeedAggregator) extractArticle(item *gofeed.Item, feedTitle string, cutoffTime time.Time) (*Article, error) {
144	// Extract metadata
145	title := item.Title
146	if title == "" {
147		title = "Untitled"
148	}
149
150	link := item.Link
151	if link == "" {
152		return nil, nil
153	}
154
155	// Parse publication date
156	published := fa.parsePublishedDate(item)
157
158	// Check if article is too old
159	if !cutoffTime.IsZero() && !published.IsZero() && published.Before(cutoffTime) {
160		if fa.progressCtx != nil {
161			fa.progressCtx.Debugf("Skipping old article: %s (%s)", title, published.Format("2006-01-02"))
162		} else {
163			fmt.Printf("Skipping old article: %s (%s)\n", title, published.Format("2006-01-02"))
164		}
165		return nil, nil
166	}
167
168	// Extract author
169	author := ""
170	if item.Author != nil {
171		author = item.Author.Name
172		if author == "" {
173			author = item.Author.Email
174		}
175	}
176	if author == "" && item.DublinCoreExt != nil && len(item.DublinCoreExt.Creator) > 0 {
177		author = item.DublinCoreExt.Creator[0]
178	}
179
180	// Get content
181	content := fa.extractContent(item, link)
182
183	// Truncate content if too long
184	if len(content) > fa.maxContentLength {
185		content = content[:fa.maxContentLength] + "... [truncated]"
186	}
187
188	return &Article{
189		Title:      title,
190		Link:       link,
191		Content:    content,
192		Published:  published,
193		Author:     author,
194		SourceFeed: feedTitle,
195	}, nil
196}
197
198func (fa *FeedAggregator) parsePublishedDate(item *gofeed.Item) time.Time {
199	if item.PublishedParsed != nil {
200		return *item.PublishedParsed
201	}
202	if item.UpdatedParsed != nil {
203		return *item.UpdatedParsed
204	}
205	// Try to parse from string
206	if item.Published != "" {
207		if t, err := time.Parse(time.RFC1123, item.Published); err == nil {
208			return t
209		}
210		if t, err := time.Parse(time.RFC1123Z, item.Published); err == nil {
211			return t
212		}
213		if t, err := time.Parse(time.RFC3339, item.Published); err == nil {
214			return t
215		}
216	}
217	return time.Time{}
218}
219
220func (fa *FeedAggregator) extractContent(item *gofeed.Item, link string) string {
221	// Priority: Content -> Description -> Fetch webpage
222	content := ""
223
224	// Try to get content from item
225	if item.Content != "" {
226		content = item.Content
227	} else if item.Description != "" {
228		content = item.Description
229	}
230
231	// If still no content or very short, fetch webpage
232	if content == "" || len(content) < 100 {
233		fetchedContent, err := fa.fetchWebpageContent(link)
234		if err == nil && fetchedContent != "" {
235			content = fetchedContent
236		}
237	}
238
239	return content
240}
241
242func (fa *FeedAggregator) fetchWebpageContent(url string) (string, error) {
243	req, err := http.NewRequest("GET", url, nil)
244	if err != nil {
245		return "", err
246	}
247	req.Header.Set("User-Agent", fa.userAgent)
248
249	resp, err := fa.client.Do(req)
250	if err != nil {
251		return "", err
252	}
253	defer resp.Body.Close()
254
255	if resp.StatusCode != http.StatusOK {
256		return "", fmt.Errorf("HTTP status %d", resp.StatusCode)
257	}
258
259	doc, err := goquery.NewDocumentFromReader(resp.Body)
260	if err != nil {
261		return "", err
262	}
263
264	// Remove script and style elements
265	doc.Find("script, style, nav, footer, header").Remove()
266
267	// Try to find article content
268	articleSelectors := []string{
269		"article",
270		"main",
271		".article-content",
272		".post-content",
273		".entry-content",
274		"#content",
275		".content",
276	}
277
278	for _, selector := range articleSelectors {
279		articleElement := doc.Find(selector).First()
280		if articleElement.Length() > 0 {
281			text := articleElement.Text()
282			text = strings.Join(strings.Fields(text), " ") // Normalise whitespace
283			if len(text) > 200 {
284				return text, nil
285			}
286		}
287	}
288
289	// Fallback: get all paragraphs
290	paragraphs := doc.Find("p")
291	if paragraphs.Length() > 0 {
292		var texts []string
293		paragraphs.Each(func(i int, s *goquery.Selection) {
294			texts = append(texts, strings.TrimSpace(s.Text()))
295		})
296		text := strings.Join(texts, " ")
297		if len(text) > 200 {
298			return text, nil
299		}
300	}
301
302	// Last resort: get all text
303	text := doc.Text()
304	text = strings.Join(strings.Fields(text), " ")
305	if len(text) > 200 {
306		return text, nil
307	}
308
309	return "", nil
310}
311