1package processor
  2
  3import (
  4	"sort"
  5	"strings"
  6	"time"
  7
  8	"codeberg.org/maxwelljensen/llm_aggregator/internal/aggregator"
  9	"codeberg.org/maxwelljensen/llm_aggregator/internal/progress"
 10	"codeberg.org/maxwelljensen/llm_aggregator/internal/tokeniser"
 11)
 12
 13// ContentProcessor processes and prepares aggregated content for LLM analysis.
 14type ContentProcessor struct {
 15	maxTotalArticles     int
 16	maxContentPerArticle int
 17	filterKeywords       []string
 18	excludeKeywords      []string
 19	logger               *progress.Context
 20}
 21
 22// NewContentProcessor creates a processor that filters and truncates articles.
 23// Keyword comparison is case-insensitive.
 24func NewContentProcessor(maxTotalArticles, maxContentPerArticle int, filterKeywords, excludeKeywords []string) *ContentProcessor {
 25	// Convert keywords to lowercase for case-insensitive matching
 26	filterLower := make([]string, len(filterKeywords))
 27	for i, kw := range filterKeywords {
 28		filterLower[i] = strings.ToLower(kw)
 29	}
 30
 31	excludeLower := make([]string, len(excludeKeywords))
 32	for i, kw := range excludeKeywords {
 33		excludeLower[i] = strings.ToLower(kw)
 34	}
 35
 36	return &ContentProcessor{
 37		maxTotalArticles:     maxTotalArticles,
 38		maxContentPerArticle: maxContentPerArticle,
 39		filterKeywords:       filterLower,
 40		excludeKeywords:      excludeLower,
 41	}
 42}
 43
 44// SetLogger sets the logger for the processor
 45func (cp *ContentProcessor) SetLogger(logger *progress.Context) {
 46	cp.logger = logger
 47}
 48
 49// ProcessArticles applies keyword filtering, sorting, and a ceiling on total count,
 50// then converts each article to a map for the LLM.
 51func (cp *ContentProcessor) ProcessArticles(articles []*aggregator.Article, sortBy string, reverse bool) []map[string]any {
 52	if len(articles) == 0 {
 53		if cp.logger != nil {
 54			cp.logger.Logf("Warning: No articles to process")
 55		}
 56		return []map[string]any{}
 57	}
 58
 59	// Filter articles
 60	filteredArticles := cp.filterArticles(articles)
 61
 62	// Sort articles
 63	sortedArticles := cp.sortArticles(filteredArticles, sortBy, reverse)
 64
 65	// Limit total articles
 66	if len(sortedArticles) > cp.maxTotalArticles {
 67		if cp.logger != nil {
 68			cp.logger.Logf("Limiting articles from %d to %d", len(sortedArticles), cp.maxTotalArticles)
 69		}
 70		sortedArticles = sortedArticles[:cp.maxTotalArticles]
 71	}
 72
 73	// Prepare articles for LLM
 74	processed := cp.prepareForLLM(sortedArticles)
 75
 76	if cp.logger != nil {
 77		cp.logger.Logf("Processed %d articles (from %d original)", len(processed), len(articles))
 78	}
 79
 80	return processed
 81}
 82
 83// filterArticles applies include/exclude keyword filters to the article list.
 84// Exclusions take priority over inclusions. If no filters are set, all articles pass.
 85func (cp *ContentProcessor) filterArticles(articles []*aggregator.Article) []*aggregator.Article {
 86	if len(cp.filterKeywords) == 0 && len(cp.excludeKeywords) == 0 {
 87		return articles
 88	}
 89
 90	filtered := []*aggregator.Article{}
 91
 92	for _, article := range articles {
 93		include := true
 94
 95		if len(cp.excludeKeywords) > 0 {
 96			articleText := strings.ToLower(article.Title + " " + article.Content)
 97			for _, keyword := range cp.excludeKeywords {
 98				if strings.Contains(articleText, keyword) {
 99					if cp.logger != nil {
100						cp.logger.Logf("Excluding article due to keyword '%s': %s", keyword, article.Title)
101					}
102					include = false
103					break
104				}
105			}
106		}
107
108		if include && len(cp.filterKeywords) > 0 {
109			articleText := strings.ToLower(article.Title + " " + article.Content)
110			include = false
111			for _, keyword := range cp.filterKeywords {
112				if strings.Contains(articleText, keyword) {
113					include = true
114					break
115				}
116			}
117		}
118
119		if include {
120			filtered = append(filtered, article)
121		}
122	}
123
124	if cp.logger != nil {
125		cp.logger.Logf(
126			"Filtered %d articles to %d (inclusion: %v, exclusion: %v)",
127			len(articles), len(filtered), cp.filterKeywords, cp.excludeKeywords,
128		)
129	}
130
131	return filtered
132}
133
134func (cp *ContentProcessor) sortArticles(articles []*aggregator.Article, sortBy string, reverse bool) []*aggregator.Article {
135	if len(articles) == 0 {
136		return articles
137	}
138
139	sortedArticles := make([]*aggregator.Article, len(articles))
140	copy(sortedArticles, articles)
141
142	switch strings.ToLower(sortBy) {
143	case "date":
144		// Zero times sort to the end when reverse=false (oldest last)
145		sort.Slice(sortedArticles, func(i, j int) bool {
146			iTime := sortedArticles[i].Published
147			jTime := sortedArticles[j].Published
148			if iTime.IsZero() {
149				iTime = time.Time{}
150			}
151			if jTime.IsZero() {
152				jTime = time.Time{}
153			}
154			if reverse {
155				return iTime.After(jTime)
156			}
157			return iTime.Before(jTime)
158		})
159	case "title":
160		sort.Slice(sortedArticles, func(i, j int) bool {
161			iTitle := strings.ToLower(sortedArticles[i].Title)
162			jTitle := strings.ToLower(sortedArticles[j].Title)
163			if reverse {
164				return iTitle > jTitle
165			}
166			return iTitle < jTitle
167		})
168	case "source":
169		sort.Slice(sortedArticles, func(i, j int) bool {
170			iSource := strings.ToLower(sortedArticles[i].SourceFeed)
171			jSource := strings.ToLower(sortedArticles[j].SourceFeed)
172			if reverse {
173				return iSource > jSource
174			}
175			return iSource < jSource
176		})
177	default:
178		// Unknown sort key: fall back to date order
179		sort.Slice(sortedArticles, func(i, j int) bool {
180			iTime := sortedArticles[i].Published
181			jTime := sortedArticles[j].Published
182			if iTime.IsZero() {
183				iTime = time.Time{}
184			}
185			if jTime.IsZero() {
186				jTime = time.Time{}
187			}
188			if reverse {
189				return iTime.After(jTime)
190			}
191			return iTime.Before(jTime)
192		})
193	}
194
195	return sortedArticles
196}
197
198func (cp *ContentProcessor) prepareForLLM(articles []*aggregator.Article) []map[string]any {
199	processed := make([]map[string]any, len(articles))
200
201	for i, article := range articles {
202		// Process content
203		content := article.Content
204		if len(content) > cp.maxContentPerArticle {
205			content = content[:cp.maxContentPerArticle] + "... [truncated]"
206		}
207
208		// Create map
209		articleMap := map[string]any{
210			"title":       article.Title,
211			"link":        article.Link,
212			"content":     content,
213			"author":      article.Author,
214			"source_feed": article.SourceFeed,
215			"summary":     article.Summary,
216		}
217
218		if !article.Published.IsZero() {
219			articleMap["published"] = article.Published
220		}
221
222		processed[i] = articleMap
223	}
224
225	return processed
226}
227
228// EstimateTokenCount uses tiktoken to estimate total token count across all articles.
229// Falls back to char÷4 on encoding errors (logged as warnings).
230func (cp *ContentProcessor) EstimateTokenCount(articles []map[string]any, model string) int {
231	totalTokens := 0
232
233	for _, article := range articles {
234		for _, field := range []string{"title", "content", "author", "source_feed"} {
235			if val, ok := article[field]; ok && val != nil {
236				if str, ok := val.(string); ok && str != "" {
237					tokens, err := tokeniser.CountTokens(str, model)
238					if err != nil {
239						// Fallback to rough estimate on error
240						if cp.logger != nil {
241							cp.logger.Logf("Warning: token count error for %s: %v", field, err)
242						}
243						totalTokens += len(str) / 4
244						continue
245					}
246					totalTokens += tokens
247				}
248			}
249		}
250	}
251
252	if cp.logger != nil {
253		cp.logger.Logf("Estimated tokens: %d", totalTokens)
254	}
255
256	return totalTokens
257}