1package processor
  2
  3import (
  4	"sort"
  5	"strings"
  6	"time"
  7
  8	"llm_aggregator/internal/aggregator"
  9	"llm_aggregator/internal/progress"
 10	"llm_aggregator/internal/tokeniser"
 11)
 12
 13// ContentProcessor processes and prepares aggregated content for LLM analysis.
 14type ContentProcessor struct {
 15	maxTotalArticles     int
 16	maxContentPerArticle int
 17	filterKeywords       []string
 18	excludeKeywords      []string
 19	logger               *progress.Context
 20}
 21
 22// NewContentProcessor creates a new ContentProcessor with the specified options.
 23func NewContentProcessor(maxTotalArticles, maxContentPerArticle int, filterKeywords, excludeKeywords []string) *ContentProcessor {
 24	// Convert keywords to lowercase for case-insensitive matching
 25	filterLower := make([]string, len(filterKeywords))
 26	for i, kw := range filterKeywords {
 27		filterLower[i] = strings.ToLower(kw)
 28	}
 29
 30	excludeLower := make([]string, len(excludeKeywords))
 31	for i, kw := range excludeKeywords {
 32		excludeLower[i] = strings.ToLower(kw)
 33	}
 34
 35	return &ContentProcessor{
 36		maxTotalArticles:     maxTotalArticles,
 37		maxContentPerArticle: maxContentPerArticle,
 38		filterKeywords:       filterLower,
 39		excludeKeywords:      excludeLower,
 40	}
 41}
 42
 43// SetLogger sets the logger for the processor
 44func (cp *ContentProcessor) SetLogger(logger *progress.Context) {
 45	cp.logger = logger
 46}
 47
 48// ProcessArticles processes articles: filter, sort, and prepare for LLM.
 49func (cp *ContentProcessor) ProcessArticles(articles []*aggregator.Article, sortBy string, reverse bool) []map[string]any {
 50	if len(articles) == 0 {
 51		if cp.logger != nil {
 52			cp.logger.Logf("Warning: No articles to process")
 53		}
 54		return []map[string]any{}
 55	}
 56
 57	// Filter articles
 58	filteredArticles := cp.filterArticles(articles)
 59
 60	// Sort articles
 61	sortedArticles := cp.sortArticles(filteredArticles, sortBy, reverse)
 62
 63	// Limit total articles
 64	if len(sortedArticles) > cp.maxTotalArticles {
 65		if cp.logger != nil {
 66			cp.logger.Logf("Limiting articles from %d to %d", len(sortedArticles), cp.maxTotalArticles)
 67		}
 68		sortedArticles = sortedArticles[:cp.maxTotalArticles]
 69	}
 70
 71	// Prepare articles for LLM
 72	processed := cp.prepareForLLM(sortedArticles)
 73
 74	if cp.logger != nil {
 75		cp.logger.Logf("Processed %d articles (from %d original)", len(processed), len(articles))
 76	}
 77
 78	return processed
 79}
 80
 81func (cp *ContentProcessor) filterArticles(articles []*aggregator.Article) []*aggregator.Article {
 82	if len(cp.filterKeywords) == 0 && len(cp.excludeKeywords) == 0 {
 83		return articles
 84	}
 85
 86	filtered := []*aggregator.Article{}
 87
 88	for _, article := range articles {
 89		include := true
 90
 91		// Check if article should be excluded
 92		if len(cp.excludeKeywords) > 0 {
 93			articleText := strings.ToLower(article.Title + " " + article.Content)
 94			for _, keyword := range cp.excludeKeywords {
 95				if strings.Contains(articleText, keyword) {
 96					if cp.logger != nil {
 97						cp.logger.Logf("Excluding article due to keyword '%s': %s", keyword, article.Title)
 98					}
 99					include = false
100					break
101				}
102			}
103		}
104
105		// Check if article should be included (only if we have inclusion filters)
106		if include && len(cp.filterKeywords) > 0 {
107			articleText := strings.ToLower(article.Title + " " + article.Content)
108			include = false
109			for _, keyword := range cp.filterKeywords {
110				if strings.Contains(articleText, keyword) {
111					include = true
112					break
113				}
114			}
115		}
116
117		if include {
118			filtered = append(filtered, article)
119		}
120	}
121
122	if cp.logger != nil {
123		cp.logger.Logf(
124			"Filtered %d articles to %d (inclusion: %v, exclusion: %v)",
125			len(articles), len(filtered), cp.filterKeywords, cp.excludeKeywords,
126		)
127	}
128
129	return filtered
130}
131
132func (cp *ContentProcessor) sortArticles(articles []*aggregator.Article, sortBy string, reverse bool) []*aggregator.Article {
133	if len(articles) == 0 {
134		return articles
135	}
136
137	// Create a copy to avoid modifying the original slice
138	sortedArticles := make([]*aggregator.Article, len(articles))
139	copy(sortedArticles, articles)
140
141	// Define sort functions
142	switch strings.ToLower(sortBy) {
143	case "date":
144		sort.Slice(sortedArticles, func(i, j int) bool {
145			iTime := sortedArticles[i].Published
146			jTime := sortedArticles[j].Published
147			if iTime.IsZero() {
148				iTime = time.Time{}
149			}
150			if jTime.IsZero() {
151				jTime = time.Time{}
152			}
153			if reverse {
154				return iTime.After(jTime)
155			}
156			return iTime.Before(jTime)
157		})
158	case "title":
159		sort.Slice(sortedArticles, func(i, j int) bool {
160			iTitle := strings.ToLower(sortedArticles[i].Title)
161			jTitle := strings.ToLower(sortedArticles[j].Title)
162			if reverse {
163				return iTitle > jTitle
164			}
165			return iTitle < jTitle
166		})
167	case "source":
168		sort.Slice(sortedArticles, func(i, j int) bool {
169			iSource := strings.ToLower(sortedArticles[i].SourceFeed)
170			jSource := strings.ToLower(sortedArticles[j].SourceFeed)
171			if reverse {
172				return iSource > jSource
173			}
174			return iSource < jSource
175		})
176	default:
177		// Default to date sorting
178		sort.Slice(sortedArticles, func(i, j int) bool {
179			iTime := sortedArticles[i].Published
180			jTime := sortedArticles[j].Published
181			if iTime.IsZero() {
182				iTime = time.Time{}
183			}
184			if jTime.IsZero() {
185				jTime = time.Time{}
186			}
187			if reverse {
188				return iTime.After(jTime)
189			}
190			return iTime.Before(jTime)
191		})
192	}
193
194	return sortedArticles
195}
196
197func (cp *ContentProcessor) prepareForLLM(articles []*aggregator.Article) []map[string]any {
198	processed := make([]map[string]any, len(articles))
199
200	for i, article := range articles {
201		// Process content
202		content := article.Content
203		if len(content) > cp.maxContentPerArticle {
204			content = content[:cp.maxContentPerArticle] + "... [truncated]"
205		}
206
207		// Create map
208		articleMap := map[string]any{
209			"title":       article.Title,
210			"link":        article.Link,
211			"content":     content,
212			"author":      article.Author,
213			"source_feed": article.SourceFeed,
214			"summary":     article.Summary,
215		}
216
217		if !article.Published.IsZero() {
218			articleMap["published"] = article.Published
219		}
220
221		processed[i] = articleMap
222	}
223
224	return processed
225}
226
227// EstimateTokenCount estimates token count for articles using tiktoken.
228// This is the accurate method using OpenAI's tokenisation.
229func (cp *ContentProcessor) EstimateTokenCount(articles []map[string]any, model string) int {
230	totalTokens := 0
231
232	for _, article := range articles {
233		for _, field := range []string{"title", "content", "author", "source_feed"} {
234			if val, ok := article[field]; ok && val != nil {
235				if str, ok := val.(string); ok && str != "" {
236					tokens, err := tokeniser.CountTokens(str, model)
237					if err != nil {
238						// Fallback to rough estimate on error
239						if cp.logger != nil {
240							cp.logger.Logf("Warning: token count error for %s: %v", field, err)
241						}
242						totalTokens += len(str) / 4
243						continue
244					}
245					totalTokens += tokens
246				}
247			}
248		}
249	}
250
251	if cp.logger != nil {
252		cp.logger.Logf("Estimated tokens: %d", totalTokens)
253	}
254
255	return totalTokens
256}