1package processor
  2
  3import (
  4	"fmt"
  5	"sort"
  6	"strings"
  7	"time"
  8
  9	"llm_aggregator/internal/aggregator"
 10	"llm_aggregator/internal/progress"
 11)
 12
 13// ContentProcessor processes and prepares aggregated content for LLM analysis.
 14type ContentProcessor struct {
 15	maxTotalArticles     int
 16	maxContentPerArticle int
 17	filterKeywords       []string
 18	excludeKeywords      []string
 19	logger               *progress.Context
 20}
 21
 22// NewContentProcessor creates a new ContentProcessor with the specified options.
 23func NewContentProcessor(maxTotalArticles, maxContentPerArticle int, filterKeywords, excludeKeywords []string) *ContentProcessor {
 24	// Convert keywords to lowercase for case-insensitive matching
 25	filterLower := make([]string, len(filterKeywords))
 26	for i, kw := range filterKeywords {
 27		filterLower[i] = strings.ToLower(kw)
 28	}
 29
 30	excludeLower := make([]string, len(excludeKeywords))
 31	for i, kw := range excludeKeywords {
 32		excludeLower[i] = strings.ToLower(kw)
 33	}
 34
 35	return &ContentProcessor{
 36		maxTotalArticles:     maxTotalArticles,
 37		maxContentPerArticle: maxContentPerArticle,
 38		filterKeywords:       filterLower,
 39		excludeKeywords:      excludeLower,
 40	}
 41}
 42
 43// SetLogger sets the logger for the processor
 44func (cp *ContentProcessor) SetLogger(logger *progress.Context) {
 45	cp.logger = logger
 46}
 47
 48// ProcessArticles processes articles: filter, sort, and prepare for LLM.
 49func (cp *ContentProcessor) ProcessArticles(articles []*aggregator.Article, sortBy string, reverse bool) []map[string]any {
 50	if len(articles) == 0 {
 51		if cp.logger != nil {
 52			cp.logger.Logf("Warning: No articles to process")
 53		}
 54		return []map[string]any{}
 55	}
 56
 57	// Filter articles
 58	filteredArticles := cp.filterArticles(articles)
 59
 60	// Sort articles
 61	sortedArticles := cp.sortArticles(filteredArticles, sortBy, reverse)
 62
 63	// Limit total articles
 64	if len(sortedArticles) > cp.maxTotalArticles {
 65		if cp.logger != nil {
 66			cp.logger.Logf("Limiting articles from %d to %d", len(sortedArticles), cp.maxTotalArticles)
 67		}
 68		sortedArticles = sortedArticles[:cp.maxTotalArticles]
 69	}
 70
 71	// Prepare articles for LLM
 72	processed := cp.prepareForLLM(sortedArticles)
 73
 74	if cp.logger != nil {
 75		cp.logger.Logf("Processed %d articles (from %d original)", len(processed), len(articles))
 76	}
 77
 78	return processed
 79}
 80
 81func (cp *ContentProcessor) filterArticles(articles []*aggregator.Article) []*aggregator.Article {
 82	if len(cp.filterKeywords) == 0 && len(cp.excludeKeywords) == 0 {
 83		return articles
 84	}
 85
 86	filtered := []*aggregator.Article{}
 87
 88	for _, article := range articles {
 89		include := true
 90
 91		// Check if article should be excluded
 92		if len(cp.excludeKeywords) > 0 {
 93			articleText := strings.ToLower(article.Title + " " + article.Content)
 94			for _, keyword := range cp.excludeKeywords {
 95				if strings.Contains(articleText, keyword) {
 96					if cp.logger != nil {
 97						cp.logger.Logf("Excluding article due to keyword '%s': %s", keyword, article.Title)
 98					}
 99					include = false
100					break
101				}
102			}
103		}
104
105		// Check if article should be included (only if we have inclusion filters)
106		if include && len(cp.filterKeywords) > 0 {
107			articleText := strings.ToLower(article.Title + " " + article.Content)
108			include = false
109			for _, keyword := range cp.filterKeywords {
110				if strings.Contains(articleText, keyword) {
111					include = true
112					break
113				}
114			}
115		}
116
117		if include {
118			filtered = append(filtered, article)
119		}
120	}
121
122	if cp.logger != nil {
123		cp.logger.Logf(
124			"Filtered %d articles to %d (inclusion: %v, exclusion: %v)",
125			len(articles), len(filtered), cp.filterKeywords, cp.excludeKeywords,
126		)
127	}
128
129	return filtered
130}
131
132func (cp *ContentProcessor) sortArticles(articles []*aggregator.Article, sortBy string, reverse bool) []*aggregator.Article {
133	if len(articles) == 0 {
134		return articles
135	}
136
137	// Create a copy to avoid modifying the original slice
138	sortedArticles := make([]*aggregator.Article, len(articles))
139	copy(sortedArticles, articles)
140
141	// Define sort functions
142	switch strings.ToLower(sortBy) {
143	case "date":
144		sort.Slice(sortedArticles, func(i, j int) bool {
145			iTime := sortedArticles[i].Published
146			jTime := sortedArticles[j].Published
147			if iTime.IsZero() {
148				iTime = time.Time{}
149			}
150			if jTime.IsZero() {
151				jTime = time.Time{}
152			}
153			if reverse {
154				return iTime.After(jTime)
155			}
156			return iTime.Before(jTime)
157		})
158	case "title":
159		sort.Slice(sortedArticles, func(i, j int) bool {
160			iTitle := strings.ToLower(sortedArticles[i].Title)
161			jTitle := strings.ToLower(sortedArticles[j].Title)
162			if reverse {
163				return iTitle > jTitle
164			}
165			return iTitle < jTitle
166		})
167	case "source":
168		sort.Slice(sortedArticles, func(i, j int) bool {
169			iSource := strings.ToLower(sortedArticles[i].SourceFeed)
170			jSource := strings.ToLower(sortedArticles[j].SourceFeed)
171			if reverse {
172				return iSource > jSource
173			}
174			return iSource < jSource
175		})
176	default:
177		// Default to date sorting
178		sort.Slice(sortedArticles, func(i, j int) bool {
179			iTime := sortedArticles[i].Published
180			jTime := sortedArticles[j].Published
181			if iTime.IsZero() {
182				iTime = time.Time{}
183			}
184			if jTime.IsZero() {
185				jTime = time.Time{}
186			}
187			if reverse {
188				return iTime.After(jTime)
189			}
190			return iTime.Before(jTime)
191		})
192	}
193
194	return sortedArticles
195}
196
197func (cp *ContentProcessor) prepareForLLM(articles []*aggregator.Article) []map[string]any {
198	processed := make([]map[string]any, len(articles))
199
200	for i, article := range articles {
201		// Process content
202		content := article.Content
203		if len(content) > cp.maxContentPerArticle {
204			content = content[:cp.maxContentPerArticle] + "... [truncated]"
205		}
206
207		// Create map
208		articleMap := map[string]any{
209			"title":       article.Title,
210			"link":        article.Link,
211			"content":     content,
212			"author":      article.Author,
213			"source_feed": article.SourceFeed,
214			"summary":     article.Summary,
215		}
216
217		if !article.Published.IsZero() {
218			articleMap["published"] = article.Published
219		}
220
221		processed[i] = articleMap
222	}
223
224	return processed
225}
226
227// EstimateTokenCount estimates token count for articles (rough approximation).
228// Note: This is a rough estimate (1 token ≈ 4 characters for English).
229// Actual tokenisation may vary.
230func (cp *ContentProcessor) EstimateTokenCount(articles []map[string]any) int {
231	totalChars := 0
232
233	for _, article := range articles {
234		for _, field := range []string{"title", "content", "author", "source_feed"} {
235			if val, ok := article[field]; ok && val != nil {
236				if str, ok := val.(string); ok {
237					totalChars += len(str)
238				}
239			}
240		}
241	}
242
243	// Rough estimate: 1 token ≈ 4 characters for English
244	estimatedTokens := totalChars / 4
245
246	if cp.logger != nil {
247		cp.logger.Logf("Estimated tokens: %d (~%d chars)", estimatedTokens, totalChars)
248	}
249
250	return estimatedTokens
251}
252
253// CreateConciseContext creates a concise context from articles, respecting token limits.
254func (cp *ContentProcessor) CreateConciseContext(articles []map[string]any, maxTotalTokens int) string {
255	if len(articles) == 0 {
256		return "No articles available."
257	}
258
259	contextParts := []string{}
260	currentTokens := 0
261
262	for i, article := range articles {
263		// Create article summary
264		articleText := fmt.Sprintf("Article %d: %s\n", i+1, article["title"])
265
266		// Add source if available
267		if source, ok := article["source_feed"].(string); ok && source != "" {
268			articleText += fmt.Sprintf("Source: %s\n", source)
269		}
270
271		// Add publication date if available
272		if published, ok := article["published"]; ok {
273			switch pub := published.(type) {
274			case time.Time:
275				if !pub.IsZero() {
276					articleText += fmt.Sprintf("Published: %s\n", pub.Format("2006-01-02"))
277				}
278			case string:
279				articleText += fmt.Sprintf("Published: %s\n", pub)
280			}
281		}
282
283		// Add content (truncated if needed)
284		content := ""
285		if c, ok := article["content"].(string); ok {
286			content = c
287		}
288
289		maxContentTokens := maxTotalTokens / len(articles) // Rough allocation
290		maxContentChars := maxContentTokens * 4
291
292		if len(content) > maxContentChars {
293			content = content[:maxContentChars] + "... [truncated]"
294		}
295
296		articleText += fmt.Sprintf("Content: %s\n", content)
297
298		// Estimate tokens for this article
299		articleTokens := len(articleText) / 4
300
301		// Check if adding this article would exceed limit
302		if currentTokens+articleTokens > maxTotalTokens {
303			if cp.logger != nil {
304				cp.logger.Logf(
305					"Reached token limit (%d). Included %d of %d articles.",
306					maxTotalTokens, i, len(articles),
307				)
308			}
309			break
310		}
311
312		contextParts = append(contextParts, articleText)
313		currentTokens += articleTokens
314	}
315
316	context := strings.Join(contextParts, "\n---\n")
317
318	if cp.logger != nil {
319		cp.logger.Logf("Created context with %d articles, ~%d tokens", len(contextParts), currentTokens)
320	}
321
322	return context
323}
324