1package processor
  2
  3import (
  4	"fmt"
  5	"sort"
  6	"strings"
  7	"time"
  8
  9	"llm_aggregator/internal/aggregator"
 10)
 11
 12// ContentProcessor processes and prepares aggregated content for LLM analysis.
 13type ContentProcessor struct {
 14	maxTotalArticles     int
 15	maxContentPerArticle int
 16	filterKeywords       []string
 17	excludeKeywords      []string
 18}
 19
 20// NewContentProcessor creates a new ContentProcessor with the specified options.
 21func NewContentProcessor(maxTotalArticles, maxContentPerArticle int, filterKeywords, excludeKeywords []string) *ContentProcessor {
 22	// Convert keywords to lowercase for case-insensitive matching
 23	filterLower := make([]string, len(filterKeywords))
 24	for i, kw := range filterKeywords {
 25		filterLower[i] = strings.ToLower(kw)
 26	}
 27
 28	excludeLower := make([]string, len(excludeKeywords))
 29	for i, kw := range excludeKeywords {
 30		excludeLower[i] = strings.ToLower(kw)
 31	}
 32
 33	return &ContentProcessor{
 34		maxTotalArticles:     maxTotalArticles,
 35		maxContentPerArticle: maxContentPerArticle,
 36		filterKeywords:       filterLower,
 37		excludeKeywords:      excludeLower,
 38	}
 39}
 40
 41// ProcessArticles processes articles: filter, sort, and prepare for LLM.
 42func (cp *ContentProcessor) ProcessArticles(articles []*aggregator.Article, sortBy string, reverse bool) []map[string]any {
 43	if len(articles) == 0 {
 44		fmt.Println("Warning: No articles to process")
 45		return []map[string]any{}
 46	}
 47
 48	// Filter articles
 49	filteredArticles := cp.filterArticles(articles)
 50
 51	// Sort articles
 52	sortedArticles := cp.sortArticles(filteredArticles, sortBy, reverse)
 53
 54	// Limit total articles
 55	if len(sortedArticles) > cp.maxTotalArticles {
 56		fmt.Printf("Limiting articles from %d to %d\n", len(sortedArticles), cp.maxTotalArticles)
 57		sortedArticles = sortedArticles[:cp.maxTotalArticles]
 58	}
 59
 60	// Prepare articles for LLM
 61	processed := cp.prepareForLLM(sortedArticles)
 62
 63	fmt.Printf("Processed %d articles (from %d original)\n", len(processed), len(articles))
 64
 65	return processed
 66}
 67
 68func (cp *ContentProcessor) filterArticles(articles []*aggregator.Article) []*aggregator.Article {
 69	if len(cp.filterKeywords) == 0 && len(cp.excludeKeywords) == 0 {
 70		return articles
 71	}
 72
 73	filtered := []*aggregator.Article{}
 74
 75	for _, article := range articles {
 76		include := true
 77
 78		// Check if article should be excluded
 79		if len(cp.excludeKeywords) > 0 {
 80			articleText := strings.ToLower(article.Title + " " + article.Content)
 81			for _, keyword := range cp.excludeKeywords {
 82				if strings.Contains(articleText, keyword) {
 83					fmt.Printf("Excluding article due to keyword '%s': %s\n", keyword, article.Title)
 84					include = false
 85					break
 86				}
 87			}
 88		}
 89
 90		// Check if article should be included (only if we have inclusion filters)
 91		if include && len(cp.filterKeywords) > 0 {
 92			articleText := strings.ToLower(article.Title + " " + article.Content)
 93			include = false
 94			for _, keyword := range cp.filterKeywords {
 95				if strings.Contains(articleText, keyword) {
 96					include = true
 97					break
 98				}
 99			}
100		}
101
102		if include {
103			filtered = append(filtered, article)
104		}
105	}
106
107	fmt.Printf(
108		"Filtered %d articles to %d (inclusion: %v, exclusion: %v)\n",
109		len(articles), len(filtered), cp.filterKeywords, cp.excludeKeywords,
110	)
111
112	return filtered
113}
114
115func (cp *ContentProcessor) sortArticles(articles []*aggregator.Article, sortBy string, reverse bool) []*aggregator.Article {
116	if len(articles) == 0 {
117		return articles
118	}
119
120	// Create a copy to avoid modifying the original slice
121	sortedArticles := make([]*aggregator.Article, len(articles))
122	copy(sortedArticles, articles)
123
124	// Define sort functions
125	switch strings.ToLower(sortBy) {
126	case "date":
127		sort.Slice(sortedArticles, func(i, j int) bool {
128			iTime := sortedArticles[i].Published
129			jTime := sortedArticles[j].Published
130			if iTime.IsZero() {
131				iTime = time.Time{}
132			}
133			if jTime.IsZero() {
134				jTime = time.Time{}
135			}
136			if reverse {
137				return iTime.After(jTime)
138			}
139			return iTime.Before(jTime)
140		})
141	case "title":
142		sort.Slice(sortedArticles, func(i, j int) bool {
143			iTitle := strings.ToLower(sortedArticles[i].Title)
144			jTitle := strings.ToLower(sortedArticles[j].Title)
145			if reverse {
146				return iTitle > jTitle
147			}
148			return iTitle < jTitle
149		})
150	case "source":
151		sort.Slice(sortedArticles, func(i, j int) bool {
152			iSource := strings.ToLower(sortedArticles[i].SourceFeed)
153			jSource := strings.ToLower(sortedArticles[j].SourceFeed)
154			if reverse {
155				return iSource > jSource
156			}
157			return iSource < jSource
158		})
159	default:
160		// Default to date sorting
161		sort.Slice(sortedArticles, func(i, j int) bool {
162			iTime := sortedArticles[i].Published
163			jTime := sortedArticles[j].Published
164			if iTime.IsZero() {
165				iTime = time.Time{}
166			}
167			if jTime.IsZero() {
168				jTime = time.Time{}
169			}
170			if reverse {
171				return iTime.After(jTime)
172			}
173			return iTime.Before(jTime)
174		})
175	}
176
177	return sortedArticles
178}
179
180func (cp *ContentProcessor) prepareForLLM(articles []*aggregator.Article) []map[string]any {
181	processed := make([]map[string]any, len(articles))
182
183	for i, article := range articles {
184		// Process content
185		content := article.Content
186		if len(content) > cp.maxContentPerArticle {
187			content = content[:cp.maxContentPerArticle] + "... [truncated]"
188		}
189
190		// Create map
191		articleMap := map[string]any{
192			"title":       article.Title,
193			"link":        article.Link,
194			"content":     content,
195			"author":      article.Author,
196			"source_feed": article.SourceFeed,
197			"summary":     article.Summary,
198		}
199
200		if !article.Published.IsZero() {
201			articleMap["published"] = article.Published
202		}
203
204		processed[i] = articleMap
205	}
206
207	return processed
208}
209
210// EstimateTokenCount estimates token count for articles (rough approximation).
211// Note: This is a rough estimate (1 token ≈ 4 characters for English).
212// Actual tokenisation may vary.
213func (cp *ContentProcessor) EstimateTokenCount(articles []map[string]any) int {
214	totalChars := 0
215
216	for _, article := range articles {
217		for _, field := range []string{"title", "content", "author", "source_feed"} {
218			if val, ok := article[field]; ok && val != nil {
219				if str, ok := val.(string); ok {
220					totalChars += len(str)
221				}
222			}
223		}
224	}
225
226	// Rough estimate: 1 token ≈ 4 characters for English
227	estimatedTokens := totalChars / 4
228
229	fmt.Printf("Estimated tokens: %d (~%d chars)\n", estimatedTokens, totalChars)
230
231	return estimatedTokens
232}
233
234// CreateConciseContext creates a concise context from articles, respecting token limits.
235func (cp *ContentProcessor) CreateConciseContext(articles []map[string]any, maxTotalTokens int) string {
236	if len(articles) == 0 {
237		return "No articles available."
238	}
239
240	contextParts := []string{}
241	currentTokens := 0
242
243	for i, article := range articles {
244		// Create article summary
245		articleText := fmt.Sprintf("Article %d: %s\n", i+1, article["title"])
246
247		// Add source if available
248		if source, ok := article["source_feed"].(string); ok && source != "" {
249			articleText += fmt.Sprintf("Source: %s\n", source)
250		}
251
252		// Add publication date if available
253		if published, ok := article["published"]; ok {
254			switch pub := published.(type) {
255			case time.Time:
256				if !pub.IsZero() {
257					articleText += fmt.Sprintf("Published: %s\n", pub.Format("2006-01-02"))
258				}
259			case string:
260				articleText += fmt.Sprintf("Published: %s\n", pub)
261			}
262		}
263
264		// Add content (truncated if needed)
265		content := ""
266		if c, ok := article["content"].(string); ok {
267			content = c
268		}
269
270		maxContentTokens := maxTotalTokens / len(articles) // Rough allocation
271		maxContentChars := maxContentTokens * 4
272
273		if len(content) > maxContentChars {
274			content = content[:maxContentChars] + "... [truncated]"
275		}
276
277		articleText += fmt.Sprintf("Content: %s\n", content)
278
279		// Estimate tokens for this article
280		articleTokens := len(articleText) / 4
281
282		// Check if adding this article would exceed limit
283		if currentTokens+articleTokens > maxTotalTokens {
284			fmt.Printf(
285				"Reached token limit (%d). Included %d of %d articles.\n",
286				maxTotalTokens, i, len(articles),
287			)
288			break
289		}
290
291		contextParts = append(contextParts, articleText)
292		currentTokens += articleTokens
293	}
294
295	context := strings.Join(contextParts, "\n---\n")
296
297	fmt.Printf("Created context with %d articles, ~%d tokens\n", len(contextParts), currentTokens)
298
299	return context
300}
301