internal/processor/processor.go (view raw)
1package processor
2
3import (
4 "fmt"
5 "sort"
6 "strings"
7 "time"
8
9 "llm_aggregator/internal/aggregator"
10 "llm_aggregator/internal/progress"
11 "llm_aggregator/internal/tokeniser"
12)
13
14// ContentProcessor processes and prepares aggregated content for LLM analysis.
15type ContentProcessor struct {
16 maxTotalArticles int
17 maxContentPerArticle int
18 filterKeywords []string
19 excludeKeywords []string
20 logger *progress.Context
21}
22
23// NewContentProcessor creates a new ContentProcessor with the specified options.
24func NewContentProcessor(maxTotalArticles, maxContentPerArticle int, filterKeywords, excludeKeywords []string) *ContentProcessor {
25 // Convert keywords to lowercase for case-insensitive matching
26 filterLower := make([]string, len(filterKeywords))
27 for i, kw := range filterKeywords {
28 filterLower[i] = strings.ToLower(kw)
29 }
30
31 excludeLower := make([]string, len(excludeKeywords))
32 for i, kw := range excludeKeywords {
33 excludeLower[i] = strings.ToLower(kw)
34 }
35
36 return &ContentProcessor{
37 maxTotalArticles: maxTotalArticles,
38 maxContentPerArticle: maxContentPerArticle,
39 filterKeywords: filterLower,
40 excludeKeywords: excludeLower,
41 }
42}
43
44// SetLogger sets the logger for the processor
45func (cp *ContentProcessor) SetLogger(logger *progress.Context) {
46 cp.logger = logger
47}
48
49// ProcessArticles processes articles: filter, sort, and prepare for LLM.
50func (cp *ContentProcessor) ProcessArticles(articles []*aggregator.Article, sortBy string, reverse bool) []map[string]any {
51 if len(articles) == 0 {
52 if cp.logger != nil {
53 cp.logger.Logf("Warning: No articles to process")
54 }
55 return []map[string]any{}
56 }
57
58 // Filter articles
59 filteredArticles := cp.filterArticles(articles)
60
61 // Sort articles
62 sortedArticles := cp.sortArticles(filteredArticles, sortBy, reverse)
63
64 // Limit total articles
65 if len(sortedArticles) > cp.maxTotalArticles {
66 if cp.logger != nil {
67 cp.logger.Logf("Limiting articles from %d to %d", len(sortedArticles), cp.maxTotalArticles)
68 }
69 sortedArticles = sortedArticles[:cp.maxTotalArticles]
70 }
71
72 // Prepare articles for LLM
73 processed := cp.prepareForLLM(sortedArticles)
74
75 if cp.logger != nil {
76 cp.logger.Logf("Processed %d articles (from %d original)", len(processed), len(articles))
77 }
78
79 return processed
80}
81
82func (cp *ContentProcessor) filterArticles(articles []*aggregator.Article) []*aggregator.Article {
83 if len(cp.filterKeywords) == 0 && len(cp.excludeKeywords) == 0 {
84 return articles
85 }
86
87 filtered := []*aggregator.Article{}
88
89 for _, article := range articles {
90 include := true
91
92 // Check if article should be excluded
93 if len(cp.excludeKeywords) > 0 {
94 articleText := strings.ToLower(article.Title + " " + article.Content)
95 for _, keyword := range cp.excludeKeywords {
96 if strings.Contains(articleText, keyword) {
97 if cp.logger != nil {
98 cp.logger.Logf("Excluding article due to keyword '%s': %s", keyword, article.Title)
99 }
100 include = false
101 break
102 }
103 }
104 }
105
106 // Check if article should be included (only if we have inclusion filters)
107 if include && len(cp.filterKeywords) > 0 {
108 articleText := strings.ToLower(article.Title + " " + article.Content)
109 include = false
110 for _, keyword := range cp.filterKeywords {
111 if strings.Contains(articleText, keyword) {
112 include = true
113 break
114 }
115 }
116 }
117
118 if include {
119 filtered = append(filtered, article)
120 }
121 }
122
123 if cp.logger != nil {
124 cp.logger.Logf(
125 "Filtered %d articles to %d (inclusion: %v, exclusion: %v)",
126 len(articles), len(filtered), cp.filterKeywords, cp.excludeKeywords,
127 )
128 }
129
130 return filtered
131}
132
133func (cp *ContentProcessor) sortArticles(articles []*aggregator.Article, sortBy string, reverse bool) []*aggregator.Article {
134 if len(articles) == 0 {
135 return articles
136 }
137
138 // Create a copy to avoid modifying the original slice
139 sortedArticles := make([]*aggregator.Article, len(articles))
140 copy(sortedArticles, articles)
141
142 // Define sort functions
143 switch strings.ToLower(sortBy) {
144 case "date":
145 sort.Slice(sortedArticles, func(i, j int) bool {
146 iTime := sortedArticles[i].Published
147 jTime := sortedArticles[j].Published
148 if iTime.IsZero() {
149 iTime = time.Time{}
150 }
151 if jTime.IsZero() {
152 jTime = time.Time{}
153 }
154 if reverse {
155 return iTime.After(jTime)
156 }
157 return iTime.Before(jTime)
158 })
159 case "title":
160 sort.Slice(sortedArticles, func(i, j int) bool {
161 iTitle := strings.ToLower(sortedArticles[i].Title)
162 jTitle := strings.ToLower(sortedArticles[j].Title)
163 if reverse {
164 return iTitle > jTitle
165 }
166 return iTitle < jTitle
167 })
168 case "source":
169 sort.Slice(sortedArticles, func(i, j int) bool {
170 iSource := strings.ToLower(sortedArticles[i].SourceFeed)
171 jSource := strings.ToLower(sortedArticles[j].SourceFeed)
172 if reverse {
173 return iSource > jSource
174 }
175 return iSource < jSource
176 })
177 default:
178 // Default to date sorting
179 sort.Slice(sortedArticles, func(i, j int) bool {
180 iTime := sortedArticles[i].Published
181 jTime := sortedArticles[j].Published
182 if iTime.IsZero() {
183 iTime = time.Time{}
184 }
185 if jTime.IsZero() {
186 jTime = time.Time{}
187 }
188 if reverse {
189 return iTime.After(jTime)
190 }
191 return iTime.Before(jTime)
192 })
193 }
194
195 return sortedArticles
196}
197
198func (cp *ContentProcessor) prepareForLLM(articles []*aggregator.Article) []map[string]any {
199 processed := make([]map[string]any, len(articles))
200
201 for i, article := range articles {
202 // Process content
203 content := article.Content
204 if len(content) > cp.maxContentPerArticle {
205 content = content[:cp.maxContentPerArticle] + "... [truncated]"
206 }
207
208 // Create map
209 articleMap := map[string]any{
210 "title": article.Title,
211 "link": article.Link,
212 "content": content,
213 "author": article.Author,
214 "source_feed": article.SourceFeed,
215 "summary": article.Summary,
216 }
217
218 if !article.Published.IsZero() {
219 articleMap["published"] = article.Published
220 }
221
222 processed[i] = articleMap
223 }
224
225 return processed
226}
227
228// EstimateTokenCount estimates token count for articles using tiktoken.
229// This is the accurate method using OpenAI's tokenisation.
230func (cp *ContentProcessor) EstimateTokenCount(articles []map[string]any, model string) int {
231 totalTokens := 0
232
233 for _, article := range articles {
234 for _, field := range []string{"title", "content", "author", "source_feed"} {
235 if val, ok := article[field]; ok && val != nil {
236 if str, ok := val.(string); ok && str != "" {
237 tokens, err := tokeniser.CountTokens(str, model)
238 if err != nil {
239 // Fallback to rough estimate on error
240 if cp.logger != nil {
241 cp.logger.Logf("Warning: token count error for %s: %v", field, err)
242 }
243 totalTokens += len(str) / 4
244 continue
245 }
246 totalTokens += tokens
247 }
248 }
249 }
250 }
251
252 if cp.logger != nil {
253 cp.logger.Logf("Estimated tokens: %d", totalTokens)
254 }
255
256 return totalTokens
257}
258
259// CreateConciseContext creates a concise context from articles, respecting token limits.
260func (cp *ContentProcessor) CreateConciseContext(articles []map[string]any, maxTotalTokens int, model string) string {
261 if len(articles) == 0 {
262 return "No articles available."
263 }
264
265 // Get the appropriate encoding for accurate token counting
266 encodingName, err := tokeniser.EncodingForModel(model)
267 if err != nil {
268 // Fallback to rough estimate if model not recognised
269 if cp.logger != nil {
270 cp.logger.Logf("Warning: unknown model %s, using rough token estimate", model)
271 }
272 return cp.createConciseContextRough(articles, maxTotalTokens)
273 }
274
275 enc, err := tokeniser.GetEncoding(encodingName)
276 if err != nil {
277 if cp.logger != nil {
278 cp.logger.Logf("Warning: failed to get encoding: %v", err)
279 }
280 return cp.createConciseContextRough(articles, maxTotalTokens)
281 }
282
283 contextParts := []string{}
284 currentTokens := 0
285
286 for i, article := range articles {
287 // Create article summary
288 articleText := fmt.Sprintf("Article %d: %s\n", i+1, article["title"])
289
290 // Add source if available
291 if source, ok := article["source_feed"].(string); ok && source != "" {
292 articleText += fmt.Sprintf("Source: %s\n", source)
293 }
294
295 // Add publication date if available
296 if published, ok := article["published"]; ok {
297 switch pub := published.(type) {
298 case time.Time:
299 if !pub.IsZero() {
300 articleText += fmt.Sprintf("Published: %s\n", pub.Format("2006-01-02"))
301 }
302 case string:
303 articleText += fmt.Sprintf("Published: %s\n", pub)
304 }
305 }
306
307 // Add content (we'll truncate based on actual tokens)
308 content := ""
309 if c, ok := article["content"].(string); ok {
310 content = c
311 }
312
313 // First, check if we can add the header tokens
314 headerTokens := len(enc.Encode(articleText, nil, nil))
315
316 // Calculate remaining tokens for this article
317 remainingTokens := maxTotalTokens - currentTokens - headerTokens - 3 // -3 for separator
318
319 if remainingTokens <= 0 {
320 if cp.logger != nil {
321 cp.logger.Logf(
322 "Reached token limit (%d). Included %d of %d articles.",
323 maxTotalTokens, i, len(articles),
324 )
325 }
326 break
327 }
328
329 // Convert token budget to approximate character limit (rough guide)
330 // Characters are typically 3-4x tokens for English
331 maxContentChars := remainingTokens * 4
332
333 if len(content) > maxContentChars {
334 // Need to encode and truncate more precisely
335 contentTokens := len(enc.Encode(content, nil, nil))
336 if contentTokens > remainingTokens {
337 // Binary search for truncation point would be more accurate,
338 // but for performance, use the rough estimate
339 content = content[:maxContentChars] + "... [truncated]"
340 }
341 }
342
343 articleText += fmt.Sprintf("Content: %s\n", content)
344
345 // Count actual tokens for this article
346 articleTokens := len(enc.Encode(articleText, nil, nil))
347
348 // Double-check we haven't exceeded limit
349 if currentTokens+articleTokens > maxTotalTokens {
350 // Truncate content more aggressively
351 maxArticleChars := (maxTotalTokens - currentTokens - headerTokens - 20) * 4
352 if maxArticleChars > 0 {
353 content = content[:maxArticleChars] + "... [truncated]"
354 articleText = fmt.Sprintf("Article %d: %s\n", i+1, article["title"])
355 if source, ok := article["source_feed"].(string); ok && source != "" {
356 articleText += fmt.Sprintf("Source: %s\n", source)
357 }
358 articleText += fmt.Sprintf("Content: %s\n", content)
359 articleTokens = len(enc.Encode(articleText, nil, nil))
360 }
361
362 if currentTokens+articleTokens > maxTotalTokens {
363 if cp.logger != nil {
364 cp.logger.Logf(
365 "Reached token limit (%d). Included %d of %d articles.",
366 maxTotalTokens, i, len(articles),
367 )
368 }
369 break
370 }
371 }
372
373 contextParts = append(contextParts, articleText)
374 currentTokens += articleTokens
375 }
376
377 context := strings.Join(contextParts, "\n---\n")
378
379 if cp.logger != nil {
380 cp.logger.Logf("Created context with %d articles, ~%d tokens", len(contextParts), currentTokens)
381 }
382
383 return context
384}
385
386// createConciseContextRough creates a concise context using rough character-based estimation.
387// Used as fallback when tiktoken is unavailable.
388func (cp *ContentProcessor) createConciseContextRough(articles []map[string]any, maxTotalTokens int) string {
389 if len(articles) == 0 {
390 return "No articles available."
391 }
392
393 contextParts := []string{}
394 currentTokens := 0
395
396 for i, article := range articles {
397 // Create article summary
398 articleText := fmt.Sprintf("Article %d: %s\n", i+1, article["title"])
399
400 // Add source if available
401 if source, ok := article["source_feed"].(string); ok && source != "" {
402 articleText += fmt.Sprintf("Source: %s\n", source)
403 }
404
405 // Add publication date if available
406 if published, ok := article["published"]; ok {
407 switch pub := published.(type) {
408 case time.Time:
409 if !pub.IsZero() {
410 articleText += fmt.Sprintf("Published: %s\n", pub.Format("2006-01-02"))
411 }
412 case string:
413 articleText += fmt.Sprintf("Published: %s\n", pub)
414 }
415 }
416
417 // Add content (truncated if needed)
418 content := ""
419 if c, ok := article["content"].(string); ok {
420 content = c
421 }
422
423 // Rough allocation per article
424 maxContentTokens := maxTotalTokens / len(articles)
425 maxContentChars := maxContentTokens * 4
426
427 if len(content) > maxContentChars {
428 content = content[:maxContentChars] + "... [truncated]"
429 }
430
431 articleText += fmt.Sprintf("Content: %s\n", content)
432
433 // Rough estimate: 1 token ≈ 4 characters
434 articleTokens := len(articleText) / 4
435
436 // Check if adding this article would exceed limit
437 if currentTokens+articleTokens > maxTotalTokens {
438 if cp.logger != nil {
439 cp.logger.Logf(
440 "Reached token limit (%d). Included %d of %d articles.",
441 maxTotalTokens, i, len(articles),
442 )
443 }
444 break
445 }
446
447 contextParts = append(contextParts, articleText)
448 currentTokens += articleTokens
449 }
450
451 context := strings.Join(contextParts, "\n---\n")
452
453 if cp.logger != nil {
454 cp.logger.Logf("Created context with %d articles, ~%d tokens (rough estimate)", len(contextParts), currentTokens)
455 }
456
457 return context
458}