internal/processor/processor_test.go (view raw)
1package processor
2
3import (
4 "strings"
5 "testing"
6 "time"
7
8 "llm_aggregator/internal/aggregator"
9)
10
11func TestNewContentProcessor(t *testing.T) {
12 tests := []struct {
13 name string
14 maxTotalArticles int
15 maxContentPerArticle int
16 filterKeywords []string
17 excludeKeywords []string
18 }{
19 {"standard config", 20, 5000, []string{"tech"}, []string{"spam"}},
20 {"zero limits", 0, 0, nil, nil},
21 {"empty keywords", 10, 1000, []string{}, []string{}},
22 {"no keywords", 15, 3000, nil, nil},
23 }
24
25 for _, tt := range tests {
26 t.Run(tt.name, func(t *testing.T) {
27 cp := NewContentProcessor(
28 tt.maxTotalArticles,
29 tt.maxContentPerArticle,
30 tt.filterKeywords,
31 tt.excludeKeywords,
32 )
33 if cp == nil {
34 t.Error("Expected non-nil ContentProcessor")
35 }
36 })
37 }
38}
39
40func TestProcessArticles(t *testing.T) {
41 articles := createTestArticles()
42
43 t.Run("process all articles", func(t *testing.T) {
44 cp := NewContentProcessor(100, 5000, nil, nil)
45 result := cp.ProcessArticles(articles, "date", false)
46
47 if len(result) != len(articles) {
48 t.Errorf("Expected %d articles, got %d", len(articles), len(result))
49 }
50 })
51
52 t.Run("limit articles", func(t *testing.T) {
53 cp := NewContentProcessor(3, 5000, nil, nil)
54 result := cp.ProcessArticles(articles, "date", false)
55
56 if len(result) != 3 {
57 t.Errorf("Expected 3 articles, got %d", len(result))
58 }
59 })
60
61 t.Run("sort by date ascending", func(t *testing.T) {
62 cp := NewContentProcessor(100, 5000, nil, nil)
63 result := cp.ProcessArticles(articles, "date", false)
64
65 // Check first article has earliest date
66 firstTitle := result[0]["title"].(string)
67 if !strings.Contains(firstTitle, "Old") {
68 t.Errorf("Expected first article to be oldest, got %s", firstTitle)
69 }
70 })
71
72 t.Run("sort by date descending", func(t *testing.T) {
73 cp := NewContentProcessor(100, 5000, nil, nil)
74 result := cp.ProcessArticles(articles, "date", true)
75
76 // Check first article has latest date
77 firstTitle := result[0]["title"].(string)
78 if !strings.Contains(firstTitle, "New") {
79 t.Errorf("Expected first article to be newest, got %s", firstTitle)
80 }
81 })
82
83 t.Run("empty articles slice", func(t *testing.T) {
84 cp := NewContentProcessor(10, 5000, nil, nil)
85 result := cp.ProcessArticles([]*aggregator.Article{}, "date", false)
86
87 if len(result) != 0 {
88 t.Errorf("Expected 0 articles, got %d", len(result))
89 }
90 })
91
92 t.Run("nil articles slice", func(t *testing.T) {
93 cp := NewContentProcessor(10, 5000, nil, nil)
94 result := cp.ProcessArticles(nil, "date", false)
95
96 if len(result) != 0 {
97 t.Errorf("Expected 0 articles, got %d", len(result))
98 }
99 })
100}
101
102func TestFilterArticlesByIncludeKeywords(t *testing.T) {
103 articles := []*aggregator.Article{
104 {Title: "Tech Article", Content: "Tech news content"},
105 {Title: "Sports Article", Content: "Sports news content"},
106 {Title: "AI Technology", Content: "Artificial intelligence content"},
107 }
108
109 cp := NewContentProcessor(100, 5000, []string{"tech", "ai"}, nil)
110 result := cp.ProcessArticles(articles, "date", false)
111
112 if len(result) != 2 {
113 t.Errorf("Expected 2 articles after filtering, got %d", len(result))
114 }
115
116 titles := make([]string, len(result))
117 for i, a := range result {
118 titles[i] = a["title"].(string)
119 }
120
121 if !containsString(titles, "Tech Article") {
122 t.Error("Expected Tech Article to be included")
123 }
124 if !containsString(titles, "AI Technology") {
125 t.Error("Expected AI Technology to be included")
126 }
127 if containsString(titles, "Sports Article") {
128 t.Error("Sports Article should be excluded")
129 }
130}
131
132func TestFilterArticlesByExcludeKeywords(t *testing.T) {
133 articles := []*aggregator.Article{
134 {Title: "Interesting Tech", Content: "Tech content"},
135 {Title: "Advertisement", Content: "Buy now! Buy now!"},
136 {Title: "More Tech", Content: "More tech content"},
137 }
138
139 cp := NewContentProcessor(100, 5000, nil, []string{"advertisement", "buy now"})
140 result := cp.ProcessArticles(articles, "date", false)
141
142 if len(result) != 2 {
143 t.Errorf("Expected 2 articles after filtering, got %d", len(result))
144 }
145
146 titles := make([]string, len(result))
147 for i, a := range result {
148 titles[i] = a["title"].(string)
149 }
150
151 if containsString(titles, "Advertisement") {
152 t.Error("Advertisement should be excluded")
153 }
154}
155
156func TestFilterArticlesCaseInsensitive(t *testing.T) {
157 articles := []*aggregator.Article{
158 {Title: "TECH NEWS", Content: "TECH CONTENT"},
159 {Title: "Tech News", Content: "Tech Content"},
160 {Title: "tech news", Content: "tech content"},
161 }
162
163 cp := NewContentProcessor(100, 5000, []string{"tech"}, nil)
164 result := cp.ProcessArticles(articles, "date", false)
165
166 if len(result) != 3 {
167 t.Errorf("Expected all 3 articles (case insensitive), got %d", len(result))
168 }
169}
170
171func TestSortArticlesByTitle(t *testing.T) {
172 articles := []*aggregator.Article{
173 {Title: "Zebra Article", Content: "Content"},
174 {Title: "Alpha Article", Content: "Content"},
175 {Title: "Middle Article", Content: "Content"},
176 }
177
178 t.Run("ascending", func(t *testing.T) {
179 cp := NewContentProcessor(100, 5000, nil, nil)
180 result := cp.ProcessArticles(articles, "title", false)
181
182 if result[0]["title"] != "Alpha Article" {
183 t.Errorf("Expected Alpha first, got %s", result[0]["title"])
184 }
185 if result[2]["title"] != "Zebra Article" {
186 t.Errorf("Expected Zebra last, got %s", result[2]["title"])
187 }
188 })
189
190 t.Run("descending", func(t *testing.T) {
191 cp := NewContentProcessor(100, 5000, nil, nil)
192 result := cp.ProcessArticles(articles, "title", true)
193
194 if result[0]["title"] != "Zebra Article" {
195 t.Errorf("Expected Zebra first, got %s", result[0]["title"])
196 }
197 })
198}
199
200func TestSortArticlesBySource(t *testing.T) {
201 articles := []*aggregator.Article{
202 {Title: "A", SourceFeed: "Zulu Feed"},
203 {Title: "B", SourceFeed: "Alpha Feed"},
204 {Title: "C", SourceFeed: "Middle Feed"},
205 }
206
207 t.Run("ascending", func(t *testing.T) {
208 cp := NewContentProcessor(100, 5000, nil, nil)
209 result := cp.ProcessArticles(articles, "source", false)
210
211 if result[0]["source_feed"] != "Alpha Feed" {
212 t.Errorf("Expected Alpha Feed first, got %s", result[0]["source_feed"])
213 }
214 })
215
216 t.Run("descending", func(t *testing.T) {
217 cp := NewContentProcessor(100, 5000, nil, nil)
218 result := cp.ProcessArticles(articles, "source", true)
219
220 if result[0]["source_feed"] != "Zulu Feed" {
221 t.Errorf("Expected Zulu Feed first, got %s", result[0]["source_feed"])
222 }
223 })
224}
225
226func TestPrepareForLLM(t *testing.T) {
227 now := time.Now()
228 article := &aggregator.Article{
229 Title: "Test Title",
230 Link: "https://example.com/test",
231 Content: "This is the article content",
232 Author: "Test Author",
233 SourceFeed: "Test Feed",
234 Summary: "Test Summary",
235 Published: now,
236 }
237
238 cp := NewContentProcessor(10, 5000, nil, nil)
239 result := cp.ProcessArticles([]*aggregator.Article{article}, "date", false)
240
241 if len(result) != 1 {
242 t.Fatalf("Expected 1 article, got %d", len(result))
243 }
244
245 // Check all expected fields are present
246 articleMap := result[0]
247 if articleMap["title"] != "Test Title" {
248 t.Errorf("Expected title 'Test Title', got %v", articleMap["title"])
249 }
250 if articleMap["link"] != "https://example.com/test" {
251 t.Errorf("Expected link, got %v", articleMap["link"])
252 }
253 if articleMap["author"] != "Test Author" {
254 t.Errorf("Expected author, got %v", articleMap["author"])
255 }
256 if articleMap["source_feed"] != "Test Feed" {
257 t.Errorf("Expected source_feed, got %v", articleMap["source_feed"])
258 }
259 if articleMap["summary"] != "Test Summary" {
260 t.Errorf("Expected summary, got %v", articleMap["summary"])
261 }
262}
263
264func TestContentTruncation(t *testing.T) {
265 article := &aggregator.Article{
266 Title: "Long Article",
267 Link: "https://example.com/long",
268 Content: strings.Repeat("x", 6000),
269 }
270
271 t.Run("truncate at max content length", func(t *testing.T) {
272 cp := NewContentProcessor(10, 1000, nil, nil)
273 result := cp.ProcessArticles([]*aggregator.Article{article}, "date", false)
274
275 content := result[0]["content"].(string)
276 if !strings.HasSuffix(content, "... [truncated]") {
277 t.Error("Expected content to be truncated")
278 }
279 if len(content) <= 1000 {
280 t.Errorf("Expected truncated content around 1000 chars, got %d", len(content))
281 }
282 })
283
284 t.Run("no truncation for short content", func(t *testing.T) {
285 shortArticle := &aggregator.Article{
286 Title: "Short Article",
287 Link: "https://example.com/short",
288 Content: "This is a short article content.",
289 }
290 cp := NewContentProcessor(10, 5000, nil, nil)
291 result := cp.ProcessArticles([]*aggregator.Article{shortArticle}, "date", false)
292
293 content := result[0]["content"].(string)
294 if strings.Contains(content, "[truncated]") {
295 t.Error("Short content should not be truncated")
296 }
297 })
298}
299
300func TestSetLogger(t *testing.T) {
301 cp := NewContentProcessor(10, 5000, nil, nil)
302
303 t.Run("nil logger is handled", func(t *testing.T) {
304 cp.SetLogger(nil)
305 // Should not panic
306 result := cp.ProcessArticles(nil, "date", false)
307 if len(result) != 0 {
308 t.Error("Expected empty result")
309 }
310 })
311}
312
313func TestUnknownSortField(t *testing.T) {
314 articles := createTestArticles()
315 cp := NewContentProcessor(100, 5000, nil, nil)
316
317 // Unknown sort field should default to date sorting
318 result := cp.ProcessArticles(articles, "unknown_field", false)
319
320 // Should not panic and return results
321 if len(result) == 0 {
322 t.Error("Expected some results for unknown sort field")
323 }
324}
325
326func TestMixedIncludeExclude(t *testing.T) {
327 articles := []*aggregator.Article{
328 {Title: "Tech Advertisement", Content: "Tech content with ads"},
329 {Title: "Tech News", Content: "Interesting tech content"},
330 {Title: "Sports Tech", Content: "Tech in sports"},
331 {Title: "Advertisement Only", Content: "Just ads"},
332 }
333
334 // Include "tech" but exclude "advertisement"
335 cp := NewContentProcessor(100, 5000, []string{"tech"}, []string{"advertisement"})
336 result := cp.ProcessArticles(articles, "date", false)
337
338 // Should only include "Tech News" and "Sports Tech"
339 if len(result) != 2 {
340 t.Errorf("Expected 2 articles, got %d", len(result))
341 }
342
343 titles := make([]string, len(result))
344 for i, a := range result {
345 titles[i] = a["title"].(string)
346 }
347
348 if !containsString(titles, "Tech News") {
349 t.Error("Expected Tech News")
350 }
351 if !containsString(titles, "Sports Tech") {
352 t.Error("Expected Sports Tech")
353 }
354}
355
356func TestEmptyKeywordLists(t *testing.T) {
357 articles := createTestArticles()
358 cp := NewContentProcessor(100, 5000, []string{}, []string{})
359
360 result := cp.ProcessArticles(articles, "date", false)
361
362 // Should include all articles when no keywords specified
363 if len(result) != len(articles) {
364 t.Errorf("Expected all %d articles, got %d", len(articles), len(result))
365 }
366}
367
368// Helper functions
369
370func createTestArticles() []*aggregator.Article {
371 now := time.Now()
372 return []*aggregator.Article{
373 {
374 Title: "New Article",
375 Link: "https://example.com/new",
376 Content: "New content",
377 Published: now,
378 SourceFeed: "News Feed",
379 },
380 {
381 Title: "Middle Article",
382 Link: "https://example.com/middle",
383 Content: "Middle content",
384 Published: now.Add(-24 * time.Hour),
385 SourceFeed: "News Feed",
386 },
387 {
388 Title: "Old Article",
389 Link: "https://example.com/old",
390 Content: "Old content",
391 Published: now.Add(-48 * time.Hour),
392 SourceFeed: "News Feed",
393 },
394 }
395}
396
397func containsString(list []string, s string) bool {
398 for _, item := range list {
399 if item == s {
400 return true
401 }
402 }
403 return false
404}