all repos — llm_aggregator @ b4f6beb3ba126b91482bcd5df0745445088d51d7

A CLI tool to aggregate RSS feeds and summarise them with LLMs

refactor: remove dead code and simplify API surface

- Remove deprecated constructors and unused functions
  - Delete `NewFeedAggregator` (only `WithProgress` variant remains)
  - Remove `DefaultConfig`, `Load`, `ViperToConfig`, `Save`,
    `ConfigExists` from config
  - Delete `NewRuntime` (runtime now built from config directly)
  - Remove unused formatter helpers (`FormatArticleList`)
  - Delete `AnalyseWithCustomPrompt` from LLM client
  - Drop `CreateConciseContext` and rough‑estimate fallback from
    processor
  - Remove token counting for chat messages (`CountMessagesTokens`,
    `Message` type)
  - Delete TUI command helpers (`Step`, `StepWithCounts`, `Error`)
  - Remove style functions (`Bold`, `Yellow`, `Highlight`,
    `ParseAndStyleURLs`, etc.)
- Add explanatory comments for concurrency limit and content extraction
  heuristics
- Update dependencies (`golang.org/x/net`, `sys`, `term`, `text`) and
  add `golang.org/x/tools/cmd/deadcode` tool directive
- Simplify `aggregator_test.go` to use `NewFeedAggregatorWithProgress`
  exclusively
- Remove obsolete test cases (`TestNewFeedAggregator`,
  `TestDefaultConfig`, `TestConfigLoadWithVariousSources`, etc.)
- Add `NOTE` comments in `config.go` about Viper singleton behaviour and
  `isZero` semantics
Maxwell Jensen 85795372+maxwelljens@users.noreply.github.com
Thu, 23 Apr 2026 15:54:10 +0200
commit

b4f6beb3ba126b91482bcd5df0745445088d51d7

parent

26eb79c063daa25ab326497558d2b3bf17a675bd

M README.mdREADME.md

@@ -30,7 +30,7 @@ (text, markdown, or JSON).

### Basic Options - -f, --feeds-file FILE Path to file containing RSS feed URLs (one per line) [required] + -f, --feeds-file FILE Path to file containing RSS feed URLs (one per line) [required] -p, --prompt PROMPT User prompt for summarisation/analysis [required] --api-key KEY API key (default: read from $LLM_AGGREGATOR_API_KEY) -m, --model MODEL Model to use (default: deepseek-chat)
M go.modgo.mod

@@ -5,11 +5,14 @@

require golang.org/x/sync v0.20.0 require ( + golang.org/x/mod v0.35.0 // indirect + golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa // indirect + golang.org/x/tools v0.44.0 // indirect +) + +require ( charm.land/glamour/v2 v2.0.0 // indirect charm.land/lipgloss/v2 v2.0.0 // indirect - github.com/charmbracelet/ultraviolet v0.0.0-20251205161215-1948445e3318 // indirect - github.com/charmbracelet/x/termios v0.1.1 // indirect - github.com/charmbracelet/x/windows v0.2.2 // indirect github.com/PuerkitoBio/goquery v1.8.0 github.com/alecthomas/chroma/v2 v2.20.0 // indirect github.com/alexflint/go-arg v1.6.1

@@ -23,10 +26,13 @@ github.com/charmbracelet/colorprofile v0.4.2 // indirect

github.com/charmbracelet/glamour v1.0.0 // indirect github.com/charmbracelet/harmonica v0.2.0 // indirect github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 + github.com/charmbracelet/ultraviolet v0.0.0-20251205161215-1948445e3318 // indirect github.com/charmbracelet/x/ansi v0.11.6 // indirect github.com/charmbracelet/x/cellbuf v0.0.15 // indirect github.com/charmbracelet/x/exp/slice v0.0.0-20250327172914-2fdc97757edf // indirect github.com/charmbracelet/x/term v0.2.2 // indirect + github.com/charmbracelet/x/termios v0.1.1 // indirect + github.com/charmbracelet/x/windows v0.2.2 // indirect github.com/clipperhouse/displaywidth v0.11.0 // indirect github.com/clipperhouse/stringish v0.1.1 // indirect github.com/clipperhouse/uax29/v2 v2.7.0 // indirect

@@ -71,8 +77,10 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect

github.com/yuin/goldmark v1.7.13 // indirect github.com/yuin/goldmark-emoji v1.0.6 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/net v0.48.0 // indirect - golang.org/x/sys v0.41.0 // indirect - golang.org/x/term v0.38.0 // indirect - golang.org/x/text v0.32.0 // indirect + golang.org/x/net v0.53.0 // indirect + golang.org/x/sys v0.43.0 // indirect + golang.org/x/term v0.42.0 // indirect + golang.org/x/text v0.36.0 // indirect ) + +tool golang.org/x/tools/cmd/deadcode
M go.sumgo.sum

@@ -167,9 +167,13 @@ go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=

go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= +golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= +golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

@@ -180,13 +184,23 @@ golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=

golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa h1:efT73AJZfAAUV7SOip6pWGkwJDzIGiKBZGVzHYa+ve4= +golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa/go.mod h1:kHjTxDEnAu6/Nl9lDkzjWpR+bmKfxeiRuSDlsMb70gE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= +golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= +golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= +golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
M internal/aggregator/aggregator.gointernal/aggregator/aggregator.go

@@ -27,11 +27,6 @@ userAgent string

progressCtx *progress.Context } -// NewFeedAggregator creates a new FeedAggregator with the specified options. -func NewFeedAggregator(maxArticlesPerFeed, maxDaysOld, maxContentLength int) *FeedAggregator { - return NewFeedAggregatorWithProgress(maxArticlesPerFeed, maxDaysOld, maxContentLength, nil) -} - // NewFeedAggregatorWithProgress creates a new FeedAggregator with progress context. func NewFeedAggregatorWithProgress(maxArticlesPerFeed, maxDaysOld, maxContentLength int, progressCtx *progress.Context) *FeedAggregator { return &FeedAggregator{

@@ -79,6 +74,8 @@ var allArticles []*Article

var feedErrors []string // Limit concurrency to avoid overwhelming servers + // NOTE: maxConcurrency is hardcoded to 10. Changing this requires a code + // modification. const maxConcurrency = 10 sem := make(chan struct{}, maxConcurrency)

@@ -240,6 +237,9 @@ }

func (fa *FeedAggregator) extractContent(item *gofeed.Item, link string) string { // Priority: Content -> Description -> Fetch webpage + // NOTE: Webpage scraping is only attempted when content is empty or <100 + // chars. This balances content quality against latency (webpage fetch adds + // ~1-2s per article). content := "" // Try to get content from item
M internal/aggregator/aggregator_test.gointernal/aggregator/aggregator_test.go

@@ -57,33 +57,6 @@ </item>

</channel> </rss>` -func TestNewFeedAggregator(t *testing.T) { - tests := []struct { - name string - maxArticlesPerFeed int - maxDaysOld int - maxContentLength int - wantNil bool - }{ - {"standard config", 10, 7, 5000, false}, - {"zero max articles", 0, 7, 5000, false}, - {"zero days old", 10, 0, 5000, false}, - {"all zeros", 0, 0, 0, false}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fa := NewFeedAggregator(tt.maxArticlesPerFeed, tt.maxDaysOld, tt.maxContentLength) - if tt.wantNil && fa != nil { - t.Error("Expected nil FeedAggregator") - } - if !tt.wantNil && fa == nil { - t.Error("Expected non-nil FeedAggregator") - } - }) - } -} - func TestNewFeedAggregatorWithProgress(t *testing.T) { t.Run("with nil progress context", func(t *testing.T) { fa := NewFeedAggregatorWithProgress(10, 7, 5000, nil)

@@ -97,7 +70,7 @@ })

} func TestParseFeedsFromFileNotFound(t *testing.T) { - fa := NewFeedAggregator(10, 7, 5000) + fa := NewFeedAggregatorWithProgress(10, 7, 5000, nil) _, err := fa.ParseFeedsFromFile("/nonexistent/path/to/feeds.txt") if err == nil {

@@ -117,7 +90,7 @@ if err := os.WriteFile(tmpFile, []byte(""), 0644); err != nil {

t.Fatalf("Failed to create test file: %v", err) } - fa := NewFeedAggregator(10, 7, 5000) + fa := NewFeedAggregatorWithProgress(10, 7, 5000, nil) articles, err := fa.ParseFeedsFromFile(tmpFile) // Empty file should not cause an error

@@ -134,7 +107,7 @@ if err := os.WriteFile(tmpFile, []byte("# Comment line\n# Another comment\n"), 0644); err != nil {

t.Fatalf("Failed to create test file: %v", err) } - fa := NewFeedAggregator(10, 7, 5000) + fa := NewFeedAggregatorWithProgress(10, 7, 5000, nil) articles, err := fa.ParseFeedsFromFile(tmpFile) if err != nil {

@@ -151,7 +124,7 @@ if err := os.WriteFile(tmpFile, []byte(content), 0644); err != nil {

t.Fatalf("Failed to create test file: %v", err) } - fa := NewFeedAggregator(10, 7, 5000) + fa := NewFeedAggregatorWithProgress(10, 7, 5000, nil) // This will fail to fetch but shouldn't crash _, err := fa.ParseFeedsFromFile(tmpFile)
M internal/cli/args.gointernal/cli/args.go

@@ -104,6 +104,11 @@ }

// ToViperMap converts Args to a map for binding to Viper. // Only non-nil values (explicitly provided CLI flags) are included. +// +// NOTE: FeedsFile and Prompt are strings (not pointers), so they're ALWAYS +// included if non-empty. This differs from optional fields which use pointer +// types to detect explicit provision vs. default zero values. See isZero() in +// config package. func (a *Args) ToViperMap() map[string]any { m := map[string]any{} if a.FeedsFile != "" {
M internal/cli/help.gointernal/cli/help.go

@@ -53,44 +53,6 @@ }

return "" } -func buildOptionLine(opt HelpOption, flagWidth int) string { - flagStr := renderFlag(opt.Short, opt.Name) - padding := max(flagWidth-lipgloss.Width(flagStr), 0) - - var line strings.Builder - - if shouldStyle() { - line.WriteString(flagStyle.Render(flagStr)) - line.WriteString(strings.Repeat(" ", padding+2)) - } else { - line.WriteString(flagStr) - line.WriteString(strings.Repeat(" ", padding+2)) - } - - value := getOptionValue(opt) - if value != "" { - if shouldStyle() { - line.WriteString(flagStyle.Render(value)) - line.WriteString(" ") - } else { - line.WriteString(value) - line.WriteString(" ") - } - } - - if opt.Required { - if shouldStyle() { - line.WriteString("[required] ") - } else { - line.WriteString("[required] ") - } - } - - line.WriteString(opt.Description) - - return line.String() -} - func maxFlagWidth(sections []HelpSection) int { maxWidth := 0 for _, section := range sections {

@@ -140,18 +102,6 @@ b.WriteString("\n")

} return b.String() -} - -var descriptionStyle = lipgloss.NewStyle() - -func getTerminalWidth() int { - if termWidth := os.Getenv("COLUMNS"); termWidth != "" { - var width int - if _, err := fmt.Sscanf(termWidth, "%d", &width); err == nil && width > 0 { - return width - } - } - return 80 } func BuildStyledHelp(args *Args) string {
M internal/config/config.gointernal/config/config.go

@@ -39,59 +39,31 @@ OutputFile string `mapstructure:"output_file"`

IncludeArticles bool `mapstructure:"include_articles"` } -// DefaultConfig returns a Config with sensible defaults. -func DefaultConfig() *Config { - return &Config{ - MaxArticlesPerFeed: defaults.DefaultMaxArticlesPerFeed, - MaxDaysOld: defaults.DefaultMaxDaysOld, - MaxTotalArticles: defaults.DefaultMaxTotalArticles, - Model: defaults.DefaultModel, - BaseURL: defaults.DefaultBaseURL, - MaxTokens: defaults.DefaultMaxTokens, - Temperature: defaults.DefaultTemperature, - SystemPrompt: defaults.DefaultSystemPrompt, - Output: defaults.DefaultOutput, - IncludeArticles: defaults.DefaultIncludeArticles, - } -} - -// Load loads configuration from TOML file, environment variables, and sets defaults. -// The precedence order is: CLI args > Environment variables > Config file > Defaults. -func Load() (*Config, error) { - // Get the global viper instance which handles precedence automatically - v := GetViper() - - // Return the current configuration as a Config struct - return ViperToConfig(v), nil -} - // GetViper returns the global viper instance with all configuration sources set up. // The precedence order is: CLI flags > Environment variables > Config file > Defaults. +// +// NOTE: Viper uses a singleton pattern. Subsequent calls to GetViper() return the SAME +// instance with all previously set values. This means defaults are only set once, +// environment vars are bound once, and the config file is read once. Do NOT call this +// expecting a fresh state — if you need a fresh instance for testing, use viper.New() directly. func GetViper() *viper.Viper { // Set defaults first (lowest priority) setDefaults() - // Initialise or get existing Viper instance v := viper.GetViper() - // Ensure defaults are set (for fresh instances) setDefaultsOn(v) - // Set environment variable prefix v.SetEnvPrefix("LLM_AGGREGATOR") v.AutomaticEnv() - // Bind environment variables to config fields bindEnvVars(v) - // Get config path from XDG configPath, err := GetConfigPath() if err == nil { // Set config file path v.SetConfigFile(configPath) - // Debug: show what config file is being used - // Try to read config file if err := v.ReadInConfig(); err != nil { // If config file doesn't exist, that's OK - we'll use defaults + env vars

@@ -102,7 +74,6 @@ }

} else { } } - return v }

@@ -118,41 +89,18 @@ // Feed aggregation defaults

v.SetDefault("max_articles_per_feed", defaults.DefaultMaxArticlesPerFeed) v.SetDefault("max_days_old", defaults.DefaultMaxDaysOld) v.SetDefault("max_total_articles", defaults.DefaultMaxTotalArticles) - // LLM API defaults v.SetDefault("base_url", defaults.DefaultBaseURL) v.SetDefault("model", defaults.DefaultModel) v.SetDefault("max_tokens", defaults.DefaultMaxTokens) v.SetDefault("temperature", defaults.DefaultTemperature) - // System prompt default v.SetDefault("system_prompt", defaults.DefaultSystemPrompt) - // Output defaults v.SetDefault("output", defaults.DefaultOutput) v.SetDefault("include_articles", defaults.DefaultIncludeArticles) } -// ViperToConfig converts viper configuration to a Config struct. -func ViperToConfig(v *viper.Viper) *Config { - return &Config{ - MaxArticlesPerFeed: v.GetInt("max_articles_per_feed"), - MaxDaysOld: v.GetInt("max_days_old"), - MaxTotalArticles: v.GetInt("max_total_articles"), - IncludeKeywords: v.GetString("include_keywords"), - ExcludeKeywords: v.GetString("exclude_keywords"), - APIKey: v.GetString("api_key"), - BaseURL: v.GetString("base_url"), - Model: v.GetString("model"), - MaxTokens: v.GetInt("max_tokens"), - Temperature: v.GetFloat64("temperature"), - SystemPrompt: v.GetString("system_prompt"), - Output: v.GetString("output"), - OutputFile: v.GetString("output_file"), - IncludeArticles: v.GetBool("include_articles"), - } -} - // BindCLIArgs binds CLI argument values to viper with highest precedence. // This should be called after parsing CLI arguments to override config file/env defaults. func BindCLIArgs(v *viper.Viper, args map[string]any) {

@@ -164,6 +112,12 @@ }

} // isZero checks if a value is the zero value for its type. +// +// NOTE: This function is critical for CLI argument handling. Pointer types +// (*string, *int, etc.) return true only when nil (not provided), NOT when +// pointing to a zero value. This allows distinguishing between "flag not +// provided" (nil) vs "flag explicitly set to zero" (e.g., --temperature 0 +// should override default, but no flag means preserve default). func isZero(v any) bool { // Handle nil interface values first (before type switch) if v == nil {

@@ -218,12 +172,10 @@ FeedsFile: feedsFile,

Prompt: prompt, Progress: &progress.NoopLogger{}, } - // Feed aggregation options rt.MaxArticlesPerFeed = v.GetInt("max_articles_per_feed") rt.MaxDaysOld = v.GetInt("max_days_old") rt.MaxTotalArticles = v.GetInt("max_total_articles") - // Keywords includeKeywords := v.GetString("include_keywords") excludeKeywords := v.GetString("exclude_keywords")

@@ -233,22 +185,18 @@ }

if excludeKeywords != "" { rt.ExcludeKeywords = cli.ParseKeywords(excludeKeywords) } - // LLM API options rt.APIKey = v.GetString("api_key") rt.BaseURL = v.GetString("base_url") rt.Model = v.GetString("model") rt.MaxTokens = v.GetInt("max_tokens") rt.Temperature = v.GetFloat64("temperature") - // System prompt rt.SystemPrompt = v.GetString("system_prompt") - // Output options rt.Output = v.GetString("output") rt.OutputFile = v.GetString("output_file") rt.IncludeArticles = v.GetBool("include_articles") - return rt }

@@ -260,17 +208,14 @@ v.BindEnv("max_days_old", "LLM_AGGREGATOR_MAX_DAYS_OLD")

v.BindEnv("max_total_articles", "LLM_AGGREGATOR_MAX_TOTAL_ARTICLES") v.BindEnv("include_keywords", "LLM_AGGREGATOR_INCLUDE_KEYWORDS") v.BindEnv("exclude_keywords", "LLM_AGGREGATOR_EXCLUDE_KEYWORDS") - // LLM API options v.BindEnv("api_key", "LLM_AGGREGATOR_API_KEY") v.BindEnv("base_url", "LLM_AGGREGATOR_BASE_URL") v.BindEnv("model", "LLM_AGGREGATOR_MODEL") v.BindEnv("max_tokens", "LLM_AGGREGATOR_MAX_TOKENS") v.BindEnv("temperature", "LLM_AGGREGATOR_TEMPERATURE") - // System prompt v.BindEnv("system_prompt", "LLM_AGGREGATOR_SYSTEM_PROMPT") - // Output options v.BindEnv("output", "LLM_AGGREGATOR_OUTPUT") v.BindEnv("output_file", "LLM_AGGREGATOR_OUTPUT_FILE")

@@ -285,71 +230,3 @@ return "", fmt.Errorf("failed to get user config directory: %w", err)

} return filepath.Join(configDir, "llm_aggregator", "config.toml"), nil } - -// ConfigExists returns true if a config file exists. -func ConfigExists() (bool, error) { - configPath, err := GetConfigPath() - if err != nil { - return false, err - } - _, err = os.Stat(configPath) - if err != nil { - if os.IsNotExist(err) { - return false, nil - } - return false, err - } - return true, nil -} - -// Save saves the configuration to the TOML file. -func (c *Config) Save() error { - v := viper.New() - v.SetConfigType("toml") - - // Set values from config struct - v.Set("max_articles_per_feed", c.MaxArticlesPerFeed) - v.Set("max_days_old", c.MaxDaysOld) - v.Set("max_total_articles", c.MaxTotalArticles) - v.Set("include_keywords", c.IncludeKeywords) - v.Set("exclude_keywords", c.ExcludeKeywords) - v.Set("api_key", c.APIKey) - v.Set("base_url", c.BaseURL) - v.Set("model", c.Model) - v.Set("max_tokens", c.MaxTokens) - v.Set("temperature", c.Temperature) - v.Set("system_prompt", c.SystemPrompt) - v.Set("output", c.Output) - v.Set("output_file", c.OutputFile) - v.Set("include_articles", c.IncludeArticles) - - // Get config path - configPath, err := GetConfigPath() - if err != nil { - return fmt.Errorf("failed to get config path: %w", err) - } - - // Create directory if it doesn't exist - configDir := filepath.Dir(configPath) - if err := os.MkdirAll(configDir, 0755); err != nil { - return fmt.Errorf("failed to create config directory: %w", err) - } - - // Write config file - if err := v.WriteConfigAs(configPath); err != nil { - return fmt.Errorf("failed to write config file: %w", err) - } - - return nil -} - -// IsLoaded returns true if the configuration was loaded from a file. -func (c *Config) IsLoaded() bool { - // Check if config file exists - if it does, values would have been loaded from it - configPath, err := GetConfigPath() - if err != nil { - return false - } - _, err = os.Stat(configPath) - return err == nil -}
M internal/config/config_test.gointernal/config/config_test.go

@@ -1,6 +1,7 @@

package config import ( + "fmt" "os" "path/filepath" "strings"

@@ -9,205 +10,8 @@

"github.com/spf13/viper" "llm_aggregator/internal/cli" - "llm_aggregator/internal/defaults" ) -func TestDefaultConfig(t *testing.T) { - cfg := DefaultConfig() - - if cfg.MaxArticlesPerFeed != 10 { - t.Errorf("Expected MaxArticlesPerFeed = 10, got %d", cfg.MaxArticlesPerFeed) - } - - if cfg.MaxDaysOld != 7 { - t.Errorf("Expected MaxDaysOld = 7, got %d", cfg.MaxDaysOld) - } - - if cfg.MaxTotalArticles != 20 { - t.Errorf("Expected MaxTotalArticles = 20, got %d", cfg.MaxTotalArticles) - } - - if cfg.Model != "deepseek-chat" { - t.Errorf("Expected Model = 'deepseek-chat', got %s", cfg.Model) - } - - if cfg.MaxTokens != 4000 { - t.Errorf("Expected MaxTokens = 4000, got %d", cfg.MaxTokens) - } - - if cfg.Temperature != 0.7 { - t.Errorf("Expected Temperature = 0.7, got %f", cfg.Temperature) - } - - expectedPrompt := `You are an expert analyst and summariser. -You analyse content from multiple sources and provide -concise, insightful summaries based on user requests. -Focus on key points, trends, and important information.` - - if cfg.SystemPrompt != expectedPrompt { - t.Errorf("SystemPrompt doesn't match expected default") - } - - if cfg.Output != "text" { - t.Errorf("Expected Output = 'text', got %s", cfg.Output) - } -} - -func TestConfigPath(t *testing.T) { - path, err := GetConfigPath() - if err != nil { - t.Fatalf("Failed to get config path: %v", err) - } - - // Should end with .config/llm_aggregator/config.toml - expectedSuffix := filepath.Join(".config", "llm_aggregator", "config.toml") - if !hasSuffix(path, expectedSuffix) { - t.Errorf("Config path %s doesn't end with %s", path, expectedSuffix) - } -} - -func TestConfigSaveAndLoad(t *testing.T) { - // Create temporary directory for test - tempDir := t.TempDir() - oldEnv := os.Getenv("XDG_CONFIG_HOME") - os.Setenv("XDG_CONFIG_HOME", tempDir) - defer os.Setenv("XDG_CONFIG_HOME", oldEnv) - - cfg := DefaultConfig() - cfg.SystemPrompt = "Test system prompt" - cfg.Model = "test-model" - cfg.MaxTokens = 1234 - - // Save config - if err := cfg.Save(); err != nil { - t.Fatalf("Failed to save config: %v", err) - } - - // Verify file exists - configPath, err := GetConfigPath() - if err != nil { - t.Fatalf("Failed to get config path: %v", err) - } - - if _, err := os.Stat(configPath); os.IsNotExist(err) { - t.Fatalf("Config file not created at %s", configPath) - } - - // Load config - loadedCfg, err := Load() - if err != nil { - t.Fatalf("Failed to load config: %v", err) - } - - if !loadedCfg.IsLoaded() { - t.Error("Config should be marked as loaded") - } - - if loadedCfg.SystemPrompt != cfg.SystemPrompt { - t.Errorf("SystemPrompt mismatch: expected %q, got %q", cfg.SystemPrompt, loadedCfg.SystemPrompt) - } - - if loadedCfg.Model != cfg.Model { - t.Errorf("Model mismatch: expected %s, got %s", cfg.Model, loadedCfg.Model) - } - - if loadedCfg.MaxTokens != cfg.MaxTokens { - t.Errorf("MaxTokens mismatch: expected %d, got %d", cfg.MaxTokens, loadedCfg.MaxTokens) - } -} - -func TestEnvironmentVariables(t *testing.T) { - // Create temporary directory for test - tempDir := t.TempDir() - oldEnv := os.Getenv("XDG_CONFIG_HOME") - os.Setenv("XDG_CONFIG_HOME", tempDir) - defer os.Setenv("XDG_CONFIG_HOME", oldEnv) - - // Set environment variables - os.Setenv("LLM_AGGREGATOR_SYSTEM_PROMPT", "Env system prompt") - os.Setenv("LLM_AGGREGATOR_MODEL", "env-model") - os.Setenv("LLM_AGGREGATOR_MAX_TOKENS", "9999") - defer func() { - os.Unsetenv("LLM_AGGREGATOR_SYSTEM_PROMPT") - os.Unsetenv("LLM_AGGREGATOR_MODEL") - os.Unsetenv("LLM_AGGREGATOR_MAX_TOKENS") - }() - - // Load config (should pick up env vars) - cfg, err := Load() - if err != nil { - t.Fatalf("Failed to load config: %v", err) - } - - if cfg.SystemPrompt != "Env system prompt" { - t.Errorf("SystemPrompt should be from env var, got %q", cfg.SystemPrompt) - } - - if cfg.Model != "env-model" { - t.Errorf("Model should be from env var, got %s", cfg.Model) - } - - if cfg.MaxTokens != 9999 { - t.Errorf("MaxTokens should be from env var, got %d", cfg.MaxTokens) - } -} - -func TestConfigExists(t *testing.T) { - // Create temporary directory for test - tempDir := t.TempDir() - oldEnv := os.Getenv("XDG_CONFIG_HOME") - os.Setenv("XDG_CONFIG_HOME", tempDir) - defer os.Setenv("XDG_CONFIG_HOME", oldEnv) - - // Get config path to verify - configPath, err := GetConfigPath() - if err != nil { - t.Fatalf("Failed to get config path: %v", err) - } - - // Ensure it's in our temp directory - if !contains(configPath, tempDir) { - t.Logf("Config path %s not in temp dir %s", configPath, tempDir) - } - - // Initially should not exist - exists, err := ConfigExists() - if err != nil { - t.Fatalf("Failed to check config existence: %v", err) - } - - if exists { - // Config exists, maybe from previous test run - t.Logf("Config exists at %s, removing...", configPath) - os.Remove(configPath) - // Check again - exists, err = ConfigExists() - if err != nil { - t.Fatalf("Failed to check config existence after removal: %v", err) - } - if exists { - t.Error("Config should not exist after removal") - } - } - - // Create config - cfg := DefaultConfig() - if err := cfg.Save(); err != nil { - t.Fatalf("Failed to save config: %v", err) - } - - // Now should exist - exists, err = ConfigExists() - if err != nil { - t.Fatalf("Failed to check config existence: %v", err) - } - - if !exists { - t.Error("Config should exist after saving") - } -} - -// TestIsZero tests the isZero helper function with various types including pointers. func TestIsZero(t *testing.T) { tests := []struct { name string

@@ -265,19 +69,22 @@ defer func() {

os.Setenv("XDG_CONFIG_HOME", oldEnv) }() - // Save a config file with known values + // Save a config file with known values (manually write TOML since DefaultConfig/Save don't exist) configuredModel := "config-file-model" configuredBaseURL := "https://config-file.example.com" configuredTemperature := 0.3 + configuredMaxTokens := 5000 - cfg := DefaultConfig() - cfg.Model = configuredModel - cfg.BaseURL = configuredBaseURL - cfg.Temperature = configuredTemperature - cfg.MaxTokens = 5000 + configContent := fmt.Sprintf(`model = "%s" +base_url = "%s" +temperature = %f +max_tokens = %d +`, configuredModel, configuredBaseURL, configuredTemperature, configuredMaxTokens) - if err := cfg.Save(); err != nil { - t.Fatalf("Failed to save config: %v", err) + configPath := filepath.Join(tempDir, "llm_aggregator", "config.toml") + os.MkdirAll(filepath.Dir(configPath), 0755) + if err := os.WriteFile(configPath, []byte(configContent), 0644); err != nil { + t.Fatalf("Failed to write config file: %v", err) } // Test 1: Config file should override defaults

@@ -521,7 +328,7 @@ os.Setenv("XDG_CONFIG_HOME", oldEnv)

}() // Create a test feeds file - feedsFile := filepath.Join(tempDir, "feeds.txt") + feedsFile := tempDir + "/feeds.txt" if err := os.WriteFile(feedsFile, []byte("https://example.com/feed.xml\nhttps://example.org/feed.xml"), 0644); err != nil { t.Fatalf("Failed to create test feeds file: %v", err) }

@@ -628,167 +435,6 @@ t.Errorf("APIKey = %q, want %q", *parsedArgs.APIKey, apiKey)

} } }) - } -} - -// TestConfigLoadWithVariousSources tests that configuration loading works -// regardless of the source (CLI, env, config file, defaults) -func TestConfigLoadWithVariousSources(t *testing.T) { - // Create temporary directory for test - tempDir := t.TempDir() - oldEnv := os.Getenv("XDG_CONFIG_HOME") - os.Setenv("XDG_CONFIG_HOME", tempDir) - defer func() { - os.Setenv("XDG_CONFIG_HOME", oldEnv) - }() - - // Test 1: Config loads with saved config file - t.Run("load with saved config file", func(t *testing.T) { - cfg := DefaultConfig() - cfg.Model = "saved-model" - cfg.MaxArticlesPerFeed = 30 - cfg.Temperature = 0.9 - - if err := cfg.Save(); err != nil { - t.Fatalf("Save() failed: %v", err) - } - - loadedCfg, err := Load() - if err != nil { - t.Fatalf("Load() failed: %v", err) - } - - if loadedCfg.Model != "saved-model" { - t.Errorf("Model = %q, want %q", loadedCfg.Model, "saved-model") - } - if loadedCfg.MaxArticlesPerFeed != 30 { - t.Errorf("MaxArticlesPerFeed = %d, want 30", loadedCfg.MaxArticlesPerFeed) - } - if loadedCfg.Temperature != 0.9 { - t.Errorf("Temperature = %f, want 0.9", loadedCfg.Temperature) - } - }) - - // Test 2: Verify DefaultConfig returns correct defaults - t.Run("default config has correct values", func(t *testing.T) { - cfg := DefaultConfig() - - if cfg.MaxArticlesPerFeed != defaults.DefaultMaxArticlesPerFeed { - t.Errorf("MaxArticlesPerFeed = %d, want %d", cfg.MaxArticlesPerFeed, defaults.DefaultMaxArticlesPerFeed) - } - if cfg.MaxDaysOld != defaults.DefaultMaxDaysOld { - t.Errorf("MaxDaysOld = %d, want %d", cfg.MaxDaysOld, defaults.DefaultMaxDaysOld) - } - if cfg.MaxTotalArticles != defaults.DefaultMaxTotalArticles { - t.Errorf("MaxTotalArticles = %d, want %d", cfg.MaxTotalArticles, defaults.DefaultMaxTotalArticles) - } - if cfg.BaseURL != defaults.DefaultBaseURL { - t.Errorf("BaseURL = %q, want %q", cfg.BaseURL, defaults.DefaultBaseURL) - } - if cfg.Model != defaults.DefaultModel { - t.Errorf("Model = %q, want %q", cfg.Model, defaults.DefaultModel) - } - if cfg.MaxTokens != defaults.DefaultMaxTokens { - t.Errorf("MaxTokens = %d, want %d", cfg.MaxTokens, defaults.DefaultMaxTokens) - } - if cfg.Temperature != defaults.DefaultTemperature { - t.Errorf("Temperature = %f, want %f", cfg.Temperature, defaults.DefaultTemperature) - } - if cfg.Output != defaults.DefaultOutput { - t.Errorf("Output = %q, want %q", cfg.Output, defaults.DefaultOutput) - } - if cfg.IncludeArticles != defaults.DefaultIncludeArticles { - t.Errorf("IncludeArticles = %v, want %v", cfg.IncludeArticles, defaults.DefaultIncludeArticles) - } - }) - - // Test 3: Verify ConfigExists works correctly - t.Run("config exists check", func(t *testing.T) { - // Should not exist initially (new temp dir) - exists, err := ConfigExists() - if err != nil { - t.Fatalf("ConfigExists() failed: %v", err) - } - - // Create a config - cfg := DefaultConfig() - if err := cfg.Save(); err != nil { - t.Fatalf("Save() failed: %v", err) - } - - // Now should exist - exists, err = ConfigExists() - if err != nil { - t.Fatalf("ConfigExists() failed after save: %v", err) - } - if !exists { - t.Error("ConfigExists() should return true after saving") - } - }) -} - -// TestViperToConfigConversion tests the ViperToConfig conversion function. -func TestViperToConfigConversion(t *testing.T) { - // Create a fresh viper instance - v := viper.New() - v.Set("max_articles_per_feed", 15) - v.Set("max_days_old", 14) - v.Set("max_total_articles", 50) - v.Set("include_keywords", "ai,ml") - v.Set("exclude_keywords", "spam") - v.Set("api_key", "sk-test-key") - v.Set("base_url", "https://api.example.com") - v.Set("model", "test-model") - v.Set("max_tokens", 3000) - v.Set("temperature", 0.5) - v.Set("system_prompt", "Test system prompt") - v.Set("output", "markdown") - v.Set("output_file", "/tmp/output.md") - v.Set("include_articles", true) - - cfg := ViperToConfig(v) - - if cfg.MaxArticlesPerFeed != 15 { - t.Errorf("MaxArticlesPerFeed = %d, want 15", cfg.MaxArticlesPerFeed) - } - if cfg.MaxDaysOld != 14 { - t.Errorf("MaxDaysOld = %d, want 14", cfg.MaxDaysOld) - } - if cfg.MaxTotalArticles != 50 { - t.Errorf("MaxTotalArticles = %d, want 50", cfg.MaxTotalArticles) - } - if cfg.IncludeKeywords != "ai,ml" { - t.Errorf("IncludeKeywords = %q, want %q", cfg.IncludeKeywords, "ai,ml") - } - if cfg.ExcludeKeywords != "spam" { - t.Errorf("ExcludeKeywords = %q, want %q", cfg.ExcludeKeywords, "spam") - } - if cfg.APIKey != "sk-test-key" { - t.Errorf("APIKey = %q, want %q", cfg.APIKey, "sk-test-key") - } - if cfg.BaseURL != "https://api.example.com" { - t.Errorf("BaseURL = %q, want %q", cfg.BaseURL, "https://api.example.com") - } - if cfg.Model != "test-model" { - t.Errorf("Model = %q, want %q", cfg.Model, "test-model") - } - if cfg.MaxTokens != 3000 { - t.Errorf("MaxTokens = %d, want 3000", cfg.MaxTokens) - } - if cfg.Temperature != 0.5 { - t.Errorf("Temperature = %f, want 0.5", cfg.Temperature) - } - if cfg.SystemPrompt != "Test system prompt" { - t.Errorf("SystemPrompt = %q, want %q", cfg.SystemPrompt, "Test system prompt") - } - if cfg.Output != "markdown" { - t.Errorf("Output = %q, want %q", cfg.Output, "markdown") - } - if cfg.OutputFile != "/tmp/output.md" { - t.Errorf("OutputFile = %q, want %q", cfg.OutputFile, "/tmp/output.md") - } - if cfg.IncludeArticles != true { - t.Errorf("IncludeArticles = %v, want true", cfg.IncludeArticles) } }
M internal/llm/llm.gointernal/llm/llm.go

@@ -229,24 +229,3 @@ }

return outputText, nil } - -// AnalyseWithCustomPrompt analyses articles with custom system and user prompts. -func (dc *LLMClient) AnalyseWithCustomPrompt( - articles []map[string]any, - systemPrompt string, - userPrompt string, -) (string, error) { - context := dc.prepareContext(articles) - - // Create messages for chat completion API - messages := []openai.ChatCompletionMessageParamUnion{ - openai.SystemMessage(systemPrompt), - openai.UserMessage(fmt.Sprintf(`Here are articles from various RSS feeds: - -%s - -%s`, context, userPrompt)), - } - - return dc.callAPIWithMessages(messages) -}
M internal/output/formatter.gointernal/output/formatter.go

@@ -5,8 +5,6 @@ "encoding/json"

"fmt" "strings" "time" - - "llm_aggregator/internal/aggregator" ) // Formatter formats output in different formats.

@@ -224,29 +222,3 @@ }

padding := (width - len(text)) / 2 return strings.Repeat(" ", padding) + text + strings.Repeat(" ", width-padding-len(text)) } - -// FormatArticleList formats a list of articles. -func FormatArticleList(articles []*aggregator.Article, formatType string, includeContent bool) (string, error) { - // Convert articles to maps - articleMaps := make([]map[string]any, len(articles)) - for i, article := range articles { - articleMap := article.ToMap() - if !includeContent { - delete(articleMap, "content") - } - articleMaps[i] = articleMap - } - - formatter, err := NewFormatter(formatType) - if err != nil { - return "", err - } - - data := map[string]any{ - "title": fmt.Sprintf("Articles List - %d articles", len(articles)), - "articles": articleMaps, - "articles_count": len(articles), - } - - return formatter.FormatData(data) -}
M internal/output/formatter_test.gointernal/output/formatter_test.go

@@ -5,8 +5,6 @@ "encoding/json"

"strings" "testing" "time" - - "llm_aggregator/internal/aggregator" ) func TestNewFormatter(t *testing.T) {

@@ -327,58 +325,6 @@ t.Errorf("centerText(%q, %d) = %q, want %q", tt.text, tt.width, got, tt.expect)

} }) } -} - -func TestFormatArticleList(t *testing.T) { - articles := []*aggregator.Article{ - { - Title: "Test Article", - Link: "https://example.com/test", - Content: "Article content here", - SourceFeed: "Test Feed", - Author: "Test Author", - }, - } - - t.Run("text format", func(t *testing.T) { - output, err := FormatArticleList(articles, "text", false) - if err != nil { - t.Errorf("FormatArticleList failed: %v", err) - } - if !strings.Contains(output, "Test Article") { - t.Error("Expected article title in output") - } - }) - - t.Run("markdown format", func(t *testing.T) { - output, err := FormatArticleList(articles, "markdown", false) - if err != nil { - t.Errorf("FormatArticleList failed: %v", err) - } - if !strings.Contains(output, "Articles List") { - t.Error("Expected articles list header") - } - }) - - t.Run("json format", func(t *testing.T) { - output, err := FormatArticleList(articles, "json", false) - if err != nil { - t.Errorf("FormatArticleList failed: %v", err) - } - if !strings.Contains(output, "Test Article") { - t.Error("Expected article title in JSON") - } - }) - - t.Run("include content", func(t *testing.T) { - output, err := FormatArticleList(articles, "text", true) - if err != nil { - t.Errorf("FormatArticleList failed: %v", err) - } - if !strings.Contains(output, "Article content here") { - t.Error("Expected content in output when includeContent=true") - } - }) } func TestJSONIndentation(t *testing.T) {
M internal/processor/processor.gointernal/processor/processor.go

@@ -1,7 +1,6 @@

package processor import ( - "fmt" "sort" "strings" "time"

@@ -255,204 +254,3 @@ }

return totalTokens } - -// CreateConciseContext creates a concise context from articles, respecting token limits. -func (cp *ContentProcessor) CreateConciseContext(articles []map[string]any, maxTotalTokens int, model string) string { - if len(articles) == 0 { - return "No articles available." - } - - // Get the appropriate encoding for accurate token counting - encodingName, err := tokeniser.EncodingForModel(model) - if err != nil { - // Fallback to rough estimate if model not recognised - if cp.logger != nil { - cp.logger.Logf("Warning: unknown model %s, using rough token estimate", model) - } - return cp.createConciseContextRough(articles, maxTotalTokens) - } - - enc, err := tokeniser.GetEncoding(encodingName) - if err != nil { - if cp.logger != nil { - cp.logger.Logf("Warning: failed to get encoding: %v", err) - } - return cp.createConciseContextRough(articles, maxTotalTokens) - } - - contextParts := []string{} - currentTokens := 0 - - for i, article := range articles { - // Create article summary - articleText := fmt.Sprintf("Article %d: %s\n", i+1, article["title"]) - - // Add source if available - if source, ok := article["source_feed"].(string); ok && source != "" { - articleText += fmt.Sprintf("Source: %s\n", source) - } - - // Add publication date if available - if published, ok := article["published"]; ok { - switch pub := published.(type) { - case time.Time: - if !pub.IsZero() { - articleText += fmt.Sprintf("Published: %s\n", pub.Format("2006-01-02")) - } - case string: - articleText += fmt.Sprintf("Published: %s\n", pub) - } - } - - // Add content (we'll truncate based on actual tokens) - content := "" - if c, ok := article["content"].(string); ok { - content = c - } - - // First, check if we can add the header tokens - headerTokens := len(enc.Encode(articleText, nil, nil)) - - // Calculate remaining tokens for this article - remainingTokens := maxTotalTokens - currentTokens - headerTokens - 3 // -3 for separator - - if remainingTokens <= 0 { - if cp.logger != nil { - cp.logger.Logf( - "Reached token limit (%d). Included %d of %d articles.", - maxTotalTokens, i, len(articles), - ) - } - break - } - - // Convert token budget to approximate character limit (rough guide) - // Characters are typically 3-4x tokens for English - maxContentChars := remainingTokens * 4 - - if len(content) > maxContentChars { - // Need to encode and truncate more precisely - contentTokens := len(enc.Encode(content, nil, nil)) - if contentTokens > remainingTokens { - // Binary search for truncation point would be more accurate, - // but for performance, use the rough estimate - content = content[:maxContentChars] + "... [truncated]" - } - } - - articleText += fmt.Sprintf("Content: %s\n", content) - - // Count actual tokens for this article - articleTokens := len(enc.Encode(articleText, nil, nil)) - - // Double-check we haven't exceeded limit - if currentTokens+articleTokens > maxTotalTokens { - // Truncate content more aggressively - maxArticleChars := (maxTotalTokens - currentTokens - headerTokens - 20) * 4 - if maxArticleChars > 0 { - content = content[:maxArticleChars] + "... [truncated]" - articleText = fmt.Sprintf("Article %d: %s\n", i+1, article["title"]) - if source, ok := article["source_feed"].(string); ok && source != "" { - articleText += fmt.Sprintf("Source: %s\n", source) - } - articleText += fmt.Sprintf("Content: %s\n", content) - articleTokens = len(enc.Encode(articleText, nil, nil)) - } - - if currentTokens+articleTokens > maxTotalTokens { - if cp.logger != nil { - cp.logger.Logf( - "Reached token limit (%d). Included %d of %d articles.", - maxTotalTokens, i, len(articles), - ) - } - break - } - } - - contextParts = append(contextParts, articleText) - currentTokens += articleTokens - } - - context := strings.Join(contextParts, "\n---\n") - - if cp.logger != nil { - cp.logger.Logf("Created context with %d articles, ~%d tokens", len(contextParts), currentTokens) - } - - return context -} - -// createConciseContextRough creates a concise context using rough character-based estimation. -// Used as fallback when tiktoken is unavailable. -func (cp *ContentProcessor) createConciseContextRough(articles []map[string]any, maxTotalTokens int) string { - if len(articles) == 0 { - return "No articles available." - } - - contextParts := []string{} - currentTokens := 0 - - for i, article := range articles { - // Create article summary - articleText := fmt.Sprintf("Article %d: %s\n", i+1, article["title"]) - - // Add source if available - if source, ok := article["source_feed"].(string); ok && source != "" { - articleText += fmt.Sprintf("Source: %s\n", source) - } - - // Add publication date if available - if published, ok := article["published"]; ok { - switch pub := published.(type) { - case time.Time: - if !pub.IsZero() { - articleText += fmt.Sprintf("Published: %s\n", pub.Format("2006-01-02")) - } - case string: - articleText += fmt.Sprintf("Published: %s\n", pub) - } - } - - // Add content (truncated if needed) - content := "" - if c, ok := article["content"].(string); ok { - content = c - } - - // Rough allocation per article - maxContentTokens := maxTotalTokens / len(articles) - maxContentChars := maxContentTokens * 4 - - if len(content) > maxContentChars { - content = content[:maxContentChars] + "... [truncated]" - } - - articleText += fmt.Sprintf("Content: %s\n", content) - - // Rough estimate: 1 token ≈ 4 characters - articleTokens := len(articleText) / 4 - - // Check if adding this article would exceed limit - if currentTokens+articleTokens > maxTotalTokens { - if cp.logger != nil { - cp.logger.Logf( - "Reached token limit (%d). Included %d of %d articles.", - maxTotalTokens, i, len(articles), - ) - } - break - } - - contextParts = append(contextParts, articleText) - currentTokens += articleTokens - } - - context := strings.Join(contextParts, "\n---\n") - - if cp.logger != nil { - cp.logger.Logf("Created context with %d articles, ~%d tokens (rough estimate)", len(contextParts), currentTokens) - } - - return context -}
M internal/runtime/runtime.gointernal/runtime/runtime.go

@@ -8,7 +8,7 @@ "os"

"time" "llm_aggregator/internal/aggregator" - "llm_aggregator/internal/defaults" + "llm_aggregator/internal/llm" "llm_aggregator/internal/output" "llm_aggregator/internal/processor"

@@ -43,21 +43,6 @@ Error error

// Logger for verbose output Progress progress.Progress -} - -// NewRuntime creates a new runtime with default values -func NewRuntime() *Runtime { - return &Runtime{ - MaxArticlesPerFeed: defaults.DefaultMaxArticlesPerFeed, - MaxDaysOld: defaults.DefaultMaxDaysOld, - MaxTotalArticles: defaults.DefaultMaxTotalArticles, - Model: defaults.DefaultModel, - BaseURL: defaults.DefaultBaseURL, - MaxTokens: defaults.DefaultMaxTokens, - Temperature: defaults.DefaultTemperature, - Output: defaults.DefaultOutput, - Progress: &progress.NoopLogger{}, - } } // Execute runs the full aggregation pipeline
M internal/style/style.gointernal/style/style.go

@@ -5,15 +5,9 @@

import ( "fmt" "os" - "strings" "github.com/charmbracelet/lipgloss" ) - -// Bold returns a bold style. -func Bold(text string) string { - return lipgloss.NewStyle().Bold(true).Render(text) -} // Italic returns an italic style. func Italic(text string) string {

@@ -31,9 +25,6 @@ Red = Colour("1")

// Green for success messages Green = Colour("2") - - // Yellow for emphasis - Yellow = Colour("3") // Blue for URLs Blue = Colour("12")

@@ -101,11 +92,6 @@ Bold(true).

Render("✓ " + text) } -// Successf returns a formatted success message. -func Successf(format string, args ...any) string { - return Success(fmt.Sprintf(format, args...)) -} - // Filepath returns a styled filepath in cyan. func Filepath(filepath string) string { if NoColor() {

@@ -150,14 +136,6 @@ func Heading(text string) string {

return lipgloss.NewStyle().Bold(true).Render(text) } -// Section returns a styled section header. -func Section(text string) string { - return lipgloss.NewStyle(). - Bold(true). - Underline(true). - Render(text) -} - // Value returns styled value text. func Value(text string) string { if NoColor() {

@@ -177,11 +155,6 @@ }

return lipgloss.NewStyle().Foreground(lipgloss.Color(Cyan)).Render("ℹ " + text) } -// Infof returns a formatted info message. -func Infof(format string, args ...any) string { - return Info(fmt.Sprintf(format, args...)) -} - // Debug returns a styled debug message in grey. func Debug(text string) string { if NoColor() {

@@ -197,57 +170,3 @@ // Debugf returns a formatted debug message.

func Debugf(format string, args ...any) string { return Debug(fmt.Sprintf(format, args...)) } - -// Highlight returns a highlighted text (yellow bold). -func Highlight(text string) string { - if NoColor() { - return lipgloss.NewStyle().Bold(true).Render(text) - } - return lipgloss.NewStyle(). - Foreground(lipgloss.Color(Yellow)). - Bold(true). - Render(text) -} - -// Highlightf returns a formatted highlighted message. -func Highlightf(format string, args ...any) string { - return Highlight(fmt.Sprintf(format, args...)) -} - -// ParseAndStyleURLs takes a string and styles any URLs found within it. -// This is useful for processing text that may contain URLs mixed with other content. -func ParseAndStyleURLs(text string) string { - // Simple URL pattern matching - matches http:// and https:// URLs - // This is a basic implementation; for production use, consider a regex library - parts := strings.Split(text, " ") - - for i, part := range parts { - if strings.HasPrefix(part, "http://") || strings.HasPrefix(part, "https://") { - parts[i] = URL(part) - } - } - - return strings.Join(parts, " ") -} - -// ParseAndStyleFilepaths takes a string and styles any filepaths found within it. -// This looks for paths that start with / or are relative paths containing /. -func ParseAndStyleFilepaths(text string) string { - // Simple detection for filepaths - words containing / but not just URLs - parts := strings.Split(text, " ") - - for i, part := range parts { - // Skip if it's already a URL (starts with http) - if strings.HasPrefix(part, "http://") || strings.HasPrefix(part, "https://") { - continue - } - - // Check if it looks like a filepath - if strings.Contains(part, "/") && !strings.Contains(part, "://") { - parts[i] = Filepath(part) - } - } - - return strings.Join(parts, " ") -} -
M internal/tokeniser/tokeniser.gointernal/tokeniser/tokeniser.go

@@ -22,10 +22,7 @@ mu sync.RWMutex

encodings map[string]*tiktoken.Tiktoken } -var ( - cache = &tokenizerCache{encodings: make(map[string]*tiktoken.Tiktoken)} - modelLock sync.RWMutex -) +var cache = &tokenizerCache{encodings: make(map[string]*tiktoken.Tiktoken)} // GetEncoding returns a Tiktoken encoding for the given encoding name. func GetEncoding(encodingName string) (*tiktoken.Tiktoken, error) {

@@ -105,57 +102,3 @@

tokens := enc.Encode(text, nil, nil) return len(tokens), nil } - -// CountMessagesTokens counts tokens for chat API messages. -// Based on OpenAI's official token counting method. -func CountMessagesTokens(messages []Message, model string) (int, error) { - encodingName, err := EncodingForModel(model) - if err != nil { - return 0, err - } - - enc, err := GetEncoding(encodingName) - if err != nil { - return 0, fmt.Errorf("failed to get encoding: %w", err) - } - - var tokensPerMessage, tokensPerName int - switch { - case strings.Contains(model, "gpt-3.5-turbo-0613"), - strings.Contains(model, "gpt-3.5-turbo-16k-0613"), - strings.Contains(model, "gpt-4-0314"), - strings.Contains(model, "gpt-4-32k-0314"), - strings.Contains(model, "gpt-4-0613"), - strings.Contains(model, "gpt-4-32k-0613"): - tokensPerMessage = 3 - tokensPerName = 1 - case strings.Contains(model, "gpt-3.5-turbo-0301"): - tokensPerMessage = 4 - tokensPerName = -1 - default: - // Use default values for other models - tokensPerMessage = 3 - tokensPerName = 1 - } - - numTokens := 0 - for _, msg := range messages { - numTokens += tokensPerMessage - numTokens += len(enc.Encode(msg.Content, nil, nil)) - numTokens += len(enc.Encode(msg.Role, nil, nil)) - numTokens += len(enc.Encode(msg.Name, nil, nil)) - if msg.Name != "" { - numTokens += tokensPerName - } - } - numTokens += 3 // Every reply is primed with <|start|>assistant<|message|> - - return numTokens, nil -} - -// Message represents a chat message for token counting. -type Message struct { - Role string - Content string - Name string -}
M internal/tui/model.gointernal/tui/model.go

@@ -401,29 +401,6 @@ type ErrorMsg struct {

Error string } -func Step(step int, status string) tea.Cmd { - return func() tea.Msg { - return StepMsg{Step: step, Status: status} - } -} - -func StepWithCounts(step int, status string, articles, processed int) tea.Cmd { - return func() tea.Msg { - return StepMsg{ - Step: step, - Status: status, - ArticlesCount: articles, - ProcessedCount: processed, - } - } -} - -func Error(err string) tea.Cmd { - return func() tea.Msg { - return ErrorMsg{Error: err} - } -} - // viewSummary renders the summary view with scrolling support. func (m *Model) viewSummary() string { var sb strings.Builder