package main import ( "fmt" "os" "strings" "time" htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown/v2" "github.com/JohannesKaufmann/html-to-markdown/v2/converter" "github.com/alexflint/go-arg" "github.com/imroc/req/v3" ) var ( version string buildDate string ) // Args defines the command-line arguments for mdget type Args struct { URL string `arg:"positional" help:"URL to fetch and convert to markdown"` Output string `arg:"-o,--output" help:"Output file (default: stdout)" default:""` Timeout int `arg:"-t,--timeout" help:"Request timeout in seconds" default:"30"` UserAgent string `arg:"-u,--user-agent" help:"Custom User-Agent header" default:"mdget/0.1.0"` } // Version implements the Versioned interface for go-arg func (Args) Version() string { return fmt.Sprintf("mdget v%s (built %s)", version, buildDate) } // Description returns the program description for help text func (Args) Description() string { return "Fetch a URL and convert its HTML content to Markdown." } // Epilogue returns the help text epilogue func (Args) Epilogue() string { return `Examples: mdget https://example.com mdget --output page.md https://example.com mdget --timeout 10 --user-agent "MyBot/1.0" example.com` } func main() { // Parse command-line arguments var args Args // Use MustParse which handles --help and --version automatically parser := arg.MustParse(&args) // Check if URL is provided (help/version flags are already handled by MustParse) if args.URL == "" { fmt.Fprintf(os.Stderr, "Error: URL is required\n\n") parser.WriteUsage(os.Stderr) os.Exit(1) } // Validate URL starts with http:// or https:// url := args.URL if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") { url = "https://" + url } // Determine output destination var output *os.File = os.Stdout if args.Output != "" { f, err := os.Create(args.Output) if err != nil { fmt.Fprintf(os.Stderr, "Error creating output file: %v\n", err) os.Exit(1) } defer f.Close() output = f } // Fetch the URL using req with timeout and custom User-Agent client := req.C(). SetTimeout(time.Duration(args.Timeout) * time.Second). SetUserAgent(args.UserAgent) resp, err := client.R().Get(url) if err != nil { fmt.Fprintf(os.Stderr, "Error fetching URL: %v\n", err) os.Exit(1) } if !resp.IsSuccessState() { fmt.Fprintf(os.Stderr, "HTTP error: %s\n", resp.Status) os.Exit(1) } // Get HTML content html := resp.String() // Convert HTML to Markdown with domain context for relative URLs markdown, err := htmltomarkdown.ConvertString( html, converter.WithDomain(url), ) if err != nil { fmt.Fprintf(os.Stderr, "Error converting HTML to Markdown: %v\n", err) os.Exit(1) } // Output Markdown to destination fmt.Fprint(output, markdown) }