package browse import ( "context" "io" "net/http" "time" ) // Defaults applied when the FetchOptions field is left zero. const ( // DefaultUserAgent announces the agent view; sites can see what is // browsing them and serve it like any other text agent. DefaultUserAgent = "LLM-Browser/0.1 (+https://git.k6n.net/mats/llm-browser; agent-view validator)" // DefaultFetchTimeout bounds one remote request, redirects included. DefaultFetchTimeout = 15 * time.Second // DefaultMaxBodySize caps how much of a response is read (2 MiB). DefaultMaxBodySize = 2 << 20 ) // FetchOptions configures an HTTPFetcher. Zero fields fall back to the // matching Default* constant. type FetchOptions struct { UserAgent string Timeout time.Duration MaxBodySize int64 } // HTTPFetcher is the production Fetcher: one shared client, a fixed // user agent, and a hard cap on response bodies. type HTTPFetcher struct { client *http.Client opts FetchOptions } // NewHTTPFetcher wires the fetcher. The client follows the standard // redirect policy (up to 10 hops); each request is bounded by opts.Timeout. func NewHTTPFetcher(opts FetchOptions) *HTTPFetcher { if opts.Timeout <= 0 { opts.Timeout = DefaultFetchTimeout } if opts.MaxBodySize <= 0 { opts.MaxBodySize = DefaultMaxBodySize } if opts.UserAgent == "" { opts.UserAgent = DefaultUserAgent } return &HTTPFetcher{ client: &http.Client{Timeout: opts.Timeout}, opts: opts, } } // Fetch performs a GET and reads at most MaxBodySize bytes. Transport // failures and read failures both come back as *FetchError. func (f *HTTPFetcher) Fetch(ctx context.Context, raw string) (*Response, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, raw, nil) if err != nil { return nil, &FetchError{URL: raw, Err: err} } req.Header.Set("User-Agent", f.opts.UserAgent) req.Header.Set("Accept", "text/html,application/xhtml+xml,text/markdown,text/plain;q=0.9,*/*;q=0.8") resp, err := f.client.Do(req) if err != nil { return nil, &FetchError{URL: raw, Err: err} } defer resp.Body.Close() body, truncated, err := readCapped(resp.Body, f.opts.MaxBodySize) if err != nil { return nil, &FetchError{URL: raw, Err: err} } return &Response{ Status: resp.StatusCode, FinalURL: resp.Request.URL.String(), ContentType: resp.Header.Get("Content-Type"), Header: resp.Header, Body: body, Truncated: truncated, }, nil } // readCapped reads r fully, stopping at max bytes; the second result // reports whether content beyond max was discarded. func readCapped(r io.Reader, max int64) ([]byte, bool, error) { body, err := io.ReadAll(io.LimitReader(r, max+1)) if err != nil { return nil, false, err } if int64(len(body)) > max { return body[:max], true, nil } return body, false, nil }