llm browser

backend/internal/browse/fetcher.go

@@ -0,0 +1,95 @@
+package browse
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"time"
+)
+
+// Defaults applied when the FetchOptions field is left zero.
+const (
+	// DefaultUserAgent announces the agent view; sites can see what is
+	// browsing them and serve it like any other text agent.
+	DefaultUserAgent = "LLM-Browser/0.1 (+https://git.k6n.net/mats/llm-browser; agent-view validator)"
+	// DefaultFetchTimeout bounds one remote request, redirects included.
+	DefaultFetchTimeout = 15 * time.Second
+	// DefaultMaxBodySize caps how much of a response is read (2 MiB).
+	DefaultMaxBodySize = 2 << 20
+)
+
+// FetchOptions configures an HTTPFetcher. Zero fields fall back to the
+// matching Default* constant.
+type FetchOptions struct {
+	UserAgent   string
+	Timeout     time.Duration
+	MaxBodySize int64
+}
+
+// HTTPFetcher is the production Fetcher: one shared client, a fixed
+// user agent, and a hard cap on response bodies.
+type HTTPFetcher struct {
+	client *http.Client
+	opts   FetchOptions
+}
+
+// NewHTTPFetcher wires the fetcher. The client follows the standard
+// redirect policy (up to 10 hops); each request is bounded by opts.Timeout.
+func NewHTTPFetcher(opts FetchOptions) *HTTPFetcher {
+	if opts.Timeout <= 0 {
+		opts.Timeout = DefaultFetchTimeout
+	}
+	if opts.MaxBodySize <= 0 {
+		opts.MaxBodySize = DefaultMaxBodySize
+	}
+	if opts.UserAgent == "" {
+		opts.UserAgent = DefaultUserAgent
+	}
+	return &HTTPFetcher{
+		client: &http.Client{Timeout: opts.Timeout},
+		opts:   opts,
+	}
+}
+
+// Fetch performs a GET and reads at most MaxBodySize bytes. Transport
+// failures and read failures both come back as *FetchError.
+func (f *HTTPFetcher) Fetch(ctx context.Context, raw string) (*Response, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, raw, nil)
+	if err != nil {
+		return nil, &FetchError{URL: raw, Err: err}
+	}
+	req.Header.Set("User-Agent", f.opts.UserAgent)
+	req.Header.Set("Accept", "text/html,application/xhtml+xml,text/markdown,text/plain;q=0.9,*/*;q=0.8")
+
+	resp, err := f.client.Do(req)
+	if err != nil {
+		return nil, &FetchError{URL: raw, Err: err}
+	}
+	defer resp.Body.Close()
+
+	body, truncated, err := readCapped(resp.Body, f.opts.MaxBodySize)
+	if err != nil {
+		return nil, &FetchError{URL: raw, Err: err}
+	}
+	return &Response{
+		Status:      resp.StatusCode,
+		FinalURL:    resp.Request.URL.String(),
+		ContentType: resp.Header.Get("Content-Type"),
+		Header:      resp.Header,
+		Body:        body,
+		Truncated:   truncated,
+	}, nil
+}
+
+// readCapped reads r fully, stopping at max bytes; the second result
+// reports whether content beyond max was discarded.
+func readCapped(r io.Reader, max int64) ([]byte, bool, error) {
+	body, err := io.ReadAll(io.LimitReader(r, max+1))
+	if err != nil {
+		return nil, false, err
+	}
+	if int64(len(body)) > max {
+		return body[:max], true, nil
+	}
+	return body, false, nil
+}