96 lines
2.7 KiB
Go
96 lines
2.7 KiB
Go
package browse
|
|
|
|
import (
|
|
"context"
|
|
"io"
|
|
"net/http"
|
|
"time"
|
|
)
|
|
|
|
// Defaults applied when the FetchOptions field is left zero.
|
|
const (
|
|
// DefaultUserAgent announces the agent view; sites can see what is
|
|
// browsing them and serve it like any other text agent.
|
|
DefaultUserAgent = "LLM-Browser/0.1 (+https://git.k6n.net/mats/llm-browser; agent-view validator)"
|
|
// DefaultFetchTimeout bounds one remote request, redirects included.
|
|
DefaultFetchTimeout = 15 * time.Second
|
|
// DefaultMaxBodySize caps how much of a response is read (2 MiB).
|
|
DefaultMaxBodySize = 2 << 20
|
|
)
|
|
|
|
// FetchOptions configures an HTTPFetcher. Zero fields fall back to the
|
|
// matching Default* constant.
|
|
type FetchOptions struct {
|
|
UserAgent string
|
|
Timeout time.Duration
|
|
MaxBodySize int64
|
|
}
|
|
|
|
// HTTPFetcher is the production Fetcher: one shared client, a fixed
|
|
// user agent, and a hard cap on response bodies.
|
|
type HTTPFetcher struct {
|
|
client *http.Client
|
|
opts FetchOptions
|
|
}
|
|
|
|
// NewHTTPFetcher wires the fetcher. The client follows the standard
|
|
// redirect policy (up to 10 hops); each request is bounded by opts.Timeout.
|
|
func NewHTTPFetcher(opts FetchOptions) *HTTPFetcher {
|
|
if opts.Timeout <= 0 {
|
|
opts.Timeout = DefaultFetchTimeout
|
|
}
|
|
if opts.MaxBodySize <= 0 {
|
|
opts.MaxBodySize = DefaultMaxBodySize
|
|
}
|
|
if opts.UserAgent == "" {
|
|
opts.UserAgent = DefaultUserAgent
|
|
}
|
|
return &HTTPFetcher{
|
|
client: &http.Client{Timeout: opts.Timeout},
|
|
opts: opts,
|
|
}
|
|
}
|
|
|
|
// Fetch performs a GET and reads at most MaxBodySize bytes. Transport
|
|
// failures and read failures both come back as *FetchError.
|
|
func (f *HTTPFetcher) Fetch(ctx context.Context, raw string) (*Response, error) {
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, raw, nil)
|
|
if err != nil {
|
|
return nil, &FetchError{URL: raw, Err: err}
|
|
}
|
|
req.Header.Set("User-Agent", f.opts.UserAgent)
|
|
req.Header.Set("Accept", "text/html,application/xhtml+xml,text/markdown,text/plain;q=0.9,*/*;q=0.8")
|
|
|
|
resp, err := f.client.Do(req)
|
|
if err != nil {
|
|
return nil, &FetchError{URL: raw, Err: err}
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
body, truncated, err := readCapped(resp.Body, f.opts.MaxBodySize)
|
|
if err != nil {
|
|
return nil, &FetchError{URL: raw, Err: err}
|
|
}
|
|
return &Response{
|
|
Status: resp.StatusCode,
|
|
FinalURL: resp.Request.URL.String(),
|
|
ContentType: resp.Header.Get("Content-Type"),
|
|
Header: resp.Header,
|
|
Body: body,
|
|
Truncated: truncated,
|
|
}, nil
|
|
}
|
|
|
|
// readCapped reads r fully, stopping at max bytes; the second result
|
|
// reports whether content beyond max was discarded.
|
|
func readCapped(r io.Reader, max int64) ([]byte, bool, error) {
|
|
body, err := io.ReadAll(io.LimitReader(r, max+1))
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
if int64(len(body)) > max {
|
|
return body[:max], true, nil
|
|
}
|
|
return body, false, nil
|
|
}
|