package main import ( "context" "encoding/json" "errors" "fmt" "io" "net/http" "os" "path/filepath" "regexp" "strings" "time" ) // userAgent identifies the crawler. The site is a public ASP.NET app and serves // the same bytes to any client; saying who we are keeps its logs readable. const userAgent = "swehockey-schedule/1.0 (+deterministic schedule crawler)" var errUnchanged = errors.New("swehockey: page unchanged") // Client fetches pages. One request, no retries, no cookies, no redirects // beyond the default limit: every byte of the output is a function of one // response. type Client struct { BaseURL string Timeout time.Duration } func (c *Client) get(ctx context.Context, path string) ([]byte, error) { hc := &http.Client{Timeout: c.Timeout} url := joinURL(c.BaseURL, path) ctx, cancel := context.WithTimeout(ctx, c.Timeout) defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return nil, err } req.Header.Set("User-Agent", userAgent) req.Header.Set("Accept", "text/html,application/xhtml+xml") req.Header.Set("Accept-Language", "en") resp, err := hc.Do(req) if err != nil { return nil, fmt.Errorf("fetch %s: %w", url, err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("fetch %s: %s", url, resp.Status) } body, err := io.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("read %s: %w", url, err) } if err := checkCharset(body); err != nil { return nil, fmt.Errorf("fetch %s: %w", url, err) } return body, nil } var metaCharsetRE = regexp.MustCompile(`(?i)]+charset\s*=\s*["']?([a-z0-9_-]+)`) // checkCharset refuses pages we cannot decode byte-exactly. Silently handing // latin-1 bytes to a UTF-8 parser would turn "Växjö" into mojibake that still // parses, which is the worst kind of crawler bug. func checkCharset(body []byte) error { m := metaCharsetRE.FindSubmatch(body) if m == nil { return nil // no declaration: html.Parse assumes UTF-8 } switch cs := strings.ToLower(string(m[1])); cs { case "utf-8", "utf8": return nil default: return fmt.Errorf("unsupported charset %q: add a decoder for it", cs) } } // state records what the last successful run saw, so a cron job can be // idempotent: an unchanged page produces no output and a zero exit code. type state struct { Path string `json:"path"` SHA256 string `json:"sha256"` LastUpdate string `json:"last_update"` SavedAt string `json:"saved_at"` } func loadState(path string) (*state, error) { b, err := os.ReadFile(path) if errors.Is(err, os.ErrNotExist) { return &state{}, nil } if err != nil { return nil, err } var st state if err := json.Unmarshal(b, &st); err != nil { return nil, fmt.Errorf("state file %s: %w", path, err) } return &st, nil } // saveState writes the state atomically so a killed run cannot leave a // truncated file that would make the next run skip real work. func saveState(path string, st *state) error { if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { return err } b, err := json.MarshalIndent(st, "", " ") if err != nil { return err } b = append(b, '\n') tmp := path + ".tmp" if err := os.WriteFile(tmp, b, 0o644); err != nil { return err } return os.Rename(tmp, path) }