Deterministic swehockey.se schedule crawler

One HTTP GET per season, no browser: the page is server-rendered, so every row
arrives in one response and the extraction becomes a pure function of the HTML.

- layouts[] declares one profile per version of the site's table (modern and
  legacy), each identified by its header row, so a new version of the page is a
  new profile plus a fixture instead of a guess.
- The table is located by header signature, not by a CSS path.
- Every cell is shape-checked, and facts the site renders twice (desktop and
  mobile date, time in the tooltip) must agree.
- Cross-field invariants: periods sum to the score, ids are unique, played games
  always have an event link and unplayed ones never do.
- Output is sorted by date, time and id, so it does not inherit the site's row
  order, and it carries no fetch timestamp: same HTML, same bytes.
- Drift is a typed error and exit code 3, with the header it actually found.
- Tests run on committed HTML fixtures and golden JSON; TestLive is opt-in.

Verified over the full SHL archive: 27/27 seasons, 2119 games, 1779 played.

fetch.go

@@ -0,0 +1,121 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"time"
+)
+
+// userAgent identifies the crawler. The site is a public ASP.NET app and serves
+// the same bytes to any client; saying who we are keeps its logs readable.
+const userAgent = "swehockey-schedule/1.0 (+deterministic schedule crawler)"
+
+var errUnchanged = errors.New("swehockey: page unchanged")
+
+// Client fetches pages. One request, no retries, no cookies, no redirects
+// beyond the default limit: every byte of the output is a function of one
+// response.
+type Client struct {
+	BaseURL string
+	Timeout time.Duration
+}
+
+func (c *Client) get(ctx context.Context, path string) ([]byte, error) {
+	hc := &http.Client{Timeout: c.Timeout}
+	url := joinURL(c.BaseURL, path)
+	ctx, cancel := context.WithTimeout(ctx, c.Timeout)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("User-Agent", userAgent)
+	req.Header.Set("Accept", "text/html,application/xhtml+xml")
+	req.Header.Set("Accept-Language", "en")
+
+	resp, err := hc.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("fetch %s: %w", url, err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("fetch %s: %s", url, resp.Status)
+	}
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("read %s: %w", url, err)
+	}
+	if err := checkCharset(body); err != nil {
+		return nil, fmt.Errorf("fetch %s: %w", url, err)
+	}
+	return body, nil
+}
+
+var metaCharsetRE = regexp.MustCompile(`(?i)<meta[^>]+charset\s*=\s*["']?([a-z0-9_-]+)`)
+
+// checkCharset refuses pages we cannot decode byte-exactly. Silently handing
+// latin-1 bytes to a UTF-8 parser would turn "Växjö" into mojibake that still
+// parses, which is the worst kind of crawler bug.
+func checkCharset(body []byte) error {
+	m := metaCharsetRE.FindSubmatch(body)
+	if m == nil {
+		return nil // no declaration: html.Parse assumes UTF-8
+	}
+	switch cs := strings.ToLower(string(m[1])); cs {
+	case "utf-8", "utf8":
+		return nil
+	default:
+		return fmt.Errorf("unsupported charset %q: add a decoder for it", cs)
+	}
+}
+
+// state records what the last successful run saw, so a cron job can be
+// idempotent: an unchanged page produces no output and a zero exit code.
+type state struct {
+	Path       string `json:"path"`
+	SHA256     string `json:"sha256"`
+	LastUpdate string `json:"last_update"`
+	SavedAt    string `json:"saved_at"`
+}
+
+func loadState(path string) (*state, error) {
+	b, err := os.ReadFile(path)
+	if errors.Is(err, os.ErrNotExist) {
+		return &state{}, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	var st state
+	if err := json.Unmarshal(b, &st); err != nil {
+		return nil, fmt.Errorf("state file %s: %w", path, err)
+	}
+	return &st, nil
+}
+
+// saveState writes the state atomically so a killed run cannot leave a
+// truncated file that would make the next run skip real work.
+func saveState(path string, st *state) error {
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		return err
+	}
+	b, err := json.MarshalIndent(st, "", "  ")
+	if err != nil {
+		return err
+	}
+	b = append(b, '\n')
+	tmp := path + ".tmp"
+	if err := os.WriteFile(tmp, b, 0o644); err != nil {
+		return err
+	}
+	return os.Rename(tmp, path)
+}