Deterministic swehockey.se schedule crawler

One HTTP GET per season, no browser: the page is server-rendered, so every row
arrives in one response and the extraction becomes a pure function of the HTML.

- layouts[] declares one profile per version of the site's table (modern and
  legacy), each identified by its header row, so a new version of the page is a
  new profile plus a fixture instead of a guess.
- The table is located by header signature, not by a CSS path.
- Every cell is shape-checked, and facts the site renders twice (desktop and
  mobile date, time in the tooltip) must agree.
- Cross-field invariants: periods sum to the score, ids are unique, played games
  always have an event link and unplayed ones never do.
- Output is sorted by date, time and id, so it does not inherit the site's row
  order, and it carries no fetch timestamp: same HTML, same bytes.
- Drift is a typed error and exit code 3, with the header it actually found.
- Tests run on committed HTML fixtures and golden JSON; TestLive is opt-in.

Verified over the full SHL archive: 27/27 seasons, 2119 games, 1779 played.

cmd_crawl.go

@@ -0,0 +1,130 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+	"time"
+)
+
+// cmdCrawl is the crawler proper: take one league, read its season selector,
+// then walk every season in ascending id order and emit one document per
+// season. The order is derived from the ids, never from map iteration or from
+// the order the site happened to list things, so two runs of the crawler
+// produce the same file even if the site's markup order changes.
+func cmdCrawl(ctx context.Context, args []string) error {
+	fs := flag.NewFlagSet("crawl", flag.ContinueOnError)
+	var c commonFlags
+	c.register(fs, "ndjson")
+	league := fs.Int("league", defaultSeason, "league id (same id space as season ids)")
+	only := fs.Int("season", 0, "crawl only this season id (0 = all seasons of the league)")
+	limit := fs.Int("limit", 0, "stop after N seasons (0 = all)")
+	delay := fs.Duration("delay", 250*time.Millisecond, "pause between season requests")
+	onDrift := fs.String("on-drift", "fail", "when a season page breaks the contract: fail or record")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	cl := c.client()
+
+	seed, err := c.read(ctx, cl, SchedulePath(*league), fmt.Sprintf("league %d", *league))
+	if errors.Is(err, errUnchanged) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	seasons, err := ParseSeasons(*league, bytes.NewReader(seed))
+	if err != nil {
+		return err
+	}
+
+	var out bytes.Buffer
+	enc := jsonEncoder(&out)
+	if c.format == "json" {
+		out.WriteString("[\n")
+	}
+	done := 0
+	for _, s := range seasons {
+		if *only != 0 && s.ID != *only {
+			continue
+		}
+		if *limit > 0 && done >= *limit {
+			break
+		}
+		if done > 0 && *delay > 0 {
+			time.Sleep(*delay)
+		}
+		page, err := cl.get(ctx, SchedulePath(s.ID))
+		if err != nil {
+			return err
+		}
+		sched, err := ParseSchedule(s.ID, bytes.NewReader(page))
+		if err != nil {
+			// A season whose page breaks the contract is either fatal or
+			// recorded, never dropped: a hole in the data has to be visible.
+			var de *DriftError
+			if *onDrift == "fail" || !errors.As(err, &de) {
+				return err
+			}
+			if !c.quiet {
+				fmt.Fprintf(os.Stderr, "season %d: %v\n", s.ID, de)
+			}
+			if c.format == "csv" {
+				fmt.Fprintf(&out, "%d,ERROR,,,,,,,,,%s\n", s.ID, de.Msg)
+			} else {
+				if c.format == "json" {
+					if done > 0 {
+						out.WriteString(",\n")
+					}
+					out.WriteString("  ")
+				}
+				if err := enc.Encode(crawlError{SeasonID: s.ID, Label: s.Label, Error: de.Msg}); err != nil {
+					return err
+				}
+			}
+			done++
+			continue
+		}
+		if c.format == "csv" {
+			b, err := renderCSV(sched)
+			if err != nil {
+				return err
+			}
+			out.Write(b)
+		} else {
+			if c.format == "json" {
+				if done > 0 {
+					out.WriteString(",\n")
+				}
+				out.WriteString("  ")
+			}
+			// Encode writes the document plus one newline: exactly the
+			// NDJSON contract, and one element of the JSON array.
+			if err := enc.Encode(sched); err != nil {
+				return err
+			}
+		}
+		done++
+		if !c.quiet {
+			fmt.Fprintf(os.Stderr, "season %d (%s): %d games, %d played\n", sched.SeasonID, s.Label, len(sched.Games), sched.Played())
+		}
+	}
+	if c.format == "json" {
+		out.WriteString("\n]\n")
+	}
+	if done == 0 {
+		return drift(*league, "", "", "crawl matched no seasons", "0 seasons", ">= 1")
+	}
+	if err := emit(c.out, c.check, out.Bytes()); err != nil {
+		return err
+	}
+	if c.state != "" {
+		if err := saveState(c.state, &state{Path: fmt.Sprintf("crawl:%d", *league), SHA256: sha256Hex(out.Bytes()), SavedAt: nowStamp()}); err != nil {
+			return err
+		}
+	}
+	return nil
+}