From 6b701819cae7588fcb5a14fdb1e803351354c612 Mon Sep 17 00:00:00 2001 From: Mats Tornberg Date: Sun, 27 Sep 2026 08:47:25 +0000 Subject: [PATCH] Deterministic swehockey.se schedule crawler One HTTP GET per season, no browser: the page is server-rendered, so every row arrives in one response and the extraction becomes a pure function of the HTML. - layouts[] declares one profile per version of the site's table (modern and legacy), each identified by its header row, so a new version of the page is a new profile plus a fixture instead of a guess. - The table is located by header signature, not by a CSS path. - Every cell is shape-checked, and facts the site renders twice (desktop and mobile date, time in the tooltip) must agree. - Cross-field invariants: periods sum to the score, ids are unique, played games always have an event link and unplayed ones never do. - Output is sorted by date, time and id, so it does not inherit the site's row order, and it carries no fetch timestamp: same HTML, same bytes. - Drift is a typed error and exit code 3, with the header it actually found. - Tests run on committed HTML fixtures and golden JSON; TestLive is opt-in. Verified over the full SHL archive: 27/27 seasons, 2119 games, 1779 played. --- .gitignore | 3 + README.md | 170 ++++++++ cmd_crawl.go | 130 ++++++ cmd_schedule.go | 103 +++++ dom.go | 129 ++++++ fetch.go | 121 ++++++ go.mod | 5 + go.sum | 2 + main.go | 160 ++++++++ output.go | 140 +++++++ parse.go | 540 +++++++++++++++++++++++++ parse_test.go | 445 ++++++++++++++++++++ seasons.go | 126 ++++++ testdata/schedule_20961.html | 46 +++ testdata/schedule_20961.json | 277 +++++++++++++ testdata/schedule_legacy.html | 39 ++ testdata/schedule_legacy.json | 203 ++++++++++ testdata/schedule_legacy_playoffs.html | 33 ++ testdata/schedule_legacy_playoffs.json | 163 ++++++++ testdata/seasons_20961.html | 41 ++ 20 files changed, 2876 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 cmd_crawl.go create mode 100644 cmd_schedule.go create mode 100644 dom.go create mode 100644 fetch.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 output.go create mode 100644 parse.go create mode 100644 parse_test.go create mode 100644 seasons.go create mode 100644 testdata/schedule_20961.html create mode 100644 testdata/schedule_20961.json create mode 100644 testdata/schedule_legacy.html create mode 100644 testdata/schedule_legacy.json create mode 100644 testdata/schedule_legacy_playoffs.html create mode 100644 testdata/schedule_legacy_playoffs.json create mode 100644 testdata/seasons_20961.html diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ea253f6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +/swehockey +*.tmp +/.codesearch/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..fa534cc --- /dev/null +++ b/README.md @@ -0,0 +1,170 @@ +# swehockey-schedule + +A deterministic crawler for the **Schedule and Results** table on +[stats.swehockey.se](https://stats.swehockey.se) — one plain HTTP GET per +season, no browser, no JavaScript, no waiting, and a byte-identical output for +identical input. + +```console +$ go build -o swehockey . +$ ./swehockey schedule -season 20961 -format csv | head -3 +season_id,game_id,event_id,date,time,home,away,home_goals,away_goals,periods,spectators,venue +20961,90001001,1109941,2026-09-19,15:15,Frölunda HC,Växjö Lakers HC,5,4,1:1-1 2:2-1 3:2-2,12044,Scandinavium +20961,90001002,1109942,2026-09-19,15:15,HV 71,IF Malmö Redhawks,4,5,1:1-0 2:1-1 3:1-3 4:1-1,6800,Husqvarna Garden +``` + +```console +$ ./swehockey crawl -league 20961 # every season of the SHL, as NDJSON +$ ./swehockey leagues # 119 competitions and their ids +$ ./swehockey schedule -fixture testdata/schedule_20961.html # offline +``` + +## How determinism is achieved + +The site's page is server-rendered: `curl` returns all 364 rows of the current +season in one response. That removes the usual sources of flakiness (timers, +`waitForSelector`, lazy rendering) and leaves exactly one risk — that the +markup changes under you. The rest of the design is about that risk. + +1. **One GET, no retries, no cookies, no state.** Same HTML in, same bytes out. + The output contains only page-derived data: no fetch timestamp, no map + iteration, fixed field order. `TestRenderIsPure` and the golden files enforce + this. +2. **The table is found by its header signature**, not by a CSS path or a + position in the document. Ads, navigation and layout changes cannot move it + out from under the crawler. +3. **A declared contract per layout, not a pile of indices.** The site has + shipped more than one version of this table, so each version is a profile in + `layouts[]` (`parse.go`), identified by the header row it declares. An + unknown shape fails with the header it actually found. +4. **Every field is shape-checked**, so a renamed column or a new cell is an + error naming the game and the cell, never a silently empty column. +5. **Facts the site renders twice must agree.** In the modern layout the date is + printed in a desktop cell *and* as a mobile `` group heading, and the + start time appears in the tooltip span and in the mobile cell. The parser + reads both and rejects the row if they disagree. Values the site prints once + per group (the date heading, the round number) are carried forward, and the + first row of a group must carry the value to inherit. +6. **Cross-field invariants**: the period scores must sum to the final score, + a played game always has an event link and an unplayed one never does, + game ids are unique, dates parse, venues are non-empty. +7. **Canonical order, imposed by us.** The site lists games in round order, and + re-scheduled or awarded games keep the round they belong to, so the page's + own date order is *not* monotonic (2011-12 has September games after October + ones). The output is sorted by date, time and id, so it is a function of the + set of rows rather than of the site's row order — `TestOrderIndependence` + swaps two rows and requires identical bytes. +8. **Fixtures, not the network, in the tests.** Real HTML slices live in + `testdata/` next to golden JSON. CI is offline and hermetic; a site change + surfaces as a failing test with a readable message. `SWEHOCKEY_LIVE=1 go + test -run TestLive` is the one opt-in test that talks to the site. +9. **Drift is a distinct failure mode**, not a crash: `*DriftError` and exit + code 3. Callers can alert on it without parsing a stack trace. +10. **Idempotent cron.** `-state ` skips the run when the page body hash + is unchanged, and `-check ` exits 3 when the output no longer + matches a committed file, so an upstream change becomes a reviewable diff. + +## The two layouts on the site today + +| profile | header | cells | seasons | +| -------- | --------------------------------------------- | ----- | ------------------ | +| `modern` | `Date Time Game Result … Spectators Venue` | 8 | 2018-19 onwards | +| `legacy` | `Round Date Game Result … Spectators Venue` | 7 | up to 2017-18 | + +They differ in more than columns: + +- `modern` renders every fact twice, for a desktop and a mobile table, and + prints the date only on the first row of each date. The time lives in + `span.lnkTooltip[title]`, which is also the row's stable id. +- `legacy` renders it once. The round is in its own column, the date and the + time share one cell, and old team cells end with an `` note + (`Eighth final 1`) that must not be swallowed into a team name. +- A played game has 3, 4, 5 or 6 period scores depending on the era's rules, so + the count is deliberately *not* pinned; the sum check is what protects the + data. + +Verified on the whole SHL archive: **27 of 27 seasons, 2119 games, 1779 played**. + +``` +$ ./swehockey crawl -league 20961 -on-drift record | jq -s '[.[] | .games | length] | add' +2119 +``` + +## Commands + +| command | what it does | +| --------- | ------------------------------------------------------------------- | +| `schedule`| one season, found by `-season` (default 20961) | +| `crawl` | every season of `-league`, in ascending id order, one document each | +| `leagues` | the competitions linked from the front page | + +Flags: `-format json|csv|ndjson`, `-out`, `-fixture`, `-state`, `-check`, +`-timeout`, `-quiet`, plus `-limit`, `-delay` and `-on-drift` for `crawl`. + +`crawl -on-drift record` emits `{"season_id":…,"error":…}` in place of a season +that breaks the contract, so a bulk run neither dies on one bad page nor hides +the hole. + +Exit codes: `0` ok, `1` error, `2` usage, `3` markup drift or golden mismatch, +`4` timeout. + +## Output + +```json +{ + "season_id": 20961, + "layout": "modern", + "title": "Schedule and Results", + "last_update": "2026-09-26 22:41", + "games": [ + { + "game_id": "90001001", + "event_id": "1109941", + "date": "2026-09-19", + "time": "15:15", + "home": "Frölunda HC", + "away": "Växjö Lakers HC", + "home_goals": 5, + "away_goals": 4, + "periods": [{"label": "1", "home": 1, "away": 1}], + "spectators": 12044, + "venue": "Scandinavium" + } + ] +} +``` + +`game_id` is the site's own stable per-fixture id and is present for scheduled +as well as played games; `event_id` comes from the `/Game/Events/` link and +appears only once a game has been played. `home_goals` is `null`, not `0`, for a +game that has not been played. `last_update` is the site's own clock, verbatim +and unparsed, so it can be compared across runs without trusting a timezone +conversion. + +## Tests and fixtures + +``` +testdata/schedule_20961.html 2026-27, 3 date groups, OT + shootout, unplayed games +testdata/schedule_legacy.html 2018-19, round carry-forward, a 6-period game +testdata/schedule_legacy_playoffs.html 2013-14, notes in the team cells +testdata/seasons_20961.html the season selector and league links +testdata/*.json golden output for the fixtures above +``` + +`TestDrift` mutates a fixture once per way the page can change shape — renamed +header, extra column, score that no longer matches the periods, a removed date +group marker, a duplicated id, a missing timestamp, an empty table — and +requires a named, actionable error for each. + +The fixtures are cut from real pages; the only thing the cutter changes is which +`` elements it keeps, so the markup around them stays real, including the +nested wrapper tables and the ad div. + +## Known limits + +- Old seasons are pruned by the site: 2018-19 serves its last 45 games, 2024-25 + only its 6 playoff games. The crawler reports exactly what the page contains. +- A handful of season pages serve more slowly than the default 30s timeout + (they stream a large table); `-timeout` covers it. +- Rows with no result and no round appear in a few old pages; those are the + reason `-on-drift record` exists. diff --git a/cmd_crawl.go b/cmd_crawl.go new file mode 100644 index 0000000..58938de --- /dev/null +++ b/cmd_crawl.go @@ -0,0 +1,130 @@ +package main + +import ( + "bytes" + "context" + "errors" + "flag" + "fmt" + "os" + "time" +) + +// cmdCrawl is the crawler proper: take one league, read its season selector, +// then walk every season in ascending id order and emit one document per +// season. The order is derived from the ids, never from map iteration or from +// the order the site happened to list things, so two runs of the crawler +// produce the same file even if the site's markup order changes. +func cmdCrawl(ctx context.Context, args []string) error { + fs := flag.NewFlagSet("crawl", flag.ContinueOnError) + var c commonFlags + c.register(fs, "ndjson") + league := fs.Int("league", defaultSeason, "league id (same id space as season ids)") + only := fs.Int("season", 0, "crawl only this season id (0 = all seasons of the league)") + limit := fs.Int("limit", 0, "stop after N seasons (0 = all)") + delay := fs.Duration("delay", 250*time.Millisecond, "pause between season requests") + onDrift := fs.String("on-drift", "fail", "when a season page breaks the contract: fail or record") + if err := fs.Parse(args); err != nil { + return err + } + cl := c.client() + + seed, err := c.read(ctx, cl, SchedulePath(*league), fmt.Sprintf("league %d", *league)) + if errors.Is(err, errUnchanged) { + return nil + } + if err != nil { + return err + } + seasons, err := ParseSeasons(*league, bytes.NewReader(seed)) + if err != nil { + return err + } + + var out bytes.Buffer + enc := jsonEncoder(&out) + if c.format == "json" { + out.WriteString("[\n") + } + done := 0 + for _, s := range seasons { + if *only != 0 && s.ID != *only { + continue + } + if *limit > 0 && done >= *limit { + break + } + if done > 0 && *delay > 0 { + time.Sleep(*delay) + } + page, err := cl.get(ctx, SchedulePath(s.ID)) + if err != nil { + return err + } + sched, err := ParseSchedule(s.ID, bytes.NewReader(page)) + if err != nil { + // A season whose page breaks the contract is either fatal or + // recorded, never dropped: a hole in the data has to be visible. + var de *DriftError + if *onDrift == "fail" || !errors.As(err, &de) { + return err + } + if !c.quiet { + fmt.Fprintf(os.Stderr, "season %d: %v\n", s.ID, de) + } + if c.format == "csv" { + fmt.Fprintf(&out, "%d,ERROR,,,,,,,,,%s\n", s.ID, de.Msg) + } else { + if c.format == "json" { + if done > 0 { + out.WriteString(",\n") + } + out.WriteString(" ") + } + if err := enc.Encode(crawlError{SeasonID: s.ID, Label: s.Label, Error: de.Msg}); err != nil { + return err + } + } + done++ + continue + } + if c.format == "csv" { + b, err := renderCSV(sched) + if err != nil { + return err + } + out.Write(b) + } else { + if c.format == "json" { + if done > 0 { + out.WriteString(",\n") + } + out.WriteString(" ") + } + // Encode writes the document plus one newline: exactly the + // NDJSON contract, and one element of the JSON array. + if err := enc.Encode(sched); err != nil { + return err + } + } + done++ + if !c.quiet { + fmt.Fprintf(os.Stderr, "season %d (%s): %d games, %d played\n", sched.SeasonID, s.Label, len(sched.Games), sched.Played()) + } + } + if c.format == "json" { + out.WriteString("\n]\n") + } + if done == 0 { + return drift(*league, "", "", "crawl matched no seasons", "0 seasons", ">= 1") + } + if err := emit(c.out, c.check, out.Bytes()); err != nil { + return err + } + if c.state != "" { + if err := saveState(c.state, &state{Path: fmt.Sprintf("crawl:%d", *league), SHA256: sha256Hex(out.Bytes()), SavedAt: nowStamp()}); err != nil { + return err + } + } + return nil +} diff --git a/cmd_schedule.go b/cmd_schedule.go new file mode 100644 index 0000000..0799115 --- /dev/null +++ b/cmd_schedule.go @@ -0,0 +1,103 @@ +package main + +import ( + "bytes" + "context" + "errors" + "flag" + "fmt" + "os" + "time" +) + +func cmdSchedule(ctx context.Context, args []string) error { + fs := flag.NewFlagSet("schedule", flag.ContinueOnError) + var c commonFlags + c.register(fs, "json") + season := fs.Int("season", defaultSeason, "season id; see the crawl or leagues command") + if err := fs.Parse(args); err != nil { + return err + } + + path := SchedulePath(*season) + page, err := c.read(ctx, c.client(), path, fmt.Sprintf("season %d", *season)) + if errors.Is(err, errUnchanged) { + return nil + } + if err != nil { + return err + } + s, err := ParseSchedule(*season, bytes.NewReader(page)) + if err != nil { + return err + } + var b []byte + if c.format == "ndjson" { + b, err = renderNDJSON(s) + } else { + b, err = render(c.format, s) + } + if err != nil { + return err + } + if err := emit(c.out, c.check, b); err != nil { + return err + } + if c.state != "" { + if err := saveState(c.state, &state{Path: path, SHA256: sha256Hex(page), LastUpdate: s.LastUpdate, SavedAt: nowStamp()}); err != nil { + return err + } + } + if !c.quiet { + fmt.Fprintf(os.Stderr, "season %d: %d games, %d played, last update %s\n", + s.SeasonID, len(s.Games), s.Played(), s.LastUpdate) + } + return nil +} + +func cmdLeagues(ctx context.Context, args []string) error { + fs := flag.NewFlagSet("leagues", flag.ContinueOnError) + var c commonFlags + c.register(fs, "json") + if err := fs.Parse(args); err != nil { + return err + } + page, err := c.read(ctx, c.client(), "/", "front page") + if errors.Is(err, errUnchanged) { + return nil + } + if err != nil { + return err + } + leagues, err := ParseLeagues(bytes.NewReader(page)) + if err != nil { + return err + } + var b []byte + if c.format == "csv" { + var out bytes.Buffer + out.WriteString("id,name,path\n") + for _, l := range leagues { + fmt.Fprintf(&out, "%d,%s,%s\n", l.ID, l.Name, l.Path) + } + b = out.Bytes() + } else { + var buf bytes.Buffer + enc := jsonEncoder(&buf) + if err := enc.Encode(leagues); err != nil { + return err + } + b = buf.Bytes() + } + return emit(c.out, c.check, b) +} + +// crawlError is what a "record" run emits in place of a season, so a broken +// page is a visible hole in the output rather than a missing line. +type crawlError struct { + SeasonID int `json:"season_id"` + Label string `json:"label"` + Error string `json:"error"` +} + +func nowStamp() string { return time.Now().UTC().Format(time.RFC3339) } diff --git a/dom.go b/dom.go new file mode 100644 index 0000000..1be430e --- /dev/null +++ b/dom.go @@ -0,0 +1,129 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "strings" + "unicode" + + "golang.org/x/net/html" +) + +// The site writes the team separator as NBSP-hyphen-NBSP wrapped in newlines, +// so every string that leaves the parser goes through normalize and downstream +// comparisons and diffs stay stable. +func normalize(s string) string { + return strings.Join(strings.FieldsFunc(s, isSpaceRune), " ") +} + +func isSpaceRune(r rune) bool { + if unicode.IsSpace(r) { + return true + } + // No-break and zero-width variants the site emits around team names. + switch r { + case '\u00a0', '\u2007', '\u202f', '\u200b', '\ufeff': + return true + } + return false +} + +// textOf returns the normalized text of a node subtree. +func textOf(n *html.Node) string { + if n == nil { + return "" + } + var b strings.Builder + var walk func(*html.Node) + walk = func(n *html.Node) { + if n.Type == html.TextNode { + b.WriteString(n.Data) + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(n) + return normalize(b.String()) +} + +// textExcluding returns the normalized text of a node subtree with the first +// descendant matching skipTag (and its subtree) left out. The schedule uses +// this to read the mobile cell's start time without its date. +func textExcluding(n *html.Node, skipTag string) string { + if n == nil { + return "" + } + var b strings.Builder + var walk func(*html.Node) bool + walk = func(n *html.Node) bool { + if n.Type == html.ElementNode && n.Data == skipTag { + return false + } + if n.Type == html.TextNode { + b.WriteString(n.Data) + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + return true + } + walk(n) + return normalize(b.String()) +} + +// childElements returns the direct children of n with the given tag name. +func childElements(n *html.Node, tag string) []*html.Node { + if n == nil { + return nil + } + var out []*html.Node + for c := n.FirstChild; c != nil; c = c.NextSibling { + if c.Type == html.ElementNode && c.Data == tag { + out = append(out, c) + } + } + return out +} + +// descendants returns every element with the given tag name, in document order. +func descendants(n *html.Node, tag string) []*html.Node { + var out []*html.Node + var walk func(*html.Node) + walk = func(n *html.Node) { + if n.Type == html.ElementNode && n.Data == tag { + out = append(out, n) + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(n) + return out +} + +func attr(n *html.Node, key string) string { + if n == nil { + return "" + } + for _, a := range n.Attr { + if a.Key == key { + return a.Val + } + } + return "" +} + +func hasClass(n *html.Node, class string) bool { + for _, f := range strings.Fields(attr(n, "class")) { + if f == class { + return true + } + } + return false +} + +func sha256Hex(b []byte) string { + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:]) +} diff --git a/fetch.go b/fetch.go new file mode 100644 index 0000000..f3c18c0 --- /dev/null +++ b/fetch.go @@ -0,0 +1,121 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "regexp" + "strings" + "time" +) + +// userAgent identifies the crawler. The site is a public ASP.NET app and serves +// the same bytes to any client; saying who we are keeps its logs readable. +const userAgent = "swehockey-schedule/1.0 (+deterministic schedule crawler)" + +var errUnchanged = errors.New("swehockey: page unchanged") + +// Client fetches pages. One request, no retries, no cookies, no redirects +// beyond the default limit: every byte of the output is a function of one +// response. +type Client struct { + BaseURL string + Timeout time.Duration +} + +func (c *Client) get(ctx context.Context, path string) ([]byte, error) { + hc := &http.Client{Timeout: c.Timeout} + url := joinURL(c.BaseURL, path) + ctx, cancel := context.WithTimeout(ctx, c.Timeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", userAgent) + req.Header.Set("Accept", "text/html,application/xhtml+xml") + req.Header.Set("Accept-Language", "en") + + resp, err := hc.Do(req) + if err != nil { + return nil, fmt.Errorf("fetch %s: %w", url, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("fetch %s: %s", url, resp.Status) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("read %s: %w", url, err) + } + if err := checkCharset(body); err != nil { + return nil, fmt.Errorf("fetch %s: %w", url, err) + } + return body, nil +} + +var metaCharsetRE = regexp.MustCompile(`(?i)]+charset\s*=\s*["']?([a-z0-9_-]+)`) + +// checkCharset refuses pages we cannot decode byte-exactly. Silently handing +// latin-1 bytes to a UTF-8 parser would turn "Växjö" into mojibake that still +// parses, which is the worst kind of crawler bug. +func checkCharset(body []byte) error { + m := metaCharsetRE.FindSubmatch(body) + if m == nil { + return nil // no declaration: html.Parse assumes UTF-8 + } + switch cs := strings.ToLower(string(m[1])); cs { + case "utf-8", "utf8": + return nil + default: + return fmt.Errorf("unsupported charset %q: add a decoder for it", cs) + } +} + +// state records what the last successful run saw, so a cron job can be +// idempotent: an unchanged page produces no output and a zero exit code. +type state struct { + Path string `json:"path"` + SHA256 string `json:"sha256"` + LastUpdate string `json:"last_update"` + SavedAt string `json:"saved_at"` +} + +func loadState(path string) (*state, error) { + b, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return &state{}, nil + } + if err != nil { + return nil, err + } + var st state + if err := json.Unmarshal(b, &st); err != nil { + return nil, fmt.Errorf("state file %s: %w", path, err) + } + return &st, nil +} + +// saveState writes the state atomically so a killed run cannot leave a +// truncated file that would make the next run skip real work. +func saveState(path string, st *state) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + b, err := json.MarshalIndent(st, "", " ") + if err != nil { + return err + } + b = append(b, '\n') + tmp := path + ".tmp" + if err := os.WriteFile(tmp, b, 0o644); err != nil { + return err + } + return os.Rename(tmp, path) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..773a187 --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module swehockey + +go 1.25.0 + +require golang.org/x/net v0.58.0 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..0a44e17 --- /dev/null +++ b/go.sum @@ -0,0 +1,2 @@ +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= diff --git a/main.go b/main.go new file mode 100644 index 0000000..7f86285 --- /dev/null +++ b/main.go @@ -0,0 +1,160 @@ +// Command swehockey-schedule extracts the "Schedule and Results" table from +// stats.swehockey.se into JSON or CSV. +// +// The extraction is deterministic by construction: +// +// - The page is server-rendered. One plain HTTP GET returns every row of the +// table, so there is no browser, no JavaScript, no waiting and no +// timing-dependent retry. Same HTML in, same bytes out. +// - The table is found by its header signature, never by a CSS path, so +// ads, navigation and layout changes cannot move it out from under us. +// - Every field is read from a documented cell index *and* shape-checked. +// Markup drift produces an error naming the game and the cell, never a +// silently truncated or mis-mapped table. +// - The site renders each fact twice (a desktop cell and a mobile cell). +// The parser requires both renderings to agree, so a responsive-layout +// change cannot corrupt half the rows unnoticed. +// - Cross-field invariants are enforced: period goals must sum to the final +// score, game ids must be unique and ascending, dates must not go back. +// - Output is a pure function of the page: no fetch timestamps, no map +// iteration, fixed field order. Diffs between runs are meaningful. +// +// Commands: +// +// swehockey-schedule schedule -season 20961 # one season -> JSON +// swehockey-schedule schedule -season 20961 -format csv +// swehockey-schedule crawl -league 20961 # every season of a league +// swehockey-schedule leagues # league ids on the front page +// swehockey-schedule schedule -fixture page.html # offline, for tests/debugging +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "os" + "time" +) + +const defaultSeason = 20961 + +func main() { + if len(os.Args) < 2 { + usage() + os.Exit(2) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + var err error + switch cmd := os.Args[1]; cmd { + case "schedule": + err = cmdSchedule(ctx, os.Args[2:]) + case "crawl": + err = cmdCrawl(ctx, os.Args[2:]) + case "leagues": + err = cmdLeagues(ctx, os.Args[2:]) + case "-h", "--help", "help": + usage() + return + default: + fmt.Fprintf(os.Stderr, "swehockey-schedule: unknown command %q\n\n", cmd) + usage() + os.Exit(2) + } + + var de *DriftError + switch { + case err == nil: + case errors.Is(err, context.DeadlineExceeded): + fmt.Fprintf(os.Stderr, "swehockey-schedule: %v\n", err) + os.Exit(4) + case errors.As(err, &de): + // Drift is not a crash: it means the site changed shape. Exit 3 so a + // cron job or CI gate can alert without a stack trace. + fmt.Fprintln(os.Stderr, de.Error()) + os.Exit(3) + default: + fmt.Fprintf(os.Stderr, "swehockey-schedule: %v\n", err) + os.Exit(1) + } +} + +func usage() { + fmt.Fprint(os.Stderr, `swehockey-schedule - deterministic stats.swehockey.se schedule crawler + +usage: + swehockey-schedule schedule [flags] extract one season's schedule table + swehockey-schedule crawl [flags] extract every season of one league (NDJSON) + swehockey-schedule leagues [flags] list league ids and names + +common flags: + -base string site root (default https://stats.swehockey.se) + -format string json|csv|ndjson (default json; crawl defaults to ndjson) + -out string output file (default stdout) + -fixture string parse a saved page instead of fetching (offline) + -state string state file; skip the run when the page is unchanged + -check string golden file; exit 3 if the output no longer matches + -timeout duration per request (default 30s) + -quiet bool suppress progress notes on stderr + +exit codes: 0 ok, 1 error, 2 usage, 3 markup drift or golden mismatch, 4 timeout +`) +} + +type commonFlags struct { + base string + format string + out string + fixture string + state string + check string + timeout time.Duration + quiet bool +} + +func (c *commonFlags) register(fs *flag.FlagSet, defFormat string) { + fs.StringVar(&c.base, "base", "https://stats.swehockey.se", "site root") + fs.StringVar(&c.format, "format", defFormat, "output format: json, csv or ndjson") + fs.StringVar(&c.out, "out", "", "write to this file instead of stdout") + fs.StringVar(&c.fixture, "fixture", "", "parse a saved HTML page instead of fetching") + fs.StringVar(&c.state, "state", "", "state file for skip-when-unchanged runs") + fs.StringVar(&c.check, "check", "", "golden file; exit 3 when the output differs") + fs.DurationVar(&c.timeout, "timeout", 30*time.Second, "per-request timeout") + fs.BoolVar(&c.quiet, "quiet", false, "suppress progress notes on stderr") +} + +func (c *commonFlags) client() *Client { + return &Client{BaseURL: c.base, Timeout: c.timeout} +} + +// read returns the page bytes for a path, honouring -fixture and -state. +func (c *commonFlags) read(ctx context.Context, cl *Client, path, what string) ([]byte, error) { + if c.fixture != "" { + b, err := os.ReadFile(c.fixture) + if err != nil { + return nil, err + } + return b, nil + } + if c.state != "" { + st, err := loadState(c.state) + if err != nil { + return nil, err + } + b, err := cl.get(ctx, path) + if err != nil { + return nil, err + } + if st.SHA256 == sha256Hex(b) && st.Path == path { + // Byte-identical page: the derived output would be identical too. + if !c.quiet { + fmt.Fprintf(os.Stderr, "swehockey-schedule: %s unchanged since %s, nothing to do\n", what, st.SavedAt) + } + return nil, errUnchanged + } + return b, nil + } + return cl.get(ctx, path) +} diff --git a/output.go b/output.go new file mode 100644 index 0000000..6e943d0 --- /dev/null +++ b/output.go @@ -0,0 +1,140 @@ +package main + +import ( + "bytes" + "encoding/csv" + "encoding/json" + "fmt" + "os" + "strconv" + "strings" +) + +// render turns a schedule into bytes. Both encoders are pure functions of the +// data: no timestamps, no maps, fixed column order, LF line endings. Running +// the crawler twice on the same page produces identical bytes, which is what +// makes -check and diff-based review meaningful. +func render(format string, s *Schedule) ([]byte, error) { + switch format { + case "json": + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) // keep team names readable + enc.SetIndent("", " ") + if err := enc.Encode(s); err != nil { + return nil, err + } + return buf.Bytes(), nil + case "csv": + return renderCSV(s) + default: + return nil, fmt.Errorf("unknown format %q: want json, csv or ndjson", format) + } +} + +func renderNDJSON(s *Schedule) ([]byte, error) { + var buf bytes.Buffer + if err := jsonEncoder(&buf).Encode(s); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// jsonEncoder keeps "ä" readable instead of \u00e4, which matters for diffs +// of team names. +func jsonEncoder(buf *bytes.Buffer) *json.Encoder { + enc := json.NewEncoder(buf) + enc.SetEscapeHTML(false) + return enc +} + +var csvHeader = []string{ + "season_id", "game_id", "event_id", "date", "time", "home", "away", + "home_goals", "away_goals", "periods", "spectators", "venue", +} + +func renderCSV(s *Schedule) ([]byte, error) { + var buf bytes.Buffer + w := csv.NewWriter(&buf) // LF, not CRLF + if err := w.Write(csvHeader); err != nil { + return nil, err + } + for _, g := range s.Games { + periods := make([]string, 0, len(g.Periods)) + for _, p := range g.Periods { + periods = append(periods, fmt.Sprintf("%s:%d-%d", p.Label, p.Home, p.Away)) + } + row := []string{ + strconv.Itoa(s.SeasonID), + g.GameID, + g.EventID, + g.Date, + g.Time, + g.Home, + g.Away, + optInt(g.HomeGoals), + optInt(g.AwayGoals), + strings.Join(periods, " "), + optInt(g.Spectators), + g.Venue, + } + if err := w.Write(row); err != nil { + return nil, err + } + } + w.Flush() + return buf.Bytes(), w.Error() +} + +func optInt(p *int) string { + if p == nil { + return "" + } + return strconv.Itoa(*p) +} + +// emit writes the bytes to -out or stdout and, when -check is set, compares +// them with a committed golden file. A mismatch exits 3 so CI turns an upstream +// change into a reviewable diff instead of a silent data change. +func emit(out, check string, b []byte) error { + if check != "" { + golden, err := os.ReadFile(check) + switch { + case err == nil && bytes.Equal(golden, b): + case err != nil: + return fmt.Errorf("golden file %s: %w", check, err) + default: + return &DriftError{Msg: fmt.Sprintf("output no longer matches %s: %s", check, firstDiff(golden, b))} + } + } + if out == "" { + _, err := os.Stdout.Write(b) + return err + } + return os.WriteFile(out, b, 0o644) +} + +// firstDiff describes the first differing line, for a readable failure. +func firstDiff(a, b []byte) string { + al, bl := strings.Split(string(a), "\n"), strings.Split(string(b), "\n") + for i := 0; i < len(al) || i < len(bl); i++ { + var x, y string + if i < len(al) { + x = al[i] + } + if i < len(bl) { + y = bl[i] + } + if x != y { + return fmt.Sprintf("line %d: %q != %q", i+1, trim(x), trim(y)) + } + } + return "documents differ only in length" +} + +func trim(s string) string { + if len(s) > 80 { + return s[:80] + "..." + } + return s +} diff --git a/parse.go b/parse.go new file mode 100644 index 0000000..ca9a635 --- /dev/null +++ b/parse.go @@ -0,0 +1,540 @@ +package main + +import ( + "fmt" + "io" + "regexp" + "slices" + "sort" + "strconv" + "strings" + + "golang.org/x/net/html" +) + +// The site has shipped more than one version of this table, and the old +// versions are still served for historical seasons. So the contract is not a +// set of hard-coded column indices: it is a set of layout profiles, each +// identified by the header row it declares. A page the crawler does not +// recognise fails loudly with the header it actually found, and adding support +// is one profile plus one fixture. +type layout struct { + name string + header []string // non-empty header labels, in order + cells int // number of in one data row + round int // -1 when the layout has no Round column + + // Modern layout: the date is rendered twice, once in the desktop "Date" + // column and once as a group heading in the mobile column, and the + // start time lives in its own cell next to the game's stable id. + // Legacy layout: no group headings; the tooltip span of one cell holds + // both "2014-03-10" and "19:00". + dateDesktop int + dateMobile int + timeCell int + grouped bool + + teams int + result int + periods int + spectators int + venue int +} + +var layouts = []*layout{ + { + name: "modern", // seasons from 2018-19 onwards + header: []string{"Date", "Time", "Game", "Result", "Spectators", "Venue"}, + cells: 8, round: -1, + dateDesktop: 0, dateMobile: 1, timeCell: 2, grouped: true, + teams: 3, result: 4, periods: 5, spectators: 6, venue: 7, + }, + { + name: "legacy", // seasons up to 2017-18 + header: []string{"Round", "Date", "Game", "Result", "Spectators", "Venue"}, + cells: 7, round: 0, + dateDesktop: -1, dateMobile: -1, timeCell: 1, grouped: false, + teams: 2, result: 3, periods: 4, spectators: 5, venue: 6, + }, +} + +var ( + dateRE = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`) + timeRE = regexp.MustCompile(`^\d{1,2}:\d{2}$`) + digitsRE = regexp.MustCompile(`^\d+$`) + scoreRE = regexp.MustCompile(`^(\d+)\s*-\s*(\d+)$`) + periodRE = regexp.MustCompile(`^(\d+)\s*-\s*(\d+)$`) + eventIDRE = regexp.MustCompile(`/Game/Events/(\d+)`) + lastUpdRE = regexp.MustCompile(`Last update:\s*(\d{4}-\d{2}-\d{2} \d{1,2}:\d{2})`) + gameIDRE = regexp.MustCompile(`^\d{5,}$`) +) + +// Period is one period (or overtime / shootout) of a played game. +type Period struct { + Label string `json:"label"` // "1", "2", ... in site order + Home int `json:"home"` + Away int `json:"away"` +} + +// Game is one row of the schedule table. +type Game struct { + GameID string `json:"game_id"` // span.lnkTooltip@title, unique per season + EventID string `json:"event_id,omitempty"` // /Game/Events/; set once the game is played + Round string `json:"round,omitempty"` // only in the legacy layout + Date string `json:"date"` // YYYY-MM-DD + Time string `json:"time"` // HH:MM local arena time + Home string `json:"home"` + Away string `json:"away"` + HomeGoals *int `json:"home_goals"` // nil until the game is played + AwayGoals *int `json:"away_goals"` // nil until the game is played + Periods []Period `json:"periods"` // empty until the game is played + Spectators *int `json:"spectators,omitempty"` // nil when not reported + Venue string `json:"venue"` +} + +func (g Game) Played() bool { return g.EventID != "" } + +// Schedule is one season page. +type Schedule struct { + SeasonID int `json:"season_id"` + Layout string `json:"layout"` + Title string `json:"title"` + LastUpdate string `json:"last_update"` // the site's own clock, verbatim + Games []Game `json:"games"` +} + +func (s *Schedule) Played() int { + n := 0 + for _, g := range s.Games { + if g.Played() { + n++ + } + } + return n +} + +// DriftError reports that the page no longer matches a known contract. It is a +// distinct type, and a distinct exit code, because it is the one failure a +// caller must not retry blindly: the site changed, not the network. +type DriftError struct { + Season int + GameID string + Column string + Got string + Want string + Msg string +} + +func (e *DriftError) Error() string { + var b strings.Builder + fmt.Fprintf(&b, "swehockey: markup drift (season %d", e.Season) + if e.GameID != "" { + fmt.Fprintf(&b, ", game %s", e.GameID) + } + b.WriteString("): ") + if e.Msg != "" { + b.WriteString(e.Msg) + } else { + b.WriteString("unexpected table shape") + } + if e.Column != "" { + fmt.Fprintf(&b, " [column %s]", e.Column) + } + if e.Got != "" { + fmt.Fprintf(&b, ": got %q, want %q", e.Got, e.Want) + } + b.WriteString(" - extend layouts[] in parse.go and add a fixture, in one commit") + return b.String() +} + +func drift(season int, gameID, column, msg, got, want string) error { + return &DriftError{Season: season, GameID: gameID, Column: column, Msg: msg, Got: got, Want: want} +} + +// ParseSchedule extracts the schedule table from one saved or fetched page. +func ParseSchedule(seasonID int, r io.Reader) (*Schedule, error) { + doc, err := html.Parse(r) + if err != nil { + return nil, fmt.Errorf("swehockey: parse page: %w", err) + } + tbl, lay := findTable(doc) + if tbl == nil { + return nil, drift(seasonID, "", "", "no table with a known header: found "+strings.Join(documentSignatures(doc), " / ")+ + ", known "+knownLayouts(), "", "") + } + + s := &Schedule{SeasonID: seasonID, Layout: lay.name, Games: []Game{}} + rows := tableRows(tbl) + headerAt, c := -1, &carry{} + for i, tr := range rows { + if len(childElements(tr, "td")) == 0 { + if slices.Equal(thLabels(tr), lay.header) { + if headerAt >= 0 { + return nil, drift(seasonID, "", "", "second header row at row "+strconv.Itoa(i), "2", "1") + } + headerAt = i + } + continue + } + if headerAt < 0 { + return nil, drift(seasonID, "", "", "data row before the header row", "", "header first") + } + g, err := parseGame(seasonID, lay, tr, c) + if err != nil { + return nil, err + } + s.Games = append(s.Games, g) + } + if headerAt < 0 { + return nil, drift(seasonID, "", "", "table has no header row", "", strings.Join(lay.header, " ")) + } + if h := descendants(tbl, "h2"); len(h) > 0 { + s.Title = textOf(h[0]) + } + if lu := lastUpdRE.FindStringSubmatch(textOf(tbl)); lu != nil { + s.LastUpdate = lu[1] + } + if s.LastUpdate == "" { + return nil, drift(seasonID, "", "", `table has no "Last update:" stamp`, "", "Last update: YYYY-MM-DD HH:MM") + } + if err := s.check(); err != nil { + return nil, err + } + // The site lists games in round order, not date order, and a re-scheduled + // or awarded game keeps the round it belongs to: 2011-12 has September + // games listed after October ones. Sorting makes the output a canonical + // function of the set of rows instead of a function of the site's row + // order, which is what makes two crawls comparable and diffs meaningful. + sort.SliceStable(s.Games, func(i, j int) bool { + a, b := s.Games[i], s.Games[j] + if a.Date != b.Date { + return a.Date < b.Date + } + if a.Time != b.Time { + return a.Time < b.Time + } + return a.GameID < b.GameID + }) + return s, nil +} + +// check enforces the invariants that hold for every season the site serves. +func (s *Schedule) check() error { + if len(s.Games) == 0 { + return drift(s.SeasonID, "", "", "table has no game rows", "0 rows", ">= 1 row") + } + // Game ids are unique, and that is the only uniqueness the page promises: + // the site assigns them per fixture, so they are neither sorted nor + // contiguous within a season. + seen := make(map[string]bool, len(s.Games)) + for _, g := range s.Games { + if _, err := strconv.Atoi(g.GameID); err != nil { + return drift(s.SeasonID, g.GameID, "", "game id is not a number", g.GameID, "90001001") + } + if seen[g.GameID] { + return drift(s.SeasonID, g.GameID, "", "duplicate game id", g.GameID, "unique") + } + seen[g.GameID] = true + if !dateRE.MatchString(g.Date) { + return drift(s.SeasonID, g.GameID, "date", "malformed date", g.Date, "2026-09-19") + } + } + return nil +} + +// carry holds the values the site prints only on the first row of a group: the +// date heading of the modern layout and the round number of the legacy one. +type carry struct{ date, round string } + +// parseGame reads one row. Where the site renders a fact twice, both +// renderings must agree before the value is accepted. +func parseGame(seasonID int, lay *layout, tr *html.Node, c *carry) (Game, error) { + tds := childElements(tr, "td") + if len(tds) != lay.cells { + return Game{}, drift(seasonID, "", "", "row width does not match the "+lay.name+" layout", + strconv.Itoa(len(tds))+" cells", strconv.Itoa(lay.cells)+" cells") + } + g := Game{Periods: []Period{}} + + // The tooltip span carries the row's stable id, so the id and the time can + // never disagree: they are the same element. + span := tooltipSpan(tds[lay.timeCell]) + if span == nil { + return Game{}, drift(seasonID, "", "time", "no game id (span.lnkTooltip[title])", textOf(tds[lay.timeCell]), "90001001") + } + g.GameID = strings.TrimSpace(attr(span, "title")) + if !gameIDRE.MatchString(g.GameID) { + return Game{}, drift(seasonID, "", "time", "malformed game id", g.GameID, "90001001") + } + + head := textOf(span) + if lay.grouped { + // Modern layout: date in two renderings, time in the span. + if !timeRE.MatchString(head) { + return Game{}, drift(seasonID, g.GameID, "time", "malformed start time", head, "19:00") + } + g.Time = head + desktop := textOf(tds[lay.dateDesktop]) + mobile := textOf(firstElement(tds[lay.dateMobile], "bold")) + switch { + case mobile != "": + if !dateRE.MatchString(mobile) { + return Game{}, drift(seasonID, g.GameID, "date", "malformed date heading", mobile, "2026-09-19") + } + if desktop != mobile { + return Game{}, drift(seasonID, g.GameID, "date", "date renderings disagree", desktop, mobile) + } + if t := textExcluding(tds[lay.dateMobile], "bold"); t != g.Time { + return Game{}, drift(seasonID, g.GameID, "date", "time renderings disagree", t, g.Time) + } + g.Date = mobile + default: + if c.date == "" { + return Game{}, drift(seasonID, g.GameID, "date", "row before any date heading", desktop, "2026-09-19") + } + if desktop != "" && desktop != g.Time { + return Game{}, drift(seasonID, g.GameID, "date", "unexpected date in a continuation row", desktop, "(empty) or "+g.Time) + } + g.Date = c.date + } + } else { + // Legacy layout: "2014-03-10 19:00" in one cell, no group headings. + d, t, ok := strings.Cut(head, " ") + switch { + case ok && dateRE.MatchString(d) && timeRE.MatchString(t): + g.Date, g.Time = d, t + case timeRE.MatchString(head): + if c.date == "" { + return Game{}, drift(seasonID, g.GameID, "date", "row before any date", head, "2026-09-19 19:00") + } + g.Date, g.Time = c.date, head + default: + return Game{}, drift(seasonID, g.GameID, "date", "cannot read date and time", head, "2026-09-19 19:00") + } + } + + if lay.round >= 0 { + // The site prints the round only on the first row of that round and + // leaves the following cells empty, exactly like the date headings. + switch round := textOf(tds[lay.round]); { + case digitsRE.MatchString(round): + g.Round = round + case round == "" && c.round != "": + g.Round = c.round + case round == "": + return Game{}, drift(seasonID, g.GameID, "round", "row before any round number", round, "1") + default: + return Game{}, drift(seasonID, g.GameID, "round", "round is not a number", round, "1") + } + } + + // Teams: the cell is "Home - Away" plus, in old seasons, an note + // such as "Eighth final 1" that is not part of a name. + teams := textExcluding(tds[lay.teams], "i") + parts := strings.Split(teams, " - ") + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return Game{}, drift(seasonID, g.GameID, "game", "cannot split teams", teams, "Home - Away") + } + g.Home, g.Away = parts[0], parts[1] + + // Result: present only for played games, and always with an event link. + if a := firstElement(tds[lay.result], "a"); a != nil { + m := scoreRE.FindStringSubmatch(textOf(a)) + if m == nil { + return Game{}, drift(seasonID, g.GameID, "result", "malformed score", textOf(a), "3 - 2") + } + ev := eventIDRE.FindStringSubmatch(attr(a, "href")) + if ev == nil { + return Game{}, drift(seasonID, g.GameID, "result", "result link without an event id", attr(a, "href"), "/Game/Events/1109941") + } + h, _ := strconv.Atoi(m[1]) + aw, _ := strconv.Atoi(m[2]) + g.HomeGoals, g.AwayGoals, g.EventID = &h, &aw, ev[1] + } + + periods, err := parsePeriods(textOf(tds[lay.periods])) + if err != nil { + return Game{}, drift(seasonID, g.GameID, "periods", err.Error(), textOf(tds[lay.periods]), "(1-1, 2-1, 2-2)") + } + g.Periods = periods + + if sp := textOf(tds[lay.spectators]); sp != "" { + if !digitsRE.MatchString(sp) { + return Game{}, drift(seasonID, g.GameID, "spectators", "spectators are not a number", sp, "10442") + } + n, _ := strconv.Atoi(sp) + g.Spectators = &n + } + g.Venue = textOf(tds[lay.venue]) + if g.Venue == "" { + return Game{}, drift(seasonID, g.GameID, "venue", "empty venue", "", "Gottde Arena") + } + c.date, c.round = g.Date, g.Round + if err := g.checkRow(); err != nil { + return Game{}, drift(seasonID, g.GameID, "", err.Error(), "", "") + } + return g, nil +} + +// checkRow enforces the per-row invariants: periods sum to the final score, +// and a game is either fully played or not played at all. +func (g Game) checkRow() error { + if !g.Played() { + if g.HomeGoals != nil || len(g.Periods) > 0 { + return fmt.Errorf("unplayed game carries a result") + } + return nil + } + if len(g.Periods) < 3 { + return fmt.Errorf("played game has %d period scores", len(g.Periods)) + } + h, a := 0, 0 + for _, p := range g.Periods { + h += p.Home + a += p.Away + } + if h != *g.HomeGoals || a != *g.AwayGoals { + return fmt.Errorf("periods sum to %d-%d but the result is %d-%d", h, a, *g.HomeGoals, *g.AwayGoals) + } + return nil +} + +// parsePeriods reads "(1-1, 2-1, 2-2)". This is the single place to extend if +// the site ever renders a period differently, e.g. a shootout as "-". +func parsePeriods(s string) ([]Period, error) { + if s == "" { + return []Period{}, nil + } + if !strings.HasPrefix(s, "(") || !strings.HasSuffix(s, ")") { + return nil, fmt.Errorf("period scores are not parenthesised") + } + tokens := strings.Split(s[1:len(s)-1], ",") + // Only the lower bound is a contract. How many periods a game lasts depends + // on the era's rules - the site serves 3, 4, 5 and 6 period breakdowns - so + // the count is not pinned; the sum check in checkRow is what protects the + // data. + if len(tokens) < 3 { + return nil, fmt.Errorf("only %d period scores", len(tokens)) + } + out := make([]Period, 0, len(tokens)) + for i, tok := range tokens { + m := periodRE.FindStringSubmatch(strings.TrimSpace(tok)) + if m == nil { + return nil, fmt.Errorf("malformed period %q", strings.TrimSpace(tok)) + } + h, _ := strconv.Atoi(m[1]) + a, _ := strconv.Atoi(m[2]) + out = append(out, Period{Label: strconv.Itoa(i + 1), Home: h, Away: a}) + } + return out, nil +} + +// findTable returns the first table that declares one of the known layouts. +// Locating the table by its header, rather than by a CSS path or an index, is +// what keeps ads, navigation and layout changes from breaking the crawler. +func findTable(root *html.Node) (*html.Node, *layout) { + var found *html.Node + var lay *layout + var walk func(*html.Node) + walk = func(n *html.Node) { + if found != nil { + return + } + if n.Type == html.ElementNode && n.Data == "table" { + for _, tr := range tableRows(n) { + labels := thLabels(tr) + for _, l := range layouts { + if slices.Equal(labels, l.header) { + found, lay = n, l + return + } + } + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(root) + return found, lay +} + +// documentSignatures describes every table on the page, so that an +// unrecognised layout is diagnosable from the error message alone. +func documentSignatures(doc *html.Node) []string { + var out []string + var walk func(*html.Node) + walk = func(n *html.Node) { + if n.Type == html.ElementNode && n.Data == "table" { + if labels := tableHeaderLabels(n); len(labels) > 0 { + cells := 0 + for _, tr := range tableRows(n) { + if c := len(childElements(tr, "td")); c > 0 { + cells = c + break + } + } + out = append(out, fmt.Sprintf("%q with %d cells", strings.Join(labels, " "), cells)) + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(doc) + if len(out) == 0 { + return []string{"no table with a header row"} + } + return out +} + +func knownLayouts() string { + names := make([]string, 0, len(layouts)) + for _, l := range layouts { + names = append(names, fmt.Sprintf("%s=%q", l.name, strings.Join(l.header, " "))) + } + return strings.Join(names, ", ") +} + +// thLabels returns the non-empty header labels of a row, or of the first row +// of a table that has one. +func thLabels(n *html.Node) []string { + trs := tableRows(n) + if n.Type == html.ElementNode && n.Data == "tr" { + trs = []*html.Node{n} + } + for _, tr := range trs { + if labels := tableHeaderLabels(tr); len(labels) > 0 { + return labels + } + } + return nil +} + +// tableHeaderLabels collects the labels of a single row. +func tableHeaderLabels(row *html.Node) []string { + var labels []string + for _, th := range childElements(row, "th") { + if l := textOf(th); l != "" { + labels = append(labels, l) + } + } + return labels +} + +func firstElement(n *html.Node, tag string) *html.Node { + if d := descendants(n, tag); len(d) > 0 { + return d[0] + } + return nil +} + +// tooltipSpan returns the span that carries a game's stable id. +func tooltipSpan(cell *html.Node) *html.Node { + for _, s := range descendants(cell, "span") { + if hasClass(s, "lnkTooltip") && attr(s, "title") != "" { + return s + } + } + return nil +} diff --git a/parse_test.go b/parse_test.go new file mode 100644 index 0000000..4eebc22 --- /dev/null +++ b/parse_test.go @@ -0,0 +1,445 @@ +package main + +import ( + "bytes" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + "time" +) + +// The tests never touch the network: they parse committed HTML slices. That is +// what makes the suite deterministic, and it is why a change in the site's +// markup shows up as a failing test with a readable message instead of as +// silently different data. + +func parseFixture(t *testing.T, name string) *Schedule { + t.Helper() + f, err := os.Open(filepath.Join("testdata", name)) + if err != nil { + t.Fatal(err) + } + defer f.Close() + s, err := ParseSchedule(20961, f) + if err != nil { + t.Fatalf("ParseSchedule(%s): %v", name, err) + } + return s +} + +// TestGolden pins the exact bytes of the rendered output. Same HTML in, same +// bytes out - that is the property the whole design exists to provide. +func TestGolden(t *testing.T) { + for _, tc := range []struct{ fixture, golden string }{ + {"schedule_20961.html", "schedule_20961.json"}, + {"schedule_legacy.html", "schedule_legacy.json"}, + {"schedule_legacy_playoffs.html", "schedule_legacy_playoffs.json"}, + } { + t.Run(tc.fixture, func(t *testing.T) { + f, err := os.Open(filepath.Join("testdata", tc.fixture)) + if err != nil { + t.Fatal(err) + } + defer f.Close() + s, err := ParseSchedule(20961, f) + if err != nil { + t.Fatalf("parse: %v", err) + } + got, err := render("json", s) + if err != nil { + t.Fatal(err) + } + want, err := os.ReadFile(filepath.Join("testdata", tc.golden)) + if err != nil { + t.Fatalf("read golden (regenerate with -update flag): %v", err) + } + if !bytes.Equal(got, want) { + t.Errorf("output drifted from the golden file:\n%s", firstDiff(want, got)) + } + }) + } +} + +func TestModernLayout(t *testing.T) { + s := parseFixture(t, "schedule_20961.html") + if s.Layout != "modern" { + t.Errorf("layout = %q, want modern", s.Layout) + } + if s.LastUpdate != "2026-09-26 22:41" { + t.Errorf("last update = %q", s.LastUpdate) + } + if len(s.Games) != 10 || s.Played() != 8 { + t.Fatalf("games = %d, played = %d; want 10/8", len(s.Games), s.Played()) + } + // Date headings appear only on the first row of a date; the rest inherit. + // A date is printed only on the first row of its group; the rest inherit it. + want := []string{"2026-09-19", "2026-09-19", "2026-09-19", "2026-09-19", "2026-09-19", "2026-09-19", + "2026-09-22", "2026-09-22", "2027-03-16", "2027-03-16"} + for i, g := range s.Games { + if g.Date != want[i] { + t.Errorf("game %d: date = %s, want %s", i, g.Date, want[i]) + } + } + // Overtime (4 periods) and a shootout (5 periods) keep every period. + if g := s.Games[1]; len(g.Periods) != 4 || *g.HomeGoals != 4 { + t.Errorf("game 2: %d periods, %d goals", len(g.Periods), *g.HomeGoals) + } + if g := s.Games[6]; len(g.Periods) != 5 || *g.HomeGoals != 3 { + t.Errorf("game 7: %d periods, %d goals", len(g.Periods), *g.HomeGoals) + } + // An unplayed game has a result of nil, not 0-0, and no periods. + if g := s.Games[len(s.Games)-1]; g.Played() || g.HomeGoals != nil || g.AwayGoals != nil || len(g.Periods) != 0 { + t.Errorf("last game should be unplayed: %+v", g) + } + if g := s.Games[0]; g.Home != "Frölunda HC" || g.Away != "Växjö Lakers HC" { + t.Errorf("teams not normalised: %q vs %q", g.Home, g.Away) + } + if g := s.Games[0]; g.EventID != "1109941" || g.Spectators == nil || *g.Spectators != 12044 { + t.Errorf("event id / spectators: %+v", g) + } +} + +func TestLegacyLayout(t *testing.T) { + s := parseFixture(t, "schedule_legacy.html") + if s.Layout != "legacy" { + t.Errorf("layout = %q, want legacy", s.Layout) + } + g := s.Games[0] + // The legacy cell holds "