commit 6b701819cae7588fcb5a14fdb1e803351354c612 Author: Mats Tornberg Date: Sun Sep 27 08:47:25 2026 +0000 Deterministic swehockey.se schedule crawler One HTTP GET per season, no browser: the page is server-rendered, so every row arrives in one response and the extraction becomes a pure function of the HTML. - layouts[] declares one profile per version of the site's table (modern and legacy), each identified by its header row, so a new version of the page is a new profile plus a fixture instead of a guess. - The table is located by header signature, not by a CSS path. - Every cell is shape-checked, and facts the site renders twice (desktop and mobile date, time in the tooltip) must agree. - Cross-field invariants: periods sum to the score, ids are unique, played games always have an event link and unplayed ones never do. - Output is sorted by date, time and id, so it does not inherit the site's row order, and it carries no fetch timestamp: same HTML, same bytes. - Drift is a typed error and exit code 3, with the header it actually found. - Tests run on committed HTML fixtures and golden JSON; TestLive is opt-in. Verified over the full SHL archive: 27/27 seasons, 2119 games, 1779 played. diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ea253f6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +/swehockey +*.tmp +/.codesearch/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..fa534cc --- /dev/null +++ b/README.md @@ -0,0 +1,170 @@ +# swehockey-schedule + +A deterministic crawler for the **Schedule and Results** table on +[stats.swehockey.se](https://stats.swehockey.se) — one plain HTTP GET per +season, no browser, no JavaScript, no waiting, and a byte-identical output for +identical input. + +```console +$ go build -o swehockey . +$ ./swehockey schedule -season 20961 -format csv | head -3 +season_id,game_id,event_id,date,time,home,away,home_goals,away_goals,periods,spectators,venue +20961,90001001,1109941,2026-09-19,15:15,Frölunda HC,Växjö Lakers HC,5,4,1:1-1 2:2-1 3:2-2,12044,Scandinavium +20961,90001002,1109942,2026-09-19,15:15,HV 71,IF Malmö Redhawks,4,5,1:1-0 2:1-1 3:1-3 4:1-1,6800,Husqvarna Garden +``` + +```console +$ ./swehockey crawl -league 20961 # every season of the SHL, as NDJSON +$ ./swehockey leagues # 119 competitions and their ids +$ ./swehockey schedule -fixture testdata/schedule_20961.html # offline +``` + +## How determinism is achieved + +The site's page is server-rendered: `curl` returns all 364 rows of the current +season in one response. That removes the usual sources of flakiness (timers, +`waitForSelector`, lazy rendering) and leaves exactly one risk — that the +markup changes under you. The rest of the design is about that risk. + +1. **One GET, no retries, no cookies, no state.** Same HTML in, same bytes out. + The output contains only page-derived data: no fetch timestamp, no map + iteration, fixed field order. `TestRenderIsPure` and the golden files enforce + this. +2. **The table is found by its header signature**, not by a CSS path or a + position in the document. Ads, navigation and layout changes cannot move it + out from under the crawler. +3. **A declared contract per layout, not a pile of indices.** The site has + shipped more than one version of this table, so each version is a profile in + `layouts[]` (`parse.go`), identified by the header row it declares. An + unknown shape fails with the header it actually found. +4. **Every field is shape-checked**, so a renamed column or a new cell is an + error naming the game and the cell, never a silently empty column. +5. **Facts the site renders twice must agree.** In the modern layout the date is + printed in a desktop cell *and* as a mobile `` group heading, and the + start time appears in the tooltip span and in the mobile cell. The parser + reads both and rejects the row if they disagree. Values the site prints once + per group (the date heading, the round number) are carried forward, and the + first row of a group must carry the value to inherit. +6. **Cross-field invariants**: the period scores must sum to the final score, + a played game always has an event link and an unplayed one never does, + game ids are unique, dates parse, venues are non-empty. +7. **Canonical order, imposed by us.** The site lists games in round order, and + re-scheduled or awarded games keep the round they belong to, so the page's + own date order is *not* monotonic (2011-12 has September games after October + ones). The output is sorted by date, time and id, so it is a function of the + set of rows rather than of the site's row order — `TestOrderIndependence` + swaps two rows and requires identical bytes. +8. **Fixtures, not the network, in the tests.** Real HTML slices live in + `testdata/` next to golden JSON. CI is offline and hermetic; a site change + surfaces as a failing test with a readable message. `SWEHOCKEY_LIVE=1 go + test -run TestLive` is the one opt-in test that talks to the site. +9. **Drift is a distinct failure mode**, not a crash: `*DriftError` and exit + code 3. Callers can alert on it without parsing a stack trace. +10. **Idempotent cron.** `-state ` skips the run when the page body hash + is unchanged, and `-check ` exits 3 when the output no longer + matches a committed file, so an upstream change becomes a reviewable diff. + +## The two layouts on the site today + +| profile | header | cells | seasons | +| -------- | --------------------------------------------- | ----- | ------------------ | +| `modern` | `Date Time Game Result … Spectators Venue` | 8 | 2018-19 onwards | +| `legacy` | `Round Date Game Result … Spectators Venue` | 7 | up to 2017-18 | + +They differ in more than columns: + +- `modern` renders every fact twice, for a desktop and a mobile table, and + prints the date only on the first row of each date. The time lives in + `span.lnkTooltip[title]`, which is also the row's stable id. +- `legacy` renders it once. The round is in its own column, the date and the + time share one cell, and old team cells end with an `` note + (`Eighth final 1`) that must not be swallowed into a team name. +- A played game has 3, 4, 5 or 6 period scores depending on the era's rules, so + the count is deliberately *not* pinned; the sum check is what protects the + data. + +Verified on the whole SHL archive: **27 of 27 seasons, 2119 games, 1779 played**. + +``` +$ ./swehockey crawl -league 20961 -on-drift record | jq -s '[.[] | .games | length] | add' +2119 +``` + +## Commands + +| command | what it does | +| --------- | ------------------------------------------------------------------- | +| `schedule`| one season, found by `-season` (default 20961) | +| `crawl` | every season of `-league`, in ascending id order, one document each | +| `leagues` | the competitions linked from the front page | + +Flags: `-format json|csv|ndjson`, `-out`, `-fixture`, `-state`, `-check`, +`-timeout`, `-quiet`, plus `-limit`, `-delay` and `-on-drift` for `crawl`. + +`crawl -on-drift record` emits `{"season_id":…,"error":…}` in place of a season +that breaks the contract, so a bulk run neither dies on one bad page nor hides +the hole. + +Exit codes: `0` ok, `1` error, `2` usage, `3` markup drift or golden mismatch, +`4` timeout. + +## Output + +```json +{ + "season_id": 20961, + "layout": "modern", + "title": "Schedule and Results", + "last_update": "2026-09-26 22:41", + "games": [ + { + "game_id": "90001001", + "event_id": "1109941", + "date": "2026-09-19", + "time": "15:15", + "home": "Frölunda HC", + "away": "Växjö Lakers HC", + "home_goals": 5, + "away_goals": 4, + "periods": [{"label": "1", "home": 1, "away": 1}], + "spectators": 12044, + "venue": "Scandinavium" + } + ] +} +``` + +`game_id` is the site's own stable per-fixture id and is present for scheduled +as well as played games; `event_id` comes from the `/Game/Events/` link and +appears only once a game has been played. `home_goals` is `null`, not `0`, for a +game that has not been played. `last_update` is the site's own clock, verbatim +and unparsed, so it can be compared across runs without trusting a timezone +conversion. + +## Tests and fixtures + +``` +testdata/schedule_20961.html 2026-27, 3 date groups, OT + shootout, unplayed games +testdata/schedule_legacy.html 2018-19, round carry-forward, a 6-period game +testdata/schedule_legacy_playoffs.html 2013-14, notes in the team cells +testdata/seasons_20961.html the season selector and league links +testdata/*.json golden output for the fixtures above +``` + +`TestDrift` mutates a fixture once per way the page can change shape — renamed +header, extra column, score that no longer matches the periods, a removed date +group marker, a duplicated id, a missing timestamp, an empty table — and +requires a named, actionable error for each. + +The fixtures are cut from real pages; the only thing the cutter changes is which +`` elements it keeps, so the markup around them stays real, including the +nested wrapper tables and the ad div. + +## Known limits + +- Old seasons are pruned by the site: 2018-19 serves its last 45 games, 2024-25 + only its 6 playoff games. The crawler reports exactly what the page contains. +- A handful of season pages serve more slowly than the default 30s timeout + (they stream a large table); `-timeout` covers it. +- Rows with no result and no round appear in a few old pages; those are the + reason `-on-drift record` exists. diff --git a/cmd_crawl.go b/cmd_crawl.go new file mode 100644 index 0000000..58938de --- /dev/null +++ b/cmd_crawl.go @@ -0,0 +1,130 @@ +package main + +import ( + "bytes" + "context" + "errors" + "flag" + "fmt" + "os" + "time" +) + +// cmdCrawl is the crawler proper: take one league, read its season selector, +// then walk every season in ascending id order and emit one document per +// season. The order is derived from the ids, never from map iteration or from +// the order the site happened to list things, so two runs of the crawler +// produce the same file even if the site's markup order changes. +func cmdCrawl(ctx context.Context, args []string) error { + fs := flag.NewFlagSet("crawl", flag.ContinueOnError) + var c commonFlags + c.register(fs, "ndjson") + league := fs.Int("league", defaultSeason, "league id (same id space as season ids)") + only := fs.Int("season", 0, "crawl only this season id (0 = all seasons of the league)") + limit := fs.Int("limit", 0, "stop after N seasons (0 = all)") + delay := fs.Duration("delay", 250*time.Millisecond, "pause between season requests") + onDrift := fs.String("on-drift", "fail", "when a season page breaks the contract: fail or record") + if err := fs.Parse(args); err != nil { + return err + } + cl := c.client() + + seed, err := c.read(ctx, cl, SchedulePath(*league), fmt.Sprintf("league %d", *league)) + if errors.Is(err, errUnchanged) { + return nil + } + if err != nil { + return err + } + seasons, err := ParseSeasons(*league, bytes.NewReader(seed)) + if err != nil { + return err + } + + var out bytes.Buffer + enc := jsonEncoder(&out) + if c.format == "json" { + out.WriteString("[\n") + } + done := 0 + for _, s := range seasons { + if *only != 0 && s.ID != *only { + continue + } + if *limit > 0 && done >= *limit { + break + } + if done > 0 && *delay > 0 { + time.Sleep(*delay) + } + page, err := cl.get(ctx, SchedulePath(s.ID)) + if err != nil { + return err + } + sched, err := ParseSchedule(s.ID, bytes.NewReader(page)) + if err != nil { + // A season whose page breaks the contract is either fatal or + // recorded, never dropped: a hole in the data has to be visible. + var de *DriftError + if *onDrift == "fail" || !errors.As(err, &de) { + return err + } + if !c.quiet { + fmt.Fprintf(os.Stderr, "season %d: %v\n", s.ID, de) + } + if c.format == "csv" { + fmt.Fprintf(&out, "%d,ERROR,,,,,,,,,%s\n", s.ID, de.Msg) + } else { + if c.format == "json" { + if done > 0 { + out.WriteString(",\n") + } + out.WriteString(" ") + } + if err := enc.Encode(crawlError{SeasonID: s.ID, Label: s.Label, Error: de.Msg}); err != nil { + return err + } + } + done++ + continue + } + if c.format == "csv" { + b, err := renderCSV(sched) + if err != nil { + return err + } + out.Write(b) + } else { + if c.format == "json" { + if done > 0 { + out.WriteString(",\n") + } + out.WriteString(" ") + } + // Encode writes the document plus one newline: exactly the + // NDJSON contract, and one element of the JSON array. + if err := enc.Encode(sched); err != nil { + return err + } + } + done++ + if !c.quiet { + fmt.Fprintf(os.Stderr, "season %d (%s): %d games, %d played\n", sched.SeasonID, s.Label, len(sched.Games), sched.Played()) + } + } + if c.format == "json" { + out.WriteString("\n]\n") + } + if done == 0 { + return drift(*league, "", "", "crawl matched no seasons", "0 seasons", ">= 1") + } + if err := emit(c.out, c.check, out.Bytes()); err != nil { + return err + } + if c.state != "" { + if err := saveState(c.state, &state{Path: fmt.Sprintf("crawl:%d", *league), SHA256: sha256Hex(out.Bytes()), SavedAt: nowStamp()}); err != nil { + return err + } + } + return nil +} diff --git a/cmd_schedule.go b/cmd_schedule.go new file mode 100644 index 0000000..0799115 --- /dev/null +++ b/cmd_schedule.go @@ -0,0 +1,103 @@ +package main + +import ( + "bytes" + "context" + "errors" + "flag" + "fmt" + "os" + "time" +) + +func cmdSchedule(ctx context.Context, args []string) error { + fs := flag.NewFlagSet("schedule", flag.ContinueOnError) + var c commonFlags + c.register(fs, "json") + season := fs.Int("season", defaultSeason, "season id; see the crawl or leagues command") + if err := fs.Parse(args); err != nil { + return err + } + + path := SchedulePath(*season) + page, err := c.read(ctx, c.client(), path, fmt.Sprintf("season %d", *season)) + if errors.Is(err, errUnchanged) { + return nil + } + if err != nil { + return err + } + s, err := ParseSchedule(*season, bytes.NewReader(page)) + if err != nil { + return err + } + var b []byte + if c.format == "ndjson" { + b, err = renderNDJSON(s) + } else { + b, err = render(c.format, s) + } + if err != nil { + return err + } + if err := emit(c.out, c.check, b); err != nil { + return err + } + if c.state != "" { + if err := saveState(c.state, &state{Path: path, SHA256: sha256Hex(page), LastUpdate: s.LastUpdate, SavedAt: nowStamp()}); err != nil { + return err + } + } + if !c.quiet { + fmt.Fprintf(os.Stderr, "season %d: %d games, %d played, last update %s\n", + s.SeasonID, len(s.Games), s.Played(), s.LastUpdate) + } + return nil +} + +func cmdLeagues(ctx context.Context, args []string) error { + fs := flag.NewFlagSet("leagues", flag.ContinueOnError) + var c commonFlags + c.register(fs, "json") + if err := fs.Parse(args); err != nil { + return err + } + page, err := c.read(ctx, c.client(), "/", "front page") + if errors.Is(err, errUnchanged) { + return nil + } + if err != nil { + return err + } + leagues, err := ParseLeagues(bytes.NewReader(page)) + if err != nil { + return err + } + var b []byte + if c.format == "csv" { + var out bytes.Buffer + out.WriteString("id,name,path\n") + for _, l := range leagues { + fmt.Fprintf(&out, "%d,%s,%s\n", l.ID, l.Name, l.Path) + } + b = out.Bytes() + } else { + var buf bytes.Buffer + enc := jsonEncoder(&buf) + if err := enc.Encode(leagues); err != nil { + return err + } + b = buf.Bytes() + } + return emit(c.out, c.check, b) +} + +// crawlError is what a "record" run emits in place of a season, so a broken +// page is a visible hole in the output rather than a missing line. +type crawlError struct { + SeasonID int `json:"season_id"` + Label string `json:"label"` + Error string `json:"error"` +} + +func nowStamp() string { return time.Now().UTC().Format(time.RFC3339) } diff --git a/dom.go b/dom.go new file mode 100644 index 0000000..1be430e --- /dev/null +++ b/dom.go @@ -0,0 +1,129 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "strings" + "unicode" + + "golang.org/x/net/html" +) + +// The site writes the team separator as NBSP-hyphen-NBSP wrapped in newlines, +// so every string that leaves the parser goes through normalize and downstream +// comparisons and diffs stay stable. +func normalize(s string) string { + return strings.Join(strings.FieldsFunc(s, isSpaceRune), " ") +} + +func isSpaceRune(r rune) bool { + if unicode.IsSpace(r) { + return true + } + // No-break and zero-width variants the site emits around team names. + switch r { + case '\u00a0', '\u2007', '\u202f', '\u200b', '\ufeff': + return true + } + return false +} + +// textOf returns the normalized text of a node subtree. +func textOf(n *html.Node) string { + if n == nil { + return "" + } + var b strings.Builder + var walk func(*html.Node) + walk = func(n *html.Node) { + if n.Type == html.TextNode { + b.WriteString(n.Data) + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(n) + return normalize(b.String()) +} + +// textExcluding returns the normalized text of a node subtree with the first +// descendant matching skipTag (and its subtree) left out. The schedule uses +// this to read the mobile cell's start time without its date. +func textExcluding(n *html.Node, skipTag string) string { + if n == nil { + return "" + } + var b strings.Builder + var walk func(*html.Node) bool + walk = func(n *html.Node) bool { + if n.Type == html.ElementNode && n.Data == skipTag { + return false + } + if n.Type == html.TextNode { + b.WriteString(n.Data) + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + return true + } + walk(n) + return normalize(b.String()) +} + +// childElements returns the direct children of n with the given tag name. +func childElements(n *html.Node, tag string) []*html.Node { + if n == nil { + return nil + } + var out []*html.Node + for c := n.FirstChild; c != nil; c = c.NextSibling { + if c.Type == html.ElementNode && c.Data == tag { + out = append(out, c) + } + } + return out +} + +// descendants returns every element with the given tag name, in document order. +func descendants(n *html.Node, tag string) []*html.Node { + var out []*html.Node + var walk func(*html.Node) + walk = func(n *html.Node) { + if n.Type == html.ElementNode && n.Data == tag { + out = append(out, n) + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(n) + return out +} + +func attr(n *html.Node, key string) string { + if n == nil { + return "" + } + for _, a := range n.Attr { + if a.Key == key { + return a.Val + } + } + return "" +} + +func hasClass(n *html.Node, class string) bool { + for _, f := range strings.Fields(attr(n, "class")) { + if f == class { + return true + } + } + return false +} + +func sha256Hex(b []byte) string { + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:]) +} diff --git a/fetch.go b/fetch.go new file mode 100644 index 0000000..f3c18c0 --- /dev/null +++ b/fetch.go @@ -0,0 +1,121 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "regexp" + "strings" + "time" +) + +// userAgent identifies the crawler. The site is a public ASP.NET app and serves +// the same bytes to any client; saying who we are keeps its logs readable. +const userAgent = "swehockey-schedule/1.0 (+deterministic schedule crawler)" + +var errUnchanged = errors.New("swehockey: page unchanged") + +// Client fetches pages. One request, no retries, no cookies, no redirects +// beyond the default limit: every byte of the output is a function of one +// response. +type Client struct { + BaseURL string + Timeout time.Duration +} + +func (c *Client) get(ctx context.Context, path string) ([]byte, error) { + hc := &http.Client{Timeout: c.Timeout} + url := joinURL(c.BaseURL, path) + ctx, cancel := context.WithTimeout(ctx, c.Timeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", userAgent) + req.Header.Set("Accept", "text/html,application/xhtml+xml") + req.Header.Set("Accept-Language", "en") + + resp, err := hc.Do(req) + if err != nil { + return nil, fmt.Errorf("fetch %s: %w", url, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("fetch %s: %s", url, resp.Status) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("read %s: %w", url, err) + } + if err := checkCharset(body); err != nil { + return nil, fmt.Errorf("fetch %s: %w", url, err) + } + return body, nil +} + +var metaCharsetRE = regexp.MustCompile(`(?i)]+charset\s*=\s*["']?([a-z0-9_-]+)`) + +// checkCharset refuses pages we cannot decode byte-exactly. Silently handing +// latin-1 bytes to a UTF-8 parser would turn "Växjö" into mojibake that still +// parses, which is the worst kind of crawler bug. +func checkCharset(body []byte) error { + m := metaCharsetRE.FindSubmatch(body) + if m == nil { + return nil // no declaration: html.Parse assumes UTF-8 + } + switch cs := strings.ToLower(string(m[1])); cs { + case "utf-8", "utf8": + return nil + default: + return fmt.Errorf("unsupported charset %q: add a decoder for it", cs) + } +} + +// state records what the last successful run saw, so a cron job can be +// idempotent: an unchanged page produces no output and a zero exit code. +type state struct { + Path string `json:"path"` + SHA256 string `json:"sha256"` + LastUpdate string `json:"last_update"` + SavedAt string `json:"saved_at"` +} + +func loadState(path string) (*state, error) { + b, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return &state{}, nil + } + if err != nil { + return nil, err + } + var st state + if err := json.Unmarshal(b, &st); err != nil { + return nil, fmt.Errorf("state file %s: %w", path, err) + } + return &st, nil +} + +// saveState writes the state atomically so a killed run cannot leave a +// truncated file that would make the next run skip real work. +func saveState(path string, st *state) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + b, err := json.MarshalIndent(st, "", " ") + if err != nil { + return err + } + b = append(b, '\n') + tmp := path + ".tmp" + if err := os.WriteFile(tmp, b, 0o644); err != nil { + return err + } + return os.Rename(tmp, path) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..773a187 --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module swehockey + +go 1.25.0 + +require golang.org/x/net v0.58.0 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..0a44e17 --- /dev/null +++ b/go.sum @@ -0,0 +1,2 @@ +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= diff --git a/main.go b/main.go new file mode 100644 index 0000000..7f86285 --- /dev/null +++ b/main.go @@ -0,0 +1,160 @@ +// Command swehockey-schedule extracts the "Schedule and Results" table from +// stats.swehockey.se into JSON or CSV. +// +// The extraction is deterministic by construction: +// +// - The page is server-rendered. One plain HTTP GET returns every row of the +// table, so there is no browser, no JavaScript, no waiting and no +// timing-dependent retry. Same HTML in, same bytes out. +// - The table is found by its header signature, never by a CSS path, so +// ads, navigation and layout changes cannot move it out from under us. +// - Every field is read from a documented cell index *and* shape-checked. +// Markup drift produces an error naming the game and the cell, never a +// silently truncated or mis-mapped table. +// - The site renders each fact twice (a desktop cell and a mobile cell). +// The parser requires both renderings to agree, so a responsive-layout +// change cannot corrupt half the rows unnoticed. +// - Cross-field invariants are enforced: period goals must sum to the final +// score, game ids must be unique and ascending, dates must not go back. +// - Output is a pure function of the page: no fetch timestamps, no map +// iteration, fixed field order. Diffs between runs are meaningful. +// +// Commands: +// +// swehockey-schedule schedule -season 20961 # one season -> JSON +// swehockey-schedule schedule -season 20961 -format csv +// swehockey-schedule crawl -league 20961 # every season of a league +// swehockey-schedule leagues # league ids on the front page +// swehockey-schedule schedule -fixture page.html # offline, for tests/debugging +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "os" + "time" +) + +const defaultSeason = 20961 + +func main() { + if len(os.Args) < 2 { + usage() + os.Exit(2) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + var err error + switch cmd := os.Args[1]; cmd { + case "schedule": + err = cmdSchedule(ctx, os.Args[2:]) + case "crawl": + err = cmdCrawl(ctx, os.Args[2:]) + case "leagues": + err = cmdLeagues(ctx, os.Args[2:]) + case "-h", "--help", "help": + usage() + return + default: + fmt.Fprintf(os.Stderr, "swehockey-schedule: unknown command %q\n\n", cmd) + usage() + os.Exit(2) + } + + var de *DriftError + switch { + case err == nil: + case errors.Is(err, context.DeadlineExceeded): + fmt.Fprintf(os.Stderr, "swehockey-schedule: %v\n", err) + os.Exit(4) + case errors.As(err, &de): + // Drift is not a crash: it means the site changed shape. Exit 3 so a + // cron job or CI gate can alert without a stack trace. + fmt.Fprintln(os.Stderr, de.Error()) + os.Exit(3) + default: + fmt.Fprintf(os.Stderr, "swehockey-schedule: %v\n", err) + os.Exit(1) + } +} + +func usage() { + fmt.Fprint(os.Stderr, `swehockey-schedule - deterministic stats.swehockey.se schedule crawler + +usage: + swehockey-schedule schedule [flags] extract one season's schedule table + swehockey-schedule crawl [flags] extract every season of one league (NDJSON) + swehockey-schedule leagues [flags] list league ids and names + +common flags: + -base string site root (default https://stats.swehockey.se) + -format string json|csv|ndjson (default json; crawl defaults to ndjson) + -out string output file (default stdout) + -fixture string parse a saved page instead of fetching (offline) + -state string state file; skip the run when the page is unchanged + -check string golden file; exit 3 if the output no longer matches + -timeout duration per request (default 30s) + -quiet bool suppress progress notes on stderr + +exit codes: 0 ok, 1 error, 2 usage, 3 markup drift or golden mismatch, 4 timeout +`) +} + +type commonFlags struct { + base string + format string + out string + fixture string + state string + check string + timeout time.Duration + quiet bool +} + +func (c *commonFlags) register(fs *flag.FlagSet, defFormat string) { + fs.StringVar(&c.base, "base", "https://stats.swehockey.se", "site root") + fs.StringVar(&c.format, "format", defFormat, "output format: json, csv or ndjson") + fs.StringVar(&c.out, "out", "", "write to this file instead of stdout") + fs.StringVar(&c.fixture, "fixture", "", "parse a saved HTML page instead of fetching") + fs.StringVar(&c.state, "state", "", "state file for skip-when-unchanged runs") + fs.StringVar(&c.check, "check", "", "golden file; exit 3 when the output differs") + fs.DurationVar(&c.timeout, "timeout", 30*time.Second, "per-request timeout") + fs.BoolVar(&c.quiet, "quiet", false, "suppress progress notes on stderr") +} + +func (c *commonFlags) client() *Client { + return &Client{BaseURL: c.base, Timeout: c.timeout} +} + +// read returns the page bytes for a path, honouring -fixture and -state. +func (c *commonFlags) read(ctx context.Context, cl *Client, path, what string) ([]byte, error) { + if c.fixture != "" { + b, err := os.ReadFile(c.fixture) + if err != nil { + return nil, err + } + return b, nil + } + if c.state != "" { + st, err := loadState(c.state) + if err != nil { + return nil, err + } + b, err := cl.get(ctx, path) + if err != nil { + return nil, err + } + if st.SHA256 == sha256Hex(b) && st.Path == path { + // Byte-identical page: the derived output would be identical too. + if !c.quiet { + fmt.Fprintf(os.Stderr, "swehockey-schedule: %s unchanged since %s, nothing to do\n", what, st.SavedAt) + } + return nil, errUnchanged + } + return b, nil + } + return cl.get(ctx, path) +} diff --git a/output.go b/output.go new file mode 100644 index 0000000..6e943d0 --- /dev/null +++ b/output.go @@ -0,0 +1,140 @@ +package main + +import ( + "bytes" + "encoding/csv" + "encoding/json" + "fmt" + "os" + "strconv" + "strings" +) + +// render turns a schedule into bytes. Both encoders are pure functions of the +// data: no timestamps, no maps, fixed column order, LF line endings. Running +// the crawler twice on the same page produces identical bytes, which is what +// makes -check and diff-based review meaningful. +func render(format string, s *Schedule) ([]byte, error) { + switch format { + case "json": + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) // keep team names readable + enc.SetIndent("", " ") + if err := enc.Encode(s); err != nil { + return nil, err + } + return buf.Bytes(), nil + case "csv": + return renderCSV(s) + default: + return nil, fmt.Errorf("unknown format %q: want json, csv or ndjson", format) + } +} + +func renderNDJSON(s *Schedule) ([]byte, error) { + var buf bytes.Buffer + if err := jsonEncoder(&buf).Encode(s); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// jsonEncoder keeps "ä" readable instead of \u00e4, which matters for diffs +// of team names. +func jsonEncoder(buf *bytes.Buffer) *json.Encoder { + enc := json.NewEncoder(buf) + enc.SetEscapeHTML(false) + return enc +} + +var csvHeader = []string{ + "season_id", "game_id", "event_id", "date", "time", "home", "away", + "home_goals", "away_goals", "periods", "spectators", "venue", +} + +func renderCSV(s *Schedule) ([]byte, error) { + var buf bytes.Buffer + w := csv.NewWriter(&buf) // LF, not CRLF + if err := w.Write(csvHeader); err != nil { + return nil, err + } + for _, g := range s.Games { + periods := make([]string, 0, len(g.Periods)) + for _, p := range g.Periods { + periods = append(periods, fmt.Sprintf("%s:%d-%d", p.Label, p.Home, p.Away)) + } + row := []string{ + strconv.Itoa(s.SeasonID), + g.GameID, + g.EventID, + g.Date, + g.Time, + g.Home, + g.Away, + optInt(g.HomeGoals), + optInt(g.AwayGoals), + strings.Join(periods, " "), + optInt(g.Spectators), + g.Venue, + } + if err := w.Write(row); err != nil { + return nil, err + } + } + w.Flush() + return buf.Bytes(), w.Error() +} + +func optInt(p *int) string { + if p == nil { + return "" + } + return strconv.Itoa(*p) +} + +// emit writes the bytes to -out or stdout and, when -check is set, compares +// them with a committed golden file. A mismatch exits 3 so CI turns an upstream +// change into a reviewable diff instead of a silent data change. +func emit(out, check string, b []byte) error { + if check != "" { + golden, err := os.ReadFile(check) + switch { + case err == nil && bytes.Equal(golden, b): + case err != nil: + return fmt.Errorf("golden file %s: %w", check, err) + default: + return &DriftError{Msg: fmt.Sprintf("output no longer matches %s: %s", check, firstDiff(golden, b))} + } + } + if out == "" { + _, err := os.Stdout.Write(b) + return err + } + return os.WriteFile(out, b, 0o644) +} + +// firstDiff describes the first differing line, for a readable failure. +func firstDiff(a, b []byte) string { + al, bl := strings.Split(string(a), "\n"), strings.Split(string(b), "\n") + for i := 0; i < len(al) || i < len(bl); i++ { + var x, y string + if i < len(al) { + x = al[i] + } + if i < len(bl) { + y = bl[i] + } + if x != y { + return fmt.Sprintf("line %d: %q != %q", i+1, trim(x), trim(y)) + } + } + return "documents differ only in length" +} + +func trim(s string) string { + if len(s) > 80 { + return s[:80] + "..." + } + return s +} diff --git a/parse.go b/parse.go new file mode 100644 index 0000000..ca9a635 --- /dev/null +++ b/parse.go @@ -0,0 +1,540 @@ +package main + +import ( + "fmt" + "io" + "regexp" + "slices" + "sort" + "strconv" + "strings" + + "golang.org/x/net/html" +) + +// The site has shipped more than one version of this table, and the old +// versions are still served for historical seasons. So the contract is not a +// set of hard-coded column indices: it is a set of layout profiles, each +// identified by the header row it declares. A page the crawler does not +// recognise fails loudly with the header it actually found, and adding support +// is one profile plus one fixture. +type layout struct { + name string + header []string // non-empty header labels, in order + cells int // number of in one data row + round int // -1 when the layout has no Round column + + // Modern layout: the date is rendered twice, once in the desktop "Date" + // column and once as a group heading in the mobile column, and the + // start time lives in its own cell next to the game's stable id. + // Legacy layout: no group headings; the tooltip span of one cell holds + // both "2014-03-10" and "19:00". + dateDesktop int + dateMobile int + timeCell int + grouped bool + + teams int + result int + periods int + spectators int + venue int +} + +var layouts = []*layout{ + { + name: "modern", // seasons from 2018-19 onwards + header: []string{"Date", "Time", "Game", "Result", "Spectators", "Venue"}, + cells: 8, round: -1, + dateDesktop: 0, dateMobile: 1, timeCell: 2, grouped: true, + teams: 3, result: 4, periods: 5, spectators: 6, venue: 7, + }, + { + name: "legacy", // seasons up to 2017-18 + header: []string{"Round", "Date", "Game", "Result", "Spectators", "Venue"}, + cells: 7, round: 0, + dateDesktop: -1, dateMobile: -1, timeCell: 1, grouped: false, + teams: 2, result: 3, periods: 4, spectators: 5, venue: 6, + }, +} + +var ( + dateRE = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`) + timeRE = regexp.MustCompile(`^\d{1,2}:\d{2}$`) + digitsRE = regexp.MustCompile(`^\d+$`) + scoreRE = regexp.MustCompile(`^(\d+)\s*-\s*(\d+)$`) + periodRE = regexp.MustCompile(`^(\d+)\s*-\s*(\d+)$`) + eventIDRE = regexp.MustCompile(`/Game/Events/(\d+)`) + lastUpdRE = regexp.MustCompile(`Last update:\s*(\d{4}-\d{2}-\d{2} \d{1,2}:\d{2})`) + gameIDRE = regexp.MustCompile(`^\d{5,}$`) +) + +// Period is one period (or overtime / shootout) of a played game. +type Period struct { + Label string `json:"label"` // "1", "2", ... in site order + Home int `json:"home"` + Away int `json:"away"` +} + +// Game is one row of the schedule table. +type Game struct { + GameID string `json:"game_id"` // span.lnkTooltip@title, unique per season + EventID string `json:"event_id,omitempty"` // /Game/Events/; set once the game is played + Round string `json:"round,omitempty"` // only in the legacy layout + Date string `json:"date"` // YYYY-MM-DD + Time string `json:"time"` // HH:MM local arena time + Home string `json:"home"` + Away string `json:"away"` + HomeGoals *int `json:"home_goals"` // nil until the game is played + AwayGoals *int `json:"away_goals"` // nil until the game is played + Periods []Period `json:"periods"` // empty until the game is played + Spectators *int `json:"spectators,omitempty"` // nil when not reported + Venue string `json:"venue"` +} + +func (g Game) Played() bool { return g.EventID != "" } + +// Schedule is one season page. +type Schedule struct { + SeasonID int `json:"season_id"` + Layout string `json:"layout"` + Title string `json:"title"` + LastUpdate string `json:"last_update"` // the site's own clock, verbatim + Games []Game `json:"games"` +} + +func (s *Schedule) Played() int { + n := 0 + for _, g := range s.Games { + if g.Played() { + n++ + } + } + return n +} + +// DriftError reports that the page no longer matches a known contract. It is a +// distinct type, and a distinct exit code, because it is the one failure a +// caller must not retry blindly: the site changed, not the network. +type DriftError struct { + Season int + GameID string + Column string + Got string + Want string + Msg string +} + +func (e *DriftError) Error() string { + var b strings.Builder + fmt.Fprintf(&b, "swehockey: markup drift (season %d", e.Season) + if e.GameID != "" { + fmt.Fprintf(&b, ", game %s", e.GameID) + } + b.WriteString("): ") + if e.Msg != "" { + b.WriteString(e.Msg) + } else { + b.WriteString("unexpected table shape") + } + if e.Column != "" { + fmt.Fprintf(&b, " [column %s]", e.Column) + } + if e.Got != "" { + fmt.Fprintf(&b, ": got %q, want %q", e.Got, e.Want) + } + b.WriteString(" - extend layouts[] in parse.go and add a fixture, in one commit") + return b.String() +} + +func drift(season int, gameID, column, msg, got, want string) error { + return &DriftError{Season: season, GameID: gameID, Column: column, Msg: msg, Got: got, Want: want} +} + +// ParseSchedule extracts the schedule table from one saved or fetched page. +func ParseSchedule(seasonID int, r io.Reader) (*Schedule, error) { + doc, err := html.Parse(r) + if err != nil { + return nil, fmt.Errorf("swehockey: parse page: %w", err) + } + tbl, lay := findTable(doc) + if tbl == nil { + return nil, drift(seasonID, "", "", "no table with a known header: found "+strings.Join(documentSignatures(doc), " / ")+ + ", known "+knownLayouts(), "", "") + } + + s := &Schedule{SeasonID: seasonID, Layout: lay.name, Games: []Game{}} + rows := tableRows(tbl) + headerAt, c := -1, &carry{} + for i, tr := range rows { + if len(childElements(tr, "td")) == 0 { + if slices.Equal(thLabels(tr), lay.header) { + if headerAt >= 0 { + return nil, drift(seasonID, "", "", "second header row at row "+strconv.Itoa(i), "2", "1") + } + headerAt = i + } + continue + } + if headerAt < 0 { + return nil, drift(seasonID, "", "", "data row before the header row", "", "header first") + } + g, err := parseGame(seasonID, lay, tr, c) + if err != nil { + return nil, err + } + s.Games = append(s.Games, g) + } + if headerAt < 0 { + return nil, drift(seasonID, "", "", "table has no header row", "", strings.Join(lay.header, " ")) + } + if h := descendants(tbl, "h2"); len(h) > 0 { + s.Title = textOf(h[0]) + } + if lu := lastUpdRE.FindStringSubmatch(textOf(tbl)); lu != nil { + s.LastUpdate = lu[1] + } + if s.LastUpdate == "" { + return nil, drift(seasonID, "", "", `table has no "Last update:" stamp`, "", "Last update: YYYY-MM-DD HH:MM") + } + if err := s.check(); err != nil { + return nil, err + } + // The site lists games in round order, not date order, and a re-scheduled + // or awarded game keeps the round it belongs to: 2011-12 has September + // games listed after October ones. Sorting makes the output a canonical + // function of the set of rows instead of a function of the site's row + // order, which is what makes two crawls comparable and diffs meaningful. + sort.SliceStable(s.Games, func(i, j int) bool { + a, b := s.Games[i], s.Games[j] + if a.Date != b.Date { + return a.Date < b.Date + } + if a.Time != b.Time { + return a.Time < b.Time + } + return a.GameID < b.GameID + }) + return s, nil +} + +// check enforces the invariants that hold for every season the site serves. +func (s *Schedule) check() error { + if len(s.Games) == 0 { + return drift(s.SeasonID, "", "", "table has no game rows", "0 rows", ">= 1 row") + } + // Game ids are unique, and that is the only uniqueness the page promises: + // the site assigns them per fixture, so they are neither sorted nor + // contiguous within a season. + seen := make(map[string]bool, len(s.Games)) + for _, g := range s.Games { + if _, err := strconv.Atoi(g.GameID); err != nil { + return drift(s.SeasonID, g.GameID, "", "game id is not a number", g.GameID, "90001001") + } + if seen[g.GameID] { + return drift(s.SeasonID, g.GameID, "", "duplicate game id", g.GameID, "unique") + } + seen[g.GameID] = true + if !dateRE.MatchString(g.Date) { + return drift(s.SeasonID, g.GameID, "date", "malformed date", g.Date, "2026-09-19") + } + } + return nil +} + +// carry holds the values the site prints only on the first row of a group: the +// date heading of the modern layout and the round number of the legacy one. +type carry struct{ date, round string } + +// parseGame reads one row. Where the site renders a fact twice, both +// renderings must agree before the value is accepted. +func parseGame(seasonID int, lay *layout, tr *html.Node, c *carry) (Game, error) { + tds := childElements(tr, "td") + if len(tds) != lay.cells { + return Game{}, drift(seasonID, "", "", "row width does not match the "+lay.name+" layout", + strconv.Itoa(len(tds))+" cells", strconv.Itoa(lay.cells)+" cells") + } + g := Game{Periods: []Period{}} + + // The tooltip span carries the row's stable id, so the id and the time can + // never disagree: they are the same element. + span := tooltipSpan(tds[lay.timeCell]) + if span == nil { + return Game{}, drift(seasonID, "", "time", "no game id (span.lnkTooltip[title])", textOf(tds[lay.timeCell]), "90001001") + } + g.GameID = strings.TrimSpace(attr(span, "title")) + if !gameIDRE.MatchString(g.GameID) { + return Game{}, drift(seasonID, "", "time", "malformed game id", g.GameID, "90001001") + } + + head := textOf(span) + if lay.grouped { + // Modern layout: date in two renderings, time in the span. + if !timeRE.MatchString(head) { + return Game{}, drift(seasonID, g.GameID, "time", "malformed start time", head, "19:00") + } + g.Time = head + desktop := textOf(tds[lay.dateDesktop]) + mobile := textOf(firstElement(tds[lay.dateMobile], "bold")) + switch { + case mobile != "": + if !dateRE.MatchString(mobile) { + return Game{}, drift(seasonID, g.GameID, "date", "malformed date heading", mobile, "2026-09-19") + } + if desktop != mobile { + return Game{}, drift(seasonID, g.GameID, "date", "date renderings disagree", desktop, mobile) + } + if t := textExcluding(tds[lay.dateMobile], "bold"); t != g.Time { + return Game{}, drift(seasonID, g.GameID, "date", "time renderings disagree", t, g.Time) + } + g.Date = mobile + default: + if c.date == "" { + return Game{}, drift(seasonID, g.GameID, "date", "row before any date heading", desktop, "2026-09-19") + } + if desktop != "" && desktop != g.Time { + return Game{}, drift(seasonID, g.GameID, "date", "unexpected date in a continuation row", desktop, "(empty) or "+g.Time) + } + g.Date = c.date + } + } else { + // Legacy layout: "2014-03-10 19:00" in one cell, no group headings. + d, t, ok := strings.Cut(head, " ") + switch { + case ok && dateRE.MatchString(d) && timeRE.MatchString(t): + g.Date, g.Time = d, t + case timeRE.MatchString(head): + if c.date == "" { + return Game{}, drift(seasonID, g.GameID, "date", "row before any date", head, "2026-09-19 19:00") + } + g.Date, g.Time = c.date, head + default: + return Game{}, drift(seasonID, g.GameID, "date", "cannot read date and time", head, "2026-09-19 19:00") + } + } + + if lay.round >= 0 { + // The site prints the round only on the first row of that round and + // leaves the following cells empty, exactly like the date headings. + switch round := textOf(tds[lay.round]); { + case digitsRE.MatchString(round): + g.Round = round + case round == "" && c.round != "": + g.Round = c.round + case round == "": + return Game{}, drift(seasonID, g.GameID, "round", "row before any round number", round, "1") + default: + return Game{}, drift(seasonID, g.GameID, "round", "round is not a number", round, "1") + } + } + + // Teams: the cell is "Home - Away" plus, in old seasons, an note + // such as "Eighth final 1" that is not part of a name. + teams := textExcluding(tds[lay.teams], "i") + parts := strings.Split(teams, " - ") + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return Game{}, drift(seasonID, g.GameID, "game", "cannot split teams", teams, "Home - Away") + } + g.Home, g.Away = parts[0], parts[1] + + // Result: present only for played games, and always with an event link. + if a := firstElement(tds[lay.result], "a"); a != nil { + m := scoreRE.FindStringSubmatch(textOf(a)) + if m == nil { + return Game{}, drift(seasonID, g.GameID, "result", "malformed score", textOf(a), "3 - 2") + } + ev := eventIDRE.FindStringSubmatch(attr(a, "href")) + if ev == nil { + return Game{}, drift(seasonID, g.GameID, "result", "result link without an event id", attr(a, "href"), "/Game/Events/1109941") + } + h, _ := strconv.Atoi(m[1]) + aw, _ := strconv.Atoi(m[2]) + g.HomeGoals, g.AwayGoals, g.EventID = &h, &aw, ev[1] + } + + periods, err := parsePeriods(textOf(tds[lay.periods])) + if err != nil { + return Game{}, drift(seasonID, g.GameID, "periods", err.Error(), textOf(tds[lay.periods]), "(1-1, 2-1, 2-2)") + } + g.Periods = periods + + if sp := textOf(tds[lay.spectators]); sp != "" { + if !digitsRE.MatchString(sp) { + return Game{}, drift(seasonID, g.GameID, "spectators", "spectators are not a number", sp, "10442") + } + n, _ := strconv.Atoi(sp) + g.Spectators = &n + } + g.Venue = textOf(tds[lay.venue]) + if g.Venue == "" { + return Game{}, drift(seasonID, g.GameID, "venue", "empty venue", "", "Gottde Arena") + } + c.date, c.round = g.Date, g.Round + if err := g.checkRow(); err != nil { + return Game{}, drift(seasonID, g.GameID, "", err.Error(), "", "") + } + return g, nil +} + +// checkRow enforces the per-row invariants: periods sum to the final score, +// and a game is either fully played or not played at all. +func (g Game) checkRow() error { + if !g.Played() { + if g.HomeGoals != nil || len(g.Periods) > 0 { + return fmt.Errorf("unplayed game carries a result") + } + return nil + } + if len(g.Periods) < 3 { + return fmt.Errorf("played game has %d period scores", len(g.Periods)) + } + h, a := 0, 0 + for _, p := range g.Periods { + h += p.Home + a += p.Away + } + if h != *g.HomeGoals || a != *g.AwayGoals { + return fmt.Errorf("periods sum to %d-%d but the result is %d-%d", h, a, *g.HomeGoals, *g.AwayGoals) + } + return nil +} + +// parsePeriods reads "(1-1, 2-1, 2-2)". This is the single place to extend if +// the site ever renders a period differently, e.g. a shootout as "-". +func parsePeriods(s string) ([]Period, error) { + if s == "" { + return []Period{}, nil + } + if !strings.HasPrefix(s, "(") || !strings.HasSuffix(s, ")") { + return nil, fmt.Errorf("period scores are not parenthesised") + } + tokens := strings.Split(s[1:len(s)-1], ",") + // Only the lower bound is a contract. How many periods a game lasts depends + // on the era's rules - the site serves 3, 4, 5 and 6 period breakdowns - so + // the count is not pinned; the sum check in checkRow is what protects the + // data. + if len(tokens) < 3 { + return nil, fmt.Errorf("only %d period scores", len(tokens)) + } + out := make([]Period, 0, len(tokens)) + for i, tok := range tokens { + m := periodRE.FindStringSubmatch(strings.TrimSpace(tok)) + if m == nil { + return nil, fmt.Errorf("malformed period %q", strings.TrimSpace(tok)) + } + h, _ := strconv.Atoi(m[1]) + a, _ := strconv.Atoi(m[2]) + out = append(out, Period{Label: strconv.Itoa(i + 1), Home: h, Away: a}) + } + return out, nil +} + +// findTable returns the first table that declares one of the known layouts. +// Locating the table by its header, rather than by a CSS path or an index, is +// what keeps ads, navigation and layout changes from breaking the crawler. +func findTable(root *html.Node) (*html.Node, *layout) { + var found *html.Node + var lay *layout + var walk func(*html.Node) + walk = func(n *html.Node) { + if found != nil { + return + } + if n.Type == html.ElementNode && n.Data == "table" { + for _, tr := range tableRows(n) { + labels := thLabels(tr) + for _, l := range layouts { + if slices.Equal(labels, l.header) { + found, lay = n, l + return + } + } + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(root) + return found, lay +} + +// documentSignatures describes every table on the page, so that an +// unrecognised layout is diagnosable from the error message alone. +func documentSignatures(doc *html.Node) []string { + var out []string + var walk func(*html.Node) + walk = func(n *html.Node) { + if n.Type == html.ElementNode && n.Data == "table" { + if labels := tableHeaderLabels(n); len(labels) > 0 { + cells := 0 + for _, tr := range tableRows(n) { + if c := len(childElements(tr, "td")); c > 0 { + cells = c + break + } + } + out = append(out, fmt.Sprintf("%q with %d cells", strings.Join(labels, " "), cells)) + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(doc) + if len(out) == 0 { + return []string{"no table with a header row"} + } + return out +} + +func knownLayouts() string { + names := make([]string, 0, len(layouts)) + for _, l := range layouts { + names = append(names, fmt.Sprintf("%s=%q", l.name, strings.Join(l.header, " "))) + } + return strings.Join(names, ", ") +} + +// thLabels returns the non-empty header labels of a row, or of the first row +// of a table that has one. +func thLabels(n *html.Node) []string { + trs := tableRows(n) + if n.Type == html.ElementNode && n.Data == "tr" { + trs = []*html.Node{n} + } + for _, tr := range trs { + if labels := tableHeaderLabels(tr); len(labels) > 0 { + return labels + } + } + return nil +} + +// tableHeaderLabels collects the labels of a single row. +func tableHeaderLabels(row *html.Node) []string { + var labels []string + for _, th := range childElements(row, "th") { + if l := textOf(th); l != "" { + labels = append(labels, l) + } + } + return labels +} + +func firstElement(n *html.Node, tag string) *html.Node { + if d := descendants(n, tag); len(d) > 0 { + return d[0] + } + return nil +} + +// tooltipSpan returns the span that carries a game's stable id. +func tooltipSpan(cell *html.Node) *html.Node { + for _, s := range descendants(cell, "span") { + if hasClass(s, "lnkTooltip") && attr(s, "title") != "" { + return s + } + } + return nil +} diff --git a/parse_test.go b/parse_test.go new file mode 100644 index 0000000..4eebc22 --- /dev/null +++ b/parse_test.go @@ -0,0 +1,445 @@ +package main + +import ( + "bytes" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + "time" +) + +// The tests never touch the network: they parse committed HTML slices. That is +// what makes the suite deterministic, and it is why a change in the site's +// markup shows up as a failing test with a readable message instead of as +// silently different data. + +func parseFixture(t *testing.T, name string) *Schedule { + t.Helper() + f, err := os.Open(filepath.Join("testdata", name)) + if err != nil { + t.Fatal(err) + } + defer f.Close() + s, err := ParseSchedule(20961, f) + if err != nil { + t.Fatalf("ParseSchedule(%s): %v", name, err) + } + return s +} + +// TestGolden pins the exact bytes of the rendered output. Same HTML in, same +// bytes out - that is the property the whole design exists to provide. +func TestGolden(t *testing.T) { + for _, tc := range []struct{ fixture, golden string }{ + {"schedule_20961.html", "schedule_20961.json"}, + {"schedule_legacy.html", "schedule_legacy.json"}, + {"schedule_legacy_playoffs.html", "schedule_legacy_playoffs.json"}, + } { + t.Run(tc.fixture, func(t *testing.T) { + f, err := os.Open(filepath.Join("testdata", tc.fixture)) + if err != nil { + t.Fatal(err) + } + defer f.Close() + s, err := ParseSchedule(20961, f) + if err != nil { + t.Fatalf("parse: %v", err) + } + got, err := render("json", s) + if err != nil { + t.Fatal(err) + } + want, err := os.ReadFile(filepath.Join("testdata", tc.golden)) + if err != nil { + t.Fatalf("read golden (regenerate with -update flag): %v", err) + } + if !bytes.Equal(got, want) { + t.Errorf("output drifted from the golden file:\n%s", firstDiff(want, got)) + } + }) + } +} + +func TestModernLayout(t *testing.T) { + s := parseFixture(t, "schedule_20961.html") + if s.Layout != "modern" { + t.Errorf("layout = %q, want modern", s.Layout) + } + if s.LastUpdate != "2026-09-26 22:41" { + t.Errorf("last update = %q", s.LastUpdate) + } + if len(s.Games) != 10 || s.Played() != 8 { + t.Fatalf("games = %d, played = %d; want 10/8", len(s.Games), s.Played()) + } + // Date headings appear only on the first row of a date; the rest inherit. + // A date is printed only on the first row of its group; the rest inherit it. + want := []string{"2026-09-19", "2026-09-19", "2026-09-19", "2026-09-19", "2026-09-19", "2026-09-19", + "2026-09-22", "2026-09-22", "2027-03-16", "2027-03-16"} + for i, g := range s.Games { + if g.Date != want[i] { + t.Errorf("game %d: date = %s, want %s", i, g.Date, want[i]) + } + } + // Overtime (4 periods) and a shootout (5 periods) keep every period. + if g := s.Games[1]; len(g.Periods) != 4 || *g.HomeGoals != 4 { + t.Errorf("game 2: %d periods, %d goals", len(g.Periods), *g.HomeGoals) + } + if g := s.Games[6]; len(g.Periods) != 5 || *g.HomeGoals != 3 { + t.Errorf("game 7: %d periods, %d goals", len(g.Periods), *g.HomeGoals) + } + // An unplayed game has a result of nil, not 0-0, and no periods. + if g := s.Games[len(s.Games)-1]; g.Played() || g.HomeGoals != nil || g.AwayGoals != nil || len(g.Periods) != 0 { + t.Errorf("last game should be unplayed: %+v", g) + } + if g := s.Games[0]; g.Home != "Frölunda HC" || g.Away != "Växjö Lakers HC" { + t.Errorf("teams not normalised: %q vs %q", g.Home, g.Away) + } + if g := s.Games[0]; g.EventID != "1109941" || g.Spectators == nil || *g.Spectators != 12044 { + t.Errorf("event id / spectators: %+v", g) + } +} + +func TestLegacyLayout(t *testing.T) { + s := parseFixture(t, "schedule_legacy.html") + if s.Layout != "legacy" { + t.Errorf("layout = %q, want legacy", s.Layout) + } + g := s.Games[0] + // The legacy cell holds "