Files
Mats Tornberg 6b701819ca Deterministic swehockey.se schedule crawler
One HTTP GET per season, no browser: the page is server-rendered, so every row
arrives in one response and the extraction becomes a pure function of the HTML.

- layouts[] declares one profile per version of the site's table (modern and
  legacy), each identified by its header row, so a new version of the page is a
  new profile plus a fixture instead of a guess.
- The table is located by header signature, not by a CSS path.
- Every cell is shape-checked, and facts the site renders twice (desktop and
  mobile date, time in the tooltip) must agree.
- Cross-field invariants: periods sum to the score, ids are unique, played games
  always have an event link and unplayed ones never do.
- Output is sorted by date, time and id, so it does not inherit the site's row
  order, and it carries no fetch timestamp: same HTML, same bytes.
- Drift is a typed error and exit code 3, with the header it actually found.
- Tests run on committed HTML fixtures and golden JSON; TestLive is opt-in.

Verified over the full SHL archive: 27/27 seasons, 2119 games, 1779 played.
2026-09-27 08:47:25 +00:00

127 lines
3.8 KiB
Go

package main
import (
"fmt"
"io"
"regexp"
"sort"
"strconv"
"strings"
"golang.org/x/net/html"
)
// League is one competition as listed on the front page.
type League struct {
ID int `json:"id"`
Name string `json:"name"`
Path string `json:"path"` // canonical path, e.g. /ScheduleAndResults/Overview/20961
}
// Season is one entry of the season <select> on a league page.
type Season struct {
ID int `json:"id"`
Label string `json:"label"` // "2026-27"
Path string `json:"path"` // a path on the site for this season
}
var (
leaguePathRE = regexp.MustCompile(`^/ScheduleAndResults/[A-Za-z]+/(\d+)$`)
seasonPathRE = regexp.MustCompile(`/ScheduleAndResults/[A-Za-z]+/(\d+)`)
)
// tableRows returns the <tr> children of a table. The HTML5 tree builder wraps
// stray rows in an implicit <tbody>, so the rows can sit one level deeper than
// the markup suggests. Both shapes are accepted: a scraper that indexes the
// raw markup, or one that trusts a hand-rolled parser, gets this wrong.
func tableRows(tbl *html.Node) []*html.Node {
if tr := childElements(tbl, "tr"); len(tr) > 0 {
return tr
}
var out []*html.Node
for _, body := range childElements(tbl, "tbody") {
out = append(out, childElements(body, "tr")...)
}
return out
}
// ParseLeagues reads the front page's league list. It is a plain anchor crawl:
// every /ScheduleAndResults/<section>/<id> link, deduplicated by id and sorted,
// so the crawl order never depends on document order.
func ParseLeagues(r io.Reader) ([]League, error) {
doc, err := html.Parse(r)
if err != nil {
return nil, fmt.Errorf("swehockey: parse page: %w", err)
}
byID := map[int]string{}
for _, a := range descendants(doc, "a") {
m := leaguePathRE.FindStringSubmatch(attr(a, "href"))
if m == nil {
continue
}
id, err := strconv.Atoi(m[1])
if err != nil {
continue
}
name := textOf(a)
if name == "" {
continue
}
if old, ok := byID[id]; ok && old != "" {
// A league is linked once per section (Live, Overview, Schedule).
// Keep the first name seen; the id is what matters.
continue
}
byID[id] = name
}
if len(byID) == 0 {
return nil, drift(0, "", "", "front page has no league links", "0 links", "/ScheduleAndResults/<section>/<id>")
}
out := make([]League, 0, len(byID))
for id, name := range byID {
out = append(out, League{ID: id, Name: name, Path: fmt.Sprintf("/ScheduleAndResults/Schedule/%d", id)})
}
sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
return out, nil
}
// ParseSeasons reads the season selector on a league page. Group headings
// ("Cups", value="-1") are skipped; the result is sorted by id.
func ParseSeasons(leagueID int, r io.Reader) ([]Season, error) {
doc, err := html.Parse(r)
if err != nil {
return nil, fmt.Errorf("swehockey: parse page: %w", err)
}
var out []Season
seen := map[int]bool{}
for _, opt := range descendants(doc, "option") {
raw := attr(opt, "value")
m := seasonPathRE.FindStringSubmatch(raw)
if m == nil {
continue // group heading such as value="-1"
}
id, err := strconv.Atoi(m[1])
if err != nil || seen[id] {
continue
}
seen[id] = true
out = append(out, Season{ID: id, Label: textOf(opt), Path: raw})
}
if len(out) == 0 {
return nil, drift(leagueID, "", "", "no season options in the selector", "0 options", "/ScheduleAndResults/<section>/<id>")
}
sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
return out, nil
}
// SchedulePath is the canonical schedule URL of a season id. Every view
// (Overview, Live, Schedule) of one season shares the id, so the crawler
// derives the schedule URL instead of trusting whichever link it arrived on.
func SchedulePath(seasonID int) string {
return fmt.Sprintf("/ScheduleAndResults/Schedule/%d", seasonID)
}
func joinURL(base, path string) string {
return strings.TrimRight(base, "/") + path
}