package main import ( "fmt" "io" "regexp" "slices" "sort" "strconv" "strings" "golang.org/x/net/html" ) // The site has shipped more than one version of this table, and the old // versions are still served for historical seasons. So the contract is not a // set of hard-coded column indices: it is a set of layout profiles, each // identified by the header row it declares. A page the crawler does not // recognise fails loudly with the header it actually found, and adding support // is one profile plus one fixture. type layout struct { name string header []string // non-empty header labels, in order cells int // number of in one data row round int // -1 when the layout has no Round column // Modern layout: the date is rendered twice, once in the desktop "Date" // column and once as a group heading in the mobile column, and the // start time lives in its own cell next to the game's stable id. // Legacy layout: no group headings; the tooltip span of one cell holds // both "2014-03-10" and "19:00". dateDesktop int dateMobile int timeCell int grouped bool teams int result int periods int spectators int venue int } var layouts = []*layout{ { name: "modern", // seasons from 2018-19 onwards header: []string{"Date", "Time", "Game", "Result", "Spectators", "Venue"}, cells: 8, round: -1, dateDesktop: 0, dateMobile: 1, timeCell: 2, grouped: true, teams: 3, result: 4, periods: 5, spectators: 6, venue: 7, }, { name: "legacy", // seasons up to 2017-18 header: []string{"Round", "Date", "Game", "Result", "Spectators", "Venue"}, cells: 7, round: 0, dateDesktop: -1, dateMobile: -1, timeCell: 1, grouped: false, teams: 2, result: 3, periods: 4, spectators: 5, venue: 6, }, } var ( dateRE = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`) timeRE = regexp.MustCompile(`^\d{1,2}:\d{2}$`) digitsRE = regexp.MustCompile(`^\d+$`) scoreRE = regexp.MustCompile(`^(\d+)\s*-\s*(\d+)$`) periodRE = regexp.MustCompile(`^(\d+)\s*-\s*(\d+)$`) eventIDRE = regexp.MustCompile(`/Game/Events/(\d+)`) lastUpdRE = regexp.MustCompile(`Last update:\s*(\d{4}-\d{2}-\d{2} \d{1,2}:\d{2})`) gameIDRE = regexp.MustCompile(`^\d{5,}$`) ) // Period is one period (or overtime / shootout) of a played game. type Period struct { Label string `json:"label"` // "1", "2", ... in site order Home int `json:"home"` Away int `json:"away"` } // Game is one row of the schedule table. type Game struct { GameID string `json:"game_id"` // span.lnkTooltip@title, unique per season EventID string `json:"event_id,omitempty"` // /Game/Events/; set once the game is played Round string `json:"round,omitempty"` // only in the legacy layout Date string `json:"date"` // YYYY-MM-DD Time string `json:"time"` // HH:MM local arena time Home string `json:"home"` Away string `json:"away"` HomeGoals *int `json:"home_goals"` // nil until the game is played AwayGoals *int `json:"away_goals"` // nil until the game is played Periods []Period `json:"periods"` // empty until the game is played Spectators *int `json:"spectators,omitempty"` // nil when not reported Venue string `json:"venue"` } func (g Game) Played() bool { return g.EventID != "" } // Schedule is one season page. type Schedule struct { SeasonID int `json:"season_id"` Layout string `json:"layout"` Title string `json:"title"` LastUpdate string `json:"last_update"` // the site's own clock, verbatim Games []Game `json:"games"` } func (s *Schedule) Played() int { n := 0 for _, g := range s.Games { if g.Played() { n++ } } return n } // DriftError reports that the page no longer matches a known contract. It is a // distinct type, and a distinct exit code, because it is the one failure a // caller must not retry blindly: the site changed, not the network. type DriftError struct { Season int GameID string Column string Got string Want string Msg string } func (e *DriftError) Error() string { var b strings.Builder fmt.Fprintf(&b, "swehockey: markup drift (season %d", e.Season) if e.GameID != "" { fmt.Fprintf(&b, ", game %s", e.GameID) } b.WriteString("): ") if e.Msg != "" { b.WriteString(e.Msg) } else { b.WriteString("unexpected table shape") } if e.Column != "" { fmt.Fprintf(&b, " [column %s]", e.Column) } if e.Got != "" { fmt.Fprintf(&b, ": got %q, want %q", e.Got, e.Want) } b.WriteString(" - extend layouts[] in parse.go and add a fixture, in one commit") return b.String() } func drift(season int, gameID, column, msg, got, want string) error { return &DriftError{Season: season, GameID: gameID, Column: column, Msg: msg, Got: got, Want: want} } // ParseSchedule extracts the schedule table from one saved or fetched page. func ParseSchedule(seasonID int, r io.Reader) (*Schedule, error) { doc, err := html.Parse(r) if err != nil { return nil, fmt.Errorf("swehockey: parse page: %w", err) } tbl, lay := findTable(doc) if tbl == nil { return nil, drift(seasonID, "", "", "no table with a known header: found "+strings.Join(documentSignatures(doc), " / ")+ ", known "+knownLayouts(), "", "") } s := &Schedule{SeasonID: seasonID, Layout: lay.name, Games: []Game{}} rows := tableRows(tbl) headerAt, c := -1, &carry{} for i, tr := range rows { if len(childElements(tr, "td")) == 0 { if slices.Equal(thLabels(tr), lay.header) { if headerAt >= 0 { return nil, drift(seasonID, "", "", "second header row at row "+strconv.Itoa(i), "2", "1") } headerAt = i } continue } if headerAt < 0 { return nil, drift(seasonID, "", "", "data row before the header row", "", "header first") } g, err := parseGame(seasonID, lay, tr, c) if err != nil { return nil, err } s.Games = append(s.Games, g) } if headerAt < 0 { return nil, drift(seasonID, "", "", "table has no header row", "", strings.Join(lay.header, " ")) } if h := descendants(tbl, "h2"); len(h) > 0 { s.Title = textOf(h[0]) } if lu := lastUpdRE.FindStringSubmatch(textOf(tbl)); lu != nil { s.LastUpdate = lu[1] } if s.LastUpdate == "" { return nil, drift(seasonID, "", "", `table has no "Last update:" stamp`, "", "Last update: YYYY-MM-DD HH:MM") } if err := s.check(); err != nil { return nil, err } // The site lists games in round order, not date order, and a re-scheduled // or awarded game keeps the round it belongs to: 2011-12 has September // games listed after October ones. Sorting makes the output a canonical // function of the set of rows instead of a function of the site's row // order, which is what makes two crawls comparable and diffs meaningful. sort.SliceStable(s.Games, func(i, j int) bool { a, b := s.Games[i], s.Games[j] if a.Date != b.Date { return a.Date < b.Date } if a.Time != b.Time { return a.Time < b.Time } return a.GameID < b.GameID }) return s, nil } // check enforces the invariants that hold for every season the site serves. func (s *Schedule) check() error { if len(s.Games) == 0 { return drift(s.SeasonID, "", "", "table has no game rows", "0 rows", ">= 1 row") } // Game ids are unique, and that is the only uniqueness the page promises: // the site assigns them per fixture, so they are neither sorted nor // contiguous within a season. seen := make(map[string]bool, len(s.Games)) for _, g := range s.Games { if _, err := strconv.Atoi(g.GameID); err != nil { return drift(s.SeasonID, g.GameID, "", "game id is not a number", g.GameID, "90001001") } if seen[g.GameID] { return drift(s.SeasonID, g.GameID, "", "duplicate game id", g.GameID, "unique") } seen[g.GameID] = true if !dateRE.MatchString(g.Date) { return drift(s.SeasonID, g.GameID, "date", "malformed date", g.Date, "2026-09-19") } } return nil } // carry holds the values the site prints only on the first row of a group: the // date heading of the modern layout and the round number of the legacy one. type carry struct{ date, round string } // parseGame reads one row. Where the site renders a fact twice, both // renderings must agree before the value is accepted. func parseGame(seasonID int, lay *layout, tr *html.Node, c *carry) (Game, error) { tds := childElements(tr, "td") if len(tds) != lay.cells { return Game{}, drift(seasonID, "", "", "row width does not match the "+lay.name+" layout", strconv.Itoa(len(tds))+" cells", strconv.Itoa(lay.cells)+" cells") } g := Game{Periods: []Period{}} // The tooltip span carries the row's stable id, so the id and the time can // never disagree: they are the same element. span := tooltipSpan(tds[lay.timeCell]) if span == nil { return Game{}, drift(seasonID, "", "time", "no game id (span.lnkTooltip[title])", textOf(tds[lay.timeCell]), "90001001") } g.GameID = strings.TrimSpace(attr(span, "title")) if !gameIDRE.MatchString(g.GameID) { return Game{}, drift(seasonID, "", "time", "malformed game id", g.GameID, "90001001") } head := textOf(span) if lay.grouped { // Modern layout: date in two renderings, time in the span. if !timeRE.MatchString(head) { return Game{}, drift(seasonID, g.GameID, "time", "malformed start time", head, "19:00") } g.Time = head desktop := textOf(tds[lay.dateDesktop]) mobile := textOf(firstElement(tds[lay.dateMobile], "bold")) switch { case mobile != "": if !dateRE.MatchString(mobile) { return Game{}, drift(seasonID, g.GameID, "date", "malformed date heading", mobile, "2026-09-19") } if desktop != mobile { return Game{}, drift(seasonID, g.GameID, "date", "date renderings disagree", desktop, mobile) } if t := textExcluding(tds[lay.dateMobile], "bold"); t != g.Time { return Game{}, drift(seasonID, g.GameID, "date", "time renderings disagree", t, g.Time) } g.Date = mobile default: if c.date == "" { return Game{}, drift(seasonID, g.GameID, "date", "row before any date heading", desktop, "2026-09-19") } if desktop != "" && desktop != g.Time { return Game{}, drift(seasonID, g.GameID, "date", "unexpected date in a continuation row", desktop, "(empty) or "+g.Time) } g.Date = c.date } } else { // Legacy layout: "2014-03-10 19:00" in one cell, no group headings. d, t, ok := strings.Cut(head, " ") switch { case ok && dateRE.MatchString(d) && timeRE.MatchString(t): g.Date, g.Time = d, t case timeRE.MatchString(head): if c.date == "" { return Game{}, drift(seasonID, g.GameID, "date", "row before any date", head, "2026-09-19 19:00") } g.Date, g.Time = c.date, head default: return Game{}, drift(seasonID, g.GameID, "date", "cannot read date and time", head, "2026-09-19 19:00") } } if lay.round >= 0 { // The site prints the round only on the first row of that round and // leaves the following cells empty, exactly like the date headings. switch round := textOf(tds[lay.round]); { case digitsRE.MatchString(round): g.Round = round case round == "" && c.round != "": g.Round = c.round case round == "": return Game{}, drift(seasonID, g.GameID, "round", "row before any round number", round, "1") default: return Game{}, drift(seasonID, g.GameID, "round", "round is not a number", round, "1") } } // Teams: the cell is "Home - Away" plus, in old seasons, an note // such as "Eighth final 1" that is not part of a name. teams := textExcluding(tds[lay.teams], "i") parts := strings.Split(teams, " - ") if len(parts) != 2 || parts[0] == "" || parts[1] == "" { return Game{}, drift(seasonID, g.GameID, "game", "cannot split teams", teams, "Home - Away") } g.Home, g.Away = parts[0], parts[1] // Result: present only for played games, and always with an event link. if a := firstElement(tds[lay.result], "a"); a != nil { m := scoreRE.FindStringSubmatch(textOf(a)) if m == nil { return Game{}, drift(seasonID, g.GameID, "result", "malformed score", textOf(a), "3 - 2") } ev := eventIDRE.FindStringSubmatch(attr(a, "href")) if ev == nil { return Game{}, drift(seasonID, g.GameID, "result", "result link without an event id", attr(a, "href"), "/Game/Events/1109941") } h, _ := strconv.Atoi(m[1]) aw, _ := strconv.Atoi(m[2]) g.HomeGoals, g.AwayGoals, g.EventID = &h, &aw, ev[1] } periods, err := parsePeriods(textOf(tds[lay.periods])) if err != nil { return Game{}, drift(seasonID, g.GameID, "periods", err.Error(), textOf(tds[lay.periods]), "(1-1, 2-1, 2-2)") } g.Periods = periods if sp := textOf(tds[lay.spectators]); sp != "" { if !digitsRE.MatchString(sp) { return Game{}, drift(seasonID, g.GameID, "spectators", "spectators are not a number", sp, "10442") } n, _ := strconv.Atoi(sp) g.Spectators = &n } g.Venue = textOf(tds[lay.venue]) if g.Venue == "" { return Game{}, drift(seasonID, g.GameID, "venue", "empty venue", "", "Gottde Arena") } c.date, c.round = g.Date, g.Round if err := g.checkRow(); err != nil { return Game{}, drift(seasonID, g.GameID, "", err.Error(), "", "") } return g, nil } // checkRow enforces the per-row invariants: periods sum to the final score, // and a game is either fully played or not played at all. func (g Game) checkRow() error { if !g.Played() { if g.HomeGoals != nil || len(g.Periods) > 0 { return fmt.Errorf("unplayed game carries a result") } return nil } if len(g.Periods) < 3 { return fmt.Errorf("played game has %d period scores", len(g.Periods)) } h, a := 0, 0 for _, p := range g.Periods { h += p.Home a += p.Away } if h != *g.HomeGoals || a != *g.AwayGoals { return fmt.Errorf("periods sum to %d-%d but the result is %d-%d", h, a, *g.HomeGoals, *g.AwayGoals) } return nil } // parsePeriods reads "(1-1, 2-1, 2-2)". This is the single place to extend if // the site ever renders a period differently, e.g. a shootout as "-". func parsePeriods(s string) ([]Period, error) { if s == "" { return []Period{}, nil } if !strings.HasPrefix(s, "(") || !strings.HasSuffix(s, ")") { return nil, fmt.Errorf("period scores are not parenthesised") } tokens := strings.Split(s[1:len(s)-1], ",") // Only the lower bound is a contract. How many periods a game lasts depends // on the era's rules - the site serves 3, 4, 5 and 6 period breakdowns - so // the count is not pinned; the sum check in checkRow is what protects the // data. if len(tokens) < 3 { return nil, fmt.Errorf("only %d period scores", len(tokens)) } out := make([]Period, 0, len(tokens)) for i, tok := range tokens { m := periodRE.FindStringSubmatch(strings.TrimSpace(tok)) if m == nil { return nil, fmt.Errorf("malformed period %q", strings.TrimSpace(tok)) } h, _ := strconv.Atoi(m[1]) a, _ := strconv.Atoi(m[2]) out = append(out, Period{Label: strconv.Itoa(i + 1), Home: h, Away: a}) } return out, nil } // findTable returns the first table that declares one of the known layouts. // Locating the table by its header, rather than by a CSS path or an index, is // what keeps ads, navigation and layout changes from breaking the crawler. func findTable(root *html.Node) (*html.Node, *layout) { var found *html.Node var lay *layout var walk func(*html.Node) walk = func(n *html.Node) { if found != nil { return } if n.Type == html.ElementNode && n.Data == "table" { for _, tr := range tableRows(n) { labels := thLabels(tr) for _, l := range layouts { if slices.Equal(labels, l.header) { found, lay = n, l return } } } } for c := n.FirstChild; c != nil; c = c.NextSibling { walk(c) } } walk(root) return found, lay } // documentSignatures describes every table on the page, so that an // unrecognised layout is diagnosable from the error message alone. func documentSignatures(doc *html.Node) []string { var out []string var walk func(*html.Node) walk = func(n *html.Node) { if n.Type == html.ElementNode && n.Data == "table" { if labels := tableHeaderLabels(n); len(labels) > 0 { cells := 0 for _, tr := range tableRows(n) { if c := len(childElements(tr, "td")); c > 0 { cells = c break } } out = append(out, fmt.Sprintf("%q with %d cells", strings.Join(labels, " "), cells)) } } for c := n.FirstChild; c != nil; c = c.NextSibling { walk(c) } } walk(doc) if len(out) == 0 { return []string{"no table with a header row"} } return out } func knownLayouts() string { names := make([]string, 0, len(layouts)) for _, l := range layouts { names = append(names, fmt.Sprintf("%s=%q", l.name, strings.Join(l.header, " "))) } return strings.Join(names, ", ") } // thLabels returns the non-empty header labels of a row, or of the first row // of a table that has one. func thLabels(n *html.Node) []string { trs := tableRows(n) if n.Type == html.ElementNode && n.Data == "tr" { trs = []*html.Node{n} } for _, tr := range trs { if labels := tableHeaderLabels(tr); len(labels) > 0 { return labels } } return nil } // tableHeaderLabels collects the labels of a single row. func tableHeaderLabels(row *html.Node) []string { var labels []string for _, th := range childElements(row, "th") { if l := textOf(th); l != "" { labels = append(labels, l) } } return labels } func firstElement(n *html.Node, tag string) *html.Node { if d := descendants(n, tag); len(d) > 0 { return d[0] } return nil } // tooltipSpan returns the span that carries a game's stable id. func tooltipSpan(cell *html.Node) *html.Node { for _, s := range descendants(cell, "span") { if hasClass(s, "lnkTooltip") && attr(s, "title") != "" { return s } } return nil }