package main
import (
"fmt"
"io"
"regexp"
"slices"
"sort"
"strconv"
"strings"
"golang.org/x/net/html"
)
// The site has shipped more than one version of this table, and the old
// versions are still served for historical seasons. So the contract is not a
// set of hard-coded column indices: it is a set of layout profiles, each
// identified by the header row it declares. A page the crawler does not
// recognise fails loudly with the header it actually found, and adding support
// is one profile plus one fixture.
type layout struct {
name string
header []string // non-empty header labels, in order
cells int // number of
in one data row
round int // -1 when the layout has no Round column
// Modern layout: the date is rendered twice, once in the desktop "Date"
// column and once as a group heading in the mobile column, and the
// start time lives in its own cell next to the game's stable id.
// Legacy layout: no group headings; the tooltip span of one cell holds
// both "2014-03-10" and "19:00".
dateDesktop int
dateMobile int
timeCell int
grouped bool
teams int
result int
periods int
spectators int
venue int
}
var layouts = []*layout{
{
name: "modern", // seasons from 2018-19 onwards
header: []string{"Date", "Time", "Game", "Result", "Spectators", "Venue"},
cells: 8, round: -1,
dateDesktop: 0, dateMobile: 1, timeCell: 2, grouped: true,
teams: 3, result: 4, periods: 5, spectators: 6, venue: 7,
},
{
name: "legacy", // seasons up to 2017-18
header: []string{"Round", "Date", "Game", "Result", "Spectators", "Venue"},
cells: 7, round: 0,
dateDesktop: -1, dateMobile: -1, timeCell: 1, grouped: false,
teams: 2, result: 3, periods: 4, spectators: 5, venue: 6,
},
}
var (
dateRE = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`)
timeRE = regexp.MustCompile(`^\d{1,2}:\d{2}$`)
digitsRE = regexp.MustCompile(`^\d+$`)
scoreRE = regexp.MustCompile(`^(\d+)\s*-\s*(\d+)$`)
periodRE = regexp.MustCompile(`^(\d+)\s*-\s*(\d+)$`)
eventIDRE = regexp.MustCompile(`/Game/Events/(\d+)`)
lastUpdRE = regexp.MustCompile(`Last update:\s*(\d{4}-\d{2}-\d{2} \d{1,2}:\d{2})`)
gameIDRE = regexp.MustCompile(`^\d{5,}$`)
)
// Period is one period (or overtime / shootout) of a played game.
type Period struct {
Label string `json:"label"` // "1", "2", ... in site order
Home int `json:"home"`
Away int `json:"away"`
}
// Game is one row of the schedule table.
type Game struct {
GameID string `json:"game_id"` // span.lnkTooltip@title, unique per season
EventID string `json:"event_id,omitempty"` // /Game/Events/; set once the game is played
Round string `json:"round,omitempty"` // only in the legacy layout
Date string `json:"date"` // YYYY-MM-DD
Time string `json:"time"` // HH:MM local arena time
Home string `json:"home"`
Away string `json:"away"`
HomeGoals *int `json:"home_goals"` // nil until the game is played
AwayGoals *int `json:"away_goals"` // nil until the game is played
Periods []Period `json:"periods"` // empty until the game is played
Spectators *int `json:"spectators,omitempty"` // nil when not reported
Venue string `json:"venue"`
}
func (g Game) Played() bool { return g.EventID != "" }
// Schedule is one season page.
type Schedule struct {
SeasonID int `json:"season_id"`
Layout string `json:"layout"`
Title string `json:"title"`
LastUpdate string `json:"last_update"` // the site's own clock, verbatim
Games []Game `json:"games"`
}
func (s *Schedule) Played() int {
n := 0
for _, g := range s.Games {
if g.Played() {
n++
}
}
return n
}
// DriftError reports that the page no longer matches a known contract. It is a
// distinct type, and a distinct exit code, because it is the one failure a
// caller must not retry blindly: the site changed, not the network.
type DriftError struct {
Season int
GameID string
Column string
Got string
Want string
Msg string
// Every layout problem ends with the same advice, so an operator reading a
// cron mail knows exactly what to do. Failures that are not about the table
// shape pass an empty Hint and get no advice.
Hint string
}
// layoutHint is the advice attached to every markup problem.
const layoutHint = " - extend layouts[] in parse.go and add a fixture, in one commit"
func (e *DriftError) Error() string {
var b strings.Builder
fmt.Fprintf(&b, "swehockey: markup drift (season %d", e.Season)
if e.GameID != "" {
fmt.Fprintf(&b, ", game %s", e.GameID)
}
b.WriteString("): ")
if e.Msg != "" {
b.WriteString(e.Msg)
} else {
b.WriteString("unexpected table shape")
}
if e.Column != "" {
fmt.Fprintf(&b, " [column %s]", e.Column)
}
if e.Got != "" {
fmt.Fprintf(&b, ": got %q, want %q", e.Got, e.Want)
}
b.WriteString(e.Hint)
return b.String()
}
func drift(season int, gameID, column, msg, got, want string) error {
return &DriftError{Season: season, GameID: gameID, Column: column, Msg: msg, Got: got, Want: want, Hint: layoutHint}
}
// ParseSchedule extracts the schedule table from one saved or fetched page.
func ParseSchedule(seasonID int, r io.Reader) (*Schedule, error) {
doc, err := html.Parse(r)
if err != nil {
return nil, fmt.Errorf("swehockey: parse page: %w", err)
}
tbl, lay := findTable(doc)
if tbl == nil {
return nil, drift(seasonID, "", "", "no table with a known header: found "+strings.Join(documentSignatures(doc), " / ")+
", known "+knownLayouts(), "", "")
}
s := &Schedule{SeasonID: seasonID, Layout: lay.name, Games: []Game{}}
rows := tableRows(tbl)
headerAt, c := -1, &carry{}
for i, tr := range rows {
if len(childElements(tr, "td")) == 0 {
if slices.Equal(thLabels(tr), lay.header) {
if headerAt >= 0 {
return nil, drift(seasonID, "", "", "second header row at row "+strconv.Itoa(i), "2", "1")
}
headerAt = i
}
continue
}
if headerAt < 0 {
return nil, drift(seasonID, "", "", "data row before the header row", "", "header first")
}
g, err := parseGame(seasonID, lay, tr, c)
if err != nil {
return nil, err
}
s.Games = append(s.Games, g)
}
if headerAt < 0 {
return nil, drift(seasonID, "", "", "table has no header row", "", strings.Join(lay.header, " "))
}
if h := descendants(tbl, "h2"); len(h) > 0 {
s.Title = textOf(h[0])
}
if lu := lastUpdRE.FindStringSubmatch(textOf(tbl)); lu != nil {
s.LastUpdate = lu[1]
}
if s.LastUpdate == "" {
return nil, drift(seasonID, "", "", `table has no "Last update:" stamp`, "", "Last update: YYYY-MM-DD HH:MM")
}
if err := s.check(); err != nil {
return nil, err
}
// The site lists games in round order, not date order, and a re-scheduled
// or awarded game keeps the round it belongs to: 2011-12 has September
// games listed after October ones. Sorting makes the output a canonical
// function of the set of rows instead of a function of the site's row
// order, which is what makes two crawls comparable and diffs meaningful.
sort.SliceStable(s.Games, func(i, j int) bool {
a, b := s.Games[i], s.Games[j]
if a.Date != b.Date {
return a.Date < b.Date
}
if a.Time != b.Time {
return a.Time < b.Time
}
return a.GameID < b.GameID
})
return s, nil
}
// check enforces the invariants that hold for every season the site serves.
func (s *Schedule) check() error {
if len(s.Games) == 0 {
return drift(s.SeasonID, "", "", "table has no game rows", "0 rows", ">= 1 row")
}
// Game ids are unique, and that is the only uniqueness the page promises:
// the site assigns them per fixture, so they are neither sorted nor
// contiguous within a season.
seen := make(map[string]bool, len(s.Games))
for _, g := range s.Games {
if _, err := strconv.Atoi(g.GameID); err != nil {
return drift(s.SeasonID, g.GameID, "", "game id is not a number", g.GameID, "90001001")
}
if seen[g.GameID] {
return drift(s.SeasonID, g.GameID, "", "duplicate game id", g.GameID, "unique")
}
seen[g.GameID] = true
if !dateRE.MatchString(g.Date) {
return drift(s.SeasonID, g.GameID, "date", "malformed date", g.Date, "2026-09-19")
}
}
return nil
}
// carry holds the values the site prints only on the first row of a group: the
// date heading of the modern layout and the round number of the legacy one.
type carry struct{ date, round string }
// parseGame reads one row. Where the site renders a fact twice, both
// renderings must agree before the value is accepted.
func parseGame(seasonID int, lay *layout, tr *html.Node, c *carry) (Game, error) {
tds := childElements(tr, "td")
if len(tds) != lay.cells {
return Game{}, drift(seasonID, "", "", "row width does not match the "+lay.name+" layout",
strconv.Itoa(len(tds))+" cells", strconv.Itoa(lay.cells)+" cells")
}
g := Game{Periods: []Period{}}
// The tooltip span carries the row's stable id, so the id and the time can
// never disagree: they are the same element.
span := tooltipSpan(tds[lay.timeCell])
if span == nil {
return Game{}, drift(seasonID, "", "time", "no game id (span.lnkTooltip[title])", textOf(tds[lay.timeCell]), "90001001")
}
g.GameID = strings.TrimSpace(attr(span, "title"))
if !gameIDRE.MatchString(g.GameID) {
return Game{}, drift(seasonID, "", "time", "malformed game id", g.GameID, "90001001")
}
head := textOf(span)
if lay.grouped {
// Modern layout: date in two renderings, time in the span.
if !timeRE.MatchString(head) {
return Game{}, drift(seasonID, g.GameID, "time", "malformed start time", head, "19:00")
}
g.Time = head
desktop := textOf(tds[lay.dateDesktop])
mobile := textOf(firstElement(tds[lay.dateMobile], "bold"))
switch {
case mobile != "":
if !dateRE.MatchString(mobile) {
return Game{}, drift(seasonID, g.GameID, "date", "malformed date heading", mobile, "2026-09-19")
}
if desktop != mobile {
return Game{}, drift(seasonID, g.GameID, "date", "date renderings disagree", desktop, mobile)
}
if t := textExcluding(tds[lay.dateMobile], "bold"); t != g.Time {
return Game{}, drift(seasonID, g.GameID, "date", "time renderings disagree", t, g.Time)
}
g.Date = mobile
default:
if c.date == "" {
return Game{}, drift(seasonID, g.GameID, "date", "row before any date heading", desktop, "2026-09-19")
}
if desktop != "" && desktop != g.Time {
return Game{}, drift(seasonID, g.GameID, "date", "unexpected date in a continuation row", desktop, "(empty) or "+g.Time)
}
g.Date = c.date
}
} else {
// Legacy layout: "2014-03-10 19:00" in one cell, no group headings.
d, t, ok := strings.Cut(head, " ")
switch {
case ok && dateRE.MatchString(d) && timeRE.MatchString(t):
g.Date, g.Time = d, t
case timeRE.MatchString(head):
if c.date == "" {
return Game{}, drift(seasonID, g.GameID, "date", "row before any date", head, "2026-09-19 19:00")
}
g.Date, g.Time = c.date, head
default:
return Game{}, drift(seasonID, g.GameID, "date", "cannot read date and time", head, "2026-09-19 19:00")
}
}
if lay.round >= 0 {
// The site prints the round only on the first row of that round and
// leaves the following cells empty, exactly like the date headings.
switch round := textOf(tds[lay.round]); {
case digitsRE.MatchString(round):
g.Round = round
case round == "" && c.round != "":
g.Round = c.round
case round == "":
return Game{}, drift(seasonID, g.GameID, "round", "row before any round number", round, "1")
default:
return Game{}, drift(seasonID, g.GameID, "round", "round is not a number", round, "1")
}
}
// Teams: the cell is "Home - Away" plus, in old seasons, an note
// such as "Eighth final 1" that is not part of a name.
teams := textExcluding(tds[lay.teams], "i")
parts := strings.Split(teams, " - ")
if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
return Game{}, drift(seasonID, g.GameID, "game", "cannot split teams", teams, "Home - Away")
}
g.Home, g.Away = parts[0], parts[1]
// Result: present only for played games, and always with an event link.
if a := firstElement(tds[lay.result], "a"); a != nil {
m := scoreRE.FindStringSubmatch(textOf(a))
if m == nil {
return Game{}, drift(seasonID, g.GameID, "result", "malformed score", textOf(a), "3 - 2")
}
ev := eventIDRE.FindStringSubmatch(attr(a, "href"))
if ev == nil {
return Game{}, drift(seasonID, g.GameID, "result", "result link without an event id", attr(a, "href"), "/Game/Events/1109941")
}
h, _ := strconv.Atoi(m[1])
aw, _ := strconv.Atoi(m[2])
g.HomeGoals, g.AwayGoals, g.EventID = &h, &aw, ev[1]
}
periods, err := parsePeriods(textOf(tds[lay.periods]))
if err != nil {
return Game{}, drift(seasonID, g.GameID, "periods", err.Error(), textOf(tds[lay.periods]), "(1-1, 2-1, 2-2)")
}
g.Periods = periods
if sp := textOf(tds[lay.spectators]); sp != "" {
if !digitsRE.MatchString(sp) {
return Game{}, drift(seasonID, g.GameID, "spectators", "spectators are not a number", sp, "10442")
}
n, _ := strconv.Atoi(sp)
g.Spectators = &n
}
g.Venue = textOf(tds[lay.venue])
if g.Venue == "" {
return Game{}, drift(seasonID, g.GameID, "venue", "empty venue", "", "Gottde Arena")
}
c.date, c.round = g.Date, g.Round
if err := g.checkRow(); err != nil {
return Game{}, drift(seasonID, g.GameID, "", err.Error(), "", "")
}
return g, nil
}
// checkRow enforces the per-row invariants: periods sum to the final score,
// and a game is either fully played or not played at all.
func (g Game) checkRow() error {
if !g.Played() {
if g.HomeGoals != nil || len(g.Periods) > 0 {
return fmt.Errorf("unplayed game carries a result")
}
return nil
}
if len(g.Periods) < 3 {
return fmt.Errorf("played game has %d period scores", len(g.Periods))
}
h, a := 0, 0
for _, p := range g.Periods {
h += p.Home
a += p.Away
}
if h != *g.HomeGoals || a != *g.AwayGoals {
return fmt.Errorf("periods sum to %d-%d but the result is %d-%d", h, a, *g.HomeGoals, *g.AwayGoals)
}
return nil
}
// parsePeriods reads "(1-1, 2-1, 2-2)". This is the single place to extend if
// the site ever renders a period differently, e.g. a shootout as "-".
func parsePeriods(s string) ([]Period, error) {
if s == "" {
return []Period{}, nil
}
if !strings.HasPrefix(s, "(") || !strings.HasSuffix(s, ")") {
return nil, fmt.Errorf("period scores are not parenthesised")
}
tokens := strings.Split(s[1:len(s)-1], ",")
// Only the lower bound is a contract. How many periods a game lasts depends
// on the era's rules - the site serves 3, 4, 5 and 6 period breakdowns - so
// the count is not pinned; the sum check in checkRow is what protects the
// data.
if len(tokens) < 3 {
return nil, fmt.Errorf("only %d period scores", len(tokens))
}
out := make([]Period, 0, len(tokens))
for i, tok := range tokens {
m := periodRE.FindStringSubmatch(strings.TrimSpace(tok))
if m == nil {
return nil, fmt.Errorf("malformed period %q", strings.TrimSpace(tok))
}
h, _ := strconv.Atoi(m[1])
a, _ := strconv.Atoi(m[2])
out = append(out, Period{Label: strconv.Itoa(i + 1), Home: h, Away: a})
}
return out, nil
}
// findTable returns the first table that declares one of the known layouts.
// Locating the table by its header, rather than by a CSS path or an index, is
// what keeps ads, navigation and layout changes from breaking the crawler.
func findTable(root *html.Node) (*html.Node, *layout) {
var found *html.Node
var lay *layout
var walk func(*html.Node)
walk = func(n *html.Node) {
if found != nil {
return
}
if n.Type == html.ElementNode && n.Data == "table" {
for _, tr := range tableRows(n) {
labels := thLabels(tr)
for _, l := range layouts {
if slices.Equal(labels, l.header) {
found, lay = n, l
return
}
}
}
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
walk(c)
}
}
walk(root)
return found, lay
}
// documentSignatures describes every table on the page, so that an
// unrecognised layout is diagnosable from the error message alone.
func documentSignatures(doc *html.Node) []string {
var out []string
var walk func(*html.Node)
walk = func(n *html.Node) {
if n.Type == html.ElementNode && n.Data == "table" {
if labels := tableHeaderLabels(n); len(labels) > 0 {
cells := 0
for _, tr := range tableRows(n) {
if c := len(childElements(tr, "td")); c > 0 {
cells = c
break
}
}
out = append(out, fmt.Sprintf("%q with %d cells", strings.Join(labels, " "), cells))
}
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
walk(c)
}
}
walk(doc)
if len(out) == 0 {
return []string{"no table with a header row"}
}
return out
}
func knownLayouts() string {
names := make([]string, 0, len(layouts))
for _, l := range layouts {
names = append(names, fmt.Sprintf("%s=%q", l.name, strings.Join(l.header, " ")))
}
return strings.Join(names, ", ")
}
// thLabels returns the non-empty header labels of a row, or of the first row
// of a table that has one.
func thLabels(n *html.Node) []string {
trs := tableRows(n)
if n.Type == html.ElementNode && n.Data == "tr" {
trs = []*html.Node{n}
}
for _, tr := range trs {
if labels := tableHeaderLabels(tr); len(labels) > 0 {
return labels
}
}
return nil
}
// tableHeaderLabels collects the | labels of a single row.
func tableHeaderLabels(row *html.Node) []string {
var labels []string
for _, th := range childElements(row, "th") {
if l := textOf(th); l != "" {
labels = append(labels, l)
}
}
return labels
}
func firstElement(n *html.Node, tag string) *html.Node {
if d := descendants(n, tag); len(d) > 0 {
return d[0]
}
return nil
}
// tooltipSpan returns the span that carries a game's stable id.
func tooltipSpan(cell *html.Node) *html.Node {
for _, s := range descendants(cell, "span") {
if hasClass(s, "lnkTooltip") && attr(s, "title") != "" {
return s
}
}
return nil
}
| |