swehockey/parse.go

package main

import (
	"fmt"
	"io"
	"regexp"
	"slices"
	"sort"
	"strconv"
	"strings"

	"golang.org/x/net/html"
)

// The site has shipped more than one version of this table, and the old
// versions are still served for historical seasons. So the contract is not a
// set of hard-coded column indices: it is a set of layout profiles, each
// identified by the header row it declares. A page the crawler does not
// recognise fails loudly with the header it actually found, and adding support
// is one profile plus one fixture.
type layout struct {
	name   string
	header []string // non-empty header labels, in order
	cells  int      // number of <td> in one data row
	round  int      // -1 when the layout has no Round column

	// Modern layout: the date is rendered twice, once in the desktop "Date"
	// column and once as a <bold> group heading in the mobile column, and the
	// start time lives in its own cell next to the game's stable id.
	// Legacy layout: no group headings; the tooltip span of one cell holds
	// both "2014-03-10" and "19:00".
	dateDesktop int
	dateMobile  int
	timeCell    int
	grouped     bool

	teams      int
	result     int
	periods    int
	spectators int
	venue      int
}

var layouts = []*layout{
	{
		name:   "modern", // seasons from 2018-19 onwards
		header: []string{"Date", "Time", "Game", "Result", "Spectators", "Venue"},
		cells:  8, round: -1,
		dateDesktop: 0, dateMobile: 1, timeCell: 2, grouped: true,
		teams: 3, result: 4, periods: 5, spectators: 6, venue: 7,
	},
	{
		name:   "legacy", // seasons up to 2017-18
		header: []string{"Round", "Date", "Game", "Result", "Spectators", "Venue"},
		cells:  7, round: 0,
		dateDesktop: -1, dateMobile: -1, timeCell: 1, grouped: false,
		teams: 2, result: 3, periods: 4, spectators: 5, venue: 6,
	},
}

var (
	dateRE    = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`)
	timeRE    = regexp.MustCompile(`^\d{1,2}:\d{2}$`)
	digitsRE  = regexp.MustCompile(`^\d+$`)
	scoreRE   = regexp.MustCompile(`^(\d+)\s*-\s*(\d+)$`)
	periodRE  = regexp.MustCompile(`^(\d+)\s*-\s*(\d+)$`)
	eventIDRE = regexp.MustCompile(`/Game/Events/(\d+)`)
	lastUpdRE = regexp.MustCompile(`Last update:\s*(\d{4}-\d{2}-\d{2} \d{1,2}:\d{2})`)
	gameIDRE  = regexp.MustCompile(`^\d{5,}$`)
)

// Period is one period (or overtime / shootout) of a played game.
type Period struct {
	Label string `json:"label"` // "1", "2", ... in site order
	Home  int    `json:"home"`
	Away  int    `json:"away"`
}

// Game is one row of the schedule table.
type Game struct {
	GameID     string   `json:"game_id"`            // span.lnkTooltip@title, unique per season
	EventID    string   `json:"event_id,omitempty"` // /Game/Events/<id>; set once the game is played
	Round      string   `json:"round,omitempty"`    // only in the legacy layout
	Date       string   `json:"date"`               // YYYY-MM-DD
	Time       string   `json:"time"`               // HH:MM local arena time
	Home       string   `json:"home"`
	Away       string   `json:"away"`
	HomeGoals  *int     `json:"home_goals"`           // nil until the game is played
	AwayGoals  *int     `json:"away_goals"`           // nil until the game is played
	Periods    []Period `json:"periods"`              // empty until the game is played
	Spectators *int     `json:"spectators,omitempty"` // nil when not reported
	Venue      string   `json:"venue"`
}

func (g Game) Played() bool { return g.EventID != "" }

// Schedule is one season page.
type Schedule struct {
	SeasonID   int    `json:"season_id"`
	Layout     string `json:"layout"`
	Title      string `json:"title"`
	LastUpdate string `json:"last_update"` // the site's own clock, verbatim
	Games      []Game `json:"games"`
}

func (s *Schedule) Played() int {
	n := 0
	for _, g := range s.Games {
		if g.Played() {
			n++
		}
	}
	return n
}

// DriftError reports that the page no longer matches a known contract. It is a
// distinct type, and a distinct exit code, because it is the one failure a
// caller must not retry blindly: the site changed, not the network.
type DriftError struct {
	Season int
	GameID string
	Column string
	Got    string
	Want   string
	Msg    string
	// Every layout problem ends with the same advice, so an operator reading a
	// cron mail knows exactly what to do. Failures that are not about the table
	// shape pass an empty Hint and get no advice.
	Hint string
}

// layoutHint is the advice attached to every markup problem.
const layoutHint = " - extend layouts[] in parse.go and add a fixture, in one commit"

func (e *DriftError) Error() string {
	var b strings.Builder
	fmt.Fprintf(&b, "swehockey: markup drift (season %d", e.Season)
	if e.GameID != "" {
		fmt.Fprintf(&b, ", game %s", e.GameID)
	}
	b.WriteString("): ")
	if e.Msg != "" {
		b.WriteString(e.Msg)
	} else {
		b.WriteString("unexpected table shape")
	}
	if e.Column != "" {
		fmt.Fprintf(&b, " [column %s]", e.Column)
	}
	if e.Got != "" {
		fmt.Fprintf(&b, ": got %q, want %q", e.Got, e.Want)
	}
	b.WriteString(e.Hint)
	return b.String()
}

func drift(season int, gameID, column, msg, got, want string) error {
	return &DriftError{Season: season, GameID: gameID, Column: column, Msg: msg, Got: got, Want: want, Hint: layoutHint}
}

// ParseSchedule extracts the schedule table from one saved or fetched page.
func ParseSchedule(seasonID int, r io.Reader) (*Schedule, error) {
	doc, err := html.Parse(r)
	if err != nil {
		return nil, fmt.Errorf("swehockey: parse page: %w", err)
	}
	tbl, lay := findTable(doc)
	if tbl == nil {
		return nil, drift(seasonID, "", "", "no table with a known header: found "+strings.Join(documentSignatures(doc), " / ")+
			", known "+knownLayouts(), "", "")
	}

	s := &Schedule{SeasonID: seasonID, Layout: lay.name, Games: []Game{}}
	rows := tableRows(tbl)
	headerAt, c := -1, &carry{}
	for i, tr := range rows {
		if len(childElements(tr, "td")) == 0 {
			if slices.Equal(thLabels(tr), lay.header) {
				if headerAt >= 0 {
					return nil, drift(seasonID, "", "", "second header row at row "+strconv.Itoa(i), "2", "1")
				}
				headerAt = i
			}
			continue
		}
		if headerAt < 0 {
			return nil, drift(seasonID, "", "", "data row before the header row", "", "header first")
		}
		g, err := parseGame(seasonID, lay, tr, c)
		if err != nil {
			return nil, err
		}
		s.Games = append(s.Games, g)
	}
	if headerAt < 0 {
		return nil, drift(seasonID, "", "", "table has no header row", "", strings.Join(lay.header, " "))
	}
	if h := descendants(tbl, "h2"); len(h) > 0 {
		s.Title = textOf(h[0])
	}
	if lu := lastUpdRE.FindStringSubmatch(textOf(tbl)); lu != nil {
		s.LastUpdate = lu[1]
	}
	if s.LastUpdate == "" {
		return nil, drift(seasonID, "", "", `table has no "Last update:" stamp`, "", "Last update: YYYY-MM-DD HH:MM")
	}
	if err := s.check(); err != nil {
		return nil, err
	}
	// The site lists games in round order, not date order, and a re-scheduled
	// or awarded game keeps the round it belongs to: 2011-12 has September
	// games listed after October ones. Sorting makes the output a canonical
	// function of the set of rows instead of a function of the site's row
	// order, which is what makes two crawls comparable and diffs meaningful.
	sort.SliceStable(s.Games, func(i, j int) bool {
		a, b := s.Games[i], s.Games[j]
		if a.Date != b.Date {
			return a.Date < b.Date
		}
		if a.Time != b.Time {
			return a.Time < b.Time
		}
		return a.GameID < b.GameID
	})
	return s, nil
}

// check enforces the invariants that hold for every season the site serves.
func (s *Schedule) check() error {
	if len(s.Games) == 0 {
		return drift(s.SeasonID, "", "", "table has no game rows", "0 rows", ">= 1 row")
	}
	// Game ids are unique, and that is the only uniqueness the page promises:
	// the site assigns them per fixture, so they are neither sorted nor
	// contiguous within a season.
	seen := make(map[string]bool, len(s.Games))
	for _, g := range s.Games {
		if _, err := strconv.Atoi(g.GameID); err != nil {
			return drift(s.SeasonID, g.GameID, "", "game id is not a number", g.GameID, "90001001")
		}
		if seen[g.GameID] {
			return drift(s.SeasonID, g.GameID, "", "duplicate game id", g.GameID, "unique")
		}
		seen[g.GameID] = true
		if !dateRE.MatchString(g.Date) {
			return drift(s.SeasonID, g.GameID, "date", "malformed date", g.Date, "2026-09-19")
		}
	}
	return nil
}

// carry holds the values the site prints only on the first row of a group: the
// date heading of the modern layout and the round number of the legacy one.
type carry struct{ date, round string }

// parseGame reads one row. Where the site renders a fact twice, both
// renderings must agree before the value is accepted.
func parseGame(seasonID int, lay *layout, tr *html.Node, c *carry) (Game, error) {
	tds := childElements(tr, "td")
	if len(tds) != lay.cells {
		return Game{}, drift(seasonID, "", "", "row width does not match the "+lay.name+" layout",
			strconv.Itoa(len(tds))+" cells", strconv.Itoa(lay.cells)+" cells")
	}
	g := Game{Periods: []Period{}}

	// The tooltip span carries the row's stable id, so the id and the time can
	// never disagree: they are the same element.
	span := tooltipSpan(tds[lay.timeCell])
	if span == nil {
		return Game{}, drift(seasonID, "", "time", "no game id (span.lnkTooltip[title])", textOf(tds[lay.timeCell]), "90001001")
	}
	g.GameID = strings.TrimSpace(attr(span, "title"))
	if !gameIDRE.MatchString(g.GameID) {
		return Game{}, drift(seasonID, "", "time", "malformed game id", g.GameID, "90001001")
	}

	head := textOf(span)
	if lay.grouped {
		// Modern layout: date in two renderings, time in the span.
		if !timeRE.MatchString(head) {
			return Game{}, drift(seasonID, g.GameID, "time", "malformed start time", head, "19:00")
		}
		g.Time = head
		desktop := textOf(tds[lay.dateDesktop])
		mobile := textOf(firstElement(tds[lay.dateMobile], "bold"))
		switch {
		case mobile != "":
			if !dateRE.MatchString(mobile) {
				return Game{}, drift(seasonID, g.GameID, "date", "malformed date heading", mobile, "2026-09-19")
			}
			if desktop != mobile {
				return Game{}, drift(seasonID, g.GameID, "date", "date renderings disagree", desktop, mobile)
			}
			if t := textExcluding(tds[lay.dateMobile], "bold"); t != g.Time {
				return Game{}, drift(seasonID, g.GameID, "date", "time renderings disagree", t, g.Time)
			}
			g.Date = mobile
		default:
			if c.date == "" {
				return Game{}, drift(seasonID, g.GameID, "date", "row before any date heading", desktop, "2026-09-19")
			}
			if desktop != "" && desktop != g.Time {
				return Game{}, drift(seasonID, g.GameID, "date", "unexpected date in a continuation row", desktop, "(empty) or "+g.Time)
			}
			g.Date = c.date
		}
	} else {
		// Legacy layout: "2014-03-10 19:00" in one cell, no group headings.
		d, t, ok := strings.Cut(head, " ")
		switch {
		case ok && dateRE.MatchString(d) && timeRE.MatchString(t):
			g.Date, g.Time = d, t
		case timeRE.MatchString(head):
			if c.date == "" {
				return Game{}, drift(seasonID, g.GameID, "date", "row before any date", head, "2026-09-19 19:00")
			}
			g.Date, g.Time = c.date, head
		default:
			return Game{}, drift(seasonID, g.GameID, "date", "cannot read date and time", head, "2026-09-19 19:00")
		}
	}

	if lay.round >= 0 {
		// The site prints the round only on the first row of that round and
		// leaves the following cells empty, exactly like the date headings.
		switch round := textOf(tds[lay.round]); {
		case digitsRE.MatchString(round):
			g.Round = round
		case round == "" && c.round != "":
			g.Round = c.round
		case round == "":
			return Game{}, drift(seasonID, g.GameID, "round", "row before any round number", round, "1")
		default:
			return Game{}, drift(seasonID, g.GameID, "round", "round is not a number", round, "1")
		}
	}

	// Teams: the cell is "Home - Away" plus, in old seasons, an <i>note</i>
	// such as "Eighth final 1" that is not part of a name.
	teams := textExcluding(tds[lay.teams], "i")
	parts := strings.Split(teams, " - ")
	if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
		return Game{}, drift(seasonID, g.GameID, "game", "cannot split teams", teams, "Home - Away")
	}
	g.Home, g.Away = parts[0], parts[1]

	// Result: present only for played games, and always with an event link.
	if a := firstElement(tds[lay.result], "a"); a != nil {
		m := scoreRE.FindStringSubmatch(textOf(a))
		if m == nil {
			return Game{}, drift(seasonID, g.GameID, "result", "malformed score", textOf(a), "3 - 2")
		}
		ev := eventIDRE.FindStringSubmatch(attr(a, "href"))
		if ev == nil {
			return Game{}, drift(seasonID, g.GameID, "result", "result link without an event id", attr(a, "href"), "/Game/Events/1109941")
		}
		h, _ := strconv.Atoi(m[1])
		aw, _ := strconv.Atoi(m[2])
		g.HomeGoals, g.AwayGoals, g.EventID = &h, &aw, ev[1]
	}

	periods, err := parsePeriods(textOf(tds[lay.periods]))
	if err != nil {
		return Game{}, drift(seasonID, g.GameID, "periods", err.Error(), textOf(tds[lay.periods]), "(1-1, 2-1, 2-2)")
	}
	g.Periods = periods

	if sp := textOf(tds[lay.spectators]); sp != "" {
		if !digitsRE.MatchString(sp) {
			return Game{}, drift(seasonID, g.GameID, "spectators", "spectators are not a number", sp, "10442")
		}
		n, _ := strconv.Atoi(sp)
		g.Spectators = &n
	}
	g.Venue = textOf(tds[lay.venue])
	if g.Venue == "" {
		return Game{}, drift(seasonID, g.GameID, "venue", "empty venue", "", "Gottde Arena")
	}
	c.date, c.round = g.Date, g.Round
	if err := g.checkRow(); err != nil {
		return Game{}, drift(seasonID, g.GameID, "", err.Error(), "", "")
	}
	return g, nil
}

// checkRow enforces the per-row invariants: periods sum to the final score,
// and a game is either fully played or not played at all.
func (g Game) checkRow() error {
	if !g.Played() {
		if g.HomeGoals != nil || len(g.Periods) > 0 {
			return fmt.Errorf("unplayed game carries a result")
		}
		return nil
	}
	if len(g.Periods) < 3 {
		return fmt.Errorf("played game has %d period scores", len(g.Periods))
	}
	h, a := 0, 0
	for _, p := range g.Periods {
		h += p.Home
		a += p.Away
	}
	if h != *g.HomeGoals || a != *g.AwayGoals {
		return fmt.Errorf("periods sum to %d-%d but the result is %d-%d", h, a, *g.HomeGoals, *g.AwayGoals)
	}
	return nil
}

// parsePeriods reads "(1-1, 2-1, 2-2)". This is the single place to extend if
// the site ever renders a period differently, e.g. a shootout as "-".
func parsePeriods(s string) ([]Period, error) {
	if s == "" {
		return []Period{}, nil
	}
	if !strings.HasPrefix(s, "(") || !strings.HasSuffix(s, ")") {
		return nil, fmt.Errorf("period scores are not parenthesised")
	}
	tokens := strings.Split(s[1:len(s)-1], ",")
	// Only the lower bound is a contract. How many periods a game lasts depends
	// on the era's rules - the site serves 3, 4, 5 and 6 period breakdowns - so
	// the count is not pinned; the sum check in checkRow is what protects the
	// data.
	if len(tokens) < 3 {
		return nil, fmt.Errorf("only %d period scores", len(tokens))
	}
	out := make([]Period, 0, len(tokens))
	for i, tok := range tokens {
		m := periodRE.FindStringSubmatch(strings.TrimSpace(tok))
		if m == nil {
			return nil, fmt.Errorf("malformed period %q", strings.TrimSpace(tok))
		}
		h, _ := strconv.Atoi(m[1])
		a, _ := strconv.Atoi(m[2])
		out = append(out, Period{Label: strconv.Itoa(i + 1), Home: h, Away: a})
	}
	return out, nil
}

// findTable returns the first table that declares one of the known layouts.
// Locating the table by its header, rather than by a CSS path or an index, is
// what keeps ads, navigation and layout changes from breaking the crawler.
func findTable(root *html.Node) (*html.Node, *layout) {
	var found *html.Node
	var lay *layout
	var walk func(*html.Node)
	walk = func(n *html.Node) {
		if found != nil {
			return
		}
		if n.Type == html.ElementNode && n.Data == "table" {
			for _, tr := range tableRows(n) {
				labels := thLabels(tr)
				for _, l := range layouts {
					if slices.Equal(labels, l.header) {
						found, lay = n, l
						return
					}
				}
			}
		}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			walk(c)
		}
	}
	walk(root)
	return found, lay
}

// documentSignatures describes every table on the page, so that an
// unrecognised layout is diagnosable from the error message alone.
func documentSignatures(doc *html.Node) []string {
	var out []string
	var walk func(*html.Node)
	walk = func(n *html.Node) {
		if n.Type == html.ElementNode && n.Data == "table" {
			if labels := tableHeaderLabels(n); len(labels) > 0 {
				cells := 0
				for _, tr := range tableRows(n) {
					if c := len(childElements(tr, "td")); c > 0 {
						cells = c
						break
					}
				}
				out = append(out, fmt.Sprintf("%q with %d cells", strings.Join(labels, " "), cells))
			}
		}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			walk(c)
		}
	}
	walk(doc)
	if len(out) == 0 {
		return []string{"no table with a header row"}
	}
	return out
}

func knownLayouts() string {
	names := make([]string, 0, len(layouts))
	for _, l := range layouts {
		names = append(names, fmt.Sprintf("%s=%q", l.name, strings.Join(l.header, " ")))
	}
	return strings.Join(names, ", ")
}

// thLabels returns the non-empty header labels of a row, or of the first row
// of a table that has one.
func thLabels(n *html.Node) []string {
	trs := tableRows(n)
	if n.Type == html.ElementNode && n.Data == "tr" {
		trs = []*html.Node{n}
	}
	for _, tr := range trs {
		if labels := tableHeaderLabels(tr); len(labels) > 0 {
			return labels
		}
	}
	return nil
}

// tableHeaderLabels collects the <th> labels of a single row.
func tableHeaderLabels(row *html.Node) []string {
	var labels []string
	for _, th := range childElements(row, "th") {
		if l := textOf(th); l != "" {
			labels = append(labels, l)
		}
	}
	return labels
}

func firstElement(n *html.Node, tag string) *html.Node {
	if d := descendants(n, tag); len(d) > 0 {
		return d[0]
	}
	return nil
}

// tooltipSpan returns the span that carries a game's stable id.
func tooltipSpan(cell *html.Node) *html.Node {
	for _, s := range descendants(cell, "span") {
		if hasClass(s, "lnkTooltip") && attr(s, "title") != "" {
			return s
		}
	}
	return nil
}