One HTTP GET per season, no browser: the page is server-rendered, so every row arrives in one response and the extraction becomes a pure function of the HTML. - layouts[] declares one profile per version of the site's table (modern and legacy), each identified by its header row, so a new version of the page is a new profile plus a fixture instead of a guess. - The table is located by header signature, not by a CSS path. - Every cell is shape-checked, and facts the site renders twice (desktop and mobile date, time in the tooltip) must agree. - Cross-field invariants: periods sum to the score, ids are unique, played games always have an event link and unplayed ones never do. - Output is sorted by date, time and id, so it does not inherit the site's row order, and it carries no fetch timestamp: same HTML, same bytes. - Drift is a typed error and exit code 3, with the header it actually found. - Tests run on committed HTML fixtures and golden JSON; TestLive is opt-in. Verified over the full SHL archive: 27/27 seasons, 2119 games, 1779 played.
127 lines
3.8 KiB
Go
127 lines
3.8 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"regexp"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"golang.org/x/net/html"
|
|
)
|
|
|
|
// League is one competition as listed on the front page.
|
|
type League struct {
|
|
ID int `json:"id"`
|
|
Name string `json:"name"`
|
|
Path string `json:"path"` // canonical path, e.g. /ScheduleAndResults/Overview/20961
|
|
}
|
|
|
|
// Season is one entry of the season <select> on a league page.
|
|
type Season struct {
|
|
ID int `json:"id"`
|
|
Label string `json:"label"` // "2026-27"
|
|
Path string `json:"path"` // a path on the site for this season
|
|
}
|
|
|
|
var (
|
|
leaguePathRE = regexp.MustCompile(`^/ScheduleAndResults/[A-Za-z]+/(\d+)$`)
|
|
seasonPathRE = regexp.MustCompile(`/ScheduleAndResults/[A-Za-z]+/(\d+)`)
|
|
)
|
|
|
|
// tableRows returns the <tr> children of a table. The HTML5 tree builder wraps
|
|
// stray rows in an implicit <tbody>, so the rows can sit one level deeper than
|
|
// the markup suggests. Both shapes are accepted: a scraper that indexes the
|
|
// raw markup, or one that trusts a hand-rolled parser, gets this wrong.
|
|
func tableRows(tbl *html.Node) []*html.Node {
|
|
if tr := childElements(tbl, "tr"); len(tr) > 0 {
|
|
return tr
|
|
}
|
|
var out []*html.Node
|
|
for _, body := range childElements(tbl, "tbody") {
|
|
out = append(out, childElements(body, "tr")...)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// ParseLeagues reads the front page's league list. It is a plain anchor crawl:
|
|
// every /ScheduleAndResults/<section>/<id> link, deduplicated by id and sorted,
|
|
// so the crawl order never depends on document order.
|
|
func ParseLeagues(r io.Reader) ([]League, error) {
|
|
doc, err := html.Parse(r)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("swehockey: parse page: %w", err)
|
|
}
|
|
byID := map[int]string{}
|
|
for _, a := range descendants(doc, "a") {
|
|
m := leaguePathRE.FindStringSubmatch(attr(a, "href"))
|
|
if m == nil {
|
|
continue
|
|
}
|
|
id, err := strconv.Atoi(m[1])
|
|
if err != nil {
|
|
continue
|
|
}
|
|
name := textOf(a)
|
|
if name == "" {
|
|
continue
|
|
}
|
|
if old, ok := byID[id]; ok && old != "" {
|
|
// A league is linked once per section (Live, Overview, Schedule).
|
|
// Keep the first name seen; the id is what matters.
|
|
continue
|
|
}
|
|
byID[id] = name
|
|
}
|
|
if len(byID) == 0 {
|
|
return nil, drift(0, "", "", "front page has no league links", "0 links", "/ScheduleAndResults/<section>/<id>")
|
|
}
|
|
out := make([]League, 0, len(byID))
|
|
for id, name := range byID {
|
|
out = append(out, League{ID: id, Name: name, Path: fmt.Sprintf("/ScheduleAndResults/Schedule/%d", id)})
|
|
}
|
|
sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
|
|
return out, nil
|
|
}
|
|
|
|
// ParseSeasons reads the season selector on a league page. Group headings
|
|
// ("Cups", value="-1") are skipped; the result is sorted by id.
|
|
func ParseSeasons(leagueID int, r io.Reader) ([]Season, error) {
|
|
doc, err := html.Parse(r)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("swehockey: parse page: %w", err)
|
|
}
|
|
var out []Season
|
|
seen := map[int]bool{}
|
|
for _, opt := range descendants(doc, "option") {
|
|
raw := attr(opt, "value")
|
|
m := seasonPathRE.FindStringSubmatch(raw)
|
|
if m == nil {
|
|
continue // group heading such as value="-1"
|
|
}
|
|
id, err := strconv.Atoi(m[1])
|
|
if err != nil || seen[id] {
|
|
continue
|
|
}
|
|
seen[id] = true
|
|
out = append(out, Season{ID: id, Label: textOf(opt), Path: raw})
|
|
}
|
|
if len(out) == 0 {
|
|
return nil, drift(leagueID, "", "", "no season options in the selector", "0 options", "/ScheduleAndResults/<section>/<id>")
|
|
}
|
|
sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
|
|
return out, nil
|
|
}
|
|
|
|
// SchedulePath is the canonical schedule URL of a season id. Every view
|
|
// (Overview, Live, Schedule) of one season shares the id, so the crawler
|
|
// derives the schedule URL instead of trusting whichever link it arrived on.
|
|
func SchedulePath(seasonID int) string {
|
|
return fmt.Sprintf("/ScheduleAndResults/Schedule/%d", seasonID)
|
|
}
|
|
|
|
func joinURL(base, path string) string {
|
|
return strings.TrimRight(base, "/") + path
|
|
}
|