One HTTP GET per season, no browser: the page is server-rendered, so every row arrives in one response and the extraction becomes a pure function of the HTML. - layouts[] declares one profile per version of the site's table (modern and legacy), each identified by its header row, so a new version of the page is a new profile plus a fixture instead of a guess. - The table is located by header signature, not by a CSS path. - Every cell is shape-checked, and facts the site renders twice (desktop and mobile date, time in the tooltip) must agree. - Cross-field invariants: periods sum to the score, ids are unique, played games always have an event link and unplayed ones never do. - Output is sorted by date, time and id, so it does not inherit the site's row order, and it carries no fetch timestamp: same HTML, same bytes. - Drift is a typed error and exit code 3, with the header it actually found. - Tests run on committed HTML fixtures and golden JSON; TestLive is opt-in. Verified over the full SHL archive: 27/27 seasons, 2119 games, 1779 played.
141 lines
3.3 KiB
Go
141 lines
3.3 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/csv"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// render turns a schedule into bytes. Both encoders are pure functions of the
|
|
// data: no timestamps, no maps, fixed column order, LF line endings. Running
|
|
// the crawler twice on the same page produces identical bytes, which is what
|
|
// makes -check and diff-based review meaningful.
|
|
func render(format string, s *Schedule) ([]byte, error) {
|
|
switch format {
|
|
case "json":
|
|
var buf bytes.Buffer
|
|
enc := json.NewEncoder(&buf)
|
|
enc.SetEscapeHTML(false) // keep team names readable
|
|
enc.SetIndent("", " ")
|
|
if err := enc.Encode(s); err != nil {
|
|
return nil, err
|
|
}
|
|
return buf.Bytes(), nil
|
|
case "csv":
|
|
return renderCSV(s)
|
|
default:
|
|
return nil, fmt.Errorf("unknown format %q: want json, csv or ndjson", format)
|
|
}
|
|
}
|
|
|
|
func renderNDJSON(s *Schedule) ([]byte, error) {
|
|
var buf bytes.Buffer
|
|
if err := jsonEncoder(&buf).Encode(s); err != nil {
|
|
return nil, err
|
|
}
|
|
return buf.Bytes(), nil
|
|
}
|
|
|
|
// jsonEncoder keeps "ä" readable instead of \u00e4, which matters for diffs
|
|
// of team names.
|
|
func jsonEncoder(buf *bytes.Buffer) *json.Encoder {
|
|
enc := json.NewEncoder(buf)
|
|
enc.SetEscapeHTML(false)
|
|
return enc
|
|
}
|
|
|
|
var csvHeader = []string{
|
|
"season_id", "game_id", "event_id", "date", "time", "home", "away",
|
|
"home_goals", "away_goals", "periods", "spectators", "venue",
|
|
}
|
|
|
|
func renderCSV(s *Schedule) ([]byte, error) {
|
|
var buf bytes.Buffer
|
|
w := csv.NewWriter(&buf) // LF, not CRLF
|
|
if err := w.Write(csvHeader); err != nil {
|
|
return nil, err
|
|
}
|
|
for _, g := range s.Games {
|
|
periods := make([]string, 0, len(g.Periods))
|
|
for _, p := range g.Periods {
|
|
periods = append(periods, fmt.Sprintf("%s:%d-%d", p.Label, p.Home, p.Away))
|
|
}
|
|
row := []string{
|
|
strconv.Itoa(s.SeasonID),
|
|
g.GameID,
|
|
g.EventID,
|
|
g.Date,
|
|
g.Time,
|
|
g.Home,
|
|
g.Away,
|
|
optInt(g.HomeGoals),
|
|
optInt(g.AwayGoals),
|
|
strings.Join(periods, " "),
|
|
optInt(g.Spectators),
|
|
g.Venue,
|
|
}
|
|
if err := w.Write(row); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
w.Flush()
|
|
return buf.Bytes(), w.Error()
|
|
}
|
|
|
|
func optInt(p *int) string {
|
|
if p == nil {
|
|
return ""
|
|
}
|
|
return strconv.Itoa(*p)
|
|
}
|
|
|
|
// emit writes the bytes to -out or stdout and, when -check is set, compares
|
|
// them with a committed golden file. A mismatch exits 3 so CI turns an upstream
|
|
// change into a reviewable diff instead of a silent data change.
|
|
func emit(out, check string, b []byte) error {
|
|
if check != "" {
|
|
golden, err := os.ReadFile(check)
|
|
switch {
|
|
case err == nil && bytes.Equal(golden, b):
|
|
case err != nil:
|
|
return fmt.Errorf("golden file %s: %w", check, err)
|
|
default:
|
|
return &DriftError{Msg: fmt.Sprintf("output no longer matches %s: %s", check, firstDiff(golden, b))}
|
|
}
|
|
}
|
|
if out == "" {
|
|
_, err := os.Stdout.Write(b)
|
|
return err
|
|
}
|
|
return os.WriteFile(out, b, 0o644)
|
|
}
|
|
|
|
// firstDiff describes the first differing line, for a readable failure.
|
|
func firstDiff(a, b []byte) string {
|
|
al, bl := strings.Split(string(a), "\n"), strings.Split(string(b), "\n")
|
|
for i := 0; i < len(al) || i < len(bl); i++ {
|
|
var x, y string
|
|
if i < len(al) {
|
|
x = al[i]
|
|
}
|
|
if i < len(bl) {
|
|
y = bl[i]
|
|
}
|
|
if x != y {
|
|
return fmt.Sprintf("line %d: %q != %q", i+1, trim(x), trim(y))
|
|
}
|
|
}
|
|
return "documents differ only in length"
|
|
}
|
|
|
|
func trim(s string) string {
|
|
if len(s) > 80 {
|
|
return s[:80] + "..."
|
|
}
|
|
return s
|
|
}
|