Mats Tornberg
6b701819ca
One HTTP GET per season, no browser: the page is server-rendered, so every row arrives in one response and the extraction becomes a pure function of the HTML. - layouts[] declares one profile per version of the site's table (modern and legacy), each identified by its header row, so a new version of the page is a new profile plus a fixture instead of a guess. - The table is located by header signature, not by a CSS path. - Every cell is shape-checked, and facts the site renders twice (desktop and mobile date, time in the tooltip) must agree. - Cross-field invariants: periods sum to the score, ids are unique, played games always have an event link and unplayed ones never do. - Output is sorted by date, time and id, so it does not inherit the site's row order, and it carries no fetch timestamp: same HTML, same bytes. - Drift is a typed error and exit code 3, with the header it actually found. - Tests run on committed HTML fixtures and golden JSON; TestLive is opt-in. Verified over the full SHL archive: 27/27 seasons, 2119 games, 1779 played.
104 lines
2.3 KiB
Go
104 lines
2.3 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"os"
|
|
"time"
|
|
)
|
|
|
|
func cmdSchedule(ctx context.Context, args []string) error {
|
|
fs := flag.NewFlagSet("schedule", flag.ContinueOnError)
|
|
var c commonFlags
|
|
c.register(fs, "json")
|
|
season := fs.Int("season", defaultSeason, "season id; see the crawl or leagues command")
|
|
if err := fs.Parse(args); err != nil {
|
|
return err
|
|
}
|
|
|
|
path := SchedulePath(*season)
|
|
page, err := c.read(ctx, c.client(), path, fmt.Sprintf("season %d", *season))
|
|
if errors.Is(err, errUnchanged) {
|
|
return nil
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
s, err := ParseSchedule(*season, bytes.NewReader(page))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var b []byte
|
|
if c.format == "ndjson" {
|
|
b, err = renderNDJSON(s)
|
|
} else {
|
|
b, err = render(c.format, s)
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := emit(c.out, c.check, b); err != nil {
|
|
return err
|
|
}
|
|
if c.state != "" {
|
|
if err := saveState(c.state, &state{Path: path, SHA256: sha256Hex(page), LastUpdate: s.LastUpdate, SavedAt: nowStamp()}); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if !c.quiet {
|
|
fmt.Fprintf(os.Stderr, "season %d: %d games, %d played, last update %s\n",
|
|
s.SeasonID, len(s.Games), s.Played(), s.LastUpdate)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func cmdLeagues(ctx context.Context, args []string) error {
|
|
fs := flag.NewFlagSet("leagues", flag.ContinueOnError)
|
|
var c commonFlags
|
|
c.register(fs, "json")
|
|
if err := fs.Parse(args); err != nil {
|
|
return err
|
|
}
|
|
page, err := c.read(ctx, c.client(), "/", "front page")
|
|
if errors.Is(err, errUnchanged) {
|
|
return nil
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
leagues, err := ParseLeagues(bytes.NewReader(page))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var b []byte
|
|
if c.format == "csv" {
|
|
var out bytes.Buffer
|
|
out.WriteString("id,name,path\n")
|
|
for _, l := range leagues {
|
|
fmt.Fprintf(&out, "%d,%s,%s\n", l.ID, l.Name, l.Path)
|
|
}
|
|
b = out.Bytes()
|
|
} else {
|
|
var buf bytes.Buffer
|
|
enc := jsonEncoder(&buf)
|
|
if err := enc.Encode(leagues); err != nil {
|
|
return err
|
|
}
|
|
b = buf.Bytes()
|
|
}
|
|
return emit(c.out, c.check, b)
|
|
}
|
|
|
|
// crawlError is what a "record" run emits in place of a season, so a broken
|
|
// page is a visible hole in the output rather than a missing line.
|
|
type crawlError struct {
|
|
SeasonID int `json:"season_id"`
|
|
Label string `json:"label"`
|
|
Error string `json:"error"`
|
|
}
|
|
|
|
func nowStamp() string { return time.Now().UTC().Format(time.RFC3339) }
|