diff --git a/.gitignore b/.gitignore index ea253f6..69408bf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ /swehockey *.tmp /.codesearch/ +__pycache__/ +*.pyc +.venv/ +*.egg-info/ diff --git a/README.md b/README.md index fa534cc..5d22cd4 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,50 @@ $ ./swehockey crawl -league 20961 -on-drift record | jq -s '[.[] | .games | leng 2119 ``` +## Two implementations, one contract + +The crawler exists twice, in Go and in Python, and that is a feature: two +independent parsers of the same page that have to agree. + +| | Go | Python | +| --- | --- | --- | +| entry point | `./swehockey schedule -season 20961` | `python3 -m swehockey.cli schedule --season 20961` | +| tests | `go test ./...` (offline) | `python3 -m unittest discover -s tests -t .` (offline) | +| live test | `SWEHOCKEY_LIVE=1 go test -run TestLive` | `SWEHOCKEY_LIVE=1 python3 -m unittest tests.test_swehockey.TestLive` | +| dependencies | `golang.org/x/net/html` | none — the tree is built on `html.parser` | + +Both share `testdata/`: the Python suite asserts its output against the **Go +golden files**, so a change in one implementation that the other does not share +fails a test. Verified on the live page as well: + +``` +$ ./swehockey schedule -season 20961 -out go.json +$ python3 -m swehockey.cli schedule --season 20961 --out py.json +$ cmp go.json py.json # 364 games, 102536 bytes, identical +$ ./swehockey schedule -season 20961 -format csv -out go.csv +$ python3 -m swehockey.cli schedule --season 20961 --format csv --out py.csv +$ cmp go.csv py.csv # 365 lines, identical +$ python3 -m swehockey.cli crawl --league 20961 | jq -s '[.[].games|length]|add' +2119 # same as the Go crawl +``` + +Same layout profiles, same invariants, same canonical order, same exit codes +(`0` ok, `1` error, `2` usage, `3` drift, `4` timeout). The only intentional +difference is in the tree builder: Python treats the site's empty `` +cells as empty elements instead of leaving them open the way a browser does, +which is what makes the two trees agree. See `python/README.md`. + +## Repository layout + +``` +main.go parse.go seasons.go fetch.go output.go the Go crawler +parse_test.go the Go tests +python/swehockey/ the Python crawler +python/tests/ the Python tests +testdata/ HTML fixtures and golden JSON, shared by both +README.md this file +``` + ## Commands | command | what it does | diff --git a/output.go b/output.go index 6e943d0..3e31823 100644 --- a/output.go +++ b/output.go @@ -104,7 +104,10 @@ func emit(out, check string, b []byte) error { case err != nil: return fmt.Errorf("golden file %s: %w", check, err) default: - return &DriftError{Msg: fmt.Sprintf("output no longer matches %s: %s", check, firstDiff(golden, b))} + // A golden mismatch is not a layout problem, so it gets its own + // advice instead of the "extend layouts[]" one. + return &DriftError{Msg: fmt.Sprintf("output no longer matches %s: %s", check, firstDiff(golden, b)), + Hint: " - regenerate the golden file only after reviewing the change"} } } if out == "" { diff --git a/parse.go b/parse.go index ca9a635..ebdc5ec 100644 --- a/parse.go +++ b/parse.go @@ -123,8 +123,15 @@ type DriftError struct { Got string Want string Msg string + // Every layout problem ends with the same advice, so an operator reading a + // cron mail knows exactly what to do. Failures that are not about the table + // shape pass an empty Hint and get no advice. + Hint string } +// layoutHint is the advice attached to every markup problem. +const layoutHint = " - extend layouts[] in parse.go and add a fixture, in one commit" + func (e *DriftError) Error() string { var b strings.Builder fmt.Fprintf(&b, "swehockey: markup drift (season %d", e.Season) @@ -143,12 +150,12 @@ func (e *DriftError) Error() string { if e.Got != "" { fmt.Fprintf(&b, ": got %q, want %q", e.Got, e.Want) } - b.WriteString(" - extend layouts[] in parse.go and add a fixture, in one commit") + b.WriteString(e.Hint) return b.String() } func drift(season int, gameID, column, msg, got, want string) error { - return &DriftError{Season: season, GameID: gameID, Column: column, Msg: msg, Got: got, Want: want} + return &DriftError{Season: season, GameID: gameID, Column: column, Msg: msg, Got: got, Want: want, Hint: layoutHint} } // ParseSchedule extracts the schedule table from one saved or fetched page. diff --git a/python/README.md b/python/README.md new file mode 100644 index 0000000..f2e0541 --- /dev/null +++ b/python/README.md @@ -0,0 +1,70 @@ +# swehockey-schedule (Python) + +The Python twin of the Go crawler in the parent directory. Same layout profiles, +same invariants, same canonical order — and, on purpose, **the same bytes**. + +```console +$ python3 -m swehockey.cli schedule --season 20961 --format csv | head -2 +season_id,game_id,event_id,date,time,home,away,home_goals,away_goals,periods,spectators,venue +20961,90001001,1109941,2026-09-19,15:15,Frölunda HC,Växjö Lakers HC,5,4,1:1-1 2:2-1 3:2-2,12044,Scandinavium + +$ python3 -m swehockey.cli crawl --league 20961 # every season, as NDJSON +$ python3 -m swehockey.cli leagues # 119 competitions +$ python3 -m unittest discover -s tests -t . # 23 offline tests +$ SWEHOCKEY_LIVE=1 python3 -m unittest discover -s tests -t . # + the site +``` + +With `pip install -e .` the same CLI is on `$PATH` as `swehockey-schedule`. + +## Why it can agree byte for byte + +- Same `LAYOUTS` profiles, keyed on the header row; same per-field shape + checks; same dual-render agreement for the modern layout; same carry-forward + for the date heading and the round number; same invariants (periods sum to + the score, ids unique, played ⇔ event link). +- Same canonical sort by `(date, time, game_id)`, because the site orders rows + by round and puts re-scheduled games where their round says. +- The output carries only page-derived data — no fetch timestamp, no unordered + iteration, fixed key order — and `json.dumps(..., indent=2, ensure_ascii=False)` + is byte-compatible with Go's `json.Encoder`. +- The tests read the **Go golden files** in `../testdata`, so the suite doubles + as a cross-implementation check. + +Verified on the live page, not just on fixtures: + +``` +$ python3 -m swehockey.cli schedule --season 20961 --out py.json +$ ../swehockey schedule -season 20961 -out go.json +$ cmp py.json go.json && echo identical +identical # 364 games, 102536 bytes + # and the same for CSV: 365 identical lines + # and for the whole archive: 27 seasons, 2119 games, 1779 played +``` + +## No runtime dependencies + +The tree is built by a ~120-line builder on `html.parser`, because the only part +of the HTML5 tree construction algorithm this crawler relies on is that stray +`` elements land in an implicit `` — which `table_rows()` handles +explicitly, exactly as in Go. + +One deliberate difference from a browser: the site writes empty cells as +``, and HTML5 ignores that slash, so a real browser leaves the +cell open and swallows its siblings. This builder treats the tag as an empty +element, which is what the author meant and what keeps `child_elements()` honest. +That is also why the Go and Python trees agree on the header row (7 `th`, 6 +non-empty labels) even though the Go parser closes the tag implicitly. + +## Modules + +| module | what it does | +| ----------------- | --------------------------------------------------------- | +| `dom.py` | the HTML tree builder and the node helpers | +| `parse.py` | layout profiles, the game reader, the invariants | +| `seasons.py` | league and season discovery | +| `fetch.py` | one GET per page, charset guard, skip-unchanged state file | +| `output.py` | JSON, NDJSON, CSV, golden-file checking | +| `cli.py` | `schedule`, `crawl`, `leagues` and the exit codes | + +Exit codes match the Go CLI: `0` ok, `1` error, `2` usage, `3` markup drift or +golden mismatch, `4` timeout. diff --git a/python/pyproject.toml b/python/pyproject.toml new file mode 100644 index 0000000..e240214 --- /dev/null +++ b/python/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "swehockey-schedule" +version = "1.0.0" +description = "Deterministic crawler for the stats.swehockey.se schedule table" +readme = "README.md" +requires-python = ">=3.10" +license = { text = "MIT" } +authors = [{ name = "Mats Tornberg" }] +keywords = ["hockey", "swehockey", "shl", "crawler", "scraper", "deterministic"] +classifiers = [ + "Programming Language :: Python :: 3", + "Topic :: Text Processing :: Markup :: HTML", +] + +# No runtime dependencies on purpose: the HTML tree is built on html.parser +# from the standard library. +dependencies = [] + +[project.scripts] +swehockey-schedule = "swehockey.cli:main" + +[project.urls] +Homepage = "https://git.k6n.net/mats/swehockey" + +[tool.setuptools] +packages = ["swehockey"] diff --git a/python/swehockey/__init__.py b/python/swehockey/__init__.py new file mode 100644 index 0000000..16a33d5 --- /dev/null +++ b/python/swehockey/__init__.py @@ -0,0 +1,65 @@ +"""A deterministic crawler for the stats.swehockey.se schedule table. + +The Python twin of the Go implementation in the parent directory: same layout +profiles, same invariants, same canonical order, same JSON bytes. See the +repository README for the design; this docstring is the whole contract: + +* one HTTP GET per page, no browser and no JavaScript, because the table is + server-rendered; +* the table is located by its header signature, not by a CSS path; +* every field is read from a declared layout profile and shape-checked; +* facts the site renders twice must agree; +* the output is a pure, sorted function of the page - no fetch timestamp, no + unordered iteration - so identical HTML always yields identical bytes; +* markup drift is a :class:`DriftError`, not a stack trace. +""" + +from .dom import Node, normalize, parse_html, table_rows, text_excluding, text_of +from .fetch import Client, FetchError, FetchTimeout, State +from .output import emit, render, render_csv, render_json, render_ndjson +from .parse import ( + LAYOUTS, + DriftError, + Game, + Layout, + Period, + Schedule, + find_table, + parse_periods, + parse_schedule, +) +from .seasons import League, Season, parse_leagues, parse_seasons, schedule_path + +__version__ = "1.0.0" + +__all__ = [ + "Client", + "DriftError", + "FetchError", + "FetchTimeout", + "Game", + "LAYOUTS", + "Layout", + "League", + "Node", + "Period", + "Schedule", + "Season", + "State", + "emit", + "find_table", + "normalize", + "parse_html", + "parse_leagues", + "parse_periods", + "parse_schedule", + "parse_seasons", + "render", + "render_csv", + "render_json", + "render_ndjson", + "schedule_path", + "table_rows", + "text_excluding", + "text_of", +] diff --git a/python/swehockey/cli.py b/python/swehockey/cli.py new file mode 100644 index 0000000..8e8e84e --- /dev/null +++ b/python/swehockey/cli.py @@ -0,0 +1,227 @@ +"""Command line interface: ``schedule``, ``crawl`` and ``leagues``. + +Same commands, flags, output and exit codes as the Go implementation. +""" + +from __future__ import annotations + +import argparse +import json +import sys +import time + +from .fetch import Client, FetchError, FetchTimeout, State, sha256_hex +from .output import emit, render, render_csv, render_ndjson +from .parse import DriftError, Schedule, parse_schedule +from .seasons import parse_leagues, parse_seasons, schedule_path + +DEFAULT_SEASON = 20961 +USER_AGENT_NOTE = "swehockey-schedule-py" + +EXIT_OK = 0 +EXIT_ERROR = 1 +EXIT_USAGE = 2 +EXIT_DRIFT = 3 +EXIT_TIMEOUT = 4 + +EPILOG = """\ +exit codes: 0 ok, 1 error, 2 usage, 3 markup drift or golden mismatch, 4 timeout +""" + + +class Unchanged(Exception): + """The page was byte-identical to the last run.""" + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="swehockey-schedule", + description="deterministic stats.swehockey.se schedule crawler", + epilog=EPILOG, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + sub = parser.add_subparsers(dest="command", required=True) + + def common(p: argparse.ArgumentParser, default_format: str) -> None: + p.add_argument("--base", default="https://stats.swehockey.se", help="site root") + p.add_argument("--format", default=default_format, choices=("json", "csv", "ndjson"), help="output format") + p.add_argument("--out", default="", help="write to this file instead of stdout") + p.add_argument("--fixture", default="", help="parse a saved HTML page instead of fetching") + p.add_argument("--state", default="", help="state file for skip-when-unchanged runs") + p.add_argument("--check", default="", help="golden file; exit 3 when the output differs") + p.add_argument("--timeout", type=float, default=30.0, help="per-request timeout in seconds") + p.add_argument("--quiet", action="store_true", help="suppress progress notes on stderr") + + schedule = sub.add_parser("schedule", help="extract one season's schedule table") + common(schedule, "json") + schedule.add_argument("--season", type=int, default=DEFAULT_SEASON, help="season id") + + crawl = sub.add_parser("crawl", help="extract every season of one league (NDJSON)") + common(crawl, "ndjson") + crawl.add_argument("--league", type=int, default=DEFAULT_SEASON, help="league id (same id space as season ids)") + crawl.add_argument("--season", type=int, default=0, help="crawl only this season id (0 = all)") + crawl.add_argument("--limit", type=int, default=0, help="stop after N seasons (0 = all)") + crawl.add_argument("--delay", type=float, default=0.25, help="pause between season requests, in seconds") + crawl.add_argument("--on-drift", default="fail", choices=("fail", "record"), help="what to do with a broken season page") + + leagues = sub.add_parser("leagues", help="list league ids and names") + common(leagues, "json") + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + if args.command == "schedule": + _cmd_schedule(args) + elif args.command == "crawl": + _cmd_crawl(args) + elif args.command == "leagues": + _cmd_leagues(args) + except Unchanged: + return EXIT_OK + except FetchTimeout as exc: + print(f"swehockey: {exc}", file=sys.stderr) + return EXIT_TIMEOUT + except DriftError as exc: + # Drift is not a crash: it means the site changed shape. Exit 3 so a + # cron job or CI gate can alert without a stack trace. The message + # already carries the "swehockey:" prefix. + print(exc, file=sys.stderr) + return EXIT_DRIFT + except (FetchError, ValueError, OSError) as exc: + print(f"swehockey: {exc}", file=sys.stderr) + return EXIT_ERROR + return EXIT_OK + + +def _read_page(args, client: Client, path: str, what: str) -> str: + """The page bytes for a path, honouring --fixture and --state.""" + if args.fixture: + with open(args.fixture, encoding="utf-8") as fh: + return fh.read() + page = client.get(path) + if args.state: + state = State.load(args.state) + if state.sha256 == sha256_hex(page.encode("utf-8")) and state.path == path: + # Byte-identical page: the derived output would be identical too. + if not args.quiet: + print(f"swehockey-schedule: {what} unchanged since {state.saved_at}, nothing to do", file=sys.stderr) + raise Unchanged() + return page + + +def _cmd_schedule(args) -> None: + path = schedule_path(args.season) + page = _read_page(args, Client(args.base, args.timeout), path, f"season {args.season}") + schedule = parse_schedule(args.season, page) + if args.format == "ndjson": + data = render_ndjson(schedule) + else: + data = render(args.format, schedule) + emit(args.out, args.check, data) + if args.state: + State( + path=path, + sha256=sha256_hex(page.encode("utf-8")), + last_update=schedule.last_update, + saved_at=_now(), + ).save(args.state) + if not args.quiet: + print( + f"season {schedule.season_id}: {len(schedule.games)} games, " + f"{schedule.played} played, last update {schedule.last_update}", + file=sys.stderr, + ) + + +def _cmd_leagues(args) -> None: + page = _read_page(args, Client(args.base, args.timeout), "/", "front page") + leagues = parse_leagues(page) + if args.format == "csv": + lines = ["id,name,path"] + lines += [f"{l.id},{l.name},{l.path}" for l in leagues] + data = ("\n".join(lines) + "\n").encode("utf-8") + else: + data = (json.dumps([l.as_dict() for l in leagues], indent=2, ensure_ascii=False) + "\n").encode("utf-8") + emit(args.out, args.check, data) + + +def _cmd_crawl(args) -> None: + """The crawler proper. + + Take one league, read its season selector, then walk every season in + ascending id order and emit one document per season. The order is derived + from the ids, never from map iteration or from the order the site happened + to list things, so two runs produce the same file even if the site's markup + order changes. + """ + client = Client(args.base, args.timeout) + seed = _read_page(args, client, schedule_path(args.league), f"league {args.league}") + seasons = parse_seasons(args.league, seed) + + out = bytearray() + first = True + if args.format == "json": + out += b"[\n" + done = 0 + for season in seasons: + if args.season and season.id != args.season: + continue + if args.limit and done >= args.limit: + break + if done and args.delay: + time.sleep(args.delay) + page = client.get(schedule_path(season.id)) + try: + schedule = parse_schedule(season.id, page) + except DriftError as exc: + # A season whose page breaks the contract is either fatal or + # recorded, never dropped: a hole in the data has to be visible. + if args.on_drift == "fail": + raise + if not args.quiet: + print(f"season {season.id}: {exc}", file=sys.stderr) + if args.format == "csv": + out += f"{season.id},ERROR,,,,,,,,,{exc}\n".encode("utf-8") + else: + if args.format == "json": + out += b"" if first else b",\n" + out += b" " + record = {"season_id": season.id, "label": season.label, "error": str(exc).split(": ", 2)[-1]} + out += (json.dumps(record, ensure_ascii=False) + "\n").encode("utf-8") + first = False + done += 1 + continue + if args.format == "csv": + out += render_csv(schedule) + else: + if args.format == "json": + out += b"" if first else b",\n" + out += b" " + # One Encode in the Go version writes the document plus exactly one + # newline: the NDJSON contract, and one element of the JSON array. + out += render_ndjson(schedule) + first = False + done += 1 + if not args.quiet: + print( + f"season {schedule.season_id} ({season.label}): {len(schedule.games)} games, {schedule.played} played", + file=sys.stderr, + ) + if args.format == "json": + out += b"\n]\n" + if not done: + raise DriftError(args.league, "crawl matched no seasons", got="0 seasons", want=">= 1") + data = bytes(out) + emit(args.out, args.check, data) + if args.state: + State(path=f"crawl:{args.league}", sha256=sha256_hex(data), saved_at=_now()).save(args.state) + + +def _now() -> str: + return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/python/swehockey/dom.py b/python/swehockey/dom.py new file mode 100644 index 0000000..cf48472 --- /dev/null +++ b/python/swehockey/dom.py @@ -0,0 +1,171 @@ +"""A tiny HTML tree builder on top of the standard library. + +The Go implementation uses golang.org/x/net/html, which implements the HTML5 +tree construction algorithm. The only part of that algorithm this crawler +depends on is that stray ```` elements end up inside an implicit ````, +which :func:`table_rows` handles explicitly. So a small builder is enough, and +it keeps the Python variant dependency-free. + +One deliberate difference from a browser: the site writes empty cells as +````. HTML5 ignores that slash, so a browser leaves the cell +open and its siblings end up nested inside it. This builder treats such a tag +as an empty element, which is what the author meant, and it keeps +``child_elements()`` honest. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from html.parser import HTMLParser + +VOID = { + "area", "base", "br", "col", "embed", "hr", "img", "input", + "link", "meta", "param", "source", "track", "wbr", +} + +# The site writes the team separator as NBSP-hyphen-NBSP wrapped in newlines, so +# every string that leaves the parser goes through normalize(). Matching the +# negated class (rather than splitting on whitespace) is what drops leading and +# trailing space, like strings.FieldsFunc does in the Go version. +_TOKEN = re.compile(r"[^\s\u00a0\u2007\u202f\u200b\ufeff]+") + + +def normalize(text: str) -> str: + """Collapse every flavour of whitespace into single spaces.""" + return " ".join(_TOKEN.findall(text)) + + +@dataclass +class Node: + tag: str | None + attrs: dict[str, str] = field(default_factory=dict) + data: str = "" + children: list["Node"] = field(default_factory=list) + parent: "Node | None" = field(default=None, repr=False) + + def walk(self): + yield self + for child in self.children: + yield from child.walk() + + +class _Builder(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.root = Node("#document") + self.stack = [self.root] + + def _append(self, node: Node) -> None: + node.parent = self.stack[-1] + self.stack[-1].children.append(node) + + def handle_starttag(self, tag, attrs): + node = Node(tag, {k: (v or "") for k, v in attrs}) + self._append(node) + if tag not in VOID: + self.stack.append(node) + + def handle_startendtag(self, tag, attrs): + # An empty cell: + self._append(Node(tag, {k: (v or "") for k, v in attrs})) + + def handle_endtag(self, tag): + for i in range(len(self.stack) - 1, 0, -1): + if self.stack[i].tag == tag: + del self.stack[i:] + return + # A stray end tag: ignore it, like a browser does. + + def handle_data(self, data): + self._append(Node(None, data=data)) + + +def parse_html(page: str) -> Node: + """Parse a document and return its root node.""" + builder = _Builder() + builder.feed(page) + builder.close() + return builder.root + + +def text_of(node: Node | None) -> str: + """The normalized text of a subtree.""" + if node is None: + return "" + return normalize("".join(n.data for n in node.walk() if n.tag is None)) + + +def text_excluding(node: Node | None, skip_tag: str) -> str: + """The normalized text of a subtree with the first skip_tag subtree removed. + + The schedule uses this to read the mobile cell's start time without its + ```` date, and the team cell without the ```` note old seasons + append to it. + """ + if node is None: + return "" + out: list[str] = [] + + def walk(n: Node) -> None: + if n.tag == skip_tag: + return + if n.tag is None: + out.append(n.data) + for child in n.children: + walk(child) + + walk(node) + return normalize("".join(out)) + + +def child_elements(node: Node | None, tag: str) -> list[Node]: + if node is None: + return [] + return [c for c in node.children if c.tag == tag] + + +def descendants(node: Node, tag: str) -> list[Node]: + return [n for n in node.walk() if n.tag == tag] + + +def first_element(node: Node | None, tag: str) -> Node | None: + if node is None: + return None + for n in node.walk(): + if n.tag == tag: + return n + return None + + +def attr(node: Node | None, name: str) -> str: + return "" if node is None else node.attrs.get(name, "") + + +def has_class(node: Node, class_name: str) -> bool: + return class_name in attr(node, "class").split() + + +def table_rows(table: Node) -> list[Node]: + """The ```` children of a table, looking inside ```` as well.""" + rows = child_elements(table, "tr") + if rows: + return rows + out: list[Node] = [] + for body in child_elements(table, "tbody"): + out.extend(child_elements(body, "tr")) + return out + + +def th_labels(node: Node) -> list[str]: + """The non-empty ```` labels of a row, or of a table's header row.""" + trs = [node] if node.tag == "tr" else table_rows(node) + for tr in trs: + labels = table_header_labels(tr) + if labels: + return labels + return [] + + +def table_header_labels(row: Node) -> list[str]: + return [t for t in (text_of(th) for th in child_elements(row, "th")) if t] diff --git a/python/swehockey/fetch.py b/python/swehockey/fetch.py new file mode 100644 index 0000000..bee24a4 --- /dev/null +++ b/python/swehockey/fetch.py @@ -0,0 +1,123 @@ +"""Fetching and the skip-unchanged state file. + +One request, no retries, no cookies, no redirect games: every byte of the output +is a function of one response. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import urllib.error +import urllib.request +from dataclasses import asdict, dataclass + +from .seasons import join_url + +# Identify the crawler. The site is a public ASP.NET app and serves the same +# bytes to any client; saying who we are keeps its logs readable. +USER_AGENT = "swehockey-schedule-py/1.0 (+deterministic schedule crawler)" + +META_CHARSET_RE = re.compile(rb"""]+charset\s*=\s*["']?([a-z0-9_-]+)""", re.IGNORECASE) + + +class FetchTimeout(Exception): + """The site did not answer in time.""" + + +class FetchError(Exception): + """Anything else that went wrong on the way to the page.""" + + +def sha256_hex(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +class Client: + def __init__(self, base_url: str = "https://stats.swehockey.se", timeout: float = 30.0) -> None: + self.base_url = base_url + self.timeout = timeout + + def get(self, path: str) -> str: + """GET one page and return it decoded as text. + + Refuses pages it cannot decode byte-exactly: handing latin-1 bytes to a + UTF-8 parser would turn "Växjö" into mojibake that still parses, which + is the worst kind of crawler bug. + """ + url = join_url(self.base_url, path) + request = urllib.request.Request( + url, + headers={ + "User-Agent": USER_AGENT, + "Accept": "text/html,application/xhtml+xml", + "Accept-Language": "en", + }, + ) + try: + with urllib.request.urlopen(request, timeout=self.timeout) as response: + if response.status != 200: + raise FetchError(f"fetch {url}: HTTP {response.status}") + body = response.read() + except urllib.error.HTTPError as exc: + raise FetchError(f"fetch {url}: {exc.code} {exc.reason}") from None + except TimeoutError as exc: + raise FetchTimeout(f"fetch {url}: timed out after {self.timeout}s") from None + except urllib.error.URLError as exc: + if isinstance(exc.reason, TimeoutError): + raise FetchTimeout(f"fetch {url}: timed out after {self.timeout}s") from None + raise FetchError(f"fetch {url}: {exc.reason}") from None + except OSError as exc: + raise FetchError(f"fetch {url}: {exc}") from None + + check_charset(body) + try: + return body.decode("utf-8") + except UnicodeDecodeError as exc: + raise FetchError(f"fetch {url}: not valid UTF-8: {exc}") from None + + +def check_charset(body: bytes) -> None: + """Refuse a page we cannot decode byte-exactly.""" + m = META_CHARSET_RE.search(body) + if m is None: + return # no declaration: UTF-8 is assumed + charset = m.group(1).decode("ascii").lower() + if charset not in ("utf-8", "utf8"): + raise FetchError(f'unsupported charset "{charset}": add a decoder for it') + + +@dataclass +class State: + """What the last successful run saw, so a cron job can be idempotent. + + An unchanged page produces no output and a zero exit code. + """ + + path: str = "" + sha256: str = "" + last_update: str = "" + saved_at: str = "" + + def load(path: str) -> "State": + try: + with open(path, encoding="utf-8") as fh: + return State(**json.load(fh)) + except FileNotFoundError: + return State() + except (OSError, TypeError, json.JSONDecodeError) as exc: + raise FetchError(f"state file {path}: {exc}") from None + + def save(self, path: str) -> None: + """Write the state atomically, so a killed run cannot leave a truncated + file that would make the next run skip real work.""" + directory = os.path.dirname(path) + if directory: + os.makedirs(directory, exist_ok=True) + tmp = path + ".tmp" + with open(tmp, "w", encoding="utf-8") as fh: + json.dump(asdict(self), fh, indent=2) + fh.write("\n") + os.replace(tmp, path) diff --git a/python/swehockey/output.py b/python/swehockey/output.py new file mode 100644 index 0000000..e4fd6a6 --- /dev/null +++ b/python/swehockey/output.py @@ -0,0 +1,123 @@ +"""Output: JSON, NDJSON and CSV. + +Both encoders are pure functions of the data: no timestamps, no unordered +iteration, fixed field order, LF line endings. The JSON writer is tuned to +produce exactly the same bytes as the Go implementation, which is what lets the +two share one set of golden files. +""" + +from __future__ import annotations + +import csv +import io +import json +import sys + +from .parse import DriftError, Schedule + +CSV_HEADER = [ + "season_id", + "game_id", + "event_id", + "date", + "time", + "home", + "away", + "home_goals", + "away_goals", + "periods", + "spectators", + "venue", +] + + +def render(fmt: str, schedule: Schedule) -> bytes: + if fmt == "json": + return render_json(schedule) + if fmt == "csv": + return render_csv(schedule) + raise ValueError(f'unknown format "{fmt}": want json, csv or ndjson') + + +def render_json(schedule: Schedule) -> bytes: + # ensure_ascii=False keeps "Växjö" readable, which matters for diffs of + # team names; indent=2 and the default separators match encoding/json. + text = json.dumps(schedule.as_dict(), indent=2, ensure_ascii=False) + return (text + "\n").encode("utf-8") + + +def render_ndjson(schedule: Schedule) -> bytes: + text = json.dumps(schedule.as_dict(), ensure_ascii=False) + return (text + "\n").encode("utf-8") + + +def _opt(value: int | None) -> str: + return "" if value is None else str(value) + + +def render_csv(schedule: Schedule) -> bytes: + buf = io.StringIO() + writer = csv.writer(buf, lineterminator="\n") # LF, not CRLF + writer.writerow(CSV_HEADER) + for game in schedule.games: + periods = " ".join(f"{p.label}:{p.home}-{p.away}" for p in game.periods) + writer.writerow( + [ + schedule.season_id, + game.game_id, + game.event_id, + game.date, + game.time, + game.home, + game.away, + _opt(game.home_goals), + _opt(game.away_goals), + periods, + _opt(game.spectators), + game.venue, + ] + ) + return buf.getvalue().encode("utf-8") + + +def emit(out: str, check: str, data: bytes) -> None: + """Write the bytes to -out or stdout and, when -check is set, compare them + with a committed golden file. + + A mismatch raises, so CI turns an upstream change into a reviewable diff + instead of a silent data change. + """ + if check: + try: + with open(check, "rb") as fh: + golden = fh.read() + except FileNotFoundError as exc: + raise DriftError(0, f"golden file {check}: {exc}", hint="") from None + if golden != data: + raise DriftError( + 0, + f"output no longer matches {check}: {first_diff(golden, data)}", + hint=" - regenerate the golden file only after reviewing the change", + ) + if out: + with open(out, "wb") as fh: + fh.write(data) + else: + sys.stdout.buffer.write(data) + sys.stdout.buffer.flush() + + +def first_diff(a: bytes, b: bytes) -> str: + """Describe the first differing line, for a readable failure.""" + left = a.decode("utf-8", "replace").split("\n") + right = b.decode("utf-8", "replace").split("\n") + for i in range(max(len(left), len(right))): + x = left[i] if i < len(left) else "" + y = right[i] if i < len(right) else "" + if x != y: + return f'line {i + 1}: "{_trim(x)}" != "{_trim(y)}"' + return "documents differ only in length" + + +def _trim(line: str) -> str: + return line if len(line) <= 80 else line[:80] + "..." diff --git a/python/swehockey/parse.py b/python/swehockey/parse.py new file mode 100644 index 0000000..d48abfc --- /dev/null +++ b/python/swehockey/parse.py @@ -0,0 +1,520 @@ +"""Extraction of the "Schedule and Results" table from stats.swehockey.se. + +This is a line-by-line counterpart of ``../parse.go``: same layout profiles, +same invariants, same canonical order, same JSON. The Python and the Go +implementation are checked against the same golden files, so they have to agree +byte for byte. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field + +from .dom import ( + Node, + attr, + child_elements, + descendants, + first_element, + has_class, + parse_html, + table_rows, + text_excluding, + text_of, + th_labels, +) + +# The site has shipped more than one version of this table, and the old +# versions are still served for historical seasons. So the contract is not a +# set of hard-coded column indices: it is a set of layout profiles, each +# identified by the header row it declares. A page the crawler does not +# recognise fails loudly with the header it actually found, and adding support +# is one profile plus one fixture. +MODERN = "modern" # seasons from 2018-19 onwards +LEGACY = "legacy" # seasons up to 2017-18 + + +@dataclass(frozen=True) +class Layout: + name: str + header: tuple[str, ...] + cells: int + round: int # -1 when the layout has no Round column + # Modern layout: the date is rendered twice, once in the desktop "Date" + # column and once as a group heading in the mobile column, and the + # start time lives in its own cell next to the game's stable id. + # Legacy layout: no group headings; the tooltip span of one cell holds + # both "2014-03-10" and "19:00". + date_desktop: int + date_mobile: int + time_cell: int + grouped: bool + teams: int + result: int + periods: int + spectators: int + venue: int + + +LAYOUTS: tuple[Layout, ...] = ( + Layout( + name=MODERN, + header=("Date", "Time", "Game", "Result", "Spectators", "Venue"), + cells=8, + round=-1, + date_desktop=0, + date_mobile=1, + time_cell=2, + grouped=True, + teams=3, + result=4, + periods=5, + spectators=6, + venue=7, + ), + Layout( + name=LEGACY, + header=("Round", "Date", "Game", "Result", "Spectators", "Venue"), + cells=7, + round=0, + date_desktop=-1, + date_mobile=-1, + time_cell=1, + grouped=False, + teams=2, + result=3, + periods=4, + spectators=5, + venue=6, + ), +) + +DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") +TIME_RE = re.compile(r"^\d{1,2}:\d{2}$") +DIGITS_RE = re.compile(r"^\d+$") +SCORE_RE = re.compile(r"^(\d+)\s*-\s*(\d+)$") +PERIOD_RE = re.compile(r"^(\d+)\s*-\s*(\d+)$") +EVENT_ID_RE = re.compile(r"/Game/Events/(\d+)") +LAST_UPDATE_RE = re.compile(r"Last update:\s*(\d{4}-\d{2}-\d{2} \d{1,2}:\d{2})") +GAME_ID_RE = re.compile(r"^\d{5,}$") + + +@dataclass +class Period: + """One period (or overtime / shootout) of a played game.""" + + label: str + home: int + away: int + + def as_dict(self) -> dict: + return {"label": self.label, "home": self.home, "away": self.away} + + +@dataclass +class Game: + """One row of the schedule table.""" + + game_id: str # span.lnkTooltip@title, unique per season + event_id: str = "" # /Game/Events/; set once the game is played + round: str = "" # only in the legacy layout + date: str = "" # YYYY-MM-DD + time: str = "" # HH:MM local arena time + home: str = "" + away: str = "" + home_goals: int | None = None # None until the game is played + away_goals: int | None = None + periods: list[Period] = field(default_factory=list) + spectators: int | None = None # None when not reported + venue: str = "" + + @property + def played(self) -> bool: + return bool(self.event_id) + + def as_dict(self) -> dict: + out = {"game_id": self.game_id} + if self.event_id: + out["event_id"] = self.event_id + if self.round: + out["round"] = self.round + out["date"] = self.date + out["time"] = self.time + out["home"] = self.home + out["away"] = self.away + out["home_goals"] = self.home_goals + out["away_goals"] = self.away_goals + out["periods"] = [p.as_dict() for p in self.periods] + if self.spectators is not None: + out["spectators"] = self.spectators + out["venue"] = self.venue + return out + + +@dataclass +class Schedule: + season_id: int + layout: str + title: str + last_update: str # the site's own clock, verbatim + games: list[Game] = field(default_factory=list) + + @property + def played(self) -> int: + return sum(1 for g in self.games if g.played) + + def as_dict(self) -> dict: + return { + "season_id": self.season_id, + "layout": self.layout, + "title": self.title, + "last_update": self.last_update, + "games": [g.as_dict() for g in self.games], + } + + +class DriftError(Exception): + """The page no longer matches a known contract. + + Its own type, and its own exit code, because it is the one failure a caller + must not retry blindly: the site changed, not the network. + """ + + #: Every layout problem ends with the same advice, so an operator reading a + #: cron mail knows exactly what to do. Failures that are not about the table + #: shape pass hint="" and get no advice. + HINT = " - extend LAYOUTS in parse.py and add a fixture, in one commit" + + def __init__( + self, + season: int, + message: str = "", + game_id: str = "", + column: str = "", + got: str = "", + want: str = "", + hint: str | None = None, + ) -> None: + self.season = season + self.game_id = game_id + self.column = column + self.got = got + self.want = want + parts = [f" (season {season}"] + if game_id: + parts.append(f", game {game_id}") + parts.append("): ") + parts.append(message or "unexpected table shape") + if column: + parts.append(f" [column {column}]") + if got: + parts.append(f': got "{got}", want "{want}"') + parts.append(self.HINT if hint is None else hint) + super().__init__("swehockey: markup drift" + "".join(parts)) + + +def _drift(season, message, game_id="", column="", got="", want=""): + return DriftError(season, message, game_id, column, got, want) + + +class _Carry: + """Values the site prints only on the first row of a group. + + The date heading of the modern layout, the round number of the legacy one. + """ + + def __init__(self) -> None: + self.date = "" + self.round = "" + + +def parse_schedule(season_id: int, page: str) -> Schedule: + """Extract the schedule table from one saved or fetched page.""" + doc = parse_html(page) + table = find_table(doc) + if table is None: + raise _drift( + season_id, + "no table with a known header: found " + + " / ".join(document_signatures(doc)) + + ", known " + + known_layouts(), + ) + node, lay = table + + schedule = Schedule(season_id, lay.name, "", "", []) + rows = table_rows(node) + header_at = -1 + carry = _Carry() + for i, tr in enumerate(rows): + if not child_elements(tr, "td"): + if tuple(th_labels(tr)) == lay.header: + if header_at >= 0: + raise _drift(season_id, f"second header row at row {i}", got="2", want="1") + header_at = i + continue + if header_at < 0: + raise _drift(season_id, "data row before the header row", want="header first") + schedule.games.append(_parse_game(season_id, lay, tr, carry)) + if header_at < 0: + raise _drift(season_id, "table has no header row", want=" ".join(lay.header)) + + heading = descendants(node, "h2") + if heading: + schedule.title = text_of(heading[0]) + found = LAST_UPDATE_RE.search(text_of(node)) + if found: + schedule.last_update = found.group(1) + if not schedule.last_update: + raise _drift(season_id, 'table has no "Last update:" stamp', want="Last update: YYYY-MM-DD HH:MM") + + _check(schedule) + # The site lists games in round order, not date order, and a re-scheduled + # or awarded game keeps the round it belongs to: 2011-12 has September + # games listed after October ones. Sorting makes the output a canonical + # function of the set of rows instead of a function of the site's row + # order, which is what makes two crawls comparable and diffs meaningful. + schedule.games.sort(key=lambda g: (g.date, g.time, g.game_id)) + return schedule + + +def _check(schedule: Schedule) -> None: + """The invariants that hold for every season the site serves.""" + if not schedule.games: + raise _drift(schedule.season_id, "table has no game rows", got="0 rows", want=">= 1 row") + # Game ids are unique, and that is the only uniqueness the page promises: + # the site assigns them per fixture, so they are neither sorted nor + # contiguous within a season. + seen: set[str] = set() + for game in schedule.games: + if not game.game_id.isdigit(): + raise _drift(schedule.season_id, "game id is not a number", game.game_id, got=game.game_id, want="90001001") + if game.game_id in seen: + raise _drift(schedule.season_id, "duplicate game id", game.game_id, got=game.game_id, want="unique") + seen.add(game.game_id) + if not DATE_RE.match(game.date): + raise _drift(schedule.season_id, "malformed date", game.game_id, "date", game.date, "2026-09-19") + + +def _parse_game(season_id: int, lay: Layout, tr: Node, carry: _Carry) -> Game: + """Read one row. Where the site renders a fact twice, both must agree.""" + tds = child_elements(tr, "td") + if len(tds) != lay.cells: + raise _drift( + season_id, + f"row width does not match the {lay.name} layout", + got=f"{len(tds)} cells", + want=f"{lay.cells} cells", + ) + game = Game(game_id="") + + # The tooltip span carries the row's stable id, so the id and the time can + # never disagree: they are the same element. + span = _tooltip_span(tds[lay.time_cell]) + if span is None: + raise _drift(season_id, "no game id (span.lnkTooltip[title])", column="time", got=text_of(tds[lay.time_cell]), want="90001001") + game.game_id = attr(span, "title").strip() + if not GAME_ID_RE.match(game.game_id): + raise _drift(season_id, "malformed game id", column="time", got=game.game_id, want="90001001") + + head = text_of(span) + if lay.grouped: + # Modern layout: date in two renderings, time in the span. + if not TIME_RE.match(head): + raise _drift(season_id, "malformed start time", game.game_id, "time", head, "19:00") + game.time = head + desktop = text_of(tds[lay.date_desktop]) + mobile = text_of(first_element(tds[lay.date_mobile], "bold")) + if mobile: + if not DATE_RE.match(mobile): + raise _drift(season_id, "malformed date heading", game.game_id, "date", mobile, "2026-09-19") + if desktop != mobile: + raise _drift(season_id, "date renderings disagree", game.game_id, "date", desktop, mobile) + if text_excluding(tds[lay.date_mobile], "bold") != game.time: + raise _drift( + season_id, + "time renderings disagree", + game.game_id, + "date", + text_excluding(tds[lay.date_mobile], "bold"), + game.time, + ) + game.date = mobile + else: + if not carry.date: + raise _drift(season_id, "row before any date heading", game.game_id, "date", desktop, "2026-09-19") + if desktop and desktop != game.time: + raise _drift( + season_id, + "unexpected date in a continuation row", + game.game_id, + "date", + desktop, + "(empty) or " + game.time, + ) + game.date = carry.date + else: + # Legacy layout: "2014-03-10 19:00" in one cell, no group headings. + pieces = head.split(" ", 1) + if len(pieces) == 2 and DATE_RE.match(pieces[0]) and TIME_RE.match(pieces[1]): + game.date, game.time = pieces + elif TIME_RE.match(head): + if not carry.date: + raise _drift(season_id, "row before any date", game.game_id, "date", head, "2026-09-19 19:00") + game.date, game.time = carry.date, head + else: + raise _drift(season_id, "cannot read date and time", game.game_id, "date", head, "2026-09-19 19:00") + + if lay.round >= 0: + # The site prints the round only on the first row of that round and + # leaves the following cells empty, exactly like the date headings. + round_text = text_of(tds[lay.round]) + if DIGITS_RE.match(round_text): + game.round = round_text + elif round_text == "" and carry.round: + game.round = carry.round + elif round_text == "": + raise _drift(season_id, "row before any round number", game.game_id, "round", round_text, "1") + else: + raise _drift(season_id, "round is not a number", game.game_id, "round", round_text, "1") + + # Teams: the cell is "Home - Away" plus, in old seasons, an note + # such as "Eighth final 1" that is not part of a name. + teams = text_excluding(tds[lay.teams], "i") + parts = teams.split(" - ") + if len(parts) != 2 or not parts[0] or not parts[1]: + raise _drift(season_id, "cannot split teams", game.game_id, "game", teams, "Home - Away") + game.home, game.away = parts + + # Result: present only for played games, and always with an event link. + link = first_element(tds[lay.result], "a") + if link is not None: + score = SCORE_RE.match(text_of(link)) + if score is None: + raise _drift(season_id, "malformed score", game.game_id, "result", text_of(link), "3 - 2") + event = EVENT_ID_RE.search(attr(link, "href")) + if event is None: + raise _drift( + season_id, + "result link without an event id", + game.game_id, + "result", + attr(link, "href"), + "/Game/Events/1109941", + ) + game.home_goals, game.away_goals = int(score.group(1)), int(score.group(2)) + game.event_id = event.group(1) + + try: + game.periods = parse_periods(text_of(tds[lay.periods])) + except ValueError as exc: + raise _drift(season_id, str(exc), game.game_id, "periods", text_of(tds[lay.periods]), "(1-1, 2-1, 2-2)") from None + + spectators = text_of(tds[lay.spectators]) + if spectators: + if not DIGITS_RE.match(spectators): + raise _drift(season_id, "spectators are not a number", game.game_id, "spectators", spectators, "10442") + game.spectators = int(spectators) + game.venue = text_of(tds[lay.venue]) + if not game.venue: + raise _drift(season_id, "empty venue", game.game_id, "venue", want="Gottde Arena") + + carry.date, carry.round = game.date, game.round + try: + _check_row(game) + except ValueError as exc: + raise _drift(season_id, str(exc), game.game_id) from None + return game + + +def _check_row(game: Game) -> None: + """Periods sum to the final score, and a game is played or it is not.""" + if not game.played: + if game.home_goals is not None or game.periods: + raise ValueError("unplayed game carries a result") + return + if len(game.periods) < 3: + raise ValueError(f"played game has {len(game.periods)} period scores") + home = sum(p.home for p in game.periods) + away = sum(p.away for p in game.periods) + if home != game.home_goals or away != game.away_goals: + raise ValueError( + f"periods sum to {home}-{away} but the result is {game.home_goals}-{game.away_goals}" + ) + + +def parse_periods(text: str) -> list[Period]: + """Read "(1-1, 2-1, 2-2)". + + This is the single place to extend if the site ever renders a period + differently, for instance a shootout as "-". + """ + if not text: + return [] + if not (text.startswith("(") and text.endswith(")")): + raise ValueError("period scores are not parenthesised") + tokens = text[1:-1].split(",") + # Only the lower bound is a contract. How many periods a game lasts depends + # on the era's rules - the site serves 3, 4, 5 and 6 period breakdowns - so + # the count is not pinned; the sum check in _check_row is what protects the + # data. + if len(tokens) < 3: + raise ValueError(f"only {len(tokens)} period scores") + out: list[Period] = [] + for i, token in enumerate(tokens): + m = PERIOD_RE.match(token.strip()) + if m is None: + raise ValueError(f'malformed period "{token.strip()}"') + out.append(Period(str(i + 1), int(m.group(1)), int(m.group(2)))) + return out + + +def find_table(doc: Node): + """The first table that declares one of the known layouts. + + Locating the table by its header, rather than by a CSS path or an index, is + what keeps ads, navigation and layout changes from breaking the crawler. + """ + for node in doc.walk(): + if node.tag != "table": + continue + for tr in table_rows(node): + labels = tuple(th_labels(tr)) + for lay in LAYOUTS: + if labels == lay.header: + return node, lay + return None + + +def document_signatures(doc: Node) -> list[str]: + """Describe every table on the page, so an unrecognised layout is + diagnosable from the error message alone.""" + out: list[str] = [] + for node in doc.walk(): + if node.tag != "table": + continue + for tr in table_rows(node): + labels = [t for t in (text_of(th) for th in child_elements(tr, "th")) if t] + if not labels: + continue + cells = 0 + for row in table_rows(node): + cells = len(child_elements(row, "td")) + if cells: + break + out.append(f'"{" ".join(labels)}" with {cells} cells') + break + return out or ["no table with a header row"] + + +def known_layouts() -> str: + return ", ".join(f'{lay.name}="{" ".join(lay.header)}"' for lay in LAYOUTS) + + +def _tooltip_span(cell: Node) -> Node | None: + """The span that carries a game's stable id.""" + for span in descendants(cell, "span"): + if has_class(span, "lnkTooltip") and attr(span, "title"): + return span + return None diff --git a/python/swehockey/seasons.py b/python/swehockey/seasons.py new file mode 100644 index 0000000..12050a0 --- /dev/null +++ b/python/swehockey/seasons.py @@ -0,0 +1,104 @@ +"""League and season discovery: the two hops of the crawl. + +The front page links every competition; each competition's page carries a +``