Add the Python variant, agreeing with Go byte for byte
A second, independent implementation of the same contract, in the same repo: - swehockey/dom.py builds the tree on html.parser, so the variant has no runtime dependencies; the only HTML5 behaviour it depends on - stray <tr> landing in an implicit <tbody> - is handled explicitly by table_rows(). - parse.py mirrors the Go layout profiles, dual-render checks, carry-forward and invariants; a DriftError is raised in the same places with the same advice. - cli.py offers the same three commands, flags and exit codes as the Go main. - tests/ assert against the Go golden files, so the suite is also a cross-implementation check. 23 offline tests plus an opt-in live test. Verified against the live site: JSON and CSV are byte-identical to the Go output (364 games, 102536 bytes / 365 lines), and the full archive crawl gives the same 27 seasons, 2119 games and 1779 played. Also split the DriftError hint out in Go, so a golden mismatch no longer tells the reader to extend layouts[].
This commit is contained in:
Mats Tornberg committed
1 parent
6b701819ca
commit
47e1811d9e
15 files changed
+1821
-3
No files matched your search
@@ -1,3 +1,7 @@
|
|||||||
/swehockey
|
/swehockey
|
||||||
*.tmp
|
*.tmp
|
||||||
/.codesearch/
|
/.codesearch/
|
||||||
|
__pycache__/
|
||||||
|
*.pyc
|
||||||
|
.venv/
|
||||||
|
*.egg-info/
|
||||||
@@ -90,6 +90,50 @@ $ ./swehockey crawl -league 20961 -on-drift record | jq -s '[.[] | .games | leng
|
|||||||
2119
|
2119
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Two implementations, one contract
|
||||||
|
|
||||||
|
The crawler exists twice, in Go and in Python, and that is a feature: two
|
||||||
|
independent parsers of the same page that have to agree.
|
||||||
|
|
||||||
|
| | Go | Python |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| entry point | `./swehockey schedule -season 20961` | `python3 -m swehockey.cli schedule --season 20961` |
|
||||||
|
| tests | `go test ./...` (offline) | `python3 -m unittest discover -s tests -t .` (offline) |
|
||||||
|
| live test | `SWEHOCKEY_LIVE=1 go test -run TestLive` | `SWEHOCKEY_LIVE=1 python3 -m unittest tests.test_swehockey.TestLive` |
|
||||||
|
| dependencies | `golang.org/x/net/html` | none — the tree is built on `html.parser` |
|
||||||
|
|
||||||
|
Both share `testdata/`: the Python suite asserts its output against the **Go
|
||||||
|
golden files**, so a change in one implementation that the other does not share
|
||||||
|
fails a test. Verified on the live page as well:
|
||||||
|
|
||||||
|
```
|
||||||
|
$ ./swehockey schedule -season 20961 -out go.json
|
||||||
|
$ python3 -m swehockey.cli schedule --season 20961 --out py.json
|
||||||
|
$ cmp go.json py.json # 364 games, 102536 bytes, identical
|
||||||
|
$ ./swehockey schedule -season 20961 -format csv -out go.csv
|
||||||
|
$ python3 -m swehockey.cli schedule --season 20961 --format csv --out py.csv
|
||||||
|
$ cmp go.csv py.csv # 365 lines, identical
|
||||||
|
$ python3 -m swehockey.cli crawl --league 20961 | jq -s '[.[].games|length]|add'
|
||||||
|
2119 # same as the Go crawl
|
||||||
|
```
|
||||||
|
|
||||||
|
Same layout profiles, same invariants, same canonical order, same exit codes
|
||||||
|
(`0` ok, `1` error, `2` usage, `3` drift, `4` timeout). The only intentional
|
||||||
|
difference is in the tree builder: Python treats the site's empty `<th … />`
|
||||||
|
cells as empty elements instead of leaving them open the way a browser does,
|
||||||
|
which is what makes the two trees agree. See `python/README.md`.
|
||||||
|
|
||||||
|
## Repository layout
|
||||||
|
|
||||||
|
```
|
||||||
|
main.go parse.go seasons.go fetch.go output.go the Go crawler
|
||||||
|
parse_test.go the Go tests
|
||||||
|
python/swehockey/ the Python crawler
|
||||||
|
python/tests/ the Python tests
|
||||||
|
testdata/ HTML fixtures and golden JSON, shared by both
|
||||||
|
README.md this file
|
||||||
|
```
|
||||||
|
|
||||||
## Commands
|
## Commands
|
||||||
|
|
||||||
| command | what it does |
|
| command | what it does |
|
||||||
|
|||||||
@@ -104,7 +104,10 @@ func emit(out, check string, b []byte) error {
|
|||||||
case err != nil:
|
case err != nil:
|
||||||
return fmt.Errorf("golden file %s: %w", check, err)
|
return fmt.Errorf("golden file %s: %w", check, err)
|
||||||
default:
|
default:
|
||||||
return &DriftError{Msg: fmt.Sprintf("output no longer matches %s: %s", check, firstDiff(golden, b))}
|
// A golden mismatch is not a layout problem, so it gets its own
|
||||||
|
// advice instead of the "extend layouts[]" one.
|
||||||
|
return &DriftError{Msg: fmt.Sprintf("output no longer matches %s: %s", check, firstDiff(golden, b)),
|
||||||
|
Hint: " - regenerate the golden file only after reviewing the change"}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if out == "" {
|
if out == "" {
|
||||||
|
|||||||
@@ -123,8 +123,15 @@ type DriftError struct {
|
|||||||
Got string
|
Got string
|
||||||
Want string
|
Want string
|
||||||
Msg string
|
Msg string
|
||||||
|
// Every layout problem ends with the same advice, so an operator reading a
|
||||||
|
// cron mail knows exactly what to do. Failures that are not about the table
|
||||||
|
// shape pass an empty Hint and get no advice.
|
||||||
|
Hint string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// layoutHint is the advice attached to every markup problem.
|
||||||
|
const layoutHint = " - extend layouts[] in parse.go and add a fixture, in one commit"
|
||||||
|
|
||||||
func (e *DriftError) Error() string {
|
func (e *DriftError) Error() string {
|
||||||
var b strings.Builder
|
var b strings.Builder
|
||||||
fmt.Fprintf(&b, "swehockey: markup drift (season %d", e.Season)
|
fmt.Fprintf(&b, "swehockey: markup drift (season %d", e.Season)
|
||||||
@@ -143,12 +150,12 @@ func (e *DriftError) Error() string {
|
|||||||
if e.Got != "" {
|
if e.Got != "" {
|
||||||
fmt.Fprintf(&b, ": got %q, want %q", e.Got, e.Want)
|
fmt.Fprintf(&b, ": got %q, want %q", e.Got, e.Want)
|
||||||
}
|
}
|
||||||
b.WriteString(" - extend layouts[] in parse.go and add a fixture, in one commit")
|
b.WriteString(e.Hint)
|
||||||
return b.String()
|
return b.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
func drift(season int, gameID, column, msg, got, want string) error {
|
func drift(season int, gameID, column, msg, got, want string) error {
|
||||||
return &DriftError{Season: season, GameID: gameID, Column: column, Msg: msg, Got: got, Want: want}
|
return &DriftError{Season: season, GameID: gameID, Column: column, Msg: msg, Got: got, Want: want, Hint: layoutHint}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParseSchedule extracts the schedule table from one saved or fetched page.
|
// ParseSchedule extracts the schedule table from one saved or fetched page.
|
||||||
|
|||||||
@@ -0,0 +1,70 @@
|
|||||||
|
# swehockey-schedule (Python)
|
||||||
|
|
||||||
|
The Python twin of the Go crawler in the parent directory. Same layout profiles,
|
||||||
|
same invariants, same canonical order — and, on purpose, **the same bytes**.
|
||||||
|
|
||||||
|
```console
|
||||||
|
$ python3 -m swehockey.cli schedule --season 20961 --format csv | head -2
|
||||||
|
season_id,game_id,event_id,date,time,home,away,home_goals,away_goals,periods,spectators,venue
|
||||||
|
20961,90001001,1109941,2026-09-19,15:15,Frölunda HC,Växjö Lakers HC,5,4,1:1-1 2:2-1 3:2-2,12044,Scandinavium
|
||||||
|
|
||||||
|
$ python3 -m swehockey.cli crawl --league 20961 # every season, as NDJSON
|
||||||
|
$ python3 -m swehockey.cli leagues # 119 competitions
|
||||||
|
$ python3 -m unittest discover -s tests -t . # 23 offline tests
|
||||||
|
$ SWEHOCKEY_LIVE=1 python3 -m unittest discover -s tests -t . # + the site
|
||||||
|
```
|
||||||
|
|
||||||
|
With `pip install -e .` the same CLI is on `$PATH` as `swehockey-schedule`.
|
||||||
|
|
||||||
|
## Why it can agree byte for byte
|
||||||
|
|
||||||
|
- Same `LAYOUTS` profiles, keyed on the header row; same per-field shape
|
||||||
|
checks; same dual-render agreement for the modern layout; same carry-forward
|
||||||
|
for the date heading and the round number; same invariants (periods sum to
|
||||||
|
the score, ids unique, played ⇔ event link).
|
||||||
|
- Same canonical sort by `(date, time, game_id)`, because the site orders rows
|
||||||
|
by round and puts re-scheduled games where their round says.
|
||||||
|
- The output carries only page-derived data — no fetch timestamp, no unordered
|
||||||
|
iteration, fixed key order — and `json.dumps(..., indent=2, ensure_ascii=False)`
|
||||||
|
is byte-compatible with Go's `json.Encoder`.
|
||||||
|
- The tests read the **Go golden files** in `../testdata`, so the suite doubles
|
||||||
|
as a cross-implementation check.
|
||||||
|
|
||||||
|
Verified on the live page, not just on fixtures:
|
||||||
|
|
||||||
|
```
|
||||||
|
$ python3 -m swehockey.cli schedule --season 20961 --out py.json
|
||||||
|
$ ../swehockey schedule -season 20961 -out go.json
|
||||||
|
$ cmp py.json go.json && echo identical
|
||||||
|
identical # 364 games, 102536 bytes
|
||||||
|
# and the same for CSV: 365 identical lines
|
||||||
|
# and for the whole archive: 27 seasons, 2119 games, 1779 played
|
||||||
|
```
|
||||||
|
|
||||||
|
## No runtime dependencies
|
||||||
|
|
||||||
|
The tree is built by a ~120-line builder on `html.parser`, because the only part
|
||||||
|
of the HTML5 tree construction algorithm this crawler relies on is that stray
|
||||||
|
`<tr>` elements land in an implicit `<tbody>` — which `table_rows()` handles
|
||||||
|
explicitly, exactly as in Go.
|
||||||
|
|
||||||
|
One deliberate difference from a browser: the site writes empty cells as
|
||||||
|
`<th class="…" />`, and HTML5 ignores that slash, so a real browser leaves the
|
||||||
|
cell open and swallows its siblings. This builder treats the tag as an empty
|
||||||
|
element, which is what the author meant and what keeps `child_elements()` honest.
|
||||||
|
That is also why the Go and Python trees agree on the header row (7 `th`, 6
|
||||||
|
non-empty labels) even though the Go parser closes the tag implicitly.
|
||||||
|
|
||||||
|
## Modules
|
||||||
|
|
||||||
|
| module | what it does |
|
||||||
|
| ----------------- | --------------------------------------------------------- |
|
||||||
|
| `dom.py` | the HTML tree builder and the node helpers |
|
||||||
|
| `parse.py` | layout profiles, the game reader, the invariants |
|
||||||
|
| `seasons.py` | league and season discovery |
|
||||||
|
| `fetch.py` | one GET per page, charset guard, skip-unchanged state file |
|
||||||
|
| `output.py` | JSON, NDJSON, CSV, golden-file checking |
|
||||||
|
| `cli.py` | `schedule`, `crawl`, `leagues` and the exit codes |
|
||||||
|
|
||||||
|
Exit codes match the Go CLI: `0` ok, `1` error, `2` usage, `3` markup drift or
|
||||||
|
golden mismatch, `4` timeout.
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
[build-system]
|
||||||
|
requires = ["setuptools>=68"]
|
||||||
|
build-backend = "setuptools.build_meta"
|
||||||
|
|
||||||
|
[project]
|
||||||
|
name = "swehockey-schedule"
|
||||||
|
version = "1.0.0"
|
||||||
|
description = "Deterministic crawler for the stats.swehockey.se schedule table"
|
||||||
|
readme = "README.md"
|
||||||
|
requires-python = ">=3.10"
|
||||||
|
license = { text = "MIT" }
|
||||||
|
authors = [{ name = "Mats Tornberg" }]
|
||||||
|
keywords = ["hockey", "swehockey", "shl", "crawler", "scraper", "deterministic"]
|
||||||
|
classifiers = [
|
||||||
|
"Programming Language :: Python :: 3",
|
||||||
|
"Topic :: Text Processing :: Markup :: HTML",
|
||||||
|
]
|
||||||
|
|
||||||
|
# No runtime dependencies on purpose: the HTML tree is built on html.parser
|
||||||
|
# from the standard library.
|
||||||
|
dependencies = []
|
||||||
|
|
||||||
|
[project.scripts]
|
||||||
|
swehockey-schedule = "swehockey.cli:main"
|
||||||
|
|
||||||
|
[project.urls]
|
||||||
|
Homepage = "https://git.k6n.net/mats/swehockey"
|
||||||
|
|
||||||
|
[tool.setuptools]
|
||||||
|
packages = ["swehockey"]
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
"""A deterministic crawler for the stats.swehockey.se schedule table.
|
||||||
|
|
||||||
|
The Python twin of the Go implementation in the parent directory: same layout
|
||||||
|
profiles, same invariants, same canonical order, same JSON bytes. See the
|
||||||
|
repository README for the design; this docstring is the whole contract:
|
||||||
|
|
||||||
|
* one HTTP GET per page, no browser and no JavaScript, because the table is
|
||||||
|
server-rendered;
|
||||||
|
* the table is located by its header signature, not by a CSS path;
|
||||||
|
* every field is read from a declared layout profile and shape-checked;
|
||||||
|
* facts the site renders twice must agree;
|
||||||
|
* the output is a pure, sorted function of the page - no fetch timestamp, no
|
||||||
|
unordered iteration - so identical HTML always yields identical bytes;
|
||||||
|
* markup drift is a :class:`DriftError`, not a stack trace.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from .dom import Node, normalize, parse_html, table_rows, text_excluding, text_of
|
||||||
|
from .fetch import Client, FetchError, FetchTimeout, State
|
||||||
|
from .output import emit, render, render_csv, render_json, render_ndjson
|
||||||
|
from .parse import (
|
||||||
|
LAYOUTS,
|
||||||
|
DriftError,
|
||||||
|
Game,
|
||||||
|
Layout,
|
||||||
|
Period,
|
||||||
|
Schedule,
|
||||||
|
find_table,
|
||||||
|
parse_periods,
|
||||||
|
parse_schedule,
|
||||||
|
)
|
||||||
|
from .seasons import League, Season, parse_leagues, parse_seasons, schedule_path
|
||||||
|
|
||||||
|
__version__ = "1.0.0"
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"Client",
|
||||||
|
"DriftError",
|
||||||
|
"FetchError",
|
||||||
|
"FetchTimeout",
|
||||||
|
"Game",
|
||||||
|
"LAYOUTS",
|
||||||
|
"Layout",
|
||||||
|
"League",
|
||||||
|
"Node",
|
||||||
|
"Period",
|
||||||
|
"Schedule",
|
||||||
|
"Season",
|
||||||
|
"State",
|
||||||
|
"emit",
|
||||||
|
"find_table",
|
||||||
|
"normalize",
|
||||||
|
"parse_html",
|
||||||
|
"parse_leagues",
|
||||||
|
"parse_periods",
|
||||||
|
"parse_schedule",
|
||||||
|
"parse_seasons",
|
||||||
|
"render",
|
||||||
|
"render_csv",
|
||||||
|
"render_json",
|
||||||
|
"render_ndjson",
|
||||||
|
"schedule_path",
|
||||||
|
"table_rows",
|
||||||
|
"text_excluding",
|
||||||
|
"text_of",
|
||||||
|
]
|
||||||
@@ -0,0 +1,227 @@
|
|||||||
|
"""Command line interface: ``schedule``, ``crawl`` and ``leagues``.
|
||||||
|
|
||||||
|
Same commands, flags, output and exit codes as the Go implementation.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
|
||||||
|
from .fetch import Client, FetchError, FetchTimeout, State, sha256_hex
|
||||||
|
from .output import emit, render, render_csv, render_ndjson
|
||||||
|
from .parse import DriftError, Schedule, parse_schedule
|
||||||
|
from .seasons import parse_leagues, parse_seasons, schedule_path
|
||||||
|
|
||||||
|
DEFAULT_SEASON = 20961
|
||||||
|
USER_AGENT_NOTE = "swehockey-schedule-py"
|
||||||
|
|
||||||
|
EXIT_OK = 0
|
||||||
|
EXIT_ERROR = 1
|
||||||
|
EXIT_USAGE = 2
|
||||||
|
EXIT_DRIFT = 3
|
||||||
|
EXIT_TIMEOUT = 4
|
||||||
|
|
||||||
|
EPILOG = """\
|
||||||
|
exit codes: 0 ok, 1 error, 2 usage, 3 markup drift or golden mismatch, 4 timeout
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class Unchanged(Exception):
|
||||||
|
"""The page was byte-identical to the last run."""
|
||||||
|
|
||||||
|
|
||||||
|
def build_parser() -> argparse.ArgumentParser:
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
prog="swehockey-schedule",
|
||||||
|
description="deterministic stats.swehockey.se schedule crawler",
|
||||||
|
epilog=EPILOG,
|
||||||
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||||
|
)
|
||||||
|
sub = parser.add_subparsers(dest="command", required=True)
|
||||||
|
|
||||||
|
def common(p: argparse.ArgumentParser, default_format: str) -> None:
|
||||||
|
p.add_argument("--base", default="https://stats.swehockey.se", help="site root")
|
||||||
|
p.add_argument("--format", default=default_format, choices=("json", "csv", "ndjson"), help="output format")
|
||||||
|
p.add_argument("--out", default="", help="write to this file instead of stdout")
|
||||||
|
p.add_argument("--fixture", default="", help="parse a saved HTML page instead of fetching")
|
||||||
|
p.add_argument("--state", default="", help="state file for skip-when-unchanged runs")
|
||||||
|
p.add_argument("--check", default="", help="golden file; exit 3 when the output differs")
|
||||||
|
p.add_argument("--timeout", type=float, default=30.0, help="per-request timeout in seconds")
|
||||||
|
p.add_argument("--quiet", action="store_true", help="suppress progress notes on stderr")
|
||||||
|
|
||||||
|
schedule = sub.add_parser("schedule", help="extract one season's schedule table")
|
||||||
|
common(schedule, "json")
|
||||||
|
schedule.add_argument("--season", type=int, default=DEFAULT_SEASON, help="season id")
|
||||||
|
|
||||||
|
crawl = sub.add_parser("crawl", help="extract every season of one league (NDJSON)")
|
||||||
|
common(crawl, "ndjson")
|
||||||
|
crawl.add_argument("--league", type=int, default=DEFAULT_SEASON, help="league id (same id space as season ids)")
|
||||||
|
crawl.add_argument("--season", type=int, default=0, help="crawl only this season id (0 = all)")
|
||||||
|
crawl.add_argument("--limit", type=int, default=0, help="stop after N seasons (0 = all)")
|
||||||
|
crawl.add_argument("--delay", type=float, default=0.25, help="pause between season requests, in seconds")
|
||||||
|
crawl.add_argument("--on-drift", default="fail", choices=("fail", "record"), help="what to do with a broken season page")
|
||||||
|
|
||||||
|
leagues = sub.add_parser("leagues", help="list league ids and names")
|
||||||
|
common(leagues, "json")
|
||||||
|
return parser
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: list[str] | None = None) -> int:
|
||||||
|
args = build_parser().parse_args(argv)
|
||||||
|
try:
|
||||||
|
if args.command == "schedule":
|
||||||
|
_cmd_schedule(args)
|
||||||
|
elif args.command == "crawl":
|
||||||
|
_cmd_crawl(args)
|
||||||
|
elif args.command == "leagues":
|
||||||
|
_cmd_leagues(args)
|
||||||
|
except Unchanged:
|
||||||
|
return EXIT_OK
|
||||||
|
except FetchTimeout as exc:
|
||||||
|
print(f"swehockey: {exc}", file=sys.stderr)
|
||||||
|
return EXIT_TIMEOUT
|
||||||
|
except DriftError as exc:
|
||||||
|
# Drift is not a crash: it means the site changed shape. Exit 3 so a
|
||||||
|
# cron job or CI gate can alert without a stack trace. The message
|
||||||
|
# already carries the "swehockey:" prefix.
|
||||||
|
print(exc, file=sys.stderr)
|
||||||
|
return EXIT_DRIFT
|
||||||
|
except (FetchError, ValueError, OSError) as exc:
|
||||||
|
print(f"swehockey: {exc}", file=sys.stderr)
|
||||||
|
return EXIT_ERROR
|
||||||
|
return EXIT_OK
|
||||||
|
|
||||||
|
|
||||||
|
def _read_page(args, client: Client, path: str, what: str) -> str:
|
||||||
|
"""The page bytes for a path, honouring --fixture and --state."""
|
||||||
|
if args.fixture:
|
||||||
|
with open(args.fixture, encoding="utf-8") as fh:
|
||||||
|
return fh.read()
|
||||||
|
page = client.get(path)
|
||||||
|
if args.state:
|
||||||
|
state = State.load(args.state)
|
||||||
|
if state.sha256 == sha256_hex(page.encode("utf-8")) and state.path == path:
|
||||||
|
# Byte-identical page: the derived output would be identical too.
|
||||||
|
if not args.quiet:
|
||||||
|
print(f"swehockey-schedule: {what} unchanged since {state.saved_at}, nothing to do", file=sys.stderr)
|
||||||
|
raise Unchanged()
|
||||||
|
return page
|
||||||
|
|
||||||
|
|
||||||
|
def _cmd_schedule(args) -> None:
|
||||||
|
path = schedule_path(args.season)
|
||||||
|
page = _read_page(args, Client(args.base, args.timeout), path, f"season {args.season}")
|
||||||
|
schedule = parse_schedule(args.season, page)
|
||||||
|
if args.format == "ndjson":
|
||||||
|
data = render_ndjson(schedule)
|
||||||
|
else:
|
||||||
|
data = render(args.format, schedule)
|
||||||
|
emit(args.out, args.check, data)
|
||||||
|
if args.state:
|
||||||
|
State(
|
||||||
|
path=path,
|
||||||
|
sha256=sha256_hex(page.encode("utf-8")),
|
||||||
|
last_update=schedule.last_update,
|
||||||
|
saved_at=_now(),
|
||||||
|
).save(args.state)
|
||||||
|
if not args.quiet:
|
||||||
|
print(
|
||||||
|
f"season {schedule.season_id}: {len(schedule.games)} games, "
|
||||||
|
f"{schedule.played} played, last update {schedule.last_update}",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _cmd_leagues(args) -> None:
|
||||||
|
page = _read_page(args, Client(args.base, args.timeout), "/", "front page")
|
||||||
|
leagues = parse_leagues(page)
|
||||||
|
if args.format == "csv":
|
||||||
|
lines = ["id,name,path"]
|
||||||
|
lines += [f"{l.id},{l.name},{l.path}" for l in leagues]
|
||||||
|
data = ("\n".join(lines) + "\n").encode("utf-8")
|
||||||
|
else:
|
||||||
|
data = (json.dumps([l.as_dict() for l in leagues], indent=2, ensure_ascii=False) + "\n").encode("utf-8")
|
||||||
|
emit(args.out, args.check, data)
|
||||||
|
|
||||||
|
|
||||||
|
def _cmd_crawl(args) -> None:
|
||||||
|
"""The crawler proper.
|
||||||
|
|
||||||
|
Take one league, read its season selector, then walk every season in
|
||||||
|
ascending id order and emit one document per season. The order is derived
|
||||||
|
from the ids, never from map iteration or from the order the site happened
|
||||||
|
to list things, so two runs produce the same file even if the site's markup
|
||||||
|
order changes.
|
||||||
|
"""
|
||||||
|
client = Client(args.base, args.timeout)
|
||||||
|
seed = _read_page(args, client, schedule_path(args.league), f"league {args.league}")
|
||||||
|
seasons = parse_seasons(args.league, seed)
|
||||||
|
|
||||||
|
out = bytearray()
|
||||||
|
first = True
|
||||||
|
if args.format == "json":
|
||||||
|
out += b"[\n"
|
||||||
|
done = 0
|
||||||
|
for season in seasons:
|
||||||
|
if args.season and season.id != args.season:
|
||||||
|
continue
|
||||||
|
if args.limit and done >= args.limit:
|
||||||
|
break
|
||||||
|
if done and args.delay:
|
||||||
|
time.sleep(args.delay)
|
||||||
|
page = client.get(schedule_path(season.id))
|
||||||
|
try:
|
||||||
|
schedule = parse_schedule(season.id, page)
|
||||||
|
except DriftError as exc:
|
||||||
|
# A season whose page breaks the contract is either fatal or
|
||||||
|
# recorded, never dropped: a hole in the data has to be visible.
|
||||||
|
if args.on_drift == "fail":
|
||||||
|
raise
|
||||||
|
if not args.quiet:
|
||||||
|
print(f"season {season.id}: {exc}", file=sys.stderr)
|
||||||
|
if args.format == "csv":
|
||||||
|
out += f"{season.id},ERROR,,,,,,,,,{exc}\n".encode("utf-8")
|
||||||
|
else:
|
||||||
|
if args.format == "json":
|
||||||
|
out += b"" if first else b",\n"
|
||||||
|
out += b" "
|
||||||
|
record = {"season_id": season.id, "label": season.label, "error": str(exc).split(": ", 2)[-1]}
|
||||||
|
out += (json.dumps(record, ensure_ascii=False) + "\n").encode("utf-8")
|
||||||
|
first = False
|
||||||
|
done += 1
|
||||||
|
continue
|
||||||
|
if args.format == "csv":
|
||||||
|
out += render_csv(schedule)
|
||||||
|
else:
|
||||||
|
if args.format == "json":
|
||||||
|
out += b"" if first else b",\n"
|
||||||
|
out += b" "
|
||||||
|
# One Encode in the Go version writes the document plus exactly one
|
||||||
|
# newline: the NDJSON contract, and one element of the JSON array.
|
||||||
|
out += render_ndjson(schedule)
|
||||||
|
first = False
|
||||||
|
done += 1
|
||||||
|
if not args.quiet:
|
||||||
|
print(
|
||||||
|
f"season {schedule.season_id} ({season.label}): {len(schedule.games)} games, {schedule.played} played",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
if args.format == "json":
|
||||||
|
out += b"\n]\n"
|
||||||
|
if not done:
|
||||||
|
raise DriftError(args.league, "crawl matched no seasons", got="0 seasons", want=">= 1")
|
||||||
|
data = bytes(out)
|
||||||
|
emit(args.out, args.check, data)
|
||||||
|
if args.state:
|
||||||
|
State(path=f"crawl:{args.league}", sha256=sha256_hex(data), saved_at=_now()).save(args.state)
|
||||||
|
|
||||||
|
|
||||||
|
def _now() -> str:
|
||||||
|
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__": # pragma: no cover
|
||||||
|
sys.exit(main())
|
||||||
@@ -0,0 +1,171 @@
|
|||||||
|
"""A tiny HTML tree builder on top of the standard library.
|
||||||
|
|
||||||
|
The Go implementation uses golang.org/x/net/html, which implements the HTML5
|
||||||
|
tree construction algorithm. The only part of that algorithm this crawler
|
||||||
|
depends on is that stray ``<tr>`` elements end up inside an implicit ``<tbody>``,
|
||||||
|
which :func:`table_rows` handles explicitly. So a small builder is enough, and
|
||||||
|
it keeps the Python variant dependency-free.
|
||||||
|
|
||||||
|
One deliberate difference from a browser: the site writes empty cells as
|
||||||
|
``<th class="..." />``. HTML5 ignores that slash, so a browser leaves the cell
|
||||||
|
open and its siblings end up nested inside it. This builder treats such a tag
|
||||||
|
as an empty element, which is what the author meant, and it keeps
|
||||||
|
``child_elements()`` honest.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from html.parser import HTMLParser
|
||||||
|
|
||||||
|
VOID = {
|
||||||
|
"area", "base", "br", "col", "embed", "hr", "img", "input",
|
||||||
|
"link", "meta", "param", "source", "track", "wbr",
|
||||||
|
}
|
||||||
|
|
||||||
|
# The site writes the team separator as NBSP-hyphen-NBSP wrapped in newlines, so
|
||||||
|
# every string that leaves the parser goes through normalize(). Matching the
|
||||||
|
# negated class (rather than splitting on whitespace) is what drops leading and
|
||||||
|
# trailing space, like strings.FieldsFunc does in the Go version.
|
||||||
|
_TOKEN = re.compile(r"[^\s\u00a0\u2007\u202f\u200b\ufeff]+")
|
||||||
|
|
||||||
|
|
||||||
|
def normalize(text: str) -> str:
|
||||||
|
"""Collapse every flavour of whitespace into single spaces."""
|
||||||
|
return " ".join(_TOKEN.findall(text))
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Node:
|
||||||
|
tag: str | None
|
||||||
|
attrs: dict[str, str] = field(default_factory=dict)
|
||||||
|
data: str = ""
|
||||||
|
children: list["Node"] = field(default_factory=list)
|
||||||
|
parent: "Node | None" = field(default=None, repr=False)
|
||||||
|
|
||||||
|
def walk(self):
|
||||||
|
yield self
|
||||||
|
for child in self.children:
|
||||||
|
yield from child.walk()
|
||||||
|
|
||||||
|
|
||||||
|
class _Builder(HTMLParser):
|
||||||
|
def __init__(self) -> None:
|
||||||
|
super().__init__(convert_charrefs=True)
|
||||||
|
self.root = Node("#document")
|
||||||
|
self.stack = [self.root]
|
||||||
|
|
||||||
|
def _append(self, node: Node) -> None:
|
||||||
|
node.parent = self.stack[-1]
|
||||||
|
self.stack[-1].children.append(node)
|
||||||
|
|
||||||
|
def handle_starttag(self, tag, attrs):
|
||||||
|
node = Node(tag, {k: (v or "") for k, v in attrs})
|
||||||
|
self._append(node)
|
||||||
|
if tag not in VOID:
|
||||||
|
self.stack.append(node)
|
||||||
|
|
||||||
|
def handle_startendtag(self, tag, attrs):
|
||||||
|
# An empty cell: <th class="tdHeader d-none d-sm-table-cell" />
|
||||||
|
self._append(Node(tag, {k: (v or "") for k, v in attrs}))
|
||||||
|
|
||||||
|
def handle_endtag(self, tag):
|
||||||
|
for i in range(len(self.stack) - 1, 0, -1):
|
||||||
|
if self.stack[i].tag == tag:
|
||||||
|
del self.stack[i:]
|
||||||
|
return
|
||||||
|
# A stray end tag: ignore it, like a browser does.
|
||||||
|
|
||||||
|
def handle_data(self, data):
|
||||||
|
self._append(Node(None, data=data))
|
||||||
|
|
||||||
|
|
||||||
|
def parse_html(page: str) -> Node:
|
||||||
|
"""Parse a document and return its root node."""
|
||||||
|
builder = _Builder()
|
||||||
|
builder.feed(page)
|
||||||
|
builder.close()
|
||||||
|
return builder.root
|
||||||
|
|
||||||
|
|
||||||
|
def text_of(node: Node | None) -> str:
|
||||||
|
"""The normalized text of a subtree."""
|
||||||
|
if node is None:
|
||||||
|
return ""
|
||||||
|
return normalize("".join(n.data for n in node.walk() if n.tag is None))
|
||||||
|
|
||||||
|
|
||||||
|
def text_excluding(node: Node | None, skip_tag: str) -> str:
|
||||||
|
"""The normalized text of a subtree with the first skip_tag subtree removed.
|
||||||
|
|
||||||
|
The schedule uses this to read the mobile cell's start time without its
|
||||||
|
``<bold>`` date, and the team cell without the ``<i>`` note old seasons
|
||||||
|
append to it.
|
||||||
|
"""
|
||||||
|
if node is None:
|
||||||
|
return ""
|
||||||
|
out: list[str] = []
|
||||||
|
|
||||||
|
def walk(n: Node) -> None:
|
||||||
|
if n.tag == skip_tag:
|
||||||
|
return
|
||||||
|
if n.tag is None:
|
||||||
|
out.append(n.data)
|
||||||
|
for child in n.children:
|
||||||
|
walk(child)
|
||||||
|
|
||||||
|
walk(node)
|
||||||
|
return normalize("".join(out))
|
||||||
|
|
||||||
|
|
||||||
|
def child_elements(node: Node | None, tag: str) -> list[Node]:
|
||||||
|
if node is None:
|
||||||
|
return []
|
||||||
|
return [c for c in node.children if c.tag == tag]
|
||||||
|
|
||||||
|
|
||||||
|
def descendants(node: Node, tag: str) -> list[Node]:
|
||||||
|
return [n for n in node.walk() if n.tag == tag]
|
||||||
|
|
||||||
|
|
||||||
|
def first_element(node: Node | None, tag: str) -> Node | None:
|
||||||
|
if node is None:
|
||||||
|
return None
|
||||||
|
for n in node.walk():
|
||||||
|
if n.tag == tag:
|
||||||
|
return n
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def attr(node: Node | None, name: str) -> str:
|
||||||
|
return "" if node is None else node.attrs.get(name, "")
|
||||||
|
|
||||||
|
|
||||||
|
def has_class(node: Node, class_name: str) -> bool:
|
||||||
|
return class_name in attr(node, "class").split()
|
||||||
|
|
||||||
|
|
||||||
|
def table_rows(table: Node) -> list[Node]:
|
||||||
|
"""The ``<tr>`` children of a table, looking inside ``<tbody>`` as well."""
|
||||||
|
rows = child_elements(table, "tr")
|
||||||
|
if rows:
|
||||||
|
return rows
|
||||||
|
out: list[Node] = []
|
||||||
|
for body in child_elements(table, "tbody"):
|
||||||
|
out.extend(child_elements(body, "tr"))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def th_labels(node: Node) -> list[str]:
|
||||||
|
"""The non-empty ``<th>`` labels of a row, or of a table's header row."""
|
||||||
|
trs = [node] if node.tag == "tr" else table_rows(node)
|
||||||
|
for tr in trs:
|
||||||
|
labels = table_header_labels(tr)
|
||||||
|
if labels:
|
||||||
|
return labels
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def table_header_labels(row: Node) -> list[str]:
|
||||||
|
return [t for t in (text_of(th) for th in child_elements(row, "th")) if t]
|
||||||
@@ -0,0 +1,123 @@
|
|||||||
|
"""Fetching and the skip-unchanged state file.
|
||||||
|
|
||||||
|
One request, no retries, no cookies, no redirect games: every byte of the output
|
||||||
|
is a function of one response.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
from dataclasses import asdict, dataclass
|
||||||
|
|
||||||
|
from .seasons import join_url
|
||||||
|
|
||||||
|
# Identify the crawler. The site is a public ASP.NET app and serves the same
|
||||||
|
# bytes to any client; saying who we are keeps its logs readable.
|
||||||
|
USER_AGENT = "swehockey-schedule-py/1.0 (+deterministic schedule crawler)"
|
||||||
|
|
||||||
|
META_CHARSET_RE = re.compile(rb"""<meta[^>]+charset\s*=\s*["']?([a-z0-9_-]+)""", re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
|
class FetchTimeout(Exception):
|
||||||
|
"""The site did not answer in time."""
|
||||||
|
|
||||||
|
|
||||||
|
class FetchError(Exception):
|
||||||
|
"""Anything else that went wrong on the way to the page."""
|
||||||
|
|
||||||
|
|
||||||
|
def sha256_hex(data: bytes) -> str:
|
||||||
|
return hashlib.sha256(data).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
class Client:
|
||||||
|
def __init__(self, base_url: str = "https://stats.swehockey.se", timeout: float = 30.0) -> None:
|
||||||
|
self.base_url = base_url
|
||||||
|
self.timeout = timeout
|
||||||
|
|
||||||
|
def get(self, path: str) -> str:
|
||||||
|
"""GET one page and return it decoded as text.
|
||||||
|
|
||||||
|
Refuses pages it cannot decode byte-exactly: handing latin-1 bytes to a
|
||||||
|
UTF-8 parser would turn "Växjö" into mojibake that still parses, which
|
||||||
|
is the worst kind of crawler bug.
|
||||||
|
"""
|
||||||
|
url = join_url(self.base_url, path)
|
||||||
|
request = urllib.request.Request(
|
||||||
|
url,
|
||||||
|
headers={
|
||||||
|
"User-Agent": USER_AGENT,
|
||||||
|
"Accept": "text/html,application/xhtml+xml",
|
||||||
|
"Accept-Language": "en",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(request, timeout=self.timeout) as response:
|
||||||
|
if response.status != 200:
|
||||||
|
raise FetchError(f"fetch {url}: HTTP {response.status}")
|
||||||
|
body = response.read()
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
raise FetchError(f"fetch {url}: {exc.code} {exc.reason}") from None
|
||||||
|
except TimeoutError as exc:
|
||||||
|
raise FetchTimeout(f"fetch {url}: timed out after {self.timeout}s") from None
|
||||||
|
except urllib.error.URLError as exc:
|
||||||
|
if isinstance(exc.reason, TimeoutError):
|
||||||
|
raise FetchTimeout(f"fetch {url}: timed out after {self.timeout}s") from None
|
||||||
|
raise FetchError(f"fetch {url}: {exc.reason}") from None
|
||||||
|
except OSError as exc:
|
||||||
|
raise FetchError(f"fetch {url}: {exc}") from None
|
||||||
|
|
||||||
|
check_charset(body)
|
||||||
|
try:
|
||||||
|
return body.decode("utf-8")
|
||||||
|
except UnicodeDecodeError as exc:
|
||||||
|
raise FetchError(f"fetch {url}: not valid UTF-8: {exc}") from None
|
||||||
|
|
||||||
|
|
||||||
|
def check_charset(body: bytes) -> None:
|
||||||
|
"""Refuse a page we cannot decode byte-exactly."""
|
||||||
|
m = META_CHARSET_RE.search(body)
|
||||||
|
if m is None:
|
||||||
|
return # no declaration: UTF-8 is assumed
|
||||||
|
charset = m.group(1).decode("ascii").lower()
|
||||||
|
if charset not in ("utf-8", "utf8"):
|
||||||
|
raise FetchError(f'unsupported charset "{charset}": add a decoder for it')
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class State:
|
||||||
|
"""What the last successful run saw, so a cron job can be idempotent.
|
||||||
|
|
||||||
|
An unchanged page produces no output and a zero exit code.
|
||||||
|
"""
|
||||||
|
|
||||||
|
path: str = ""
|
||||||
|
sha256: str = ""
|
||||||
|
last_update: str = ""
|
||||||
|
saved_at: str = ""
|
||||||
|
|
||||||
|
def load(path: str) -> "State":
|
||||||
|
try:
|
||||||
|
with open(path, encoding="utf-8") as fh:
|
||||||
|
return State(**json.load(fh))
|
||||||
|
except FileNotFoundError:
|
||||||
|
return State()
|
||||||
|
except (OSError, TypeError, json.JSONDecodeError) as exc:
|
||||||
|
raise FetchError(f"state file {path}: {exc}") from None
|
||||||
|
|
||||||
|
def save(self, path: str) -> None:
|
||||||
|
"""Write the state atomically, so a killed run cannot leave a truncated
|
||||||
|
file that would make the next run skip real work."""
|
||||||
|
directory = os.path.dirname(path)
|
||||||
|
if directory:
|
||||||
|
os.makedirs(directory, exist_ok=True)
|
||||||
|
tmp = path + ".tmp"
|
||||||
|
with open(tmp, "w", encoding="utf-8") as fh:
|
||||||
|
json.dump(asdict(self), fh, indent=2)
|
||||||
|
fh.write("\n")
|
||||||
|
os.replace(tmp, path)
|
||||||
@@ -0,0 +1,123 @@
|
|||||||
|
"""Output: JSON, NDJSON and CSV.
|
||||||
|
|
||||||
|
Both encoders are pure functions of the data: no timestamps, no unordered
|
||||||
|
iteration, fixed field order, LF line endings. The JSON writer is tuned to
|
||||||
|
produce exactly the same bytes as the Go implementation, which is what lets the
|
||||||
|
two share one set of golden files.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import csv
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from .parse import DriftError, Schedule
|
||||||
|
|
||||||
|
CSV_HEADER = [
|
||||||
|
"season_id",
|
||||||
|
"game_id",
|
||||||
|
"event_id",
|
||||||
|
"date",
|
||||||
|
"time",
|
||||||
|
"home",
|
||||||
|
"away",
|
||||||
|
"home_goals",
|
||||||
|
"away_goals",
|
||||||
|
"periods",
|
||||||
|
"spectators",
|
||||||
|
"venue",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def render(fmt: str, schedule: Schedule) -> bytes:
|
||||||
|
if fmt == "json":
|
||||||
|
return render_json(schedule)
|
||||||
|
if fmt == "csv":
|
||||||
|
return render_csv(schedule)
|
||||||
|
raise ValueError(f'unknown format "{fmt}": want json, csv or ndjson')
|
||||||
|
|
||||||
|
|
||||||
|
def render_json(schedule: Schedule) -> bytes:
|
||||||
|
# ensure_ascii=False keeps "Växjö" readable, which matters for diffs of
|
||||||
|
# team names; indent=2 and the default separators match encoding/json.
|
||||||
|
text = json.dumps(schedule.as_dict(), indent=2, ensure_ascii=False)
|
||||||
|
return (text + "\n").encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def render_ndjson(schedule: Schedule) -> bytes:
|
||||||
|
text = json.dumps(schedule.as_dict(), ensure_ascii=False)
|
||||||
|
return (text + "\n").encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _opt(value: int | None) -> str:
|
||||||
|
return "" if value is None else str(value)
|
||||||
|
|
||||||
|
|
||||||
|
def render_csv(schedule: Schedule) -> bytes:
|
||||||
|
buf = io.StringIO()
|
||||||
|
writer = csv.writer(buf, lineterminator="\n") # LF, not CRLF
|
||||||
|
writer.writerow(CSV_HEADER)
|
||||||
|
for game in schedule.games:
|
||||||
|
periods = " ".join(f"{p.label}:{p.home}-{p.away}" for p in game.periods)
|
||||||
|
writer.writerow(
|
||||||
|
[
|
||||||
|
schedule.season_id,
|
||||||
|
game.game_id,
|
||||||
|
game.event_id,
|
||||||
|
game.date,
|
||||||
|
game.time,
|
||||||
|
game.home,
|
||||||
|
game.away,
|
||||||
|
_opt(game.home_goals),
|
||||||
|
_opt(game.away_goals),
|
||||||
|
periods,
|
||||||
|
_opt(game.spectators),
|
||||||
|
game.venue,
|
||||||
|
]
|
||||||
|
)
|
||||||
|
return buf.getvalue().encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def emit(out: str, check: str, data: bytes) -> None:
|
||||||
|
"""Write the bytes to -out or stdout and, when -check is set, compare them
|
||||||
|
with a committed golden file.
|
||||||
|
|
||||||
|
A mismatch raises, so CI turns an upstream change into a reviewable diff
|
||||||
|
instead of a silent data change.
|
||||||
|
"""
|
||||||
|
if check:
|
||||||
|
try:
|
||||||
|
with open(check, "rb") as fh:
|
||||||
|
golden = fh.read()
|
||||||
|
except FileNotFoundError as exc:
|
||||||
|
raise DriftError(0, f"golden file {check}: {exc}", hint="") from None
|
||||||
|
if golden != data:
|
||||||
|
raise DriftError(
|
||||||
|
0,
|
||||||
|
f"output no longer matches {check}: {first_diff(golden, data)}",
|
||||||
|
hint=" - regenerate the golden file only after reviewing the change",
|
||||||
|
)
|
||||||
|
if out:
|
||||||
|
with open(out, "wb") as fh:
|
||||||
|
fh.write(data)
|
||||||
|
else:
|
||||||
|
sys.stdout.buffer.write(data)
|
||||||
|
sys.stdout.buffer.flush()
|
||||||
|
|
||||||
|
|
||||||
|
def first_diff(a: bytes, b: bytes) -> str:
|
||||||
|
"""Describe the first differing line, for a readable failure."""
|
||||||
|
left = a.decode("utf-8", "replace").split("\n")
|
||||||
|
right = b.decode("utf-8", "replace").split("\n")
|
||||||
|
for i in range(max(len(left), len(right))):
|
||||||
|
x = left[i] if i < len(left) else ""
|
||||||
|
y = right[i] if i < len(right) else ""
|
||||||
|
if x != y:
|
||||||
|
return f'line {i + 1}: "{_trim(x)}" != "{_trim(y)}"'
|
||||||
|
return "documents differ only in length"
|
||||||
|
|
||||||
|
|
||||||
|
def _trim(line: str) -> str:
|
||||||
|
return line if len(line) <= 80 else line[:80] + "..."
|
||||||
@@ -0,0 +1,520 @@
|
|||||||
|
"""Extraction of the "Schedule and Results" table from stats.swehockey.se.
|
||||||
|
|
||||||
|
This is a line-by-line counterpart of ``../parse.go``: same layout profiles,
|
||||||
|
same invariants, same canonical order, same JSON. The Python and the Go
|
||||||
|
implementation are checked against the same golden files, so they have to agree
|
||||||
|
byte for byte.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
|
||||||
|
from .dom import (
|
||||||
|
Node,
|
||||||
|
attr,
|
||||||
|
child_elements,
|
||||||
|
descendants,
|
||||||
|
first_element,
|
||||||
|
has_class,
|
||||||
|
parse_html,
|
||||||
|
table_rows,
|
||||||
|
text_excluding,
|
||||||
|
text_of,
|
||||||
|
th_labels,
|
||||||
|
)
|
||||||
|
|
||||||
|
# The site has shipped more than one version of this table, and the old
|
||||||
|
# versions are still served for historical seasons. So the contract is not a
|
||||||
|
# set of hard-coded column indices: it is a set of layout profiles, each
|
||||||
|
# identified by the header row it declares. A page the crawler does not
|
||||||
|
# recognise fails loudly with the header it actually found, and adding support
|
||||||
|
# is one profile plus one fixture.
|
||||||
|
MODERN = "modern" # seasons from 2018-19 onwards
|
||||||
|
LEGACY = "legacy" # seasons up to 2017-18
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Layout:
|
||||||
|
name: str
|
||||||
|
header: tuple[str, ...]
|
||||||
|
cells: int
|
||||||
|
round: int # -1 when the layout has no Round column
|
||||||
|
# Modern layout: the date is rendered twice, once in the desktop "Date"
|
||||||
|
# column and once as a <bold> group heading in the mobile column, and the
|
||||||
|
# start time lives in its own cell next to the game's stable id.
|
||||||
|
# Legacy layout: no group headings; the tooltip span of one cell holds
|
||||||
|
# both "2014-03-10" and "19:00".
|
||||||
|
date_desktop: int
|
||||||
|
date_mobile: int
|
||||||
|
time_cell: int
|
||||||
|
grouped: bool
|
||||||
|
teams: int
|
||||||
|
result: int
|
||||||
|
periods: int
|
||||||
|
spectators: int
|
||||||
|
venue: int
|
||||||
|
|
||||||
|
|
||||||
|
LAYOUTS: tuple[Layout, ...] = (
|
||||||
|
Layout(
|
||||||
|
name=MODERN,
|
||||||
|
header=("Date", "Time", "Game", "Result", "Spectators", "Venue"),
|
||||||
|
cells=8,
|
||||||
|
round=-1,
|
||||||
|
date_desktop=0,
|
||||||
|
date_mobile=1,
|
||||||
|
time_cell=2,
|
||||||
|
grouped=True,
|
||||||
|
teams=3,
|
||||||
|
result=4,
|
||||||
|
periods=5,
|
||||||
|
spectators=6,
|
||||||
|
venue=7,
|
||||||
|
),
|
||||||
|
Layout(
|
||||||
|
name=LEGACY,
|
||||||
|
header=("Round", "Date", "Game", "Result", "Spectators", "Venue"),
|
||||||
|
cells=7,
|
||||||
|
round=0,
|
||||||
|
date_desktop=-1,
|
||||||
|
date_mobile=-1,
|
||||||
|
time_cell=1,
|
||||||
|
grouped=False,
|
||||||
|
teams=2,
|
||||||
|
result=3,
|
||||||
|
periods=4,
|
||||||
|
spectators=5,
|
||||||
|
venue=6,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
|
||||||
|
TIME_RE = re.compile(r"^\d{1,2}:\d{2}$")
|
||||||
|
DIGITS_RE = re.compile(r"^\d+$")
|
||||||
|
SCORE_RE = re.compile(r"^(\d+)\s*-\s*(\d+)$")
|
||||||
|
PERIOD_RE = re.compile(r"^(\d+)\s*-\s*(\d+)$")
|
||||||
|
EVENT_ID_RE = re.compile(r"/Game/Events/(\d+)")
|
||||||
|
LAST_UPDATE_RE = re.compile(r"Last update:\s*(\d{4}-\d{2}-\d{2} \d{1,2}:\d{2})")
|
||||||
|
GAME_ID_RE = re.compile(r"^\d{5,}$")
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Period:
|
||||||
|
"""One period (or overtime / shootout) of a played game."""
|
||||||
|
|
||||||
|
label: str
|
||||||
|
home: int
|
||||||
|
away: int
|
||||||
|
|
||||||
|
def as_dict(self) -> dict:
|
||||||
|
return {"label": self.label, "home": self.home, "away": self.away}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Game:
|
||||||
|
"""One row of the schedule table."""
|
||||||
|
|
||||||
|
game_id: str # span.lnkTooltip@title, unique per season
|
||||||
|
event_id: str = "" # /Game/Events/<id>; set once the game is played
|
||||||
|
round: str = "" # only in the legacy layout
|
||||||
|
date: str = "" # YYYY-MM-DD
|
||||||
|
time: str = "" # HH:MM local arena time
|
||||||
|
home: str = ""
|
||||||
|
away: str = ""
|
||||||
|
home_goals: int | None = None # None until the game is played
|
||||||
|
away_goals: int | None = None
|
||||||
|
periods: list[Period] = field(default_factory=list)
|
||||||
|
spectators: int | None = None # None when not reported
|
||||||
|
venue: str = ""
|
||||||
|
|
||||||
|
@property
|
||||||
|
def played(self) -> bool:
|
||||||
|
return bool(self.event_id)
|
||||||
|
|
||||||
|
def as_dict(self) -> dict:
|
||||||
|
out = {"game_id": self.game_id}
|
||||||
|
if self.event_id:
|
||||||
|
out["event_id"] = self.event_id
|
||||||
|
if self.round:
|
||||||
|
out["round"] = self.round
|
||||||
|
out["date"] = self.date
|
||||||
|
out["time"] = self.time
|
||||||
|
out["home"] = self.home
|
||||||
|
out["away"] = self.away
|
||||||
|
out["home_goals"] = self.home_goals
|
||||||
|
out["away_goals"] = self.away_goals
|
||||||
|
out["periods"] = [p.as_dict() for p in self.periods]
|
||||||
|
if self.spectators is not None:
|
||||||
|
out["spectators"] = self.spectators
|
||||||
|
out["venue"] = self.venue
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Schedule:
|
||||||
|
season_id: int
|
||||||
|
layout: str
|
||||||
|
title: str
|
||||||
|
last_update: str # the site's own clock, verbatim
|
||||||
|
games: list[Game] = field(default_factory=list)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def played(self) -> int:
|
||||||
|
return sum(1 for g in self.games if g.played)
|
||||||
|
|
||||||
|
def as_dict(self) -> dict:
|
||||||
|
return {
|
||||||
|
"season_id": self.season_id,
|
||||||
|
"layout": self.layout,
|
||||||
|
"title": self.title,
|
||||||
|
"last_update": self.last_update,
|
||||||
|
"games": [g.as_dict() for g in self.games],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class DriftError(Exception):
|
||||||
|
"""The page no longer matches a known contract.
|
||||||
|
|
||||||
|
Its own type, and its own exit code, because it is the one failure a caller
|
||||||
|
must not retry blindly: the site changed, not the network.
|
||||||
|
"""
|
||||||
|
|
||||||
|
#: Every layout problem ends with the same advice, so an operator reading a
|
||||||
|
#: cron mail knows exactly what to do. Failures that are not about the table
|
||||||
|
#: shape pass hint="" and get no advice.
|
||||||
|
HINT = " - extend LAYOUTS in parse.py and add a fixture, in one commit"
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
season: int,
|
||||||
|
message: str = "",
|
||||||
|
game_id: str = "",
|
||||||
|
column: str = "",
|
||||||
|
got: str = "",
|
||||||
|
want: str = "",
|
||||||
|
hint: str | None = None,
|
||||||
|
) -> None:
|
||||||
|
self.season = season
|
||||||
|
self.game_id = game_id
|
||||||
|
self.column = column
|
||||||
|
self.got = got
|
||||||
|
self.want = want
|
||||||
|
parts = [f" (season {season}"]
|
||||||
|
if game_id:
|
||||||
|
parts.append(f", game {game_id}")
|
||||||
|
parts.append("): ")
|
||||||
|
parts.append(message or "unexpected table shape")
|
||||||
|
if column:
|
||||||
|
parts.append(f" [column {column}]")
|
||||||
|
if got:
|
||||||
|
parts.append(f': got "{got}", want "{want}"')
|
||||||
|
parts.append(self.HINT if hint is None else hint)
|
||||||
|
super().__init__("swehockey: markup drift" + "".join(parts))
|
||||||
|
|
||||||
|
|
||||||
|
def _drift(season, message, game_id="", column="", got="", want=""):
|
||||||
|
return DriftError(season, message, game_id, column, got, want)
|
||||||
|
|
||||||
|
|
||||||
|
class _Carry:
|
||||||
|
"""Values the site prints only on the first row of a group.
|
||||||
|
|
||||||
|
The date heading of the modern layout, the round number of the legacy one.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.date = ""
|
||||||
|
self.round = ""
|
||||||
|
|
||||||
|
|
||||||
|
def parse_schedule(season_id: int, page: str) -> Schedule:
|
||||||
|
"""Extract the schedule table from one saved or fetched page."""
|
||||||
|
doc = parse_html(page)
|
||||||
|
table = find_table(doc)
|
||||||
|
if table is None:
|
||||||
|
raise _drift(
|
||||||
|
season_id,
|
||||||
|
"no table with a known header: found "
|
||||||
|
+ " / ".join(document_signatures(doc))
|
||||||
|
+ ", known "
|
||||||
|
+ known_layouts(),
|
||||||
|
)
|
||||||
|
node, lay = table
|
||||||
|
|
||||||
|
schedule = Schedule(season_id, lay.name, "", "", [])
|
||||||
|
rows = table_rows(node)
|
||||||
|
header_at = -1
|
||||||
|
carry = _Carry()
|
||||||
|
for i, tr in enumerate(rows):
|
||||||
|
if not child_elements(tr, "td"):
|
||||||
|
if tuple(th_labels(tr)) == lay.header:
|
||||||
|
if header_at >= 0:
|
||||||
|
raise _drift(season_id, f"second header row at row {i}", got="2", want="1")
|
||||||
|
header_at = i
|
||||||
|
continue
|
||||||
|
if header_at < 0:
|
||||||
|
raise _drift(season_id, "data row before the header row", want="header first")
|
||||||
|
schedule.games.append(_parse_game(season_id, lay, tr, carry))
|
||||||
|
if header_at < 0:
|
||||||
|
raise _drift(season_id, "table has no header row", want=" ".join(lay.header))
|
||||||
|
|
||||||
|
heading = descendants(node, "h2")
|
||||||
|
if heading:
|
||||||
|
schedule.title = text_of(heading[0])
|
||||||
|
found = LAST_UPDATE_RE.search(text_of(node))
|
||||||
|
if found:
|
||||||
|
schedule.last_update = found.group(1)
|
||||||
|
if not schedule.last_update:
|
||||||
|
raise _drift(season_id, 'table has no "Last update:" stamp', want="Last update: YYYY-MM-DD HH:MM")
|
||||||
|
|
||||||
|
_check(schedule)
|
||||||
|
# The site lists games in round order, not date order, and a re-scheduled
|
||||||
|
# or awarded game keeps the round it belongs to: 2011-12 has September
|
||||||
|
# games listed after October ones. Sorting makes the output a canonical
|
||||||
|
# function of the set of rows instead of a function of the site's row
|
||||||
|
# order, which is what makes two crawls comparable and diffs meaningful.
|
||||||
|
schedule.games.sort(key=lambda g: (g.date, g.time, g.game_id))
|
||||||
|
return schedule
|
||||||
|
|
||||||
|
|
||||||
|
def _check(schedule: Schedule) -> None:
|
||||||
|
"""The invariants that hold for every season the site serves."""
|
||||||
|
if not schedule.games:
|
||||||
|
raise _drift(schedule.season_id, "table has no game rows", got="0 rows", want=">= 1 row")
|
||||||
|
# Game ids are unique, and that is the only uniqueness the page promises:
|
||||||
|
# the site assigns them per fixture, so they are neither sorted nor
|
||||||
|
# contiguous within a season.
|
||||||
|
seen: set[str] = set()
|
||||||
|
for game in schedule.games:
|
||||||
|
if not game.game_id.isdigit():
|
||||||
|
raise _drift(schedule.season_id, "game id is not a number", game.game_id, got=game.game_id, want="90001001")
|
||||||
|
if game.game_id in seen:
|
||||||
|
raise _drift(schedule.season_id, "duplicate game id", game.game_id, got=game.game_id, want="unique")
|
||||||
|
seen.add(game.game_id)
|
||||||
|
if not DATE_RE.match(game.date):
|
||||||
|
raise _drift(schedule.season_id, "malformed date", game.game_id, "date", game.date, "2026-09-19")
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_game(season_id: int, lay: Layout, tr: Node, carry: _Carry) -> Game:
|
||||||
|
"""Read one row. Where the site renders a fact twice, both must agree."""
|
||||||
|
tds = child_elements(tr, "td")
|
||||||
|
if len(tds) != lay.cells:
|
||||||
|
raise _drift(
|
||||||
|
season_id,
|
||||||
|
f"row width does not match the {lay.name} layout",
|
||||||
|
got=f"{len(tds)} cells",
|
||||||
|
want=f"{lay.cells} cells",
|
||||||
|
)
|
||||||
|
game = Game(game_id="")
|
||||||
|
|
||||||
|
# The tooltip span carries the row's stable id, so the id and the time can
|
||||||
|
# never disagree: they are the same element.
|
||||||
|
span = _tooltip_span(tds[lay.time_cell])
|
||||||
|
if span is None:
|
||||||
|
raise _drift(season_id, "no game id (span.lnkTooltip[title])", column="time", got=text_of(tds[lay.time_cell]), want="90001001")
|
||||||
|
game.game_id = attr(span, "title").strip()
|
||||||
|
if not GAME_ID_RE.match(game.game_id):
|
||||||
|
raise _drift(season_id, "malformed game id", column="time", got=game.game_id, want="90001001")
|
||||||
|
|
||||||
|
head = text_of(span)
|
||||||
|
if lay.grouped:
|
||||||
|
# Modern layout: date in two renderings, time in the span.
|
||||||
|
if not TIME_RE.match(head):
|
||||||
|
raise _drift(season_id, "malformed start time", game.game_id, "time", head, "19:00")
|
||||||
|
game.time = head
|
||||||
|
desktop = text_of(tds[lay.date_desktop])
|
||||||
|
mobile = text_of(first_element(tds[lay.date_mobile], "bold"))
|
||||||
|
if mobile:
|
||||||
|
if not DATE_RE.match(mobile):
|
||||||
|
raise _drift(season_id, "malformed date heading", game.game_id, "date", mobile, "2026-09-19")
|
||||||
|
if desktop != mobile:
|
||||||
|
raise _drift(season_id, "date renderings disagree", game.game_id, "date", desktop, mobile)
|
||||||
|
if text_excluding(tds[lay.date_mobile], "bold") != game.time:
|
||||||
|
raise _drift(
|
||||||
|
season_id,
|
||||||
|
"time renderings disagree",
|
||||||
|
game.game_id,
|
||||||
|
"date",
|
||||||
|
text_excluding(tds[lay.date_mobile], "bold"),
|
||||||
|
game.time,
|
||||||
|
)
|
||||||
|
game.date = mobile
|
||||||
|
else:
|
||||||
|
if not carry.date:
|
||||||
|
raise _drift(season_id, "row before any date heading", game.game_id, "date", desktop, "2026-09-19")
|
||||||
|
if desktop and desktop != game.time:
|
||||||
|
raise _drift(
|
||||||
|
season_id,
|
||||||
|
"unexpected date in a continuation row",
|
||||||
|
game.game_id,
|
||||||
|
"date",
|
||||||
|
desktop,
|
||||||
|
"(empty) or " + game.time,
|
||||||
|
)
|
||||||
|
game.date = carry.date
|
||||||
|
else:
|
||||||
|
# Legacy layout: "2014-03-10 19:00" in one cell, no group headings.
|
||||||
|
pieces = head.split(" ", 1)
|
||||||
|
if len(pieces) == 2 and DATE_RE.match(pieces[0]) and TIME_RE.match(pieces[1]):
|
||||||
|
game.date, game.time = pieces
|
||||||
|
elif TIME_RE.match(head):
|
||||||
|
if not carry.date:
|
||||||
|
raise _drift(season_id, "row before any date", game.game_id, "date", head, "2026-09-19 19:00")
|
||||||
|
game.date, game.time = carry.date, head
|
||||||
|
else:
|
||||||
|
raise _drift(season_id, "cannot read date and time", game.game_id, "date", head, "2026-09-19 19:00")
|
||||||
|
|
||||||
|
if lay.round >= 0:
|
||||||
|
# The site prints the round only on the first row of that round and
|
||||||
|
# leaves the following cells empty, exactly like the date headings.
|
||||||
|
round_text = text_of(tds[lay.round])
|
||||||
|
if DIGITS_RE.match(round_text):
|
||||||
|
game.round = round_text
|
||||||
|
elif round_text == "" and carry.round:
|
||||||
|
game.round = carry.round
|
||||||
|
elif round_text == "":
|
||||||
|
raise _drift(season_id, "row before any round number", game.game_id, "round", round_text, "1")
|
||||||
|
else:
|
||||||
|
raise _drift(season_id, "round is not a number", game.game_id, "round", round_text, "1")
|
||||||
|
|
||||||
|
# Teams: the cell is "Home - Away" plus, in old seasons, an <i>note</i>
|
||||||
|
# such as "Eighth final 1" that is not part of a name.
|
||||||
|
teams = text_excluding(tds[lay.teams], "i")
|
||||||
|
parts = teams.split(" - ")
|
||||||
|
if len(parts) != 2 or not parts[0] or not parts[1]:
|
||||||
|
raise _drift(season_id, "cannot split teams", game.game_id, "game", teams, "Home - Away")
|
||||||
|
game.home, game.away = parts
|
||||||
|
|
||||||
|
# Result: present only for played games, and always with an event link.
|
||||||
|
link = first_element(tds[lay.result], "a")
|
||||||
|
if link is not None:
|
||||||
|
score = SCORE_RE.match(text_of(link))
|
||||||
|
if score is None:
|
||||||
|
raise _drift(season_id, "malformed score", game.game_id, "result", text_of(link), "3 - 2")
|
||||||
|
event = EVENT_ID_RE.search(attr(link, "href"))
|
||||||
|
if event is None:
|
||||||
|
raise _drift(
|
||||||
|
season_id,
|
||||||
|
"result link without an event id",
|
||||||
|
game.game_id,
|
||||||
|
"result",
|
||||||
|
attr(link, "href"),
|
||||||
|
"/Game/Events/1109941",
|
||||||
|
)
|
||||||
|
game.home_goals, game.away_goals = int(score.group(1)), int(score.group(2))
|
||||||
|
game.event_id = event.group(1)
|
||||||
|
|
||||||
|
try:
|
||||||
|
game.periods = parse_periods(text_of(tds[lay.periods]))
|
||||||
|
except ValueError as exc:
|
||||||
|
raise _drift(season_id, str(exc), game.game_id, "periods", text_of(tds[lay.periods]), "(1-1, 2-1, 2-2)") from None
|
||||||
|
|
||||||
|
spectators = text_of(tds[lay.spectators])
|
||||||
|
if spectators:
|
||||||
|
if not DIGITS_RE.match(spectators):
|
||||||
|
raise _drift(season_id, "spectators are not a number", game.game_id, "spectators", spectators, "10442")
|
||||||
|
game.spectators = int(spectators)
|
||||||
|
game.venue = text_of(tds[lay.venue])
|
||||||
|
if not game.venue:
|
||||||
|
raise _drift(season_id, "empty venue", game.game_id, "venue", want="Gottde Arena")
|
||||||
|
|
||||||
|
carry.date, carry.round = game.date, game.round
|
||||||
|
try:
|
||||||
|
_check_row(game)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise _drift(season_id, str(exc), game.game_id) from None
|
||||||
|
return game
|
||||||
|
|
||||||
|
|
||||||
|
def _check_row(game: Game) -> None:
|
||||||
|
"""Periods sum to the final score, and a game is played or it is not."""
|
||||||
|
if not game.played:
|
||||||
|
if game.home_goals is not None or game.periods:
|
||||||
|
raise ValueError("unplayed game carries a result")
|
||||||
|
return
|
||||||
|
if len(game.periods) < 3:
|
||||||
|
raise ValueError(f"played game has {len(game.periods)} period scores")
|
||||||
|
home = sum(p.home for p in game.periods)
|
||||||
|
away = sum(p.away for p in game.periods)
|
||||||
|
if home != game.home_goals or away != game.away_goals:
|
||||||
|
raise ValueError(
|
||||||
|
f"periods sum to {home}-{away} but the result is {game.home_goals}-{game.away_goals}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_periods(text: str) -> list[Period]:
|
||||||
|
"""Read "(1-1, 2-1, 2-2)".
|
||||||
|
|
||||||
|
This is the single place to extend if the site ever renders a period
|
||||||
|
differently, for instance a shootout as "-".
|
||||||
|
"""
|
||||||
|
if not text:
|
||||||
|
return []
|
||||||
|
if not (text.startswith("(") and text.endswith(")")):
|
||||||
|
raise ValueError("period scores are not parenthesised")
|
||||||
|
tokens = text[1:-1].split(",")
|
||||||
|
# Only the lower bound is a contract. How many periods a game lasts depends
|
||||||
|
# on the era's rules - the site serves 3, 4, 5 and 6 period breakdowns - so
|
||||||
|
# the count is not pinned; the sum check in _check_row is what protects the
|
||||||
|
# data.
|
||||||
|
if len(tokens) < 3:
|
||||||
|
raise ValueError(f"only {len(tokens)} period scores")
|
||||||
|
out: list[Period] = []
|
||||||
|
for i, token in enumerate(tokens):
|
||||||
|
m = PERIOD_RE.match(token.strip())
|
||||||
|
if m is None:
|
||||||
|
raise ValueError(f'malformed period "{token.strip()}"')
|
||||||
|
out.append(Period(str(i + 1), int(m.group(1)), int(m.group(2))))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def find_table(doc: Node):
|
||||||
|
"""The first table that declares one of the known layouts.
|
||||||
|
|
||||||
|
Locating the table by its header, rather than by a CSS path or an index, is
|
||||||
|
what keeps ads, navigation and layout changes from breaking the crawler.
|
||||||
|
"""
|
||||||
|
for node in doc.walk():
|
||||||
|
if node.tag != "table":
|
||||||
|
continue
|
||||||
|
for tr in table_rows(node):
|
||||||
|
labels = tuple(th_labels(tr))
|
||||||
|
for lay in LAYOUTS:
|
||||||
|
if labels == lay.header:
|
||||||
|
return node, lay
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def document_signatures(doc: Node) -> list[str]:
|
||||||
|
"""Describe every table on the page, so an unrecognised layout is
|
||||||
|
diagnosable from the error message alone."""
|
||||||
|
out: list[str] = []
|
||||||
|
for node in doc.walk():
|
||||||
|
if node.tag != "table":
|
||||||
|
continue
|
||||||
|
for tr in table_rows(node):
|
||||||
|
labels = [t for t in (text_of(th) for th in child_elements(tr, "th")) if t]
|
||||||
|
if not labels:
|
||||||
|
continue
|
||||||
|
cells = 0
|
||||||
|
for row in table_rows(node):
|
||||||
|
cells = len(child_elements(row, "td"))
|
||||||
|
if cells:
|
||||||
|
break
|
||||||
|
out.append(f'"{" ".join(labels)}" with {cells} cells')
|
||||||
|
break
|
||||||
|
return out or ["no table with a header row"]
|
||||||
|
|
||||||
|
|
||||||
|
def known_layouts() -> str:
|
||||||
|
return ", ".join(f'{lay.name}="{" ".join(lay.header)}"' for lay in LAYOUTS)
|
||||||
|
|
||||||
|
|
||||||
|
def _tooltip_span(cell: Node) -> Node | None:
|
||||||
|
"""The span that carries a game's stable id."""
|
||||||
|
for span in descendants(cell, "span"):
|
||||||
|
if has_class(span, "lnkTooltip") and attr(span, "title"):
|
||||||
|
return span
|
||||||
|
return None
|
||||||
@@ -0,0 +1,104 @@
|
|||||||
|
"""League and season discovery: the two hops of the crawl.
|
||||||
|
|
||||||
|
The front page links every competition; each competition's page carries a
|
||||||
|
``<select>`` with one option per season. Both lists are sorted by id, so the
|
||||||
|
crawl order never depends on document order.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
from .dom import attr, descendants, parse_html, text_of
|
||||||
|
from .parse import DriftError
|
||||||
|
|
||||||
|
LEAGUE_PATH_RE = re.compile(r"^/ScheduleAndResults/[A-Za-z]+/(\d+)$")
|
||||||
|
SEASON_PATH_RE = re.compile(r"/ScheduleAndResults/[A-Za-z]+/(\d+)")
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class League:
|
||||||
|
"""One competition as listed on the front page."""
|
||||||
|
|
||||||
|
id: int
|
||||||
|
name: str
|
||||||
|
path: str # canonical path, e.g. /ScheduleAndResults/Schedule/20961
|
||||||
|
|
||||||
|
def as_dict(self) -> dict:
|
||||||
|
return {"id": self.id, "name": self.name, "path": self.path}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Season:
|
||||||
|
"""One entry of the season selector on a league page."""
|
||||||
|
|
||||||
|
id: int
|
||||||
|
label: str # "2026-27"
|
||||||
|
path: str # a path on the site for this season
|
||||||
|
|
||||||
|
def as_dict(self) -> dict:
|
||||||
|
return {"id": self.id, "label": self.label, "path": self.path}
|
||||||
|
|
||||||
|
|
||||||
|
def parse_leagues(page: str) -> list[League]:
|
||||||
|
"""Read the front page's league list.
|
||||||
|
|
||||||
|
A plain anchor crawl: every /ScheduleAndResults/<section>/<id> link,
|
||||||
|
deduplicated by id and sorted. A league is linked once per section (Live,
|
||||||
|
Overview, Schedule), so only the first name seen is kept.
|
||||||
|
"""
|
||||||
|
doc = parse_html(page)
|
||||||
|
names: dict[int, str] = {}
|
||||||
|
for a in descendants(doc, "a"):
|
||||||
|
m = LEAGUE_PATH_RE.match(attr(a, "href"))
|
||||||
|
if m is None:
|
||||||
|
continue
|
||||||
|
name = text_of(a)
|
||||||
|
if not name:
|
||||||
|
continue
|
||||||
|
names.setdefault(int(m.group(1)), name)
|
||||||
|
if not names:
|
||||||
|
raise DriftError(0, "front page has no league links", got="0 links", want="/ScheduleAndResults/<section>/<id>")
|
||||||
|
return [
|
||||||
|
League(id=league_id, name=name, path=schedule_path(league_id))
|
||||||
|
for league_id, name in sorted(names.items())
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def parse_seasons(league_id: int, page: str) -> list[Season]:
|
||||||
|
"""Read the season selector on a league page.
|
||||||
|
|
||||||
|
Group headings ("Cups", value="-1") are skipped; the result is sorted by id.
|
||||||
|
"""
|
||||||
|
doc = parse_html(page)
|
||||||
|
out: list[Season] = []
|
||||||
|
for opt in descendants(doc, "option"):
|
||||||
|
m = SEASON_PATH_RE.search(attr(opt, "value"))
|
||||||
|
if m is None:
|
||||||
|
continue
|
||||||
|
out.append(Season(id=int(m.group(1)), label=text_of(opt), path=attr(opt, "value")))
|
||||||
|
if not out:
|
||||||
|
raise DriftError(league_id, "no season options in the selector", got="0 options", want="/ScheduleAndResults/<section>/<id>")
|
||||||
|
seen: set[int] = set()
|
||||||
|
unique = []
|
||||||
|
for season in sorted(out, key=lambda s: s.id):
|
||||||
|
if season.id in seen:
|
||||||
|
continue
|
||||||
|
seen.add(season.id)
|
||||||
|
unique.append(season)
|
||||||
|
return unique
|
||||||
|
|
||||||
|
|
||||||
|
def schedule_path(season_id: int) -> str:
|
||||||
|
"""The canonical schedule URL of a season id.
|
||||||
|
|
||||||
|
Every view (Overview, Live, Schedule) of one season shares the id, so the
|
||||||
|
crawler derives the schedule URL instead of trusting whichever link it
|
||||||
|
arrived on.
|
||||||
|
"""
|
||||||
|
return f"/ScheduleAndResults/Schedule/{season_id}"
|
||||||
|
|
||||||
|
|
||||||
|
def join_url(base: str, path: str) -> str:
|
||||||
|
return base.rstrip("/") + path
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
"""Tests for the Python variant."""
|
||||||
@@ -0,0 +1,326 @@
|
|||||||
|
"""The Python implementation has to agree with the Go one, byte for byte.
|
||||||
|
|
||||||
|
The golden files in ``testdata/`` are the ones committed for the Go
|
||||||
|
implementation, so every test below is also a cross-implementation check: same
|
||||||
|
fixtures in, same bytes out. The suite is hermetic; the one test that talks to
|
||||||
|
the site is opt-in.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2] # repository root
|
||||||
|
FIXTURES = ROOT / "testdata"
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||||
|
|
||||||
|
from swehockey import ( # noqa: E402
|
||||||
|
Client,
|
||||||
|
DriftError,
|
||||||
|
parse_leagues,
|
||||||
|
parse_schedule,
|
||||||
|
parse_seasons,
|
||||||
|
render,
|
||||||
|
schedule_path,
|
||||||
|
)
|
||||||
|
from swehockey.fetch import check_charset # noqa: E402
|
||||||
|
from swehockey.output import first_diff # noqa: E402
|
||||||
|
|
||||||
|
MODERN_GOLDEN = [
|
||||||
|
("schedule_20961.html", "schedule_20961.json"),
|
||||||
|
("schedule_legacy.html", "schedule_legacy.json"),
|
||||||
|
("schedule_legacy_playoffs.html", "schedule_legacy_playoffs.json"),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def read_fixture(name: str) -> str:
|
||||||
|
return (FIXTURES / name).read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def parse_fixture(name: str):
|
||||||
|
return parse_schedule(20961, read_fixture(name))
|
||||||
|
|
||||||
|
|
||||||
|
def render_fixture(name: str) -> bytes:
|
||||||
|
return render("json", parse_fixture(name))
|
||||||
|
|
||||||
|
|
||||||
|
class TestGolden(unittest.TestCase):
|
||||||
|
"""Identical HTML must give identical bytes - and the same bytes the Go
|
||||||
|
implementation produces."""
|
||||||
|
|
||||||
|
def test_matches_the_go_goldens(self):
|
||||||
|
for fixture, golden in MODERN_GOLDEN:
|
||||||
|
with self.subTest(fixture=fixture):
|
||||||
|
got = render_fixture(fixture)
|
||||||
|
want = (FIXTURES / golden).read_bytes()
|
||||||
|
self.assertEqual(
|
||||||
|
got,
|
||||||
|
want,
|
||||||
|
f"Python output drifted from the Go golden:\n{first_diff(want, got)}",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_render_is_pure(self):
|
||||||
|
a = render_fixture("schedule_20961.html")
|
||||||
|
b = render_fixture("schedule_20961.html")
|
||||||
|
self.assertEqual(a, b, "two runs over the same page produced different bytes")
|
||||||
|
for forbidden in (b"fetched_at", b"generated_at", b"timestamp"):
|
||||||
|
self.assertNotIn(forbidden, a, "output must be a pure function of the page")
|
||||||
|
|
||||||
|
def test_csv(self):
|
||||||
|
data = render("csv", parse_fixture("schedule_20961.html")).decode("utf-8")
|
||||||
|
lines = data.rstrip("\n").split("\n")
|
||||||
|
self.assertEqual(len(lines), 11, f"csv has {len(lines)} lines, want 11")
|
||||||
|
self.assertEqual(
|
||||||
|
lines[0],
|
||||||
|
"season_id,game_id,event_id,date,time,home,away,home_goals,away_goals,periods,spectators,venue",
|
||||||
|
)
|
||||||
|
self.assertIn("1:1-1 2:2-1 3:2-2", lines[1])
|
||||||
|
# An unplayed game leaves the score cells empty rather than writing 0-0.
|
||||||
|
self.assertNotIn(",0,0,", lines[-1])
|
||||||
|
|
||||||
|
|
||||||
|
class TestModernLayout(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.schedule = parse_fixture("schedule_20961.html")
|
||||||
|
|
||||||
|
def test_shape(self):
|
||||||
|
self.assertEqual(self.schedule.layout, "modern")
|
||||||
|
self.assertEqual(self.schedule.last_update, "2026-09-26 22:41")
|
||||||
|
self.assertEqual(len(self.schedule.games), 10)
|
||||||
|
self.assertEqual(self.schedule.played, 8)
|
||||||
|
|
||||||
|
def test_date_carry_forward(self):
|
||||||
|
# A date is printed only on the first row of its group; the rest inherit.
|
||||||
|
want = ["2026-09-19"] * 6 + ["2026-09-22"] * 2 + ["2027-03-16"] * 2
|
||||||
|
self.assertEqual([g.date for g in self.schedule.games], want)
|
||||||
|
|
||||||
|
def test_periods_of_every_era(self):
|
||||||
|
self.assertEqual(len(self.schedule.games[1].periods), 4) # overtime
|
||||||
|
self.assertEqual(len(self.schedule.games[6].periods), 5) # shootout
|
||||||
|
|
||||||
|
def test_values(self):
|
||||||
|
first = self.schedule.games[0]
|
||||||
|
self.assertEqual((first.home, first.away), ("Frölunda HC", "Växjö Lakers HC"))
|
||||||
|
self.assertEqual(first.event_id, "1109941")
|
||||||
|
self.assertEqual(first.spectators, 12044)
|
||||||
|
self.assertEqual((first.home_goals, first.away_goals), (5, 4))
|
||||||
|
last = self.schedule.games[-1]
|
||||||
|
self.assertFalse(last.played)
|
||||||
|
self.assertIsNone(last.home_goals)
|
||||||
|
self.assertEqual(last.periods, [])
|
||||||
|
|
||||||
|
|
||||||
|
class TestLegacyLayout(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.schedule = parse_fixture("schedule_legacy.html")
|
||||||
|
|
||||||
|
def test_date_and_time_share_a_cell(self):
|
||||||
|
first = self.schedule.games[0]
|
||||||
|
self.assertEqual((first.date, first.time), ("2019-03-30", "15:15"))
|
||||||
|
self.assertEqual(self.schedule.layout, "legacy")
|
||||||
|
|
||||||
|
def test_round_carry_forward(self):
|
||||||
|
# The site prints the round only on the first row of that round.
|
||||||
|
self.assertEqual([g.round for g in self.schedule.games[:4]], ["8", "8", "8", "8"])
|
||||||
|
self.assertEqual([g.round for g in self.schedule.games[4:]], ["22", "23"])
|
||||||
|
|
||||||
|
def test_six_period_game(self):
|
||||||
|
# How many periods a game has depends on the era; the sum is the check.
|
||||||
|
first = self.schedule.games[0]
|
||||||
|
self.assertEqual(len(first.periods), 6)
|
||||||
|
self.assertEqual(first.home_goals, 3)
|
||||||
|
|
||||||
|
|
||||||
|
class TestLegacyPlayoffs(unittest.TestCase):
|
||||||
|
def test_team_cell_notes_are_dropped(self):
|
||||||
|
# Old team cells end with <i>Eighth final 1</i>, which is not a name.
|
||||||
|
first = parse_fixture("schedule_legacy_playoffs.html").games[0]
|
||||||
|
self.assertEqual((first.home, first.away), ("HV 71", "Leksands IF"))
|
||||||
|
self.assertEqual((first.date, first.time, first.round), ("2014-03-10", "19:00", "1"))
|
||||||
|
|
||||||
|
|
||||||
|
def mutate(name: str, pattern: str, repl: str) -> str:
|
||||||
|
"""Apply a regexp replacement to a fixture.
|
||||||
|
|
||||||
|
The pattern must match, otherwise the test would silently become vacuous.
|
||||||
|
"""
|
||||||
|
page = read_fixture(name)
|
||||||
|
new, count = re.subn(pattern, repl, page, flags=re.S)
|
||||||
|
if count == 0:
|
||||||
|
raise AssertionError(f"mutation pattern {pattern!r} does not match the fixture any more")
|
||||||
|
return new
|
||||||
|
|
||||||
|
|
||||||
|
class TestDrift(unittest.TestCase):
|
||||||
|
"""Every way the page can change shape has to produce a named, actionable
|
||||||
|
error instead of a plausible-looking table."""
|
||||||
|
|
||||||
|
CASES = [
|
||||||
|
("renamed header", r">Spectators<", ">Publik<", "no table with a known header"),
|
||||||
|
(
|
||||||
|
"extra column",
|
||||||
|
r'(<tr><td class="tdOdd[^>]*>2026-09-19</td>.*?)</tr>',
|
||||||
|
r"\1<td>x</td></tr>",
|
||||||
|
"row width does not match the modern layout",
|
||||||
|
),
|
||||||
|
("score no longer matches the periods", r">5\xa0-\xa04<", ">6\xa0-\xa04<", "periods sum to"),
|
||||||
|
("date group marker removed", r"<bold>2026-09-22</bold><br />", "2026-09-22<br />", "unexpected date in a continuation row"),
|
||||||
|
("duplicate game id", 'title="90001002"', 'title="90001001"', "duplicate game id"),
|
||||||
|
("no last-update stamp", "Last update:", "Last update was:", 'no "Last update:" stamp'),
|
||||||
|
("empty table", r"<tr><td class=\"tdOdd.*", "", "no game rows"),
|
||||||
|
("the start time stops parsing", r">15:15</span>", ">15h15</span>", "malformed start time"),
|
||||||
|
("the venue column is empty", r">Scandinavium</td>", "></td>", "empty venue"),
|
||||||
|
]
|
||||||
|
|
||||||
|
def test_each_mutation_is_an_actionable_error(self):
|
||||||
|
for name, pattern, repl, want in self.CASES:
|
||||||
|
with self.subTest(case=name):
|
||||||
|
page = mutate("schedule_20961.html", pattern, repl)
|
||||||
|
with self.assertRaises(DriftError) as caught:
|
||||||
|
parse_schedule(20961, page)
|
||||||
|
message = str(caught.exception)
|
||||||
|
self.assertIn(want, message)
|
||||||
|
self.assertIn("parse.py", message, "the error should say what to do")
|
||||||
|
|
||||||
|
def test_unknown_layout_quotes_the_header_it_found(self):
|
||||||
|
page = mutate("schedule_20961.html", r">Game<", ">Match<")
|
||||||
|
with self.assertRaises(DriftError) as caught:
|
||||||
|
parse_schedule(20961, page)
|
||||||
|
message = str(caught.exception)
|
||||||
|
self.assertIn("no table with a known header", message)
|
||||||
|
self.assertIn("Date Time Game Result Spectators Venue", message)
|
||||||
|
|
||||||
|
def test_round_trip_of_a_broken_page(self):
|
||||||
|
# A plain Exception subclass is what a caller needs to branch on.
|
||||||
|
with self.assertRaises(DriftError):
|
||||||
|
parse_schedule(20961, mutate("schedule_20961.html", r">Spectators<", ">Publik<"))
|
||||||
|
|
||||||
|
|
||||||
|
class TestOrderIndependence(unittest.TestCase):
|
||||||
|
def test_swapping_two_rows_changes_nothing(self):
|
||||||
|
# The site orders rows by round and by whatever it felt like printing,
|
||||||
|
# so the crawler imposes its own canonical order instead.
|
||||||
|
raw = read_fixture("schedule_20961.html")
|
||||||
|
|
||||||
|
def row_of(game_id: str) -> str:
|
||||||
|
i = raw.index(f'title="{game_id}"')
|
||||||
|
start = raw.rindex("<tr>", 0, i)
|
||||||
|
end = raw.index("</tr>", i) + len("</tr>")
|
||||||
|
return raw[start:end]
|
||||||
|
|
||||||
|
row_a, row_b = row_of("90001003"), row_of("90001004")
|
||||||
|
swapped = raw.replace(row_b, "@@ROW@@").replace(row_a, row_b).replace("@@ROW@@", row_a)
|
||||||
|
self.assertNotEqual(swapped, raw, "the swap did not change anything")
|
||||||
|
self.assertEqual(render_fixture("schedule_20961.html"), render("json", parse_schedule(20961, swapped)))
|
||||||
|
|
||||||
|
|
||||||
|
class TestDiscovery(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.page = read_fixture("seasons_20961.html")
|
||||||
|
|
||||||
|
def test_seasons(self):
|
||||||
|
seasons = parse_seasons(20961, self.page)
|
||||||
|
self.assertGreaterEqual(len(seasons), 2)
|
||||||
|
ids = [s.id for s in seasons]
|
||||||
|
self.assertEqual(ids, sorted(ids), "seasons must be sorted by id")
|
||||||
|
self.assertTrue(all(s.id > 0 for s in seasons), "group headings leaked into the list")
|
||||||
|
by_id = {s.id: s for s in seasons}
|
||||||
|
self.assertEqual(by_id[20961].label, "2026-27")
|
||||||
|
self.assertIn(18263, by_id)
|
||||||
|
|
||||||
|
def test_leagues(self):
|
||||||
|
leagues = parse_leagues(self.page)
|
||||||
|
self.assertTrue(leagues)
|
||||||
|
ids = [l.id for l in leagues]
|
||||||
|
self.assertEqual(ids, sorted(ids), "leagues must be sorted and deduplicated")
|
||||||
|
|
||||||
|
def test_schedule_path(self):
|
||||||
|
self.assertEqual(schedule_path(20961), "/ScheduleAndResults/Schedule/20961")
|
||||||
|
|
||||||
|
|
||||||
|
class TestFetch(unittest.TestCase):
|
||||||
|
def test_charset_guard(self):
|
||||||
|
with self.assertRaises(Exception):
|
||||||
|
check_charset(b'<meta charset="iso-8859-1">')
|
||||||
|
check_charset(b'<meta charset="utf-8">')
|
||||||
|
check_charset(b"<html><body>no declaration</body></html>")
|
||||||
|
|
||||||
|
|
||||||
|
class TestCli(unittest.TestCase):
|
||||||
|
def test_exit_code_zero_on_a_fixture(self):
|
||||||
|
from swehockey.cli import main
|
||||||
|
|
||||||
|
out = str(FIXTURES / "_cli_test.json")
|
||||||
|
try:
|
||||||
|
code = main(
|
||||||
|
[
|
||||||
|
"schedule",
|
||||||
|
"--fixture",
|
||||||
|
str(FIXTURES / "schedule_20961.html"),
|
||||||
|
"--out",
|
||||||
|
out,
|
||||||
|
"--quiet",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
self.assertEqual(code, 0)
|
||||||
|
self.assertEqual(Path(out).read_bytes(), (FIXTURES / "schedule_20961.json").read_bytes())
|
||||||
|
finally:
|
||||||
|
if os.path.exists(out):
|
||||||
|
os.unlink(out)
|
||||||
|
|
||||||
|
def test_exit_code_three_on_drift(self):
|
||||||
|
from swehockey.cli import main
|
||||||
|
|
||||||
|
broken = FIXTURES / "_cli_drift.html"
|
||||||
|
broken.write_text(mutate("schedule_20961.html", r">Spectators<", ">Publik<"), encoding="utf-8")
|
||||||
|
try:
|
||||||
|
code = main(["schedule", "--fixture", str(broken), "--quiet"])
|
||||||
|
self.assertEqual(code, 3, "markup drift must be exit code 3, not a crash")
|
||||||
|
finally:
|
||||||
|
os.unlink(broken)
|
||||||
|
|
||||||
|
def test_check_flag_reports_a_mismatch(self):
|
||||||
|
from swehockey.cli import main
|
||||||
|
|
||||||
|
wrong = FIXTURES / "_cli_wrong.json"
|
||||||
|
wrong.write_bytes(b'{"season_id":1}\n')
|
||||||
|
try:
|
||||||
|
code = main(
|
||||||
|
[
|
||||||
|
"schedule",
|
||||||
|
"--fixture",
|
||||||
|
str(FIXTURES / "schedule_20961.html"),
|
||||||
|
"--check",
|
||||||
|
str(wrong),
|
||||||
|
"--quiet",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
self.assertEqual(code, 3)
|
||||||
|
finally:
|
||||||
|
os.unlink(wrong)
|
||||||
|
|
||||||
|
|
||||||
|
@unittest.skipUnless(os.environ.get("SWEHOCKEY_LIVE"), "set SWEHOCKEY_LIVE=1 to hit the real site")
|
||||||
|
class TestLive(unittest.TestCase):
|
||||||
|
"""The one test that talks to the site, and how you find out about drift
|
||||||
|
the day it happens rather than the day a fixture is regenerated."""
|
||||||
|
|
||||||
|
CASES = [(20961, "modern"), (18263, "modern"), (17556, "legacy")]
|
||||||
|
|
||||||
|
def test_live_seasons(self):
|
||||||
|
client = Client(timeout=60.0)
|
||||||
|
for season_id, layout in self.CASES:
|
||||||
|
with self.subTest(season=season_id):
|
||||||
|
schedule = parse_schedule(season_id, client.get(schedule_path(season_id)))
|
||||||
|
self.assertEqual(schedule.layout, layout)
|
||||||
|
self.assertTrue(schedule.games)
|
||||||
|
self.assertTrue(schedule.last_update)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main(verbosity=2)
|
||||||
Reference in new issue
Block a user