Mats Tornberg
47e1811d9e
A second, independent implementation of the same contract, in the same repo: - swehockey/dom.py builds the tree on html.parser, so the variant has no runtime dependencies; the only HTML5 behaviour it depends on - stray <tr> landing in an implicit <tbody> - is handled explicitly by table_rows(). - parse.py mirrors the Go layout profiles, dual-render checks, carry-forward and invariants; a DriftError is raised in the same places with the same advice. - cli.py offers the same three commands, flags and exit codes as the Go main. - tests/ assert against the Go golden files, so the suite is also a cross-implementation check. 23 offline tests plus an opt-in live test. Verified against the live site: JSON and CSV are byte-identical to the Go output (364 games, 102536 bytes / 365 lines), and the full archive crawl gives the same 27 seasons, 2119 games and 1779 played. Also split the DriftError hint out in Go, so a golden mismatch no longer tells the reader to extend layouts[].
105 lines
3.2 KiB
Python
105 lines
3.2 KiB
Python
"""League and season discovery: the two hops of the crawl.
|
|
|
|
The front page links every competition; each competition's page carries a
|
|
``<select>`` with one option per season. Both lists are sorted by id, so the
|
|
crawl order never depends on document order.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from dataclasses import dataclass
|
|
|
|
from .dom import attr, descendants, parse_html, text_of
|
|
from .parse import DriftError
|
|
|
|
LEAGUE_PATH_RE = re.compile(r"^/ScheduleAndResults/[A-Za-z]+/(\d+)$")
|
|
SEASON_PATH_RE = re.compile(r"/ScheduleAndResults/[A-Za-z]+/(\d+)")
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class League:
|
|
"""One competition as listed on the front page."""
|
|
|
|
id: int
|
|
name: str
|
|
path: str # canonical path, e.g. /ScheduleAndResults/Schedule/20961
|
|
|
|
def as_dict(self) -> dict:
|
|
return {"id": self.id, "name": self.name, "path": self.path}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Season:
|
|
"""One entry of the season selector on a league page."""
|
|
|
|
id: int
|
|
label: str # "2026-27"
|
|
path: str # a path on the site for this season
|
|
|
|
def as_dict(self) -> dict:
|
|
return {"id": self.id, "label": self.label, "path": self.path}
|
|
|
|
|
|
def parse_leagues(page: str) -> list[League]:
|
|
"""Read the front page's league list.
|
|
|
|
A plain anchor crawl: every /ScheduleAndResults/<section>/<id> link,
|
|
deduplicated by id and sorted. A league is linked once per section (Live,
|
|
Overview, Schedule), so only the first name seen is kept.
|
|
"""
|
|
doc = parse_html(page)
|
|
names: dict[int, str] = {}
|
|
for a in descendants(doc, "a"):
|
|
m = LEAGUE_PATH_RE.match(attr(a, "href"))
|
|
if m is None:
|
|
continue
|
|
name = text_of(a)
|
|
if not name:
|
|
continue
|
|
names.setdefault(int(m.group(1)), name)
|
|
if not names:
|
|
raise DriftError(0, "front page has no league links", got="0 links", want="/ScheduleAndResults/<section>/<id>")
|
|
return [
|
|
League(id=league_id, name=name, path=schedule_path(league_id))
|
|
for league_id, name in sorted(names.items())
|
|
]
|
|
|
|
|
|
def parse_seasons(league_id: int, page: str) -> list[Season]:
|
|
"""Read the season selector on a league page.
|
|
|
|
Group headings ("Cups", value="-1") are skipped; the result is sorted by id.
|
|
"""
|
|
doc = parse_html(page)
|
|
out: list[Season] = []
|
|
for opt in descendants(doc, "option"):
|
|
m = SEASON_PATH_RE.search(attr(opt, "value"))
|
|
if m is None:
|
|
continue
|
|
out.append(Season(id=int(m.group(1)), label=text_of(opt), path=attr(opt, "value")))
|
|
if not out:
|
|
raise DriftError(league_id, "no season options in the selector", got="0 options", want="/ScheduleAndResults/<section>/<id>")
|
|
seen: set[int] = set()
|
|
unique = []
|
|
for season in sorted(out, key=lambda s: s.id):
|
|
if season.id in seen:
|
|
continue
|
|
seen.add(season.id)
|
|
unique.append(season)
|
|
return unique
|
|
|
|
|
|
def schedule_path(season_id: int) -> str:
|
|
"""The canonical schedule URL of a season id.
|
|
|
|
Every view (Overview, Live, Schedule) of one season shares the id, so the
|
|
crawler derives the schedule URL instead of trusting whichever link it
|
|
arrived on.
|
|
"""
|
|
return f"/ScheduleAndResults/Schedule/{season_id}"
|
|
|
|
|
|
def join_url(base: str, path: str) -> str:
|
|
return base.rstrip("/") + path
|