Add the Python variant, agreeing with Go byte for byte
A second, independent implementation of the same contract, in the same repo: - swehockey/dom.py builds the tree on html.parser, so the variant has no runtime dependencies; the only HTML5 behaviour it depends on - stray <tr> landing in an implicit <tbody> - is handled explicitly by table_rows(). - parse.py mirrors the Go layout profiles, dual-render checks, carry-forward and invariants; a DriftError is raised in the same places with the same advice. - cli.py offers the same three commands, flags and exit codes as the Go main. - tests/ assert against the Go golden files, so the suite is also a cross-implementation check. 23 offline tests plus an opt-in live test. Verified against the live site: JSON and CSV are byte-identical to the Go output (364 games, 102536 bytes / 365 lines), and the full archive crawl gives the same 27 seasons, 2119 games and 1779 played. Also split the DriftError hint out in Go, so a golden mismatch no longer tells the reader to extend layouts[].
This commit is contained in:
Mats Tornberg committed
1 parent
6b701819ca
commit
47e1811d9e
15 files changed
+1821
-3
No files matched your search
@@ -0,0 +1 @@
|
||||
"""Tests for the Python variant."""
|
||||
@@ -0,0 +1,326 @@
|
||||
"""The Python implementation has to agree with the Go one, byte for byte.
|
||||
|
||||
The golden files in ``testdata/`` are the ones committed for the Go
|
||||
implementation, so every test below is also a cross-implementation check: same
|
||||
fixtures in, same bytes out. The suite is hermetic; the one test that talks to
|
||||
the site is opt-in.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2] # repository root
|
||||
FIXTURES = ROOT / "testdata"
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from swehockey import ( # noqa: E402
|
||||
Client,
|
||||
DriftError,
|
||||
parse_leagues,
|
||||
parse_schedule,
|
||||
parse_seasons,
|
||||
render,
|
||||
schedule_path,
|
||||
)
|
||||
from swehockey.fetch import check_charset # noqa: E402
|
||||
from swehockey.output import first_diff # noqa: E402
|
||||
|
||||
MODERN_GOLDEN = [
|
||||
("schedule_20961.html", "schedule_20961.json"),
|
||||
("schedule_legacy.html", "schedule_legacy.json"),
|
||||
("schedule_legacy_playoffs.html", "schedule_legacy_playoffs.json"),
|
||||
]
|
||||
|
||||
|
||||
def read_fixture(name: str) -> str:
|
||||
return (FIXTURES / name).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def parse_fixture(name: str):
|
||||
return parse_schedule(20961, read_fixture(name))
|
||||
|
||||
|
||||
def render_fixture(name: str) -> bytes:
|
||||
return render("json", parse_fixture(name))
|
||||
|
||||
|
||||
class TestGolden(unittest.TestCase):
|
||||
"""Identical HTML must give identical bytes - and the same bytes the Go
|
||||
implementation produces."""
|
||||
|
||||
def test_matches_the_go_goldens(self):
|
||||
for fixture, golden in MODERN_GOLDEN:
|
||||
with self.subTest(fixture=fixture):
|
||||
got = render_fixture(fixture)
|
||||
want = (FIXTURES / golden).read_bytes()
|
||||
self.assertEqual(
|
||||
got,
|
||||
want,
|
||||
f"Python output drifted from the Go golden:\n{first_diff(want, got)}",
|
||||
)
|
||||
|
||||
def test_render_is_pure(self):
|
||||
a = render_fixture("schedule_20961.html")
|
||||
b = render_fixture("schedule_20961.html")
|
||||
self.assertEqual(a, b, "two runs over the same page produced different bytes")
|
||||
for forbidden in (b"fetched_at", b"generated_at", b"timestamp"):
|
||||
self.assertNotIn(forbidden, a, "output must be a pure function of the page")
|
||||
|
||||
def test_csv(self):
|
||||
data = render("csv", parse_fixture("schedule_20961.html")).decode("utf-8")
|
||||
lines = data.rstrip("\n").split("\n")
|
||||
self.assertEqual(len(lines), 11, f"csv has {len(lines)} lines, want 11")
|
||||
self.assertEqual(
|
||||
lines[0],
|
||||
"season_id,game_id,event_id,date,time,home,away,home_goals,away_goals,periods,spectators,venue",
|
||||
)
|
||||
self.assertIn("1:1-1 2:2-1 3:2-2", lines[1])
|
||||
# An unplayed game leaves the score cells empty rather than writing 0-0.
|
||||
self.assertNotIn(",0,0,", lines[-1])
|
||||
|
||||
|
||||
class TestModernLayout(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.schedule = parse_fixture("schedule_20961.html")
|
||||
|
||||
def test_shape(self):
|
||||
self.assertEqual(self.schedule.layout, "modern")
|
||||
self.assertEqual(self.schedule.last_update, "2026-09-26 22:41")
|
||||
self.assertEqual(len(self.schedule.games), 10)
|
||||
self.assertEqual(self.schedule.played, 8)
|
||||
|
||||
def test_date_carry_forward(self):
|
||||
# A date is printed only on the first row of its group; the rest inherit.
|
||||
want = ["2026-09-19"] * 6 + ["2026-09-22"] * 2 + ["2027-03-16"] * 2
|
||||
self.assertEqual([g.date for g in self.schedule.games], want)
|
||||
|
||||
def test_periods_of_every_era(self):
|
||||
self.assertEqual(len(self.schedule.games[1].periods), 4) # overtime
|
||||
self.assertEqual(len(self.schedule.games[6].periods), 5) # shootout
|
||||
|
||||
def test_values(self):
|
||||
first = self.schedule.games[0]
|
||||
self.assertEqual((first.home, first.away), ("Frölunda HC", "Växjö Lakers HC"))
|
||||
self.assertEqual(first.event_id, "1109941")
|
||||
self.assertEqual(first.spectators, 12044)
|
||||
self.assertEqual((first.home_goals, first.away_goals), (5, 4))
|
||||
last = self.schedule.games[-1]
|
||||
self.assertFalse(last.played)
|
||||
self.assertIsNone(last.home_goals)
|
||||
self.assertEqual(last.periods, [])
|
||||
|
||||
|
||||
class TestLegacyLayout(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.schedule = parse_fixture("schedule_legacy.html")
|
||||
|
||||
def test_date_and_time_share_a_cell(self):
|
||||
first = self.schedule.games[0]
|
||||
self.assertEqual((first.date, first.time), ("2019-03-30", "15:15"))
|
||||
self.assertEqual(self.schedule.layout, "legacy")
|
||||
|
||||
def test_round_carry_forward(self):
|
||||
# The site prints the round only on the first row of that round.
|
||||
self.assertEqual([g.round for g in self.schedule.games[:4]], ["8", "8", "8", "8"])
|
||||
self.assertEqual([g.round for g in self.schedule.games[4:]], ["22", "23"])
|
||||
|
||||
def test_six_period_game(self):
|
||||
# How many periods a game has depends on the era; the sum is the check.
|
||||
first = self.schedule.games[0]
|
||||
self.assertEqual(len(first.periods), 6)
|
||||
self.assertEqual(first.home_goals, 3)
|
||||
|
||||
|
||||
class TestLegacyPlayoffs(unittest.TestCase):
|
||||
def test_team_cell_notes_are_dropped(self):
|
||||
# Old team cells end with <i>Eighth final 1</i>, which is not a name.
|
||||
first = parse_fixture("schedule_legacy_playoffs.html").games[0]
|
||||
self.assertEqual((first.home, first.away), ("HV 71", "Leksands IF"))
|
||||
self.assertEqual((first.date, first.time, first.round), ("2014-03-10", "19:00", "1"))
|
||||
|
||||
|
||||
def mutate(name: str, pattern: str, repl: str) -> str:
|
||||
"""Apply a regexp replacement to a fixture.
|
||||
|
||||
The pattern must match, otherwise the test would silently become vacuous.
|
||||
"""
|
||||
page = read_fixture(name)
|
||||
new, count = re.subn(pattern, repl, page, flags=re.S)
|
||||
if count == 0:
|
||||
raise AssertionError(f"mutation pattern {pattern!r} does not match the fixture any more")
|
||||
return new
|
||||
|
||||
|
||||
class TestDrift(unittest.TestCase):
|
||||
"""Every way the page can change shape has to produce a named, actionable
|
||||
error instead of a plausible-looking table."""
|
||||
|
||||
CASES = [
|
||||
("renamed header", r">Spectators<", ">Publik<", "no table with a known header"),
|
||||
(
|
||||
"extra column",
|
||||
r'(<tr><td class="tdOdd[^>]*>2026-09-19</td>.*?)</tr>',
|
||||
r"\1<td>x</td></tr>",
|
||||
"row width does not match the modern layout",
|
||||
),
|
||||
("score no longer matches the periods", r">5\xa0-\xa04<", ">6\xa0-\xa04<", "periods sum to"),
|
||||
("date group marker removed", r"<bold>2026-09-22</bold><br />", "2026-09-22<br />", "unexpected date in a continuation row"),
|
||||
("duplicate game id", 'title="90001002"', 'title="90001001"', "duplicate game id"),
|
||||
("no last-update stamp", "Last update:", "Last update was:", 'no "Last update:" stamp'),
|
||||
("empty table", r"<tr><td class=\"tdOdd.*", "", "no game rows"),
|
||||
("the start time stops parsing", r">15:15</span>", ">15h15</span>", "malformed start time"),
|
||||
("the venue column is empty", r">Scandinavium</td>", "></td>", "empty venue"),
|
||||
]
|
||||
|
||||
def test_each_mutation_is_an_actionable_error(self):
|
||||
for name, pattern, repl, want in self.CASES:
|
||||
with self.subTest(case=name):
|
||||
page = mutate("schedule_20961.html", pattern, repl)
|
||||
with self.assertRaises(DriftError) as caught:
|
||||
parse_schedule(20961, page)
|
||||
message = str(caught.exception)
|
||||
self.assertIn(want, message)
|
||||
self.assertIn("parse.py", message, "the error should say what to do")
|
||||
|
||||
def test_unknown_layout_quotes_the_header_it_found(self):
|
||||
page = mutate("schedule_20961.html", r">Game<", ">Match<")
|
||||
with self.assertRaises(DriftError) as caught:
|
||||
parse_schedule(20961, page)
|
||||
message = str(caught.exception)
|
||||
self.assertIn("no table with a known header", message)
|
||||
self.assertIn("Date Time Game Result Spectators Venue", message)
|
||||
|
||||
def test_round_trip_of_a_broken_page(self):
|
||||
# A plain Exception subclass is what a caller needs to branch on.
|
||||
with self.assertRaises(DriftError):
|
||||
parse_schedule(20961, mutate("schedule_20961.html", r">Spectators<", ">Publik<"))
|
||||
|
||||
|
||||
class TestOrderIndependence(unittest.TestCase):
|
||||
def test_swapping_two_rows_changes_nothing(self):
|
||||
# The site orders rows by round and by whatever it felt like printing,
|
||||
# so the crawler imposes its own canonical order instead.
|
||||
raw = read_fixture("schedule_20961.html")
|
||||
|
||||
def row_of(game_id: str) -> str:
|
||||
i = raw.index(f'title="{game_id}"')
|
||||
start = raw.rindex("<tr>", 0, i)
|
||||
end = raw.index("</tr>", i) + len("</tr>")
|
||||
return raw[start:end]
|
||||
|
||||
row_a, row_b = row_of("90001003"), row_of("90001004")
|
||||
swapped = raw.replace(row_b, "@@ROW@@").replace(row_a, row_b).replace("@@ROW@@", row_a)
|
||||
self.assertNotEqual(swapped, raw, "the swap did not change anything")
|
||||
self.assertEqual(render_fixture("schedule_20961.html"), render("json", parse_schedule(20961, swapped)))
|
||||
|
||||
|
||||
class TestDiscovery(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.page = read_fixture("seasons_20961.html")
|
||||
|
||||
def test_seasons(self):
|
||||
seasons = parse_seasons(20961, self.page)
|
||||
self.assertGreaterEqual(len(seasons), 2)
|
||||
ids = [s.id for s in seasons]
|
||||
self.assertEqual(ids, sorted(ids), "seasons must be sorted by id")
|
||||
self.assertTrue(all(s.id > 0 for s in seasons), "group headings leaked into the list")
|
||||
by_id = {s.id: s for s in seasons}
|
||||
self.assertEqual(by_id[20961].label, "2026-27")
|
||||
self.assertIn(18263, by_id)
|
||||
|
||||
def test_leagues(self):
|
||||
leagues = parse_leagues(self.page)
|
||||
self.assertTrue(leagues)
|
||||
ids = [l.id for l in leagues]
|
||||
self.assertEqual(ids, sorted(ids), "leagues must be sorted and deduplicated")
|
||||
|
||||
def test_schedule_path(self):
|
||||
self.assertEqual(schedule_path(20961), "/ScheduleAndResults/Schedule/20961")
|
||||
|
||||
|
||||
class TestFetch(unittest.TestCase):
|
||||
def test_charset_guard(self):
|
||||
with self.assertRaises(Exception):
|
||||
check_charset(b'<meta charset="iso-8859-1">')
|
||||
check_charset(b'<meta charset="utf-8">')
|
||||
check_charset(b"<html><body>no declaration</body></html>")
|
||||
|
||||
|
||||
class TestCli(unittest.TestCase):
|
||||
def test_exit_code_zero_on_a_fixture(self):
|
||||
from swehockey.cli import main
|
||||
|
||||
out = str(FIXTURES / "_cli_test.json")
|
||||
try:
|
||||
code = main(
|
||||
[
|
||||
"schedule",
|
||||
"--fixture",
|
||||
str(FIXTURES / "schedule_20961.html"),
|
||||
"--out",
|
||||
out,
|
||||
"--quiet",
|
||||
]
|
||||
)
|
||||
self.assertEqual(code, 0)
|
||||
self.assertEqual(Path(out).read_bytes(), (FIXTURES / "schedule_20961.json").read_bytes())
|
||||
finally:
|
||||
if os.path.exists(out):
|
||||
os.unlink(out)
|
||||
|
||||
def test_exit_code_three_on_drift(self):
|
||||
from swehockey.cli import main
|
||||
|
||||
broken = FIXTURES / "_cli_drift.html"
|
||||
broken.write_text(mutate("schedule_20961.html", r">Spectators<", ">Publik<"), encoding="utf-8")
|
||||
try:
|
||||
code = main(["schedule", "--fixture", str(broken), "--quiet"])
|
||||
self.assertEqual(code, 3, "markup drift must be exit code 3, not a crash")
|
||||
finally:
|
||||
os.unlink(broken)
|
||||
|
||||
def test_check_flag_reports_a_mismatch(self):
|
||||
from swehockey.cli import main
|
||||
|
||||
wrong = FIXTURES / "_cli_wrong.json"
|
||||
wrong.write_bytes(b'{"season_id":1}\n')
|
||||
try:
|
||||
code = main(
|
||||
[
|
||||
"schedule",
|
||||
"--fixture",
|
||||
str(FIXTURES / "schedule_20961.html"),
|
||||
"--check",
|
||||
str(wrong),
|
||||
"--quiet",
|
||||
]
|
||||
)
|
||||
self.assertEqual(code, 3)
|
||||
finally:
|
||||
os.unlink(wrong)
|
||||
|
||||
|
||||
@unittest.skipUnless(os.environ.get("SWEHOCKEY_LIVE"), "set SWEHOCKEY_LIVE=1 to hit the real site")
|
||||
class TestLive(unittest.TestCase):
|
||||
"""The one test that talks to the site, and how you find out about drift
|
||||
the day it happens rather than the day a fixture is regenerated."""
|
||||
|
||||
CASES = [(20961, "modern"), (18263, "modern"), (17556, "legacy")]
|
||||
|
||||
def test_live_seasons(self):
|
||||
client = Client(timeout=60.0)
|
||||
for season_id, layout in self.CASES:
|
||||
with self.subTest(season=season_id):
|
||||
schedule = parse_schedule(season_id, client.get(schedule_path(season_id)))
|
||||
self.assertEqual(schedule.layout, layout)
|
||||
self.assertTrue(schedule.games)
|
||||
self.assertTrue(schedule.last_update)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
Reference in new issue
Block a user