Mats Tornberg
6b701819ca
One HTTP GET per season, no browser: the page is server-rendered, so every row arrives in one response and the extraction becomes a pure function of the HTML. - layouts[] declares one profile per version of the site's table (modern and legacy), each identified by its header row, so a new version of the page is a new profile plus a fixture instead of a guess. - The table is located by header signature, not by a CSS path. - Every cell is shape-checked, and facts the site renders twice (desktop and mobile date, time in the tooltip) must agree. - Cross-field invariants: periods sum to the score, ids are unique, played games always have an event link and unplayed ones never do. - Output is sorted by date, time and id, so it does not inherit the site's row order, and it carries no fetch timestamp: same HTML, same bytes. - Drift is a typed error and exit code 3, with the header it actually found. - Tests run on committed HTML fixtures and golden JSON; TestLive is opt-in. Verified over the full SHL archive: 27/27 seasons, 2119 games, 1779 played.
122 lines
3.2 KiB
Go
122 lines
3.2 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// userAgent identifies the crawler. The site is a public ASP.NET app and serves
|
|
// the same bytes to any client; saying who we are keeps its logs readable.
|
|
const userAgent = "swehockey-schedule/1.0 (+deterministic schedule crawler)"
|
|
|
|
var errUnchanged = errors.New("swehockey: page unchanged")
|
|
|
|
// Client fetches pages. One request, no retries, no cookies, no redirects
|
|
// beyond the default limit: every byte of the output is a function of one
|
|
// response.
|
|
type Client struct {
|
|
BaseURL string
|
|
Timeout time.Duration
|
|
}
|
|
|
|
func (c *Client) get(ctx context.Context, path string) ([]byte, error) {
|
|
hc := &http.Client{Timeout: c.Timeout}
|
|
url := joinURL(c.BaseURL, path)
|
|
ctx, cancel := context.WithTimeout(ctx, c.Timeout)
|
|
defer cancel()
|
|
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
req.Header.Set("User-Agent", userAgent)
|
|
req.Header.Set("Accept", "text/html,application/xhtml+xml")
|
|
req.Header.Set("Accept-Language", "en")
|
|
|
|
resp, err := hc.Do(req)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("fetch %s: %w", url, err)
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
return nil, fmt.Errorf("fetch %s: %s", url, resp.Status)
|
|
}
|
|
body, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read %s: %w", url, err)
|
|
}
|
|
if err := checkCharset(body); err != nil {
|
|
return nil, fmt.Errorf("fetch %s: %w", url, err)
|
|
}
|
|
return body, nil
|
|
}
|
|
|
|
var metaCharsetRE = regexp.MustCompile(`(?i)<meta[^>]+charset\s*=\s*["']?([a-z0-9_-]+)`)
|
|
|
|
// checkCharset refuses pages we cannot decode byte-exactly. Silently handing
|
|
// latin-1 bytes to a UTF-8 parser would turn "Växjö" into mojibake that still
|
|
// parses, which is the worst kind of crawler bug.
|
|
func checkCharset(body []byte) error {
|
|
m := metaCharsetRE.FindSubmatch(body)
|
|
if m == nil {
|
|
return nil // no declaration: html.Parse assumes UTF-8
|
|
}
|
|
switch cs := strings.ToLower(string(m[1])); cs {
|
|
case "utf-8", "utf8":
|
|
return nil
|
|
default:
|
|
return fmt.Errorf("unsupported charset %q: add a decoder for it", cs)
|
|
}
|
|
}
|
|
|
|
// state records what the last successful run saw, so a cron job can be
|
|
// idempotent: an unchanged page produces no output and a zero exit code.
|
|
type state struct {
|
|
Path string `json:"path"`
|
|
SHA256 string `json:"sha256"`
|
|
LastUpdate string `json:"last_update"`
|
|
SavedAt string `json:"saved_at"`
|
|
}
|
|
|
|
func loadState(path string) (*state, error) {
|
|
b, err := os.ReadFile(path)
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
return &state{}, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var st state
|
|
if err := json.Unmarshal(b, &st); err != nil {
|
|
return nil, fmt.Errorf("state file %s: %w", path, err)
|
|
}
|
|
return &st, nil
|
|
}
|
|
|
|
// saveState writes the state atomically so a killed run cannot leave a
|
|
// truncated file that would make the next run skip real work.
|
|
func saveState(path string, st *state) error {
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
|
return err
|
|
}
|
|
b, err := json.MarshalIndent(st, "", " ")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
b = append(b, '\n')
|
|
tmp := path + ".tmp"
|
|
if err := os.WriteFile(tmp, b, 0o644); err != nil {
|
|
return err
|
|
}
|
|
return os.Rename(tmp, path)
|
|
}
|