swehockey/cmd_crawl.go

package main

import (
	"bytes"
	"context"
	"errors"
	"flag"
	"fmt"
	"os"
	"time"
)

// cmdCrawl is the crawler proper: take one league, read its season selector,
// then walk every season in ascending id order and emit one document per
// season. The order is derived from the ids, never from map iteration or from
// the order the site happened to list things, so two runs of the crawler
// produce the same file even if the site's markup order changes.
func cmdCrawl(ctx context.Context, args []string) error {
	fs := flag.NewFlagSet("crawl", flag.ContinueOnError)
	var c commonFlags
	c.register(fs, "ndjson")
	league := fs.Int("league", defaultSeason, "league id (same id space as season ids)")
	only := fs.Int("season", 0, "crawl only this season id (0 = all seasons of the league)")
	limit := fs.Int("limit", 0, "stop after N seasons (0 = all)")
	delay := fs.Duration("delay", 250*time.Millisecond, "pause between season requests")
	onDrift := fs.String("on-drift", "fail", "when a season page breaks the contract: fail or record")
	if err := fs.Parse(args); err != nil {
		return err
	}
	cl := c.client()

	seed, err := c.read(ctx, cl, SchedulePath(*league), fmt.Sprintf("league %d", *league))
	if errors.Is(err, errUnchanged) {
		return nil
	}
	if err != nil {
		return err
	}
	seasons, err := ParseSeasons(*league, bytes.NewReader(seed))
	if err != nil {
		return err
	}

	var out bytes.Buffer
	enc := jsonEncoder(&out)
	if c.format == "json" {
		out.WriteString("[\n")
	}
	done := 0
	for _, s := range seasons {
		if *only != 0 && s.ID != *only {
			continue
		}
		if *limit > 0 && done >= *limit {
			break
		}
		if done > 0 && *delay > 0 {
			time.Sleep(*delay)
		}
		page, err := cl.get(ctx, SchedulePath(s.ID))
		if err != nil {
			return err
		}
		sched, err := ParseSchedule(s.ID, bytes.NewReader(page))
		if err != nil {
			// A season whose page breaks the contract is either fatal or
			// recorded, never dropped: a hole in the data has to be visible.
			var de *DriftError
			if *onDrift == "fail" || !errors.As(err, &de) {
				return err
			}
			if !c.quiet {
				fmt.Fprintf(os.Stderr, "season %d: %v\n", s.ID, de)
			}
			if c.format == "csv" {
				fmt.Fprintf(&out, "%d,ERROR,,,,,,,,,%s\n", s.ID, de.Msg)
			} else {
				if c.format == "json" {
					if done > 0 {
						out.WriteString(",\n")
					}
					out.WriteString("  ")
				}
				if err := enc.Encode(crawlError{SeasonID: s.ID, Label: s.Label, Error: de.Msg}); err != nil {
					return err
				}
			}
			done++
			continue
		}
		if c.format == "csv" {
			b, err := renderCSV(sched)
			if err != nil {
				return err
			}
			out.Write(b)
		} else {
			if c.format == "json" {
				if done > 0 {
					out.WriteString(",\n")
				}
				out.WriteString("  ")
			}
			// Encode writes the document plus one newline: exactly the
			// NDJSON contract, and one element of the JSON array.
			if err := enc.Encode(sched); err != nil {
				return err
			}
		}
		done++
		if !c.quiet {
			fmt.Fprintf(os.Stderr, "season %d (%s): %d games, %d played\n", sched.SeasonID, s.Label, len(sched.Games), sched.Played())
		}
	}
	if c.format == "json" {
		out.WriteString("\n]\n")
	}
	if done == 0 {
		return drift(*league, "", "", "crawl matched no seasons", "0 seasons", ">= 1")
	}
	if err := emit(c.out, c.check, out.Bytes()); err != nil {
		return err
	}
	if c.state != "" {
		if err := saveState(c.state, &state{Path: fmt.Sprintf("crawl:%d", *league), SHA256: sha256Hex(out.Bytes()), SavedAt: nowStamp()}); err != nil {
			return err
		}
	}
	return nil
}