Phase 6: sync jobs, inquiry and order forms

- Sync scheduler in the content service with a per-job cache (sync_cache), sync_runs reporting and Healthchecks pings. Jobs: Archidekt, BoardGameGeek, cover art (Open Library, TMDB, AniList), Suwayomi and Kavita reading progress, Bambuddy filament and Homebox material stock. A note's own value always wins over a synced one. - PersonalReader: updates only Library notes with a matching anilistId, changes to Greg's notes are held for review; Reading series with no note show as suggestions on the admin Overview. - /api/inquiries and /api/orders: Turnstile, rate limits, a saved copy in the content service (intake token) plus email over SMTP. Orders are re-checked against the live notes: accepted only when the category's ordersOpen is true and the line has capacity. - Admin Inbox for saved submissions. Options' inStock is optional so synced stock can fill it. All settings from process.env; .env.example per service. - Sources could not be reached from the build environment; parsers are tested against fixtures labelled as not live. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012aSbo2D1qP8cMFQCykf7kx
2026-11-08 04:08:24 +00:00 · 2026-10-11 23:50:06 +00:00
parent 60dfeecd7f
commit e3b0cf60c5
76 changed files with 6050 additions and 86 deletions
@@ -0,0 +1,165 @@
+/**
+ * Each source's parsing and its job, against the hand-written sample payloads
+ * in ./fixtures (documented shapes, not live responses; see fixtures/README.md).
+ */
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { archidektJob, parseDeck, scryfallArt, type RawDeck } from "../../src/sync/jobs/archidekt.ts";
+import { bggJob, parseThings } from "../../src/sync/jobs/bgg.ts";
+import { coversJob, parseAniList, parseOpenLibrary, parseTmdb } from "../../src/sync/jobs/covers.ts";
+import { Scheduler } from "../../src/sync/scheduler.ts";
+import type { SyncJob } from "../../src/sync/types.ts";
+import { book, rig, type Rig } from "../helpers.ts";
+
+const fixture = (name: string) => readFileSync(join(import.meta.dirname, "fixtures", name), "utf8");
+const json = (name: string) => JSON.parse(fixture(name));
+
+type Call = { url: string; init?: RequestInit };
+function fakeFetch(answer: (url: string, init?: RequestInit) => Response | Promise<Response>) {
+  const calls: Call[] = [];
+  const fn = (async (input: string | URL | Request, init?: RequestInit) => {
+    const url = String(input);
+    calls.push({ url, init });
+    return answer(url, init);
+  }) as typeof fetch;
+  return { fn, calls };
+}
+const ok = (body: unknown, status = 200) =>
+  new Response(typeof body === "string" ? body : JSON.stringify(body), { status });
+
+describe("Archidekt", () => {
+  it("parses commander art, color identity and a list summary", () => {
+    const v = parseDeck(json("archidekt-deck.sample.json") as RawDeck);
+    expect(v).toEqual({
+      deckUrl: "https://archidekt.com/decks/1234567",
+      commanderArt: scryfallArt("d0d33d52-3d28-4635-b985-51e126289259"),
+      colors: ["W", "U", "B", "G"],
+      commanders: ["Atraxa, Praetors' Voice"],
+      deckSize: 36,
+      format: "Commander",
+    });
+    expect(v.commanderArt).toBe("https://cards.scryfall.io/art_crop/front/d/0/d0d33d52-3d28-4635-b985-51e126289259.jpg");
+  });
+
+  it("falls back to the featured image and the whole list's colors without a commander", () => {
+    const v = parseDeck({
+      id: 5,
+      featured: "http://example.com/f.jpg",
+      cards: [{ quantity: 4, categories: ["Main"], card: { oracleCard: { name: "Bolt", colorIdentity: ["R"] } } }],
+    });
+    expect(v).toMatchObject({ commanderArt: "https://example.com/f.jpg", colors: ["R"], commanders: [], deckSize: 4 });
+  });
+});
+
+describe("BoardGameGeek", () => {
+  it("parses players, play time and box art per id", () => {
+    const m = parseThings(fixture("bgg-thing.sample.xml"));
+    expect(m.get(266192)).toEqual({
+      minPlayers: 1,
+      maxPlayers: 5,
+      playTime: 70,
+      minPlayTime: 40,
+      maxPlayTime: 70,
+      cover: "https://cf.geekdo-images.com/sample__original/img/wingspan.jpg",
+    });
+    expect(m.get(13)?.cover).toBe("https://cf.geekdo-images.com/sample__original/img/catan.jpg?a=1&b=2");
+    // Zeros mean BGG has no value: left out, so nothing wrong shows.
+    expect(m.get(99)).toEqual({});
+  });
+});
+
+describe("cover art", () => {
+  it("parses Open Library, TMDB and AniList answers", () => {
+    expect(parseOpenLibrary(json("openlibrary-search.sample.json"))).toBe("https://covers.openlibrary.org/b/id/8221255-L.jpg");
+    expect(parseTmdb(json("tmdb-search.sample.json"))).toBe("https://image.tmdb.org/t/p/w500/x2FJsf1ElAgr63Y3PNPtJrcmpoe.jpg");
+    expect(parseAniList(json("anilist-media.sample.json"))).toBe(
+      "https://s4.anilist.co/file/anilistcdn/media/manga/cover/large/sample-xl.jpg",
+    );
+    expect(parseOpenLibrary({ docs: [] })).toBeNull();
+    expect(parseAniList({ data: { Media: null } })).toBeNull();
+  });
+});
+
+// ---------------------------------------------------------------- the jobs, end to end
+
+let r: Rig;
+beforeEach(() => {
+  r = rig();
+});
+afterEach(() => r.done());
+
+const scheduler = (job: SyncJob, fetchFn: typeof fetch) =>
+  new Scheduler(r.service, [job], { fetch: fetchFn, now: () => new Date(r.clock.t), log: () => {} });
+
+describe("jobs", () => {
+  it("Archidekt syncs each deck with an id and forgets decks that lose it", async () => {
+    r.put({ type: "card-game", title: "Magic", publish: true, stance: "Main event" });
+    const atraxa = r.put({ type: "deck", title: "Atraxa", game: "[[Magic]]", archidektId: 1234567, publish: true });
+    r.put({ type: "deck", title: "Paper only", game: "[[Magic]]", listUrl: "https://example.com/list", publish: true });
+    r.service.scan();
+    const f = fakeFetch(() => ok(fixture("archidekt-deck.sample.json")));
+    const report = await scheduler(archidektJob({ delayMs: 0 }), f.fn).runNow("archidekt");
+    expect(report).toMatchObject({ ok: true, counts: { decks: 1, synced: 1, failed: 0 } });
+    expect(f.calls.map((c) => c.url)).toEqual(["https://archidekt.com/api/decks/1234567/"]);
+    expect(r.store.syncCache.forPath(atraxa)).toMatchObject({ colors: ["W", "U", "B", "G"] });
+  });
+
+  it("BoardGameGeek batches 20 ids a request, sends the token and waits out a 202", async () => {
+    for (let i = 1; i <= 25; i++) r.put({ type: "board-game", title: `Game ${i}`, bggId: i === 1 ? 266192 : 1000 + i, blurb: "Fun.", publish: true });
+    r.service.scan();
+    let queued = true;
+    const f = fakeFetch(() => {
+      if (queued) {
+        queued = false;
+        return ok("", 202);
+      }
+      return ok(fixture("bgg-thing.sample.xml"));
+    });
+    const report = await scheduler(bggJob({ token: "app-token", delayMs: 0 }), f.fn).runNow("bgg");
+    expect(f.calls).toHaveLength(3); // 202, then the first 20, then the last 5
+    const ids = (c: Call) => new URL(c.url).searchParams.get("id")!.split(",");
+    expect(f.calls[1]!.url).toMatch(/^https:\/\/boardgamegeek\.com\/xmlapi2\/thing\?id=/);
+    expect([ids(f.calls[1]!).length, ids(f.calls[2]!).length]).toEqual([20, 5]);
+    expect(new Set([...ids(f.calls[1]!), ...ids(f.calls[2]!)]).size).toBe(25);
+    expect((f.calls[0]!.init?.headers as Record<string, string>).authorization).toBe("Bearer app-token");
+    // Only Wingspan is in the sample answer; the rest count as not synced.
+    expect(report).toMatchObject({ ok: false, counts: { games: 25, synced: 1, failed: 24 } });
+    expect(r.store.syncCache.forPath("Shelf/Game 1.md")).toMatchObject({ minPlayers: 1, maxPlayers: 5, playTime: 70 });
+  });
+
+  it("covers: picks the source by medium, skips TMDB without a key, keeps a found cover on a miss", async () => {
+    const b = r.put(book("The Way of Kings", { creator: "Brandon Sanderson" }));
+    const m = r.put(book("One Piece", { medium: "manga", anilistId: 30013 }));
+    r.put(book("Arrival", { medium: "movie" }));
+    r.put(book("Has its own", { cover: "https://example.com/own.jpg" }));
+    r.service.scan();
+    const f = fakeFetch((url) =>
+      url.startsWith("https://openlibrary.org/") ? ok(fixture("openlibrary-search.sample.json")) : ok(fixture("anilist-media.sample.json")),
+    );
+    const s = scheduler(coversJob({ delayMs: 0 }), f.fn);
+    expect(await s.runNow("covers")).toMatchObject({ ok: true, counts: { checked: 2, found: 2 } });
+    expect(f.calls.map((c) => new URL(c.url).host).sort()).toEqual(["graphql.anilist.co", "openlibrary.org"]);
+    expect(f.calls.find((c) => c.url.includes("openlibrary"))!.url).toContain("author=Brandon+Sanderson");
+    expect(JSON.parse(String(f.calls.find((c) => c.url.includes("anilist"))!.init?.body)).variables).toEqual({ type: "MANGA", id: 30013 });
+    expect(r.store.syncCache.forPath(b).cover).toBe("https://covers.openlibrary.org/b/id/8221255-L.jpg");
+
+    // Nothing is due again until its retry time.
+    expect(await s.runNow("covers")).toMatchObject({ counts: { checked: 0 } });
+
+    // A month on, the sources find nothing: the earlier covers stay.
+    r.clock.advance(31 * 24 * 60);
+    const empty = fakeFetch(() => ok({ docs: [], data: { Media: null } }));
+    await scheduler(coversJob({ delayMs: 0 }), empty.fn).runNow("covers");
+    expect(r.store.syncCache.forPath(m).cover).toMatch(/anilist/);
+  });
+
+  it("covers: TMDB with a key", async () => {
+    const a = r.put(book("Arrival", { medium: "movie", year: 2016 }));
+    r.service.scan();
+    const f = fakeFetch(() => ok(fixture("tmdb-search.sample.json")));
+    await scheduler(coversJob({ delayMs: 0, tmdbApiKey: "k".repeat(32) }), f.fn).runNow("covers");
+    expect(f.calls[0]!.url).toMatch(/^https:\/\/api\.themoviedb\.org\/3\/search\/movie\?query=Arrival&include_adult=false&year=2016&api_key=k+$/);
+    expect(r.store.syncCache.forPath(a).cover).toMatch(/image\.tmdb\.org/);
+  });
+});