mirror of
https://github.com/Patiun/com.gregkilmer.www.git
synced 2026-11-08 04:08:24 +00:00
Phase 6: sync jobs, inquiry and order forms
- Sync scheduler in the content service with a per-job cache (sync_cache), sync_runs reporting and Healthchecks pings. Jobs: Archidekt, BoardGameGeek, cover art (Open Library, TMDB, AniList), Suwayomi and Kavita reading progress, Bambuddy filament and Homebox material stock. A note's own value always wins over a synced one. - PersonalReader: updates only Library notes with a matching anilistId, changes to Greg's notes are held for review; Reading series with no note show as suggestions on the admin Overview. - /api/inquiries and /api/orders: Turnstile, rate limits, a saved copy in the content service (intake token) plus email over SMTP. Orders are re-checked against the live notes: accepted only when the category's ordersOpen is true and the line has capacity. - Admin Inbox for saved submissions. Options' inStock is optional so synced stock can fill it. All settings from process.env; .env.example per service. - Sources could not be reached from the build environment; parsers are tested against fixtures labelled as not live. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012aSbo2D1qP8cMFQCykf7kx
This commit is contained in:
Claude
@@ -0,0 +1,176 @@
|
||||
/**
|
||||
* Cover art for Library items that have no `cover` of their own:
|
||||
*
|
||||
* - books: Open Library search (no key);
|
||||
* - films and TV: TMDB search (needs TMDB_API_KEY; skipped without it);
|
||||
* - anime, manga, manhwa, light novels and webnovels: AniList (no key),
|
||||
* by the note's `anilistId` when it has one, else by title.
|
||||
*
|
||||
* Video games have no source yet. Only the image address is stored; nothing
|
||||
* is downloaded. The job looks every 15 minutes, so a new item gets its
|
||||
* cover soon after it appears, and it re-checks old answers slowly: an item
|
||||
* with no cover found is asked about again after a day, one with a cover
|
||||
* after 30 days, at most `perRun` items a run. This follows the legacy
|
||||
* scripts in legacy/scripts/fetch-{book-covers,tmdb-art,anime-art}.mjs.
|
||||
*/
|
||||
import type { StoredNote } from "../../store.ts";
|
||||
import type { SyncContext, SyncJob } from "../types.ts";
|
||||
|
||||
export const OPEN_LIBRARY_SEARCH = "https://openlibrary.org/search.json";
|
||||
export const OPEN_LIBRARY_COVERS = "https://covers.openlibrary.org/b";
|
||||
export const TMDB_API = "https://api.themoviedb.org/3";
|
||||
export const TMDB_IMAGES = "https://image.tmdb.org/t/p/w500";
|
||||
export const ANILIST_API = "https://graphql.anilist.co";
|
||||
|
||||
const DAY = 24 * 60 * 60_000;
|
||||
const RETRY_FOUND_MS = 30 * DAY;
|
||||
const RETRY_MISSING_MS = DAY;
|
||||
|
||||
type Source = "openlibrary" | "tmdb" | "anilist";
|
||||
|
||||
export function sourceFor(medium: string): Source | null {
|
||||
if (medium === "book") return "openlibrary";
|
||||
if (medium === "movie" || medium === "TV") return "tmdb";
|
||||
if (["anime", "manga", "manhwa", "light novel", "webnovel"].includes(medium)) return "anilist";
|
||||
return null;
|
||||
}
|
||||
|
||||
class RateLimited extends Error {}
|
||||
|
||||
// ---------------------------------------------------------------- parsing
|
||||
|
||||
/** Open Library search.json: the first result with a cover. */
|
||||
export function parseOpenLibrary(json: unknown): string | null {
|
||||
const docs = ((json as { docs?: { cover_i?: number; cover_edition_key?: string }[] })?.docs ?? []);
|
||||
const withId = docs.find((d) => typeof d.cover_i === "number" && d.cover_i > 0);
|
||||
if (withId) return `${OPEN_LIBRARY_COVERS}/id/${withId.cover_i}-L.jpg`;
|
||||
const withEdition = docs.find((d) => d.cover_edition_key);
|
||||
return withEdition ? `${OPEN_LIBRARY_COVERS}/olid/${withEdition.cover_edition_key}-L.jpg` : null;
|
||||
}
|
||||
|
||||
/** TMDB search/movie or search/tv: the first result with a poster. */
|
||||
export function parseTmdb(json: unknown): string | null {
|
||||
const results = (json as { results?: { poster_path?: string | null }[] })?.results ?? [];
|
||||
const hit = results.find((r) => r.poster_path);
|
||||
return hit ? `${TMDB_IMAGES}${hit.poster_path}` : null;
|
||||
}
|
||||
|
||||
/** AniList Media query: the largest cover. */
|
||||
export function parseAniList(json: unknown): string | null {
|
||||
const media = (json as { data?: { Media?: { coverImage?: { extraLarge?: string; large?: string } } | null } })?.data?.Media;
|
||||
const url = media?.coverImage?.extraLarge ?? media?.coverImage?.large;
|
||||
return url && url.startsWith("https://") ? url : null;
|
||||
}
|
||||
|
||||
export const ANILIST_QUERY = `query ($id: Int, $search: String, $type: MediaType, $format: MediaFormat) {
|
||||
Media(id: $id, search: $search, type: $type, format: $format) {
|
||||
id
|
||||
title { english romaji }
|
||||
coverImage { extraLarge large }
|
||||
}
|
||||
}`;
|
||||
|
||||
// ---------------------------------------------------------------- lookups
|
||||
|
||||
async function json(ctx: SyncContext, url: string, init: RequestInit = {}) {
|
||||
const res = await ctx.fetch(url, { ...init, headers: { accept: "application/json", ...(init.headers as Record<string, string>) } });
|
||||
if (res.status === 429) throw new RateLimited("rate limited (429)");
|
||||
if (res.status === 404) return null;
|
||||
if (!res.ok) throw new Error(`answered ${res.status}`);
|
||||
return res.json();
|
||||
}
|
||||
|
||||
export async function lookup(ctx: SyncContext, note: StoredNote, tmdbKey?: string): Promise<string | null> {
|
||||
const d = note.data as { medium: string; creator?: string; year?: number; anilistId?: number };
|
||||
const title = note.title;
|
||||
switch (sourceFor(d.medium)) {
|
||||
case "openlibrary": {
|
||||
const p = new URLSearchParams({ title, limit: "5", fields: "title,cover_i,cover_edition_key" });
|
||||
if (d.creator) p.set("author", d.creator);
|
||||
return parseOpenLibrary(await json(ctx, `${OPEN_LIBRARY_SEARCH}?${p}`));
|
||||
}
|
||||
case "tmdb": {
|
||||
if (!tmdbKey) return null;
|
||||
const kind = d.medium === "movie" ? "movie" : "tv";
|
||||
const p = new URLSearchParams({ query: title, include_adult: "false" });
|
||||
if (d.year) p.set(kind === "movie" ? "year" : "first_air_date_year", String(d.year));
|
||||
// A v4 read access token (a JWT) goes in the header; a v3 key in the query.
|
||||
const bearer = tmdbKey.startsWith("eyJ");
|
||||
if (!bearer) p.set("api_key", tmdbKey);
|
||||
return parseTmdb(await json(ctx, `${TMDB_API}/search/${kind}?${p}`, bearer ? { headers: { authorization: `Bearer ${tmdbKey}` } } : {}));
|
||||
}
|
||||
case "anilist": {
|
||||
const variables: Record<string, unknown> = { type: d.medium === "anime" ? "ANIME" : "MANGA" };
|
||||
if (d.anilistId) variables.id = d.anilistId;
|
||||
else {
|
||||
variables.search = title;
|
||||
if (d.medium === "light novel" || d.medium === "webnovel") variables.format = "NOVEL";
|
||||
}
|
||||
return parseAniList(
|
||||
await json(ctx, ANILIST_API, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ query: ANILIST_QUERY, variables }),
|
||||
}),
|
||||
);
|
||||
}
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- the job
|
||||
|
||||
export function coversJob(
|
||||
o: { everyMinutes?: number; healthcheckUrl?: string; tmdbApiKey?: string; perRun?: number; delayMs?: number } = {},
|
||||
): SyncJob {
|
||||
return {
|
||||
name: "covers",
|
||||
everyMinutes: o.everyMinutes ?? 15,
|
||||
healthcheckUrl: o.healthcheckUrl,
|
||||
async run(ctx) {
|
||||
const now = ctx.now().getTime();
|
||||
const items = ctx.notes("library-item").filter((n) => !n.data.cover && sourceFor(String(n.data.medium)) !== null);
|
||||
const usable = items.filter((n) => sourceFor(String(n.data.medium)) !== "tmdb" || o.tmdbApiKey);
|
||||
// New items first, then the oldest answers that are due again.
|
||||
const due = usable
|
||||
.map((n) => ({ n, c: ctx.cached(n.path) }))
|
||||
.filter(({ c }) => !c || now - Date.parse(c.fetchedAt) >= (c.values.cover ? RETRY_FOUND_MS : RETRY_MISSING_MS))
|
||||
.sort((a, b) => (a.c ? Date.parse(a.c.fetchedAt) : 0) - (b.c ? Date.parse(b.c.fetchedAt) : 0))
|
||||
.slice(0, o.perRun ?? 20);
|
||||
|
||||
let found = 0;
|
||||
let missing = 0;
|
||||
let failed = 0;
|
||||
const errors: string[] = [];
|
||||
for (const [i, { n, c }] of due.entries()) {
|
||||
if (i > 0) await ctx.sleep(o.delayMs ?? 1000);
|
||||
if (ctx.signal.aborted) throw ctx.signal.reason;
|
||||
try {
|
||||
const cover = await lookup(ctx, n, o.tmdbApiKey);
|
||||
if (cover) {
|
||||
ctx.set(n.path, { cover });
|
||||
found++;
|
||||
} else {
|
||||
// Nothing found: keep a cover found earlier, else remember the miss.
|
||||
ctx.set(n.path, c?.values.cover ? c.values : {});
|
||||
missing++;
|
||||
}
|
||||
} catch (err) {
|
||||
if (ctx.signal.aborted) throw ctx.signal.reason;
|
||||
failed++;
|
||||
errors.push(`${n.title}: ${err instanceof Error ? err.message : String(err)}`);
|
||||
if (err instanceof RateLimited) break;
|
||||
}
|
||||
}
|
||||
ctx.retain(items.map((n) => n.path));
|
||||
return {
|
||||
ok: failed === 0,
|
||||
message: failed
|
||||
? `${failed} lookups failed: ${errors.slice(0, 3).join("; ")}`
|
||||
: `${due.length} checked: ${found} covers found, ${missing} not found`,
|
||||
counts: { checked: due.length, found, missing, failed, waiting: usable.length - due.length },
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user