mirror of
https://github.com/Patiun/com.gregkilmer.www.git
synced 2026-11-08 04:08:24 +00:00
Phase 6: sync jobs, inquiry and order forms
- Sync scheduler in the content service with a per-job cache (sync_cache), sync_runs reporting and Healthchecks pings. Jobs: Archidekt, BoardGameGeek, cover art (Open Library, TMDB, AniList), Suwayomi and Kavita reading progress, Bambuddy filament and Homebox material stock. A note's own value always wins over a synced one. - PersonalReader: updates only Library notes with a matching anilistId, changes to Greg's notes are held for review; Reading series with no note show as suggestions on the admin Overview. - /api/inquiries and /api/orders: Turnstile, rate limits, a saved copy in the content service (intake token) plus email over SMTP. Orders are re-checked against the live notes: accepted only when the category's ordersOpen is true and the line has capacity. - Admin Inbox for saved submissions. Options' inStock is optional so synced stock can fill it. All settings from process.env; .env.example per service. - Sources could not be reached from the build environment; parsers are tested against fixtures labelled as not live. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012aSbo2D1qP8cMFQCykf7kx
This commit is contained in:
Claude
@@ -0,0 +1,68 @@
|
||||
/**
|
||||
* The sync cache: what each job last fetched for each note, as JSON, with
|
||||
* when it was fetched. The site reads it through the notes API (`synced`),
|
||||
* so a source being down never takes a page down: the last good values stay
|
||||
* until a later run replaces them.
|
||||
*
|
||||
* It lives in content.db but can be rebuilt by running the jobs again.
|
||||
*/
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import type { CacheEntry, SyncedValues } from "./types.ts";
|
||||
|
||||
type Row = { job: string; path: string; vals: string; fetched_at: string };
|
||||
|
||||
export class SyncCache {
|
||||
constructor(private readonly db: DatabaseSync) {
|
||||
db.exec(`create table if not exists sync_cache (
|
||||
job text not null,
|
||||
path text not null,
|
||||
vals text not null,
|
||||
fetched_at text not null,
|
||||
primary key (job, path)
|
||||
);`);
|
||||
}
|
||||
|
||||
get(job: string, path: string): CacheEntry | null {
|
||||
const r = this.db.prepare("select * from sync_cache where job = ? and path = ?").get(job, path) as Row | undefined;
|
||||
return r ? { values: JSON.parse(r.vals), fetchedAt: r.fetched_at } : null;
|
||||
}
|
||||
|
||||
set(job: string, path: string, values: SyncedValues, fetchedAt: string) {
|
||||
this.db
|
||||
.prepare(
|
||||
`insert into sync_cache (job, path, vals, fetched_at) values (?, ?, ?, ?)
|
||||
on conflict(job, path) do update set vals = excluded.vals, fetched_at = excluded.fetched_at`,
|
||||
)
|
||||
.run(job, path, JSON.stringify(values), fetchedAt);
|
||||
}
|
||||
|
||||
/** Drop a job's rows for notes outside `paths`. */
|
||||
retain(job: string, paths: Iterable<string>) {
|
||||
const keep = new Set(paths);
|
||||
const del = this.db.prepare("delete from sync_cache where job = ? and path = ?");
|
||||
for (const r of this.db.prepare("select path from sync_cache where job = ?").all(job) as { path: string }[]) {
|
||||
if (!keep.has(r.path)) del.run(job, r.path);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Every note's synced values, each job's merged in job-name order. Jobs
|
||||
* should not set the same key for one note; if two do, the later name wins.
|
||||
*/
|
||||
all(): Map<string, SyncedValues> {
|
||||
const out = new Map<string, SyncedValues>();
|
||||
for (const r of this.db.prepare("select * from sync_cache order by job").all() as Row[]) {
|
||||
out.set(r.path, { ...out.get(r.path), ...JSON.parse(r.vals) });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** One note's synced values, merged across jobs. */
|
||||
forPath(path: string): SyncedValues {
|
||||
let out: SyncedValues = {};
|
||||
for (const r of this.db.prepare("select * from sync_cache where path = ? order by job").all(path) as Row[]) {
|
||||
out = { ...out, ...JSON.parse(r.vals) };
|
||||
}
|
||||
return out;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
/**
|
||||
* The sync jobs this service can run, and which of them are enabled. To add
|
||||
* a source, write a `SyncJob` (see types.ts and docs/content-service.md,
|
||||
* "Sync jobs") and add it here behind the settings it needs.
|
||||
*/
|
||||
import { archidektJob } from "./jobs/archidekt.ts";
|
||||
import { bggJob } from "./jobs/bgg.ts";
|
||||
import { coversJob } from "./jobs/covers.ts";
|
||||
import { homelabJobs } from "./jobs/homelab.ts";
|
||||
import type { SyncSettings } from "./settings.ts";
|
||||
import type { SyncJob } from "./types.ts";
|
||||
|
||||
export { Scheduler } from "./scheduler.ts";
|
||||
export { SyncCache } from "./cache.ts";
|
||||
export { syncSettings, type SyncSettings } from "./settings.ts";
|
||||
export type { SyncJob, SyncContext, SyncResult, SyncedValues } from "./types.ts";
|
||||
|
||||
/** Public sources need nothing but their name in SYNC_JOBS. */
|
||||
export function enabledJobs(s: SyncSettings): SyncJob[] {
|
||||
const common = (name: string) => ({ everyMinutes: s.minutes[name], healthcheckUrl: s.healthchecks[name] });
|
||||
const jobs: SyncJob[] = [];
|
||||
if (s.enabled.has("archidekt")) jobs.push(archidektJob(common("archidekt")));
|
||||
if (s.enabled.has("bgg")) jobs.push(bggJob({ ...common("bgg"), token: s.bggToken }));
|
||||
if (s.enabled.has("covers")) jobs.push(coversJob({ ...common("covers"), tmdbApiKey: s.tmdbApiKey }));
|
||||
if (s.homelab) jobs.push(...homelabJobs(s.homelab, common));
|
||||
return jobs;
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
/**
|
||||
* Archidekt: for every deck note with an `archidektId`, the deck's commander
|
||||
* art, color identity, a short list summary and its address. Hourly.
|
||||
*
|
||||
* API: GET https://archidekt.com/api/decks/<id>/ (public JSON, no key). The
|
||||
* legacy site used the owner's deck list (/api/decks/v3/?ownerUsername=);
|
||||
* the single-deck endpoint adds the cards, so commanders and color identity
|
||||
* come from the list itself.
|
||||
*/
|
||||
import type { SyncJob, SyncedValues } from "../types.ts";
|
||||
|
||||
export const ARCHIDEKT_API = "https://archidekt.com/api/decks";
|
||||
const COLOR_ORDER = ["W", "U", "B", "R", "G"] as const;
|
||||
const COLOR_WORDS: Record<string, string> = { white: "W", blue: "U", black: "B", red: "R", green: "G" };
|
||||
|
||||
/** Archidekt deckFormat ids to labels, as the legacy site had them (legacy/src/lib/archidekt.ts). */
|
||||
const FORMATS: Record<number, string> = {
|
||||
1: "Casual",
|
||||
2: "Modern",
|
||||
3: "Commander",
|
||||
5: "Standard",
|
||||
7: "Legacy",
|
||||
10: "Pauper",
|
||||
14: "Historic",
|
||||
16: "Pioneer",
|
||||
37: "Oathbreaker",
|
||||
45: "Brawl",
|
||||
};
|
||||
|
||||
interface RawCard {
|
||||
quantity?: number;
|
||||
categories?: string[] | null;
|
||||
card?: {
|
||||
uid?: string;
|
||||
oracleCard?: { name?: string; colorIdentity?: string[] };
|
||||
};
|
||||
}
|
||||
export interface RawDeck {
|
||||
id: number;
|
||||
name?: string;
|
||||
deckFormat?: number;
|
||||
featured?: string;
|
||||
customFeatured?: string;
|
||||
private?: boolean;
|
||||
categories?: { name: string; includedInDeck?: boolean; isPremier?: boolean }[];
|
||||
cards?: RawCard[];
|
||||
}
|
||||
|
||||
export interface DeckValues extends SyncedValues {
|
||||
deckUrl: string;
|
||||
/** Art crop of the (first) commander, else the deck's featured image. */
|
||||
commanderArt?: string;
|
||||
/** Color identity as WUBRG letters, in that order. Empty means colorless. */
|
||||
colors: string[];
|
||||
commanders: string[];
|
||||
/** Cards in the deck, not counting the maybeboard or other excluded categories. */
|
||||
deckSize: number;
|
||||
format?: string;
|
||||
}
|
||||
|
||||
export const deckAddress = (id: number) => `https://archidekt.com/decks/${id}`;
|
||||
|
||||
/** Scryfall's art crop for a card by its Scryfall id (Archidekt's `uid`). */
|
||||
export const scryfallArt = (uid: string) => `https://cards.scryfall.io/art_crop/front/${uid[0]}/${uid[1]}/${uid}.jpg`;
|
||||
|
||||
const https = (url?: string) => (url && /^https?:\/\//.test(url) ? url.replace(/^http:/, "https:") : undefined);
|
||||
|
||||
const colorLetter = (c: string) => COLOR_WORDS[c.toLowerCase()] ?? (COLOR_ORDER.includes(c.toUpperCase() as "W") ? c.toUpperCase() : undefined);
|
||||
|
||||
/** Turn one Archidekt deck response into the values the site shows. */
|
||||
export function parseDeck(raw: RawDeck): DeckValues {
|
||||
const excluded = new Set((raw.categories ?? []).filter((c) => c.includedInDeck === false).map((c) => c.name));
|
||||
const premier = new Set((raw.categories ?? []).filter((c) => c.isPremier).map((c) => c.name));
|
||||
premier.add("Commander");
|
||||
const inDeck = (raw.cards ?? []).filter((c) => !(c.categories ?? []).some((cat) => excluded.has(cat)));
|
||||
const commanderCards = inDeck.filter((c) => (c.categories ?? []).some((cat) => premier.has(cat)));
|
||||
const identitySource = commanderCards.length ? commanderCards : inDeck;
|
||||
const letters = new Set(identitySource.flatMap((c) => (c.card?.oracleCard?.colorIdentity ?? []).map(colorLetter)));
|
||||
const uid = commanderCards.find((c) => c.card?.uid)?.card?.uid;
|
||||
|
||||
const values: DeckValues = {
|
||||
deckUrl: deckAddress(raw.id),
|
||||
colors: COLOR_ORDER.filter((l) => letters.has(l)),
|
||||
commanders: commanderCards.map((c) => c.card?.oracleCard?.name).filter((n): n is string => !!n),
|
||||
deckSize: inDeck.reduce((n, c) => n + (c.quantity ?? 1), 0),
|
||||
};
|
||||
const art = uid ? scryfallArt(uid) : https(raw.customFeatured) ?? https(raw.featured);
|
||||
if (art) values.commanderArt = art;
|
||||
const format = raw.deckFormat !== undefined ? FORMATS[raw.deckFormat] : undefined;
|
||||
if (format) values.format = format;
|
||||
return values;
|
||||
}
|
||||
|
||||
export function archidektJob(o: { everyMinutes?: number; healthcheckUrl?: string; delayMs?: number } = {}): SyncJob {
|
||||
return {
|
||||
name: "archidekt",
|
||||
everyMinutes: o.everyMinutes ?? 60,
|
||||
healthcheckUrl: o.healthcheckUrl,
|
||||
async run(ctx) {
|
||||
const decks = ctx.notes("deck").filter((n) => typeof n.data.archidektId === "number");
|
||||
let synced = 0;
|
||||
let missing = 0;
|
||||
let failed = 0;
|
||||
const errors: string[] = [];
|
||||
for (const [i, note] of decks.entries()) {
|
||||
if (ctx.signal.aborted) throw ctx.signal.reason;
|
||||
if (i > 0) await ctx.sleep(o.delayMs ?? 1000);
|
||||
const id = note.data.archidektId as number;
|
||||
try {
|
||||
const res = await ctx.fetch(`${ARCHIDEKT_API}/${id}/`, { headers: { accept: "application/json" } });
|
||||
if (res.status === 404 || res.status === 403) {
|
||||
missing++;
|
||||
ctx.log(`${note.path}: deck ${id} is private or gone (${res.status})`);
|
||||
continue;
|
||||
}
|
||||
if (!res.ok) throw new Error(`Archidekt answered ${res.status}`);
|
||||
ctx.set(note.path, parseDeck((await res.json()) as RawDeck));
|
||||
synced++;
|
||||
} catch (err) {
|
||||
if (ctx.signal.aborted) throw ctx.signal.reason;
|
||||
failed++;
|
||||
errors.push(`${id}: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
ctx.retain(decks.map((d) => d.path));
|
||||
return {
|
||||
ok: failed === 0,
|
||||
message: failed ? `${failed} of ${decks.length} decks failed: ${errors.slice(0, 3).join("; ")}` : `${synced} decks synced`,
|
||||
counts: { decks: decks.length, synced, missing, failed },
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
/**
|
||||
* Bambuddy: filament colors in stock, for the Store's Filament options.
|
||||
* Hourly. Read only.
|
||||
*
|
||||
* UNVERIFIED: Bambuddy's API is not documented anywhere this project has, and
|
||||
* no request has been made to a real Bambuddy. The endpoint path is a setting
|
||||
* (BAMBUDDY_SPOOLS_PATH) and everything that depends on the response's shape
|
||||
* is in `mapBambuddySpools` below, so fixing it against the real API is a
|
||||
* change to one function. Check it against a live Bambuddy before relying on
|
||||
* it (docs/content-service.md, "Sync jobs").
|
||||
*/
|
||||
import type { SyncJob } from "../types.ts";
|
||||
import { mergeOptions, normalizeHex, storeOnLines, type StockOption } from "./stock.ts";
|
||||
|
||||
export const BAMBUDDY_DEFAULT_PATH = "/api/v1/inventory/spools";
|
||||
|
||||
export interface BambuddyOptions {
|
||||
url: string;
|
||||
/** Path of the spool or filament inventory endpoint. */
|
||||
path?: string;
|
||||
/** Sent as `Authorization: Bearer`, or in `authHeader` when that is set. */
|
||||
apiKey?: string;
|
||||
authHeader?: string;
|
||||
/** Option group the colors feed. Default Filament. */
|
||||
group?: string;
|
||||
/** Only these materials (like PLA, PETG), when set. */
|
||||
materials?: string[];
|
||||
/** A spool with less filament than this, in grams, counts as out. Default 50. */
|
||||
minGrams?: number;
|
||||
everyMinutes?: number;
|
||||
healthcheckUrl?: string;
|
||||
}
|
||||
|
||||
export interface Spool {
|
||||
name: string;
|
||||
hex?: string;
|
||||
material?: string;
|
||||
/** Grams left, when the source says. */
|
||||
remaining?: number;
|
||||
}
|
||||
|
||||
const pick = (o: Record<string, unknown>, keys: string[]) => {
|
||||
for (const k of keys) {
|
||||
const v = o[k];
|
||||
if (v !== undefined && v !== null && v !== "") return v;
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
|
||||
/**
|
||||
* THE ONE PLACE that knows Bambuddy's response shape (a guess; see above).
|
||||
* Accepts a list, or an object holding one under items, spools, data or
|
||||
* results, and reads each spool's color name, hex, material and grams left
|
||||
* under the common spellings. Archived or empty-named entries are skipped.
|
||||
*/
|
||||
export function mapBambuddySpools(json: unknown): Spool[] {
|
||||
const root = json as Record<string, unknown> | unknown[];
|
||||
const list = Array.isArray(root)
|
||||
? root
|
||||
: (["items", "spools", "data", "results"].map((k) => (root as Record<string, unknown>)?.[k]).find(Array.isArray) as unknown[] | undefined);
|
||||
if (!list) throw new Error("Bambuddy answered with no list of spools");
|
||||
const out: Spool[] = [];
|
||||
for (const raw of list) {
|
||||
if (!raw || typeof raw !== "object") continue;
|
||||
const o = raw as Record<string, unknown>;
|
||||
if (o.archived === true || o.deleted === true) continue;
|
||||
const name = pick(o, ["color_name", "colorName", "color", "name"]);
|
||||
if (typeof name !== "string" || !name.trim()) continue;
|
||||
const s: Spool = { name: name.trim() };
|
||||
const hex = normalizeHex(pick(o, ["color_hex", "colorHex", "hex", "rgba", "color_code"]));
|
||||
if (hex) s.hex = hex;
|
||||
const material = pick(o, ["material", "filament_type", "type"]);
|
||||
if (typeof material === "string") s.material = material.trim();
|
||||
const grams = Number(pick(o, ["remaining_weight", "remainingWeight", "weight_remaining", "remaining", "remaining_grams"]));
|
||||
if (Number.isFinite(grams)) s.remaining = grams;
|
||||
out.push(s);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Spools as Filament options: in stock when some spool of that color has enough left. */
|
||||
export function filamentOptions(spools: Spool[], o: Pick<BambuddyOptions, "materials" | "minGrams"> = {}): StockOption[] {
|
||||
const materials = o.materials?.length ? new Set(o.materials.map((m) => m.toLowerCase())) : null;
|
||||
const min = o.minGrams ?? 50;
|
||||
return mergeOptions(
|
||||
spools
|
||||
.filter((s) => !materials || (s.material && materials.has(s.material.toLowerCase())))
|
||||
.map((s) => ({ name: s.name, ...(s.hex ? { hex: s.hex } : {}), inStock: s.remaining === undefined || s.remaining >= min })),
|
||||
);
|
||||
}
|
||||
|
||||
export function bambuddyJob(o: BambuddyOptions): SyncJob {
|
||||
const group = o.group ?? "Filament";
|
||||
return {
|
||||
name: "bambuddy",
|
||||
everyMinutes: o.everyMinutes ?? 60,
|
||||
healthcheckUrl: o.healthcheckUrl,
|
||||
async run(ctx) {
|
||||
const headers: Record<string, string> = { accept: "application/json" };
|
||||
if (o.apiKey) headers[o.authHeader?.toLowerCase() || "authorization"] = o.authHeader ? o.apiKey : `Bearer ${o.apiKey}`;
|
||||
const url = `${o.url.replace(/\/$/, "")}/${(o.path ?? BAMBUDDY_DEFAULT_PATH).replace(/^\//, "")}`;
|
||||
const res = await ctx.fetch(url, { headers });
|
||||
if (res.status === 401 || res.status === 403) throw new Error(`Bambuddy refused the request (${res.status}); check BAMBUDDY_API_KEY`);
|
||||
if (!res.ok) throw new Error(`Bambuddy answered ${res.status}; check BAMBUDDY_SPOOLS_PATH`);
|
||||
const options = filamentOptions(mapBambuddySpools(await res.json()), o);
|
||||
const lines = storeOnLines(ctx, "filamentStock", [{ group, options }]);
|
||||
const inStock = options.filter((x) => x.inStock).length;
|
||||
return { message: `${inStock} of ${options.length} colors in stock, on ${lines} lines`, counts: { colors: options.length, inStock, lines } };
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
/**
|
||||
* BoardGameGeek: for every board-game note, its player count, play time and
|
||||
* box art, from the game's `bggId`. Daily.
|
||||
*
|
||||
* API: XML API2 `thing`, GET https://boardgamegeek.com/xmlapi2/thing?id=1,2,3
|
||||
* Up to 20 ids per request, with a pause between requests. BGG may answer
|
||||
* 202 (request queued: ask again shortly) or 429 (slow down). BGG now asks
|
||||
* API users to register an application; when BGG_TOKEN is set it is sent as
|
||||
* `Authorization: Bearer <token>`.
|
||||
*/
|
||||
import type { SyncJob, SyncedValues } from "../types.ts";
|
||||
|
||||
export const BGG_API = "https://boardgamegeek.com/xmlapi2";
|
||||
const BATCH = 20;
|
||||
|
||||
export interface GameValues extends SyncedValues {
|
||||
minPlayers?: number;
|
||||
maxPlayers?: number;
|
||||
/** BGG's playing time in minutes, with its range when given. */
|
||||
playTime?: number;
|
||||
minPlayTime?: number;
|
||||
maxPlayTime?: number;
|
||||
/** BGG's box art. */
|
||||
cover?: string;
|
||||
}
|
||||
|
||||
const decode = (s: string) =>
|
||||
s
|
||||
.replace(/&#(\d+);/g, (_, n) => String.fromCharCode(Number(n)))
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/&/g, "&");
|
||||
|
||||
const attr = (tag: string, name: string) => {
|
||||
const m = new RegExp(`\\b${name}="([^"]*)"`).exec(tag);
|
||||
return m ? decode(m[1]!) : undefined;
|
||||
};
|
||||
|
||||
/** `<name value="3"/>` as a positive number. */
|
||||
const valueOf = (xml: string, el: string) => {
|
||||
const m = new RegExp(`<${el}\\b[^>]*>`).exec(xml);
|
||||
const n = m ? Number(attr(m[0], "value")) : NaN;
|
||||
return Number.isFinite(n) && n > 0 ? n : undefined;
|
||||
};
|
||||
|
||||
/** Parse a `thing` response into values keyed by BGG id. */
|
||||
export function parseThings(xml: string): Map<number, GameValues> {
|
||||
const out = new Map<number, GameValues>();
|
||||
for (const m of xml.matchAll(/<item\b([^>]*)>([\s\S]*?)<\/item>/g)) {
|
||||
const id = Number(attr(m[1]!, "id"));
|
||||
if (!Number.isInteger(id)) continue;
|
||||
const body = m[2]!;
|
||||
const v: GameValues = {};
|
||||
const set = (k: keyof GameValues, n: number | undefined) => {
|
||||
if (n !== undefined) v[k] = n;
|
||||
};
|
||||
set("minPlayers", valueOf(body, "minplayers"));
|
||||
set("maxPlayers", valueOf(body, "maxplayers"));
|
||||
set("playTime", valueOf(body, "playingtime"));
|
||||
set("minPlayTime", valueOf(body, "minplaytime"));
|
||||
set("maxPlayTime", valueOf(body, "maxplaytime"));
|
||||
const image = /<image>\s*([^<]+?)\s*<\/image>/.exec(body)?.[1];
|
||||
if (image) {
|
||||
const url = decode(image).replace(/^\/\//, "https://").replace(/^http:/, "https:");
|
||||
if (url.startsWith("https://")) v.cover = url;
|
||||
}
|
||||
out.set(id, v);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function bggJob(o: { everyMinutes?: number; healthcheckUrl?: string; token?: string; delayMs?: number } = {}): SyncJob {
|
||||
const delay = o.delayMs ?? 5000;
|
||||
return {
|
||||
name: "bgg",
|
||||
everyMinutes: o.everyMinutes ?? 24 * 60,
|
||||
healthcheckUrl: o.healthcheckUrl,
|
||||
timeoutMs: 30 * 60_000,
|
||||
async run(ctx) {
|
||||
const games = ctx.notes("board-game").filter((n) => typeof n.data.bggId === "number");
|
||||
const byId = new Map<number, string[]>();
|
||||
for (const g of games) byId.set(g.data.bggId as number, [...(byId.get(g.data.bggId as number) ?? []), g.path]);
|
||||
const ids = [...byId.keys()];
|
||||
const headers: Record<string, string> = { accept: "application/xml" };
|
||||
if (o.token) headers.authorization = `Bearer ${o.token}`;
|
||||
|
||||
let synced = 0;
|
||||
let failed = 0;
|
||||
const errors: string[] = [];
|
||||
for (let i = 0; i < ids.length; i += BATCH) {
|
||||
if (i > 0) await ctx.sleep(delay);
|
||||
if (ctx.signal.aborted) throw ctx.signal.reason;
|
||||
const batch = ids.slice(i, i + BATCH);
|
||||
const url = `${BGG_API}/thing?id=${batch.join(",")}&type=boardgame,boardgameexpansion`;
|
||||
let xml: string | null = null;
|
||||
try {
|
||||
for (let attempt = 0; attempt < 4 && xml === null; attempt++) {
|
||||
const res = await ctx.fetch(url, { headers });
|
||||
if (res.status === 202) {
|
||||
await ctx.sleep(delay * (attempt + 1));
|
||||
continue;
|
||||
}
|
||||
if (res.status === 429) throw new Error("BoardGameGeek asked us to slow down (429)");
|
||||
if (res.status === 401 || res.status === 403) throw new Error(`BoardGameGeek refused the request (${res.status}); set BGG_TOKEN`);
|
||||
if (!res.ok) throw new Error(`BoardGameGeek answered ${res.status}`);
|
||||
xml = await res.text();
|
||||
}
|
||||
if (xml === null) throw new Error("BoardGameGeek kept the request queued");
|
||||
} catch (err) {
|
||||
if (ctx.signal.aborted) throw ctx.signal.reason;
|
||||
failed += batch.length;
|
||||
errors.push(err instanceof Error ? err.message : String(err));
|
||||
if (/429|refused/.test(errors.at(-1)!)) break;
|
||||
continue;
|
||||
}
|
||||
const found = parseThings(xml);
|
||||
for (const id of batch) {
|
||||
const v = found.get(id);
|
||||
if (!v) {
|
||||
failed++;
|
||||
errors.push(`no game ${id}`);
|
||||
continue;
|
||||
}
|
||||
for (const path of byId.get(id) ?? []) ctx.set(path, v);
|
||||
synced++;
|
||||
}
|
||||
}
|
||||
ctx.retain(games.map((g) => g.path));
|
||||
const unsynced = ids.length - synced;
|
||||
return {
|
||||
ok: unsynced === 0,
|
||||
message: unsynced ? `${unsynced} of ${ids.length} games not synced: ${[...new Set(errors)].slice(0, 3).join("; ")}` : `${synced} games synced`,
|
||||
counts: { games: ids.length, synced, failed: unsynced },
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
/**
|
||||
* Cover art for Library items that have no `cover` of their own:
|
||||
*
|
||||
* - books: Open Library search (no key);
|
||||
* - films and TV: TMDB search (needs TMDB_API_KEY; skipped without it);
|
||||
* - anime, manga, manhwa, light novels and webnovels: AniList (no key),
|
||||
* by the note's `anilistId` when it has one, else by title.
|
||||
*
|
||||
* Video games have no source yet. Only the image address is stored; nothing
|
||||
* is downloaded. The job looks every 15 minutes, so a new item gets its
|
||||
* cover soon after it appears, and it re-checks old answers slowly: an item
|
||||
* with no cover found is asked about again after a day, one with a cover
|
||||
* after 30 days, at most `perRun` items a run. This follows the legacy
|
||||
* scripts in legacy/scripts/fetch-{book-covers,tmdb-art,anime-art}.mjs.
|
||||
*/
|
||||
import type { StoredNote } from "../../store.ts";
|
||||
import type { SyncContext, SyncJob } from "../types.ts";
|
||||
|
||||
export const OPEN_LIBRARY_SEARCH = "https://openlibrary.org/search.json";
|
||||
export const OPEN_LIBRARY_COVERS = "https://covers.openlibrary.org/b";
|
||||
export const TMDB_API = "https://api.themoviedb.org/3";
|
||||
export const TMDB_IMAGES = "https://image.tmdb.org/t/p/w500";
|
||||
export const ANILIST_API = "https://graphql.anilist.co";
|
||||
|
||||
const DAY = 24 * 60 * 60_000;
|
||||
const RETRY_FOUND_MS = 30 * DAY;
|
||||
const RETRY_MISSING_MS = DAY;
|
||||
|
||||
type Source = "openlibrary" | "tmdb" | "anilist";
|
||||
|
||||
export function sourceFor(medium: string): Source | null {
|
||||
if (medium === "book") return "openlibrary";
|
||||
if (medium === "movie" || medium === "TV") return "tmdb";
|
||||
if (["anime", "manga", "manhwa", "light novel", "webnovel"].includes(medium)) return "anilist";
|
||||
return null;
|
||||
}
|
||||
|
||||
class RateLimited extends Error {}
|
||||
|
||||
// ---------------------------------------------------------------- parsing
|
||||
|
||||
/** Open Library search.json: the first result with a cover. */
|
||||
export function parseOpenLibrary(json: unknown): string | null {
|
||||
const docs = ((json as { docs?: { cover_i?: number; cover_edition_key?: string }[] })?.docs ?? []);
|
||||
const withId = docs.find((d) => typeof d.cover_i === "number" && d.cover_i > 0);
|
||||
if (withId) return `${OPEN_LIBRARY_COVERS}/id/${withId.cover_i}-L.jpg`;
|
||||
const withEdition = docs.find((d) => d.cover_edition_key);
|
||||
return withEdition ? `${OPEN_LIBRARY_COVERS}/olid/${withEdition.cover_edition_key}-L.jpg` : null;
|
||||
}
|
||||
|
||||
/** TMDB search/movie or search/tv: the first result with a poster. */
|
||||
export function parseTmdb(json: unknown): string | null {
|
||||
const results = (json as { results?: { poster_path?: string | null }[] })?.results ?? [];
|
||||
const hit = results.find((r) => r.poster_path);
|
||||
return hit ? `${TMDB_IMAGES}${hit.poster_path}` : null;
|
||||
}
|
||||
|
||||
/** AniList Media query: the largest cover. */
|
||||
export function parseAniList(json: unknown): string | null {
|
||||
const media = (json as { data?: { Media?: { coverImage?: { extraLarge?: string; large?: string } } | null } })?.data?.Media;
|
||||
const url = media?.coverImage?.extraLarge ?? media?.coverImage?.large;
|
||||
return url && url.startsWith("https://") ? url : null;
|
||||
}
|
||||
|
||||
export const ANILIST_QUERY = `query ($id: Int, $search: String, $type: MediaType, $format: MediaFormat) {
|
||||
Media(id: $id, search: $search, type: $type, format: $format) {
|
||||
id
|
||||
title { english romaji }
|
||||
coverImage { extraLarge large }
|
||||
}
|
||||
}`;
|
||||
|
||||
// ---------------------------------------------------------------- lookups
|
||||
|
||||
async function json(ctx: SyncContext, url: string, init: RequestInit = {}) {
|
||||
const res = await ctx.fetch(url, { ...init, headers: { accept: "application/json", ...(init.headers as Record<string, string>) } });
|
||||
if (res.status === 429) throw new RateLimited("rate limited (429)");
|
||||
if (res.status === 404) return null;
|
||||
if (!res.ok) throw new Error(`answered ${res.status}`);
|
||||
return res.json();
|
||||
}
|
||||
|
||||
export async function lookup(ctx: SyncContext, note: StoredNote, tmdbKey?: string): Promise<string | null> {
|
||||
const d = note.data as { medium: string; creator?: string; year?: number; anilistId?: number };
|
||||
const title = note.title;
|
||||
switch (sourceFor(d.medium)) {
|
||||
case "openlibrary": {
|
||||
const p = new URLSearchParams({ title, limit: "5", fields: "title,cover_i,cover_edition_key" });
|
||||
if (d.creator) p.set("author", d.creator);
|
||||
return parseOpenLibrary(await json(ctx, `${OPEN_LIBRARY_SEARCH}?${p}`));
|
||||
}
|
||||
case "tmdb": {
|
||||
if (!tmdbKey) return null;
|
||||
const kind = d.medium === "movie" ? "movie" : "tv";
|
||||
const p = new URLSearchParams({ query: title, include_adult: "false" });
|
||||
if (d.year) p.set(kind === "movie" ? "year" : "first_air_date_year", String(d.year));
|
||||
// A v4 read access token (a JWT) goes in the header; a v3 key in the query.
|
||||
const bearer = tmdbKey.startsWith("eyJ");
|
||||
if (!bearer) p.set("api_key", tmdbKey);
|
||||
return parseTmdb(await json(ctx, `${TMDB_API}/search/${kind}?${p}`, bearer ? { headers: { authorization: `Bearer ${tmdbKey}` } } : {}));
|
||||
}
|
||||
case "anilist": {
|
||||
const variables: Record<string, unknown> = { type: d.medium === "anime" ? "ANIME" : "MANGA" };
|
||||
if (d.anilistId) variables.id = d.anilistId;
|
||||
else {
|
||||
variables.search = title;
|
||||
if (d.medium === "light novel" || d.medium === "webnovel") variables.format = "NOVEL";
|
||||
}
|
||||
return parseAniList(
|
||||
await json(ctx, ANILIST_API, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ query: ANILIST_QUERY, variables }),
|
||||
}),
|
||||
);
|
||||
}
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- the job
|
||||
|
||||
export function coversJob(
|
||||
o: { everyMinutes?: number; healthcheckUrl?: string; tmdbApiKey?: string; perRun?: number; delayMs?: number } = {},
|
||||
): SyncJob {
|
||||
return {
|
||||
name: "covers",
|
||||
everyMinutes: o.everyMinutes ?? 15,
|
||||
healthcheckUrl: o.healthcheckUrl,
|
||||
async run(ctx) {
|
||||
const now = ctx.now().getTime();
|
||||
const items = ctx.notes("library-item").filter((n) => !n.data.cover && sourceFor(String(n.data.medium)) !== null);
|
||||
const usable = items.filter((n) => sourceFor(String(n.data.medium)) !== "tmdb" || o.tmdbApiKey);
|
||||
// New items first, then the oldest answers that are due again.
|
||||
const due = usable
|
||||
.map((n) => ({ n, c: ctx.cached(n.path) }))
|
||||
.filter(({ c }) => !c || now - Date.parse(c.fetchedAt) >= (c.values.cover ? RETRY_FOUND_MS : RETRY_MISSING_MS))
|
||||
.sort((a, b) => (a.c ? Date.parse(a.c.fetchedAt) : 0) - (b.c ? Date.parse(b.c.fetchedAt) : 0))
|
||||
.slice(0, o.perRun ?? 20);
|
||||
|
||||
let found = 0;
|
||||
let missing = 0;
|
||||
let failed = 0;
|
||||
const errors: string[] = [];
|
||||
for (const [i, { n, c }] of due.entries()) {
|
||||
if (i > 0) await ctx.sleep(o.delayMs ?? 1000);
|
||||
if (ctx.signal.aborted) throw ctx.signal.reason;
|
||||
try {
|
||||
const cover = await lookup(ctx, n, o.tmdbApiKey);
|
||||
if (cover) {
|
||||
ctx.set(n.path, { cover });
|
||||
found++;
|
||||
} else {
|
||||
// Nothing found: keep a cover found earlier, else remember the miss.
|
||||
ctx.set(n.path, c?.values.cover ? c.values : {});
|
||||
missing++;
|
||||
}
|
||||
} catch (err) {
|
||||
if (ctx.signal.aborted) throw ctx.signal.reason;
|
||||
failed++;
|
||||
errors.push(`${n.title}: ${err instanceof Error ? err.message : String(err)}`);
|
||||
if (err instanceof RateLimited) break;
|
||||
}
|
||||
}
|
||||
ctx.retain(items.map((n) => n.path));
|
||||
return {
|
||||
ok: failed === 0,
|
||||
message: failed
|
||||
? `${failed} lookups failed: ${errors.slice(0, 3).join("; ")}`
|
||||
: `${due.length} checked: ${found} covers found, ${missing} not found`,
|
||||
counts: { checked: due.length, found, missing, failed, waiting: usable.length - due.length },
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
/**
|
||||
* Homebox: leather and paper materials in stock, for the Store's Leather and
|
||||
* Paper options. Hourly. Read only.
|
||||
*
|
||||
* API (Homebox's REST API, v1):
|
||||
* - `POST /api/v1/users/login` with username and password gives a token
|
||||
* (already prefixed `Bearer `), sent as `Authorization`.
|
||||
* - `GET /api/v1/labels` and `GET /api/v1/locations` name the ids to filter on.
|
||||
* - `GET /api/v1/items?labels=<id>&page=&pageSize=` (or `locations=<id>`)
|
||||
* lists the items, each with name, quantity and description.
|
||||
*
|
||||
* Which items count is a setting: HOMEBOX_LABELS (like `leather,paper`) and
|
||||
* HOMEBOX_LOCATIONS, each entry `name` or `name=Group`. Without `=Group` the
|
||||
* option group is the name capitalized (leather: Leather). An item is in
|
||||
* stock when its quantity is above zero; a `#rrggbb` in its description
|
||||
* becomes the swatch color.
|
||||
*
|
||||
* Written from Homebox's API documentation; not checked against a live
|
||||
* server from here (see docs/content-service.md).
|
||||
*/
|
||||
import type { SyncContext, SyncJob } from "../types.ts";
|
||||
import { mergeOptions, normalizeHex, storeOnLines, type StockGroup, type StockOption } from "./stock.ts";
|
||||
|
||||
export interface HomeboxFilter {
|
||||
/** Label or location name in Homebox, matched case-insensitively. */
|
||||
name: string;
|
||||
/** Store option group it feeds. */
|
||||
group: string;
|
||||
}
|
||||
|
||||
export interface HomeboxOptions {
|
||||
url: string;
|
||||
user: string;
|
||||
pass: string;
|
||||
labels: HomeboxFilter[];
|
||||
locations?: HomeboxFilter[];
|
||||
everyMinutes?: number;
|
||||
healthcheckUrl?: string;
|
||||
}
|
||||
|
||||
/** `leather,paper=Paper goods` as filters. */
|
||||
export function parseFilters(raw: string | undefined): HomeboxFilter[] {
|
||||
return (raw ?? "")
|
||||
.split(",")
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean)
|
||||
.map((s) => {
|
||||
const [name, group] = s.split("=").map((x) => x.trim());
|
||||
return { name: name!, group: group || name!.charAt(0).toUpperCase() + name!.slice(1) };
|
||||
});
|
||||
}
|
||||
|
||||
export interface HomeboxItem {
|
||||
name: string;
|
||||
quantity: number;
|
||||
description?: string;
|
||||
}
|
||||
|
||||
/** `GET /api/v1/items`: `{ items, page, pageSize, total }`. */
|
||||
export function parseItems(json: unknown): { items: HomeboxItem[]; total: number } {
|
||||
const r = json as { items?: { name?: string; quantity?: number; description?: string; archived?: boolean }[]; total?: number };
|
||||
if (!Array.isArray(r?.items)) throw new Error("Homebox answered with no list of items");
|
||||
const items = r.items
|
||||
.filter((i) => i && typeof i.name === "string" && i.name.trim() && i.archived !== true)
|
||||
.map((i) => ({ name: i.name!.trim(), quantity: Number(i.quantity ?? 0) || 0, ...(i.description ? { description: i.description } : {}) }));
|
||||
return { items, total: Number(r.total ?? items.length) };
|
||||
}
|
||||
|
||||
/** `GET /api/v1/labels` or `/locations`: `[{ id, name }]` to a name to id map. */
|
||||
export function parseNamed(json: unknown): Map<string, string> {
|
||||
const list = Array.isArray(json) ? json : ((json as { items?: unknown[] })?.items ?? []);
|
||||
const out = new Map<string, string>();
|
||||
for (const x of list as { id?: string; name?: string }[]) if (x?.id && x.name) out.set(x.name.trim().toLowerCase(), x.id);
|
||||
return out;
|
||||
}
|
||||
|
||||
export function materialOptions(items: HomeboxItem[]): StockOption[] {
|
||||
return mergeOptions(
|
||||
items.map((i) => {
|
||||
const hex = normalizeHex(/#[0-9a-f]{6}\b/i.exec(i.description ?? "")?.[0]);
|
||||
return { name: i.name, ...(hex ? { hex } : {}), inStock: i.quantity > 0 };
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async function call(ctx: SyncContext, url: string, init: RequestInit = {}) {
|
||||
const res = await ctx.fetch(url, init);
|
||||
if (res.status === 401 || res.status === 403) throw new Error(`Homebox refused the request (${res.status}); check HOMEBOX_USER and HOMEBOX_PASS`);
|
||||
if (!res.ok) throw new Error(`Homebox answered ${res.status}`);
|
||||
return res.json() as Promise<unknown>;
|
||||
}
|
||||
|
||||
export function homeboxJob(o: HomeboxOptions): SyncJob {
|
||||
const api = (path: string) => `${o.url.replace(/\/$/, "")}/api/v1${path}`;
|
||||
return {
|
||||
name: "homebox",
|
||||
everyMinutes: o.everyMinutes ?? 60,
|
||||
healthcheckUrl: o.healthcheckUrl,
|
||||
async run(ctx) {
|
||||
const login = (await call(ctx, api("/users/login"), {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", accept: "application/json" },
|
||||
body: JSON.stringify({ username: o.user, password: o.pass, stayLoggedIn: false }),
|
||||
})) as { token?: string };
|
||||
if (!login?.token) throw new Error("Homebox login gave no token");
|
||||
const headers = { accept: "application/json", authorization: /^Bearer /i.test(login.token) ? login.token : `Bearer ${login.token}` };
|
||||
|
||||
const filters: { kind: "labels" | "locations"; f: HomeboxFilter }[] = [
|
||||
...o.labels.map((f) => ({ kind: "labels" as const, f })),
|
||||
...(o.locations ?? []).map((f) => ({ kind: "locations" as const, f })),
|
||||
];
|
||||
const ids = { labels: new Map<string, string>(), locations: new Map<string, string>() };
|
||||
for (const kind of ["labels", "locations"] as const) {
|
||||
if (filters.some((x) => x.kind === kind)) ids[kind] = parseNamed(await call(ctx, api(`/${kind}`), { headers }));
|
||||
}
|
||||
|
||||
const byGroup = new Map<string, HomeboxItem[]>();
|
||||
const unknown: string[] = [];
|
||||
for (const { kind, f } of filters) {
|
||||
const id = ids[kind].get(f.name.toLowerCase());
|
||||
if (!id) {
|
||||
unknown.push(f.name);
|
||||
continue;
|
||||
}
|
||||
const items = byGroup.get(f.group) ?? [];
|
||||
for (let page = 1; page <= 20; page++) {
|
||||
const p = new URLSearchParams({ [kind]: id, page: String(page), pageSize: "100" });
|
||||
const r = parseItems(await call(ctx, api(`/items?${p}`), { headers }));
|
||||
items.push(...r.items);
|
||||
if (r.items.length < 100 || page * 100 >= r.total) break;
|
||||
}
|
||||
byGroup.set(f.group, items);
|
||||
}
|
||||
// A filter naming nothing in Homebox is a setup mistake: fail rather than mark everything out.
|
||||
if (unknown.length) throw new Error(`No Homebox label or location named ${unknown.join(", ")}`);
|
||||
|
||||
const groups: StockGroup[] = [...byGroup].map(([group, items]) => ({ group, options: materialOptions(items) }));
|
||||
const lines = storeOnLines(ctx, "materialStock", groups);
|
||||
const counts: Record<string, number> = { lines };
|
||||
for (const g of groups) counts[g.group.toLowerCase()] = g.options.filter((x) => x.inStock).length;
|
||||
return { message: `${groups.map((g) => `${g.group}: ${g.options.filter((x) => x.inStock).length} in stock`).join(", ")}; on ${lines} lines`, counts };
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
/**
|
||||
* Settings and jobs for the homelab sources on Cosmos: Suwayomi and Kavita
|
||||
* (PersonalReader), Bambuddy and Homebox (Store stock). Each is enabled by
|
||||
* its own URL and credentials alone, not by SYNC_JOBS. All are read only
|
||||
* against their source.
|
||||
*/
|
||||
import type { SyncJob } from "../types.ts";
|
||||
import { bambuddyJob, type BambuddyOptions } from "./bambuddy.ts";
|
||||
import { homeboxJob, parseFilters, type HomeboxOptions } from "./homebox.ts";
|
||||
import { kavitaJob, type KavitaOptions } from "./kavita.ts";
|
||||
import { suwayomiJob, type SuwayomiAuth, type SuwayomiOptions } from "./suwayomi.ts";
|
||||
|
||||
type Base<T> = Omit<T, "everyMinutes" | "healthcheckUrl">;
|
||||
|
||||
export interface HomelabSettings {
|
||||
suwayomi?: Base<SuwayomiOptions>;
|
||||
kavita?: Base<KavitaOptions>;
|
||||
bambuddy?: Base<BambuddyOptions>;
|
||||
homebox?: Base<HomeboxOptions>;
|
||||
}
|
||||
|
||||
const list = (v: string | undefined) => (v ?? "").split(",").map((s) => s.trim()).filter(Boolean);
|
||||
|
||||
export function homelabSettings(env: NodeJS.ProcessEnv): HomelabSettings {
|
||||
const s: HomelabSettings = {};
|
||||
if (env.SUWAYOMI_URL) {
|
||||
const auth = env.SUWAYOMI_AUTH?.trim().toLowerCase().replace("basic_auth", "basic");
|
||||
if (auth && !["ui_login", "basic", "none"].includes(auth)) throw new Error(`SUWAYOMI_AUTH: "${auth}" is not ui_login, basic or none`);
|
||||
s.suwayomi = {
|
||||
url: env.SUWAYOMI_URL,
|
||||
user: env.SUWAYOMI_USER || undefined,
|
||||
pass: env.SUWAYOMI_PASS || undefined,
|
||||
auth: (auth as SuwayomiAuth) || undefined,
|
||||
};
|
||||
}
|
||||
if (env.KAVITA_URL && env.KAVITA_API_KEY) s.kavita = { url: env.KAVITA_URL, apiKey: env.KAVITA_API_KEY };
|
||||
if (env.BAMBUDDY_URL) {
|
||||
s.bambuddy = {
|
||||
url: env.BAMBUDDY_URL,
|
||||
path: env.BAMBUDDY_SPOOLS_PATH || undefined,
|
||||
apiKey: env.BAMBUDDY_API_KEY || undefined,
|
||||
authHeader: env.BAMBUDDY_AUTH_HEADER || undefined,
|
||||
group: env.BAMBUDDY_GROUP || undefined,
|
||||
materials: list(env.BAMBUDDY_MATERIALS),
|
||||
minGrams: env.BAMBUDDY_MIN_GRAMS ? Number(env.BAMBUDDY_MIN_GRAMS) : undefined,
|
||||
};
|
||||
}
|
||||
if (env.HOMEBOX_URL && env.HOMEBOX_USER && env.HOMEBOX_PASS) {
|
||||
s.homebox = {
|
||||
url: env.HOMEBOX_URL,
|
||||
user: env.HOMEBOX_USER,
|
||||
pass: env.HOMEBOX_PASS,
|
||||
labels: parseFilters(env.HOMEBOX_LABELS ?? "leather,paper"),
|
||||
locations: parseFilters(env.HOMEBOX_LOCATIONS),
|
||||
};
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
export function homelabJobs(s: HomelabSettings, common: (name: string) => { everyMinutes?: number; healthcheckUrl?: string }): SyncJob[] {
|
||||
const jobs: SyncJob[] = [];
|
||||
if (s.suwayomi) jobs.push(suwayomiJob({ ...s.suwayomi, ...common("suwayomi") }));
|
||||
if (s.kavita) jobs.push(kavitaJob({ ...s.kavita, ...common("kavita") }));
|
||||
if (s.bambuddy) jobs.push(bambuddyJob({ ...s.bambuddy, ...common("bambuddy") }));
|
||||
if (s.homebox) jobs.push(homeboxJob({ ...s.homebox, ...common("homebox") }));
|
||||
return jobs;
|
||||
}
|
||||
@@ -0,0 +1,238 @@
|
||||
/**
|
||||
* Kavita (PersonalReader's light novels and books): reading progress for
|
||||
* Library items linked to a Kavita series. Daily. Read only against Kavita.
|
||||
*
|
||||
* A Library item is linked only explicitly, never by title:
|
||||
* - `kavitaSeriesId` on the note names the Kavita series; or
|
||||
* - a book, light novel or webnovel note's `anilistId` equals the AniList id
|
||||
* in a Kavita series' metadata (an anilist.co address in its web links).
|
||||
*
|
||||
* Auth: the API key is exchanged for a token at
|
||||
* `POST /api/Plugin/authenticate?apiKey=…&pluginName=…`, sent as
|
||||
* `Authorization: Bearer`. Series come from `POST /api/Series/all-v2` (only
|
||||
* when an AniList match is needed) or `GET /api/Series/<id>`, metadata from
|
||||
* `GET /api/Series/metadata?seriesId=`, the Want to Read list from
|
||||
* `POST /api/want-to-read/v2`.
|
||||
*
|
||||
* State: fully read is finished (or caught up while the series is still
|
||||
* ongoing), partly read is now, unread and on Want to Read is up next;
|
||||
* anything else leaves the note alone. Cached per note: `bookProgress`
|
||||
* (title, percent read, when). Nothing that links into Kavita.
|
||||
*
|
||||
* Written from Kavita's API documentation; not checked against a live server
|
||||
* from here (see docs/content-service.md).
|
||||
*/
|
||||
import type { StoredNote } from "../../store.ts";
|
||||
import type { SyncContext, SyncJob, SyncResult } from "../types.ts";
|
||||
import { errorText, isoTime, MEMO_KEY, StateProposer, targetState, type LibraryState } from "./reader.ts";
|
||||
|
||||
export interface KavitaOptions {
|
||||
url: string;
|
||||
apiKey: string;
|
||||
everyMinutes?: number;
|
||||
healthcheckUrl?: string;
|
||||
delayMs?: number;
|
||||
}
|
||||
|
||||
/** Media that may be linked to Kavita by AniList id. */
|
||||
export const KAVITA_MEDIA = ["book", "light novel", "webnovel"];
|
||||
|
||||
/** Kavita's PublicationStatus: OnGoing 0, Hiatus 1, Completed 2, Cancelled 3, Ended 4. */
|
||||
const DONE_STATUSES = new Set([2, 4]);
|
||||
|
||||
export interface KavitaSeries {
|
||||
id: number;
|
||||
name: string;
|
||||
pages: number;
|
||||
pagesRead: number;
|
||||
latestReadDate?: string;
|
||||
}
|
||||
|
||||
export interface KavitaMetadata {
|
||||
anilistId?: number;
|
||||
publicationStatus?: number;
|
||||
}
|
||||
|
||||
export function parseSeries(raw: unknown): KavitaSeries | null {
|
||||
const s = raw as { id?: number; name?: string; localizedName?: string; pages?: number; pagesRead?: number; latestReadDate?: string };
|
||||
if (typeof s?.id !== "number" || !s.name) return null;
|
||||
const out: KavitaSeries = { id: s.id, name: s.localizedName || s.name, pages: Math.max(0, Number(s.pages) || 0), pagesRead: Math.max(0, Number(s.pagesRead) || 0) };
|
||||
const at = isoTime(s.latestReadDate);
|
||||
if (at) out.latestReadDate = at;
|
||||
return out;
|
||||
}
|
||||
|
||||
const ANILIST_LINK = /anilist\.co\/(?:manga|anime)\/(\d+)/i;
|
||||
|
||||
/** AniList id from the series' web links (comma separated addresses), and its publication status. */
|
||||
export function parseMetadata(raw: unknown): KavitaMetadata {
|
||||
const m = raw as { webLinks?: string; publicationStatus?: number };
|
||||
const out: KavitaMetadata = {};
|
||||
const id = ANILIST_LINK.exec(m?.webLinks ?? "")?.[1];
|
||||
if (id) out.anilistId = Number(id);
|
||||
if (typeof m?.publicationStatus === "number") out.publicationStatus = m.publicationStatus;
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The Library state Kavita's progress means, or null to leave the note alone. */
|
||||
export function kavitaState(s: KavitaSeries, meta: KavitaMetadata, wantToRead: boolean): LibraryState | null {
|
||||
if (s.pages > 0 && s.pagesRead >= s.pages) return DONE_STATUSES.has(meta.publicationStatus ?? -1) ? "finished" : "caught up";
|
||||
if (s.pagesRead > 0) return "now";
|
||||
return wantToRead ? "up next" : null;
|
||||
}
|
||||
|
||||
export const percentRead = (s: KavitaSeries) => (s.pages > 0 ? Math.min(100, Math.floor((s.pagesRead / s.pages) * 100)) : 0);
|
||||
|
||||
// ---------------------------------------------------------------- requests
|
||||
|
||||
const FILTER = { statements: [], combination: 1, limitTo: 0, sortOptions: { sortField: 1, isAscending: true } };
|
||||
|
||||
class Kavita {
|
||||
private headers: Record<string, string> = {};
|
||||
constructor(private readonly ctx: SyncContext, private readonly o: KavitaOptions) {}
|
||||
|
||||
private url = (path: string) => `${this.o.url.replace(/\/$/, "")}${path}`;
|
||||
|
||||
async login() {
|
||||
const p = new URLSearchParams({ apiKey: this.o.apiKey, pluginName: "GregKilmer.com" });
|
||||
const res = await this.ctx.fetch(this.url(`/api/Plugin/authenticate?${p}`), { method: "POST", headers: { accept: "application/json" } });
|
||||
if (res.status === 401 || res.status === 403) throw new Error(`Kavita refused the API key (${res.status})`);
|
||||
if (!res.ok) throw new Error(`Kavita login answered ${res.status}`);
|
||||
const token = ((await res.json()) as { token?: string })?.token;
|
||||
if (!token) throw new Error("Kavita login gave no token");
|
||||
this.headers = { authorization: `Bearer ${token}` };
|
||||
}
|
||||
|
||||
async call(path: string, body?: unknown): Promise<unknown> {
|
||||
const res = await this.ctx.fetch(this.url(path), {
|
||||
method: body === undefined ? "GET" : "POST",
|
||||
headers: { accept: "application/json", ...(body === undefined ? {} : { "content-type": "application/json" }), ...this.headers },
|
||||
body: body === undefined ? undefined : JSON.stringify(body),
|
||||
});
|
||||
if (res.status === 404) return null;
|
||||
if (!res.ok) throw new Error(`Kavita answered ${res.status} for ${path.split("?")[0]}`);
|
||||
return res.json();
|
||||
}
|
||||
|
||||
async allSeries(): Promise<KavitaSeries[]> {
|
||||
const out: KavitaSeries[] = [];
|
||||
for (let page = 1; page <= 50; page++) {
|
||||
const list = (await this.call(`/api/Series/all-v2?PageNumber=${page}&PageSize=200`, FILTER)) as unknown[] | null;
|
||||
if (!Array.isArray(list) || !list.length) break;
|
||||
for (const r of list) {
|
||||
const s = parseSeries(r);
|
||||
if (s) out.push(s);
|
||||
}
|
||||
if (list.length < 200) break;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async series(id: number) {
|
||||
return parseSeries(await this.call(`/api/Series/${id}`));
|
||||
}
|
||||
|
||||
async metadata(id: number) {
|
||||
return parseMetadata(await this.call(`/api/Series/metadata?seriesId=${id}`));
|
||||
}
|
||||
|
||||
async wantToRead(): Promise<Set<number>> {
|
||||
try {
|
||||
const list = (await this.call("/api/want-to-read/v2?PageNumber=1&PageSize=1000", FILTER)) as { id?: number }[] | null;
|
||||
return new Set((list ?? []).map((s) => s.id).filter((id): id is number => typeof id === "number"));
|
||||
} catch (err) {
|
||||
if (this.ctx.signal.aborted) throw this.ctx.signal.reason;
|
||||
this.ctx.log(`Want to Read list not read: ${errorText(err)}`);
|
||||
return new Set();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- the job
|
||||
|
||||
export function kavitaJob(o: KavitaOptions): SyncJob {
|
||||
const delay = o.delayMs ?? 250;
|
||||
return {
|
||||
name: "kavita",
|
||||
everyMinutes: o.everyMinutes ?? 24 * 60,
|
||||
healthcheckUrl: o.healthcheckUrl,
|
||||
timeoutMs: 30 * 60_000,
|
||||
async run(ctx): Promise<SyncResult> {
|
||||
const items = ctx.notes("library-item");
|
||||
const byKavita = items.filter((n) => typeof n.data.kavitaSeriesId === "number");
|
||||
const byAniList = items.filter(
|
||||
(n) => typeof n.data.kavitaSeriesId !== "number" && typeof n.data.anilistId === "number" && KAVITA_MEDIA.includes(String(n.data.medium)),
|
||||
);
|
||||
if (!byKavita.length && !byAniList.length) {
|
||||
ctx.retain([MEMO_KEY]);
|
||||
return { message: "No Library item is linked to Kavita", counts: { linked: 0 } };
|
||||
}
|
||||
|
||||
const k = new Kavita(ctx, o);
|
||||
await k.login();
|
||||
|
||||
// Pairs of (note, Kavita series id), with the metadata already read.
|
||||
const pairs: { note: StoredNote; id: number }[] = byKavita.map((note) => ({ note, id: note.data.kavitaSeriesId as number }));
|
||||
const meta = new Map<number, KavitaMetadata>();
|
||||
const known = new Map<number, KavitaSeries>();
|
||||
if (byAniList.length) {
|
||||
const wanted = new Set(byAniList.map((n) => n.data.anilistId as number));
|
||||
for (const [i, s] of (await k.allSeries()).entries()) {
|
||||
if (i > 0) await ctx.sleep(delay);
|
||||
if (ctx.signal.aborted) throw ctx.signal.reason;
|
||||
const m = await k.metadata(s.id);
|
||||
if (!m.anilistId || !wanted.has(m.anilistId)) continue;
|
||||
meta.set(s.id, m);
|
||||
known.set(s.id, s);
|
||||
for (const note of byAniList.filter((n) => n.data.anilistId === m.anilistId)) pairs.push({ note, id: s.id });
|
||||
}
|
||||
}
|
||||
const wantToRead = await k.wantToRead();
|
||||
|
||||
const proposer = new StateProposer(ctx);
|
||||
const kept: string[] = [MEMO_KEY];
|
||||
let synced = 0;
|
||||
let missing = 0;
|
||||
let failed = 0;
|
||||
const errors: string[] = [];
|
||||
for (const { note, id } of pairs) {
|
||||
if (ctx.signal.aborted) throw ctx.signal.reason;
|
||||
kept.push(note.path);
|
||||
try {
|
||||
let s = known.get(id) ?? null;
|
||||
if (!s) {
|
||||
await ctx.sleep(delay);
|
||||
s = await k.series(id);
|
||||
}
|
||||
if (!s) {
|
||||
missing++;
|
||||
ctx.log(`${note.path}: no Kavita series ${id}`);
|
||||
continue;
|
||||
}
|
||||
const m = meta.get(id) ?? (await k.metadata(id));
|
||||
const progress: Record<string, unknown> = { title: s.name, percent: percentRead(s) };
|
||||
if (s.latestReadDate) progress.readAt = s.latestReadDate;
|
||||
ctx.set(note.path, { bookProgress: progress });
|
||||
synced++;
|
||||
proposer.propose(note, targetState(String(note.data.state), kavitaState(s, m, wantToRead.has(id))));
|
||||
} catch (err) {
|
||||
if (ctx.signal.aborted) throw ctx.signal.reason;
|
||||
failed++;
|
||||
errors.push(`${note.title}: ${errorText(err)}`);
|
||||
}
|
||||
}
|
||||
proposer.save(kept);
|
||||
ctx.retain(kept);
|
||||
|
||||
const c = proposer.counts;
|
||||
const problems = [...errors, ...proposer.errors];
|
||||
return {
|
||||
ok: problems.length === 0,
|
||||
message: problems.length
|
||||
? `${problems.length} problems: ${problems.slice(0, 3).join("; ")}`
|
||||
: `${synced} series synced: ${c.accepted} updated, ${c.held} held for review`,
|
||||
counts: { linked: pairs.length, synced, missing, failed, accepted: c.accepted, held: c.held, waiting: c.waiting, rejected: c.rejected },
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
/**
|
||||
* What the two PersonalReader jobs (Suwayomi and Kavita) share: the state
|
||||
* mapping, proposing a state change through `ctx.apply` without nagging, and
|
||||
* the job-private cache entries.
|
||||
*
|
||||
* PersonalReader is private. A job updates a Library item only when the note
|
||||
* carries the series' id (`anilistId`, or `kavitaSeriesId` for Kavita), and it
|
||||
* stores nothing that names a source site, a scan group or a link into the
|
||||
* reader: only title, chapter or percent, when it was read, and an AniList
|
||||
* cover.
|
||||
*/
|
||||
import { libraryStates } from "@gk/schema";
|
||||
import type { StoredNote } from "../../store.ts";
|
||||
import type { SyncContext, SyncedValues } from "../types.ts";
|
||||
|
||||
export type LibraryState = (typeof libraryStates)[number];
|
||||
|
||||
/**
|
||||
* Job-private cache entries. They are keyed like note paths but are not notes
|
||||
* (no `.md`), so the notes API never shows them to the site.
|
||||
*/
|
||||
export const MEMO_KEY = "_memo";
|
||||
export const SUGGESTIONS_KEY = "_suggestions";
|
||||
|
||||
/** Suwayomi category to Library state. `null`: leave the note alone (Dropped). */
|
||||
const CATEGORY_STATES: Record<string, LibraryState | null> = {
|
||||
reading: "now",
|
||||
"plan to read": "up next",
|
||||
completed: "finished",
|
||||
dropped: null,
|
||||
};
|
||||
|
||||
/** The Library state a reader category maps to; undefined for a category the site does not know. */
|
||||
export function stateForCategory(category: string): LibraryState | null | undefined {
|
||||
const key = category.trim().toLowerCase().replace(/\s+/g, " ");
|
||||
return key in CATEGORY_STATES ? CATEGORY_STATES[key] : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* The state to propose for a note, or null to leave it. Two refinements keep
|
||||
* a distinction the reader cannot make: "caught up" (read to the latest of an
|
||||
* ongoing series) is not turned back into "now", and "finished" is not
|
||||
* turned into "caught up".
|
||||
*/
|
||||
export function targetState(current: string, mapped: LibraryState | null | undefined): LibraryState | null {
|
||||
if (!mapped || mapped === current) return null;
|
||||
if (mapped === "now" && current === "caught up") return null;
|
||||
if (mapped === "caught up" && current === "finished") return null;
|
||||
return mapped;
|
||||
}
|
||||
|
||||
export type ProposeResult = "accepted" | "held" | "waiting" | "rejected" | "skipped";
|
||||
|
||||
interface Memo {
|
||||
/** The state last proposed for each note path. */
|
||||
proposed: Record<string, string>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Proposes state changes, once each. A change Greg rejected is not proposed
|
||||
* again until the reader says something different, so the review queue is not
|
||||
* refilled every hour.
|
||||
*/
|
||||
export class StateProposer {
|
||||
private readonly memo: Memo;
|
||||
readonly counts = { accepted: 0, held: 0, waiting: 0, rejected: 0 };
|
||||
readonly errors: string[] = [];
|
||||
|
||||
constructor(private readonly ctx: SyncContext) {
|
||||
const m = ctx.cached(MEMO_KEY)?.values as Partial<Memo> | undefined;
|
||||
this.memo = { proposed: { ...(m?.proposed ?? {}) } };
|
||||
}
|
||||
|
||||
propose(note: StoredNote, target: LibraryState | null): ProposeResult {
|
||||
const path = note.path;
|
||||
if (!target) {
|
||||
delete this.memo.proposed[path];
|
||||
return "skipped";
|
||||
}
|
||||
if (this.memo.proposed[path] === target) return "skipped";
|
||||
const o = this.ctx.apply({ op: "update", path, fields: { state: target } });
|
||||
if (o.status === "accepted" || o.status === "unchanged") {
|
||||
this.counts.accepted++;
|
||||
this.memo.proposed[path] = target;
|
||||
return "accepted";
|
||||
}
|
||||
if (o.status === "held") {
|
||||
this.counts.held++;
|
||||
this.memo.proposed[path] = target;
|
||||
return "held";
|
||||
}
|
||||
// A change already waits for review: try again next run.
|
||||
if (o.reasons.some((r) => r.code === "waiting")) {
|
||||
this.counts.waiting++;
|
||||
return "waiting";
|
||||
}
|
||||
this.counts.rejected++;
|
||||
this.errors.push(`${note.title}: ${o.reasons.map((r) => r.message).join("; ")}`);
|
||||
return "rejected";
|
||||
}
|
||||
|
||||
/** Save the memo, forgetting notes no longer in scope. */
|
||||
save(paths: Iterable<string>) {
|
||||
const keep = new Set(paths);
|
||||
const proposed = Object.fromEntries(Object.entries(this.memo.proposed).filter(([p]) => keep.has(p)));
|
||||
this.ctx.set(MEMO_KEY, { proposed } satisfies SyncedValues);
|
||||
}
|
||||
}
|
||||
|
||||
/** Epoch seconds or milliseconds (or an ISO string) as ISO; undefined for zero or junk. */
|
||||
export function isoTime(v: unknown): string | undefined {
|
||||
if (v === null || v === undefined || v === "") return undefined;
|
||||
if (typeof v === "string" && !/^\d+$/.test(v)) {
|
||||
const t = Date.parse(v);
|
||||
// Kavita says 0001-01-01 for "never".
|
||||
return Number.isFinite(t) && t > 0 ? new Date(t).toISOString() : undefined;
|
||||
}
|
||||
const n = Number(v);
|
||||
if (!Number.isFinite(n) || n <= 0) return undefined;
|
||||
return new Date(n < 1e12 ? n * 1000 : n).toISOString();
|
||||
}
|
||||
|
||||
export const errorText = (err: unknown) => (err instanceof Error ? err.message : String(err));
|
||||
@@ -0,0 +1,64 @@
|
||||
/**
|
||||
* What the two Store stock jobs (Bambuddy filament, Homebox leather and
|
||||
* paper) share: the synced shape and how it lands on product lines.
|
||||
*
|
||||
* A job reads what is in stock and stores it on every product-line note that
|
||||
* has an option in a group the job feeds (Filament for Bambuddy; the groups
|
||||
* named in HOMEBOX_LABELS for Homebox). The site lays it under the note's own
|
||||
* options (services/site/src/lib/store.ts): a note's option always wins, a
|
||||
* note option that leaves `inStock` or `hex` out takes the synced value, and
|
||||
* in-stock colors the note does not list are added after the note's own.
|
||||
*/
|
||||
import type { SyncContext, SyncedValues } from "../types.ts";
|
||||
|
||||
export interface StockOption {
|
||||
name: string;
|
||||
hex?: string;
|
||||
inStock: boolean;
|
||||
}
|
||||
|
||||
export interface StockGroup {
|
||||
group: string;
|
||||
options: StockOption[];
|
||||
}
|
||||
|
||||
/** `#rrggbb` from `rrggbb`, `#RRGGBB` or `#rrggbbaa`; undefined otherwise. */
|
||||
export function normalizeHex(v: unknown): string | undefined {
|
||||
if (typeof v !== "string") return undefined;
|
||||
const m = /^#?([0-9a-f]{6})(?:[0-9a-f]{2})?$/i.exec(v.trim());
|
||||
return m ? `#${m[1]!.toLowerCase()}` : undefined;
|
||||
}
|
||||
|
||||
/** One entry per name (case-insensitive): in stock when any entry is, keeping the first hex seen. */
|
||||
export function mergeOptions(options: StockOption[]): StockOption[] {
|
||||
const out = new Map<string, StockOption>();
|
||||
for (const o of options) {
|
||||
const key = o.name.trim().toLowerCase();
|
||||
if (!key) continue;
|
||||
const prev = out.get(key);
|
||||
if (!prev) out.set(key, { ...o, name: o.name.trim() });
|
||||
else {
|
||||
prev.inStock ||= o.inStock;
|
||||
prev.hex ??= o.hex;
|
||||
}
|
||||
}
|
||||
return [...out.values()].sort((a, b) => Number(b.inStock) - Number(a.inStock) || a.name.localeCompare(b.name));
|
||||
}
|
||||
|
||||
/**
|
||||
* Store `groups` under `key` on every product line that uses one of them,
|
||||
* forgetting lines that no longer do. Returns how many lines were updated.
|
||||
*/
|
||||
export function storeOnLines(ctx: SyncContext, key: string, groups: StockGroup[]): number {
|
||||
const fed = new Map(groups.map((g) => [g.group.toLowerCase(), g]));
|
||||
const kept: string[] = [];
|
||||
for (const line of ctx.notes("product-line")) {
|
||||
const used = new Set(((line.data.options as { group?: string }[] | undefined) ?? []).map((o) => String(o.group ?? "").toLowerCase()));
|
||||
const mine = [...used].map((g) => fed.get(g)).filter((g): g is StockGroup => !!g);
|
||||
if (!mine.length) continue;
|
||||
ctx.set(line.path, { [key]: mine } satisfies SyncedValues);
|
||||
kept.push(line.path);
|
||||
}
|
||||
ctx.retain(kept);
|
||||
return kept.length;
|
||||
}
|
||||
@@ -0,0 +1,313 @@
|
||||
/**
|
||||
* Suwayomi (PersonalReader's manga and manhwa): reading progress for Library
|
||||
* items that carry the series' `anilistId`. Hourly. Read only against
|
||||
* Suwayomi.
|
||||
*
|
||||
* - Logs in with its own read-only account. With Suwayomi's `ui_login` auth
|
||||
* mode that is the GraphQL `login` mutation, whose access token is sent as
|
||||
* `Authorization: Bearer`; with `basic_auth` it is HTTP basic auth.
|
||||
* - Reads the library over GraphQL (`<SUWAYOMI_URL>/api/graphql`): each
|
||||
* category (Reading, Plan to read, Completed, Dropped) with its series, the
|
||||
* furthest chapter read and when, and the AniList tracker record, whose
|
||||
* remote id is the AniList id.
|
||||
* - Maps the category to the note's state (Reading: now, Plan to read: up
|
||||
* next, Completed: finished, Dropped: left alone) through `ctx.apply` as
|
||||
* `sync`; Greg's notes are held for review.
|
||||
* - Caches `chapterProgress` (title, chapter, when) and the AniList `cover`.
|
||||
* Never a source name, scan group, thumbnail or any Suwayomi address.
|
||||
* - A series in Reading with no Library item becomes a suggestion on the
|
||||
* admin page.
|
||||
*
|
||||
* Written from Suwayomi-Server's GraphQL schema; not checked against a live
|
||||
* server from here (see docs/content-service.md).
|
||||
*/
|
||||
import type { StoredNote } from "../../store.ts";
|
||||
import type { SyncContext, SyncJob } from "../types.ts";
|
||||
import { ANILIST_API } from "./covers.ts";
|
||||
import { errorText, isoTime, StateProposer, stateForCategory, SUGGESTIONS_KEY, MEMO_KEY, targetState } from "./reader.ts";
|
||||
|
||||
/** Suwayomi's (and Mihon's) tracker id for AniList. */
|
||||
export const ANILIST_TRACKER_ID = 2;
|
||||
|
||||
export type SuwayomiAuth = "ui_login" | "basic" | "none";
|
||||
|
||||
export interface SuwayomiOptions {
|
||||
url: string;
|
||||
user?: string;
|
||||
pass?: string;
|
||||
/** Default `ui_login` when a user is set, else `none`. */
|
||||
auth?: SuwayomiAuth;
|
||||
everyMinutes?: number;
|
||||
healthcheckUrl?: string;
|
||||
}
|
||||
|
||||
export const LOGIN_MUTATION = `mutation GkLogin($username: String!, $password: String!) {
|
||||
login(input: { username: $username, password: $password }) { accessToken }
|
||||
}`;
|
||||
|
||||
/** Only what the site needs: no ids, urls, thumbnails, sources or scanlators. */
|
||||
export const LIBRARY_QUERY = `query GkLibrary {
|
||||
categories {
|
||||
nodes {
|
||||
name
|
||||
mangas {
|
||||
nodes {
|
||||
title
|
||||
inLibrary
|
||||
latestReadChapter { chapterNumber lastReadAt }
|
||||
trackRecords { nodes { trackerId remoteId title lastChapterRead } }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}`;
|
||||
|
||||
/** Batched AniList lookup for the cover and what a suggestion needs. */
|
||||
export const ANILIST_BATCH_QUERY = `query ($ids: [Int]) {
|
||||
Page(perPage: 50) {
|
||||
media(id_in: $ids, type: MANGA) {
|
||||
id
|
||||
title { english romaji }
|
||||
format
|
||||
countryOfOrigin
|
||||
coverImage { extraLarge large }
|
||||
}
|
||||
}
|
||||
}`;
|
||||
|
||||
export interface Series {
|
||||
title: string;
|
||||
category: string;
|
||||
anilistId?: number;
|
||||
/** Furthest chapter read. */
|
||||
chapter?: number;
|
||||
readAt?: string;
|
||||
}
|
||||
|
||||
export interface ChapterProgress {
|
||||
title: string;
|
||||
/** Library state the category maps to, or the category itself for Dropped. */
|
||||
status: string;
|
||||
chapter?: number;
|
||||
readAt?: string;
|
||||
}
|
||||
|
||||
export interface AniListInfo {
|
||||
title?: string;
|
||||
medium: "manga" | "manhwa" | "light novel";
|
||||
cover?: string;
|
||||
}
|
||||
|
||||
const CATEGORY_ORDER = ["reading", "plan to read", "completed", "dropped"];
|
||||
|
||||
interface RawChapter {
|
||||
chapterNumber?: number | string | null;
|
||||
lastReadAt?: number | string | null;
|
||||
}
|
||||
interface RawManga {
|
||||
title?: string;
|
||||
inLibrary?: boolean;
|
||||
latestReadChapter?: RawChapter | null;
|
||||
trackRecords?: { nodes?: { trackerId?: number; remoteId?: number | string; title?: string; lastChapterRead?: number | null }[] } | null;
|
||||
}
|
||||
interface RawLibrary {
|
||||
data?: { categories?: { nodes?: { name?: string; mangas?: { nodes?: RawManga[] } | null }[] } | null } | null;
|
||||
errors?: { message?: string }[];
|
||||
}
|
||||
|
||||
const positive = (v: unknown) => {
|
||||
const n = Number(v);
|
||||
return Number.isFinite(n) && n > 0 ? n : undefined;
|
||||
};
|
||||
|
||||
/**
|
||||
* The library response as series. A series in several known categories takes
|
||||
* the first of Reading, Plan to read, Completed, Dropped; series only in
|
||||
* categories the site does not know are left out.
|
||||
*/
|
||||
export function parseLibrary(json: unknown): Series[] {
|
||||
const raw = json as RawLibrary;
|
||||
if (raw?.errors?.length) throw new Error(`Suwayomi: ${raw.errors.map((e) => e.message).join("; ")}`);
|
||||
const cats = raw?.data?.categories?.nodes;
|
||||
if (!Array.isArray(cats)) throw new Error("Suwayomi answered without a library");
|
||||
const best = new Map<string, Series & { rank: number }>();
|
||||
for (const cat of cats) {
|
||||
const name = (cat.name ?? "").trim();
|
||||
const rank = CATEGORY_ORDER.indexOf(name.toLowerCase().replace(/\s+/g, " "));
|
||||
if (rank < 0 || stateForCategory(name) === undefined) continue;
|
||||
for (const m of cat.mangas?.nodes ?? []) {
|
||||
if (m.inLibrary === false || !m.title) continue;
|
||||
const track = (m.trackRecords?.nodes ?? []).find((t) => t.trackerId === ANILIST_TRACKER_ID);
|
||||
const anilistId = positive(track?.remoteId);
|
||||
const s: Series & { rank: number } = { title: (track?.title || m.title).trim(), category: name, rank };
|
||||
if (anilistId && Number.isInteger(anilistId)) s.anilistId = anilistId;
|
||||
const chapter = positive(m.latestReadChapter?.chapterNumber) ?? positive(track?.lastChapterRead);
|
||||
if (chapter !== undefined) s.chapter = chapter;
|
||||
const readAt = isoTime(m.latestReadChapter?.lastReadAt);
|
||||
if (readAt) s.readAt = readAt;
|
||||
// Key by AniList id when there is one, else by title.
|
||||
const key = anilistId ? `a${anilistId}` : `t${m.title}`;
|
||||
const prev = best.get(key);
|
||||
if (!prev || rank < prev.rank) best.set(key, s);
|
||||
}
|
||||
}
|
||||
return [...best.values()].map(({ rank: _, ...s }) => s);
|
||||
}
|
||||
|
||||
/** AniList's batched Media answer, by id. */
|
||||
export function parseAniListBatch(json: unknown): Map<number, AniListInfo> {
|
||||
type M = { id: number; title?: { english?: string; romaji?: string }; format?: string; countryOfOrigin?: string; coverImage?: { extraLarge?: string; large?: string } };
|
||||
const media = (json as { data?: { Page?: { media?: M[] } } })?.data?.Page?.media ?? [];
|
||||
const out = new Map<number, AniListInfo>();
|
||||
for (const m of media) {
|
||||
if (typeof m?.id !== "number") continue;
|
||||
const cover = m.coverImage?.extraLarge ?? m.coverImage?.large;
|
||||
const info: AniListInfo = { medium: m.format === "NOVEL" ? "light novel" : m.countryOfOrigin === "KR" ? "manhwa" : "manga" };
|
||||
const title = m.title?.english || m.title?.romaji;
|
||||
if (title) info.title = title;
|
||||
if (cover?.startsWith("https://")) info.cover = cover;
|
||||
out.set(m.id, info);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- requests
|
||||
|
||||
const endpoint = (base: string) => `${base.replace(/\/$/, "")}/api/graphql`;
|
||||
|
||||
async function graphql(ctx: SyncContext, url: string, headers: Record<string, string>, query: string, variables?: Record<string, unknown>) {
|
||||
const res = await ctx.fetch(url, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", accept: "application/json", ...headers },
|
||||
body: JSON.stringify({ query, variables }),
|
||||
});
|
||||
if (res.status === 401 || res.status === 403) throw new Error(`Suwayomi refused the login (${res.status})`);
|
||||
if (!res.ok) throw new Error(`Suwayomi answered ${res.status}`);
|
||||
return res.json() as Promise<unknown>;
|
||||
}
|
||||
|
||||
/** The headers that authenticate one run. */
|
||||
export async function authHeaders(ctx: SyncContext, o: SuwayomiOptions): Promise<Record<string, string>> {
|
||||
const mode = o.auth ?? (o.user ? "ui_login" : "none");
|
||||
if (mode === "none") return {};
|
||||
if (!o.user || !o.pass) throw new Error("SUWAYOMI_USER and SUWAYOMI_PASS are required for this auth mode");
|
||||
if (mode === "basic") return { authorization: `Basic ${Buffer.from(`${o.user}:${o.pass}`).toString("base64")}` };
|
||||
const json = (await graphql(ctx, endpoint(o.url), {}, LOGIN_MUTATION, { username: o.user, password: o.pass })) as {
|
||||
data?: { login?: { accessToken?: string } };
|
||||
errors?: { message?: string }[];
|
||||
};
|
||||
const token = json?.data?.login?.accessToken;
|
||||
if (!token) throw new Error(`Suwayomi login failed${json?.errors?.length ? `: ${json.errors.map((e) => e.message).join("; ")}` : ""}`);
|
||||
return { authorization: `Bearer ${token}` };
|
||||
}
|
||||
|
||||
async function aniList(ctx: SyncContext, ids: number[]): Promise<Map<number, AniListInfo>> {
|
||||
const out = new Map<number, AniListInfo>();
|
||||
for (let i = 0; i < ids.length; i += 50) {
|
||||
if (i > 0) await ctx.sleep(1000);
|
||||
const res = await ctx.fetch(ANILIST_API, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", accept: "application/json" },
|
||||
body: JSON.stringify({ query: ANILIST_BATCH_QUERY, variables: { ids: ids.slice(i, i + 50) } }),
|
||||
});
|
||||
if (!res.ok) throw new Error(`AniList answered ${res.status}`);
|
||||
for (const [k, v] of parseAniListBatch(await res.json())) out.set(k, v);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- the job
|
||||
|
||||
export interface Suggestion {
|
||||
anilistId: number;
|
||||
title: string;
|
||||
medium: AniListInfo["medium"];
|
||||
cover?: string;
|
||||
}
|
||||
|
||||
export function suwayomiJob(o: SuwayomiOptions): SyncJob {
|
||||
return {
|
||||
name: "suwayomi",
|
||||
everyMinutes: o.everyMinutes ?? 60,
|
||||
healthcheckUrl: o.healthcheckUrl,
|
||||
async run(ctx) {
|
||||
const headers = await authHeaders(ctx, o);
|
||||
const series = parseLibrary(await graphql(ctx, endpoint(o.url), headers, LIBRARY_QUERY));
|
||||
|
||||
const items = ctx.notes("library-item");
|
||||
const byAniList = new Map<number, StoredNote[]>();
|
||||
for (const n of items) {
|
||||
const id = n.data.anilistId;
|
||||
if (typeof id === "number") byAniList.set(id, [...(byAniList.get(id) ?? []), n]);
|
||||
}
|
||||
|
||||
// AniList: covers for matched series, titles and media for suggestions.
|
||||
const tracked = series.filter((s) => s.anilistId);
|
||||
const wanted = tracked.filter((s) => byAniList.has(s.anilistId!) || stateForCategory(s.category) === "now").map((s) => s.anilistId!);
|
||||
let info = new Map<number, AniListInfo>();
|
||||
let anilistError: string | null = null;
|
||||
try {
|
||||
if (wanted.length) info = await aniList(ctx, [...new Set(wanted)]);
|
||||
} catch (err) {
|
||||
if (ctx.signal.aborted) throw ctx.signal.reason;
|
||||
anilistError = errorText(err);
|
||||
}
|
||||
|
||||
const proposer = new StateProposer(ctx);
|
||||
const kept: string[] = [MEMO_KEY, SUGGESTIONS_KEY];
|
||||
const suggestions: Suggestion[] = [];
|
||||
const oldSuggestions = (ctx.cached(SUGGESTIONS_KEY)?.values.items as Suggestion[] | undefined) ?? [];
|
||||
let matched = 0;
|
||||
|
||||
for (const s of series) {
|
||||
const mapped = stateForCategory(s.category);
|
||||
const notes = s.anilistId ? (byAniList.get(s.anilistId) ?? []) : [];
|
||||
if (!notes.length) {
|
||||
if (mapped === "now" && s.anilistId) {
|
||||
const a = info.get(s.anilistId);
|
||||
const old = oldSuggestions.find((x) => x.anilistId === s.anilistId);
|
||||
const sug: Suggestion = { anilistId: s.anilistId, title: a?.title ?? old?.title ?? s.title, medium: a?.medium ?? old?.medium ?? "manga" };
|
||||
const cover = a?.cover ?? old?.cover;
|
||||
if (cover) sug.cover = cover;
|
||||
suggestions.push(sug);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
for (const note of notes) {
|
||||
matched++;
|
||||
kept.push(note.path);
|
||||
const progress: Record<string, unknown> = { title: s.title, status: mapped ?? s.category.toLowerCase() };
|
||||
if (s.chapter !== undefined) progress.chapter = s.chapter;
|
||||
if (s.readAt) progress.readAt = s.readAt;
|
||||
// Keep the last good cover when AniList could not be asked.
|
||||
const cover = info.get(s.anilistId!)?.cover ?? (ctx.cached(note.path)?.values.cover as string | undefined);
|
||||
ctx.set(note.path, cover ? { chapterProgress: progress, cover } : { chapterProgress: progress });
|
||||
proposer.propose(note, targetState(String(note.data.state), mapped));
|
||||
}
|
||||
}
|
||||
|
||||
ctx.set(SUGGESTIONS_KEY, { items: suggestions });
|
||||
proposer.save(kept);
|
||||
ctx.retain(kept);
|
||||
|
||||
const c = proposer.counts;
|
||||
const problems = [...(anilistError ? [`AniList: ${anilistError}`] : []), ...proposer.errors];
|
||||
return {
|
||||
ok: problems.length === 0,
|
||||
message: problems.length
|
||||
? problems.slice(0, 3).join("; ")
|
||||
: `${matched} series synced: ${c.accepted} updated, ${c.held} held for review, ${suggestions.length} suggested`,
|
||||
counts: {
|
||||
series: series.length,
|
||||
matched,
|
||||
untracked: series.length - tracked.length,
|
||||
accepted: c.accepted,
|
||||
held: c.held,
|
||||
waiting: c.waiting,
|
||||
rejected: c.rejected,
|
||||
suggestions: suggestions.length,
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
/**
|
||||
* Runs each sync job on its interval: once shortly after start (staggered so
|
||||
* the jobs do not all hit the network together), then every `everyMinutes`.
|
||||
* A job never runs twice at once: a tick that finds it still running is
|
||||
* skipped. A run longer than its timeout is aborted and reported as failed.
|
||||
*
|
||||
* Every run is reported to sync_runs (shown on /api/syncs and the admin
|
||||
* Overview) and pinged to the job's Healthchecks URL when it has one.
|
||||
*/
|
||||
import type { ContentService } from "../service.ts";
|
||||
import type { SyncCache } from "./cache.ts";
|
||||
import type { SyncContext, SyncJob, SyncResult } from "./types.ts";
|
||||
|
||||
export const SYNC_USER_AGENT = "GregKilmer.com content sync (+https://gregkilmer.com)";
|
||||
const DEFAULT_TIMEOUT_MS = 10 * 60_000;
|
||||
|
||||
export interface SchedulerOptions {
|
||||
/** Delay between the jobs' first runs. Default 20 seconds. */
|
||||
staggerMs?: number;
|
||||
/** Injectable for tests. */
|
||||
fetch?: typeof fetch;
|
||||
now?: () => Date;
|
||||
log?: (message: string) => void;
|
||||
}
|
||||
|
||||
export interface RunReport {
|
||||
job: string;
|
||||
ok: boolean;
|
||||
message: string;
|
||||
counts: Record<string, number>;
|
||||
/** True when the run was skipped because the job was already running. */
|
||||
skipped?: boolean;
|
||||
}
|
||||
|
||||
export class Scheduler {
|
||||
readonly cache: SyncCache;
|
||||
private readonly running = new Map<string, Promise<RunReport>>();
|
||||
private readonly timers = new Set<NodeJS.Timeout>();
|
||||
private readonly stopping = new AbortController();
|
||||
private readonly fetchFn: typeof fetch;
|
||||
private readonly now: () => Date;
|
||||
private readonly log: (message: string) => void;
|
||||
|
||||
constructor(
|
||||
private readonly service: ContentService,
|
||||
readonly jobs: SyncJob[],
|
||||
private readonly o: SchedulerOptions = {},
|
||||
) {
|
||||
this.cache = service.store.syncCache;
|
||||
this.fetchFn = o.fetch ?? fetch;
|
||||
this.now = o.now ?? (() => new Date());
|
||||
this.log = o.log ?? ((m) => console.log(m));
|
||||
}
|
||||
|
||||
/** Run every job once, staggered, then on its interval. */
|
||||
start() {
|
||||
const stagger = this.o.staggerMs ?? 20_000;
|
||||
this.jobs.forEach((job, i) => {
|
||||
const loop = () => {
|
||||
void this.runNow(job.name);
|
||||
this.later(loop, job.everyMinutes * 60_000);
|
||||
};
|
||||
this.later(loop, i * stagger);
|
||||
});
|
||||
}
|
||||
|
||||
stop() {
|
||||
for (const t of this.timers) clearTimeout(t);
|
||||
this.timers.clear();
|
||||
this.stopping.abort();
|
||||
}
|
||||
|
||||
/** Run one job now, unless it is already running. */
|
||||
runNow(name: string): Promise<RunReport> {
|
||||
const job = this.jobs.find((j) => j.name === name);
|
||||
if (!job) return Promise.resolve({ job: name, ok: false, message: "No such job", counts: {} });
|
||||
if (this.running.has(name)) {
|
||||
return Promise.resolve({ job: name, ok: false, message: "Already running", counts: {}, skipped: true });
|
||||
}
|
||||
const run = this.execute(job);
|
||||
this.running.set(name, run);
|
||||
// Released only when the job's promise settles, even after a timeout, so
|
||||
// a job that ignores its abort signal still never overlaps itself.
|
||||
return run;
|
||||
}
|
||||
|
||||
isRunning(name: string) {
|
||||
return this.running.has(name);
|
||||
}
|
||||
|
||||
private later(fn: () => void, ms: number) {
|
||||
const t = setTimeout(() => {
|
||||
this.timers.delete(t);
|
||||
fn();
|
||||
}, ms);
|
||||
t.unref?.();
|
||||
this.timers.add(t);
|
||||
}
|
||||
|
||||
private async execute(job: SyncJob): Promise<RunReport> {
|
||||
const abort = new AbortController();
|
||||
const onStop = () => abort.abort(new Error("The service is stopping"));
|
||||
this.stopping.signal.addEventListener("abort", onStop);
|
||||
const timeoutMs = job.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
||||
this.ping(job, "/start");
|
||||
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
const work = job.run(this.context(job, abort.signal));
|
||||
const timedOut = new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => {
|
||||
const err = new Error(`Timed out after ${Math.round(timeoutMs / 1000)} seconds`);
|
||||
abort.abort(err);
|
||||
reject(err);
|
||||
}, timeoutMs);
|
||||
});
|
||||
|
||||
let report: RunReport;
|
||||
try {
|
||||
const result: SyncResult = await Promise.race([work, timedOut]);
|
||||
const ok = result.ok !== false;
|
||||
report = { job: job.name, ok, message: result.message ?? (ok ? "OK" : "Some items failed"), counts: result.counts ?? {} };
|
||||
} catch (err) {
|
||||
report = { job: job.name, ok: false, message: err instanceof Error ? err.message : String(err), counts: {} };
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
this.stopping.signal.removeEventListener("abort", onStop);
|
||||
}
|
||||
|
||||
this.service.store.reportSync({ ...report, message: report.message.slice(0, 2000), at: this.now().toISOString() });
|
||||
this.ping(job, report.ok ? "" : "/fail", report.message);
|
||||
if (!report.ok) this.log(`Sync ${job.name} failed: ${report.message}`);
|
||||
|
||||
// Hold the running flag until the job itself has finished.
|
||||
work.then(
|
||||
() => this.running.delete(job.name),
|
||||
() => this.running.delete(job.name),
|
||||
);
|
||||
return report;
|
||||
}
|
||||
|
||||
private context(job: SyncJob, signal: AbortSignal): SyncContext {
|
||||
const { service, cache } = this;
|
||||
return {
|
||||
signal,
|
||||
now: this.now,
|
||||
notes: (type) => service.store.list({ type }),
|
||||
cached: (path) => cache.get(job.name, path),
|
||||
set: (path, values) => cache.set(job.name, path, values, this.now().toISOString()),
|
||||
retain: (paths) => cache.retain(job.name, paths),
|
||||
apply: (change) => service.apply(change, "sync"),
|
||||
fetch: (url, init = {}) => {
|
||||
const { timeoutMs = 30_000, headers, ...rest } = init;
|
||||
return this.fetchFn(url, {
|
||||
...rest,
|
||||
headers: { "user-agent": SYNC_USER_AGENT, ...(headers as Record<string, string> | undefined) },
|
||||
signal: AbortSignal.any([signal, AbortSignal.timeout(timeoutMs)]),
|
||||
});
|
||||
},
|
||||
sleep: (ms) =>
|
||||
new Promise<void>((resolve) => {
|
||||
if (signal.aborted) return resolve();
|
||||
const t = setTimeout(done, ms);
|
||||
function done() {
|
||||
clearTimeout(t);
|
||||
signal.removeEventListener("abort", done);
|
||||
resolve();
|
||||
}
|
||||
signal.addEventListener("abort", done);
|
||||
}),
|
||||
log: (m) => this.log(`Sync ${job.name}: ${m}`),
|
||||
};
|
||||
}
|
||||
|
||||
/** Healthchecks: `<url>/start`, `<url>` on success, `<url>/fail` with the message. */
|
||||
private ping(job: SyncJob, suffix: string, body?: string) {
|
||||
if (!job.healthcheckUrl) return;
|
||||
const url = job.healthcheckUrl.replace(/\/$/, "") + suffix;
|
||||
this.fetchFn(url, { method: "POST", body: body ?? "", signal: AbortSignal.timeout(10_000) }).catch(() => {});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
/**
|
||||
* Sync settings from the environment. A job is enabled only when its config
|
||||
* is present: the public sources (no credentials) are named in SYNC_JOBS;
|
||||
* jobs that need a URL or a login are enabled by those variables alone.
|
||||
*/
|
||||
import { homelabSettings, type HomelabSettings } from "./jobs/homelab.ts";
|
||||
|
||||
export interface SyncSettings {
|
||||
/** Public jobs switched on in SYNC_JOBS, e.g. "archidekt,bgg,covers". */
|
||||
enabled: Set<string>;
|
||||
/** SYNC_<JOB>_MINUTES overrides a job's interval. */
|
||||
minutes: Record<string, number>;
|
||||
/** SYNC_<JOB>_HEALTHCHECK_URL. */
|
||||
healthchecks: Record<string, string>;
|
||||
bggToken?: string;
|
||||
tmdbApiKey?: string;
|
||||
/** Suwayomi, Kavita, Bambuddy and Homebox: enabled by their own URL and login. */
|
||||
homelab: HomelabSettings;
|
||||
}
|
||||
|
||||
const envKey = (job: string) => job.toUpperCase().replace(/[^A-Z0-9]/g, "_");
|
||||
|
||||
export function syncSettings(env: NodeJS.ProcessEnv = process.env): SyncSettings {
|
||||
const enabled = new Set((env.SYNC_JOBS ?? "").split(",").map((s) => s.trim().toLowerCase()).filter(Boolean));
|
||||
const minutes: Record<string, number> = {};
|
||||
const healthchecks: Record<string, string> = {};
|
||||
for (const [k, v] of Object.entries(env)) {
|
||||
if (!v) continue;
|
||||
const m = /^SYNC_([A-Z0-9_]+)_(MINUTES|HEALTHCHECK_URL)$/.exec(k);
|
||||
if (!m) continue;
|
||||
const job = m[1]!.toLowerCase().replace(/_/g, "-");
|
||||
if (m[2] === "MINUTES" && Number(v) > 0) minutes[job] = Number(v);
|
||||
if (m[2] === "HEALTHCHECK_URL") healthchecks[job] = v;
|
||||
}
|
||||
return {
|
||||
enabled,
|
||||
minutes,
|
||||
healthchecks,
|
||||
bggToken: env.BGG_TOKEN || undefined,
|
||||
tmdbApiKey: env.TMDB_API_KEY || undefined,
|
||||
homelab: homelabSettings(env),
|
||||
};
|
||||
}
|
||||
|
||||
/** The env var name for one job's setting, for docs and messages. */
|
||||
export const syncEnvName = (job: string, what: "MINUTES" | "HEALTHCHECK_URL") => `SYNC_${envKey(job)}_${what}`;
|
||||
@@ -0,0 +1,73 @@
|
||||
/**
|
||||
* The sync job interface. A job reads notes, asks one outside source about
|
||||
* them, and either stores what it learned in the sync cache (shown on the
|
||||
* site under the note's own values) or, for jobs that must change a note,
|
||||
* proposes the change through `ContentService.apply` as `sync`.
|
||||
*
|
||||
* A job never touches the vault, the store's tables or another job's cache
|
||||
* directly: everything goes through the context, so a sync can do no more
|
||||
* than an agent can. See docs/content-service.md, "Sync jobs".
|
||||
*/
|
||||
import type { Change, Outcome } from "../service.ts";
|
||||
import type { StoredNote } from "../store.ts";
|
||||
|
||||
/** Synced values for one note: plain JSON. */
|
||||
export type SyncedValues = Record<string, unknown>;
|
||||
|
||||
export interface CacheEntry {
|
||||
values: SyncedValues;
|
||||
/** When these values were fetched. */
|
||||
fetchedAt: string;
|
||||
}
|
||||
|
||||
export interface SyncContext {
|
||||
/** Aborted when the run times out or the service stops. Pass it to every request. */
|
||||
readonly signal: AbortSignal;
|
||||
/** The current time (injectable for tests). */
|
||||
now(): Date;
|
||||
/** Live notes of one type, published or not. Read only. */
|
||||
notes(type: string): StoredNote[];
|
||||
/** This job's cached values for a note, if any. */
|
||||
cached(path: string): CacheEntry | null;
|
||||
/**
|
||||
* Store values fetched for a note, replacing this job's previous values for
|
||||
* it. Only call this with values that were fetched successfully: whatever
|
||||
* is not set keeps its last good values.
|
||||
*/
|
||||
set(path: string, values: SyncedValues): void;
|
||||
/** Forget this job's values for every note not in `paths` (notes that left the job's scope). */
|
||||
retain(paths: Iterable<string>): void;
|
||||
/**
|
||||
* Propose a note change as `sync`. It runs the same rules as an agent's
|
||||
* change: contact details, secrets, text loss, notes the sync did not write
|
||||
* (unless `agentEdits: true`), publishing and the batch limit all hold it
|
||||
* for review. A sync cannot delete, move or rename a note.
|
||||
*/
|
||||
apply(change: Change): Outcome;
|
||||
/** fetch with the sync User-Agent, the run's abort signal and a per-request timeout. */
|
||||
fetch(url: string, init?: RequestInit & { timeoutMs?: number }): Promise<Response>;
|
||||
/** A polite pause between requests; ends early if the run is aborted. */
|
||||
sleep(ms: number): Promise<void>;
|
||||
log(message: string): void;
|
||||
}
|
||||
|
||||
export interface SyncResult {
|
||||
/** Shown on the admin Overview. */
|
||||
message?: string;
|
||||
counts?: Record<string, number>;
|
||||
/** Set false when the run finished but some items failed; the run is reported as failed. */
|
||||
ok?: boolean;
|
||||
}
|
||||
|
||||
export interface SyncJob {
|
||||
/** Lowercase id, as in sync_runs and the cache: `archidekt`, `bgg`, `covers`. */
|
||||
name: string;
|
||||
/** Minutes between runs. */
|
||||
everyMinutes: number;
|
||||
/** A run longer than this is aborted and reported as failed. Default 10 minutes. */
|
||||
timeoutMs?: number;
|
||||
/** Healthchecks ping URL: `/start` before, the URL on success, `/fail` on failure. */
|
||||
healthcheckUrl?: string;
|
||||
/** One run. Throwing marks the run failed; the cache keeps its last good values. */
|
||||
run(ctx: SyncContext): Promise<SyncResult>;
|
||||
}
|
||||
Reference in New Issue
Block a user