mirror of
https://github.com/Patiun/com.gregkilmer.www.git
synced 2026-11-08 04:08:24 +00:00
Phase 6: sync jobs, inquiry and order forms
- Sync scheduler in the content service with a per-job cache (sync_cache), sync_runs reporting and Healthchecks pings. Jobs: Archidekt, BoardGameGeek, cover art (Open Library, TMDB, AniList), Suwayomi and Kavita reading progress, Bambuddy filament and Homebox material stock. A note's own value always wins over a synced one. - PersonalReader: updates only Library notes with a matching anilistId, changes to Greg's notes are held for review; Reading series with no note show as suggestions on the admin Overview. - /api/inquiries and /api/orders: Turnstile, rate limits, a saved copy in the content service (intake token) plus email over SMTP. Orders are re-checked against the live notes: accepted only when the category's ordersOpen is true and the line has capacity. - Admin Inbox for saved submissions. Options' inStock is optional so synced stock can fill it. All settings from process.env; .env.example per service. - Sources could not be reached from the build environment; parsers are tested against fixtures labelled as not live. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012aSbo2D1qP8cMFQCykf7kx
This commit is contained in:
Claude
@@ -0,0 +1,55 @@
|
||||
# Content service settings. Copy to .env (never commit it) or set them in the
|
||||
# container's environment. See docs/content-service.md, "Running it".
|
||||
|
||||
# --- Sync jobs (docs/content-service.md, "Sync jobs") ------------------------
|
||||
# Public sources to run: archidekt (hourly), bgg (daily), covers (every 15 min).
|
||||
SYNC_JOBS=archidekt,bgg,covers
|
||||
# Optional per-job interval override, in minutes.
|
||||
# SYNC_ARCHIDEKT_MINUTES=60
|
||||
# SYNC_BGG_MINUTES=1440
|
||||
# SYNC_COVERS_MINUTES=15
|
||||
# Healthchecks ping URLs, one per job (start, success and fail are pinged).
|
||||
SYNC_ARCHIDEKT_HEALTHCHECK_URL=
|
||||
SYNC_BGG_HEALTHCHECK_URL=
|
||||
SYNC_COVERS_HEALTHCHECK_URL=
|
||||
# BoardGameGeek application token (sent as Authorization: Bearer), if BGG asks for one.
|
||||
BGG_TOKEN=
|
||||
# TMDB v3 API key or v4 read access token, for film and TV covers.
|
||||
TMDB_API_KEY=
|
||||
|
||||
# --- Homelab syncs on Cosmos (docs/content-service.md, "Homelab sources") ----
|
||||
# Each is enabled by its own URL and login; all are read only against the source.
|
||||
# PersonalReader, manga and manhwa (hourly). Its own read-only Suwayomi account.
|
||||
SUWAYOMI_URL=
|
||||
SUWAYOMI_USER=
|
||||
SUWAYOMI_PASS=
|
||||
# ui_login (default with a user), basic, or none.
|
||||
# SUWAYOMI_AUTH=ui_login
|
||||
# PersonalReader, light novels and books (daily).
|
||||
KAVITA_URL=
|
||||
KAVITA_API_KEY=
|
||||
# Filament in stock (hourly). UNVERIFIED API: check the path and mapping first.
|
||||
BAMBUDDY_URL=
|
||||
# BAMBUDDY_SPOOLS_PATH=/api/v1/inventory/spools
|
||||
# BAMBUDDY_API_KEY=
|
||||
# BAMBUDDY_AUTH_HEADER=
|
||||
# BAMBUDDY_GROUP=Filament
|
||||
# BAMBUDDY_MATERIALS=PLA,PETG
|
||||
# BAMBUDDY_MIN_GRAMS=50
|
||||
# Leather and paper in stock (hourly). Entries are name or name=Group.
|
||||
HOMEBOX_URL=
|
||||
HOMEBOX_USER=
|
||||
HOMEBOX_PASS=
|
||||
HOMEBOX_LABELS=leather,paper
|
||||
# HOMEBOX_LOCATIONS=
|
||||
SYNC_SUWAYOMI_HEALTHCHECK_URL=
|
||||
SYNC_KAVITA_HEALTHCHECK_URL=
|
||||
SYNC_BAMBUDDY_HEALTHCHECK_URL=
|
||||
SYNC_HOMEBOX_HEALTHCHECK_URL=
|
||||
|
||||
# --- Form submissions (docs/content-service.md, "Form submissions") ----------
|
||||
# Add an intake token for the site's forms to CONTENT_TOKENS, e.g.
|
||||
# CONTENT_TOKENS=reader:…,admin:…,intake:<24+ random characters>
|
||||
# The site uses the same value as CONTENT_INTAKE_TOKEN. It can only file submissions.
|
||||
# Largest attachment a submission may carry, in MB. Files go to DATA_DIR/submissions.
|
||||
SUBMISSION_MAX_MB=10
|
||||
@@ -4,8 +4,10 @@
|
||||
* cannot change who may write or what the rules allow.
|
||||
*/
|
||||
import { resolve } from "node:path";
|
||||
import { syncSettings, type SyncSettings } from "./sync/settings.ts";
|
||||
|
||||
export type Role = "reader" | "agent" | "editor" | "admin";
|
||||
/** `intake` can only file submissions (the site's forms); it can read nothing. */
|
||||
export type Role = "reader" | "agent" | "editor" | "admin" | "intake";
|
||||
|
||||
export interface Config {
|
||||
/** The vault's Site folder. */
|
||||
@@ -22,9 +24,13 @@ export interface Config {
|
||||
notifyWebhookUrl?: string;
|
||||
allowedEmails: string[];
|
||||
scanMinutes: number;
|
||||
/** Which sync jobs run, their intervals, Healthchecks URLs and keys. */
|
||||
sync: SyncSettings;
|
||||
/** Largest attachment a submission may carry, in bytes. */
|
||||
submissionMaxBytes: number;
|
||||
}
|
||||
|
||||
const roles = new Set<Role>(["reader", "agent", "editor", "admin"]);
|
||||
const roles = new Set<Role>(["reader", "agent", "editor", "admin", "intake"]);
|
||||
|
||||
/** CONTENT_TOKENS is a comma separated list of role:token pairs. */
|
||||
export function parseTokens(raw: string | undefined): Map<string, Role> {
|
||||
@@ -53,12 +59,14 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config {
|
||||
notifyWebhookUrl: env.NOTIFY_WEBHOOK_URL || undefined,
|
||||
allowedEmails: (env.ALLOWED_EMAILS ?? "").split(",").map((s) => s.trim().toLowerCase()).filter(Boolean),
|
||||
scanMinutes: Number(env.SCAN_MINUTES ?? 5),
|
||||
sync: syncSettings(env),
|
||||
submissionMaxBytes: Math.round(Number(env.SUBMISSION_MAX_MB ?? 10) * 1024 * 1024),
|
||||
};
|
||||
}
|
||||
|
||||
/** What each role may do. Checked by the HTTP API and the MCP server alike. */
|
||||
export const can = {
|
||||
readDrafts: (r: Role) => r !== "reader",
|
||||
readDrafts: (r: Role) => r !== "reader" && r !== "intake",
|
||||
write: (r: Role) => r === "agent" || r === "editor" || r === "admin",
|
||||
review: (r: Role) => r === "editor" || r === "admin",
|
||||
rollback: (r: Role) => r === "admin",
|
||||
|
||||
@@ -13,6 +13,8 @@ import { can, type Config, type Role } from "./config.ts";
|
||||
import { handleMcp } from "./mcp.ts";
|
||||
import type { Change, ContentService } from "./service.ts";
|
||||
import type { Author, StoredNote } from "./store.ts";
|
||||
import { handleSubmissions } from "./submissions.ts";
|
||||
import { handleSuggestions } from "./suggestions.ts";
|
||||
import { heldView, noteView, outcomeView, revisionView, summaryView } from "./views.ts";
|
||||
|
||||
const MAX_BODY = 1_000_000;
|
||||
@@ -66,7 +68,7 @@ export function schemaList() {
|
||||
}));
|
||||
}
|
||||
|
||||
export function createHttpServer(service: ContentService, config: Pick<Config, "tokens">) {
|
||||
export function createHttpServer(service: ContentService, config: Pick<Config, "tokens"> & Partial<Pick<Config, "submissionMaxBytes">>) {
|
||||
const { store } = service;
|
||||
|
||||
const visible = (n: StoredNote | null, role: Role) => (n && !n.removed && (n.publish || can.readDrafts(role)) ? n : null);
|
||||
@@ -87,6 +89,12 @@ export function createHttpServer(service: ContentService, config: Pick<Config, "
|
||||
if (!ok) throw new HttpError(403, `The ${role} token cannot do this`);
|
||||
};
|
||||
|
||||
// Form submissions have their own rules; an intake token can do nothing else.
|
||||
if (await handleSubmissions(req, res, { path, method, role, url, submissions: store.submissions, maxBytes: config.submissionMaxBytes })) return;
|
||||
if (role === "intake") throw new HttpError(403, "The intake token can only file submissions");
|
||||
// Library suggestions from PersonalReader (see suggestions.ts).
|
||||
if (await handleSuggestions(req, res, { path, method, role, store })) return;
|
||||
|
||||
if (path === "/mcp") {
|
||||
need(can.write(role));
|
||||
const body = method === "POST" ? await readJson(req) : undefined;
|
||||
@@ -105,8 +113,11 @@ export function createHttpServer(service: ContentService, config: Pick<Config, "
|
||||
const notes = query
|
||||
? store.search(query, { publishedOnly, limit }).filter((n) => !type || n.type === type)
|
||||
: store.list({ type, publishedOnly, limit, offset: Number(q("offset") ?? 0) });
|
||||
// full=true returns whole notes, so a list page needs one request.
|
||||
return send(res, 200, notes.map((n) => (q("full") === "true" ? noteView(n) : summaryView(n))));
|
||||
// full=true returns whole notes, so a list page needs one request,
|
||||
// each with `synced`: what the sync jobs fetched for it (see docs).
|
||||
if (q("full") !== "true") return send(res, 200, notes.map(summaryView));
|
||||
const synced = store.syncCache.all();
|
||||
return send(res, 200, notes.map((n) => ({ ...noteView(n), synced: synced.get(n.path) ?? {} })));
|
||||
}
|
||||
|
||||
if (path === "/api/note") {
|
||||
@@ -129,6 +140,7 @@ export function createHttpServer(service: ContentService, config: Pick<Config, "
|
||||
if (!found) throw new HttpError(404, "No such note");
|
||||
return send(res, 200, {
|
||||
...noteView(found, { text: can.readDrafts(role) }),
|
||||
synced: store.syncCache.forPath(found.path),
|
||||
backlinks: store.backlinks(found.name).filter((b) => visible(b, role)).map(summaryView),
|
||||
...(can.readDrafts(role) ? { pending: store.pendingFor(found.path) ? heldView(store.pendingFor(found.path)!) : null } : {}),
|
||||
});
|
||||
|
||||
@@ -6,6 +6,7 @@ import { loadConfig } from "./config.ts";
|
||||
import { createHttpServer } from "./http.ts";
|
||||
import { ContentService } from "./service.ts";
|
||||
import { Store } from "./store.ts";
|
||||
import { enabledJobs, Scheduler } from "./sync/index.ts";
|
||||
import { Vault } from "./vault.ts";
|
||||
import { Watcher } from "./watcher.ts";
|
||||
import { heldView } from "./views.ts";
|
||||
@@ -41,6 +42,11 @@ const watcher = new Watcher(service, config.siteDir, {
|
||||
});
|
||||
watcher.start();
|
||||
|
||||
// Sync jobs: each enabled source on its own schedule, into the sync cache.
|
||||
const syncs = new Scheduler(service, enabledJobs(config.sync));
|
||||
if (syncs.jobs.length) console.log(`Sync jobs: ${syncs.jobs.map((j) => `${j.name} every ${j.everyMinutes} min`).join(", ")}`);
|
||||
syncs.start();
|
||||
|
||||
const server = createHttpServer(service, config);
|
||||
server.listen(config.port, config.host, () => {
|
||||
const c = store.count();
|
||||
@@ -49,6 +55,7 @@ server.listen(config.port, config.host, () => {
|
||||
|
||||
const stop = () => {
|
||||
watcher.stop();
|
||||
syncs.stop();
|
||||
server.close();
|
||||
store.close();
|
||||
process.exit(0);
|
||||
|
||||
@@ -52,6 +52,12 @@ export interface RuleContext {
|
||||
settings: RuleSettings;
|
||||
}
|
||||
|
||||
/**
|
||||
* Authors held to the agent rules. A sync job writes as `sync` and may do
|
||||
* nothing an agent cannot: the same contact, publish, edit and batch limits.
|
||||
*/
|
||||
export const isLimited = (a: Author) => a === "agent" || a === "sync";
|
||||
|
||||
export type Verdict = { decision: "accept" } | { decision: "hold"; reasons: HeldReason[] };
|
||||
|
||||
/** Strings that look like credentials. Matching is on new text only. */
|
||||
@@ -125,26 +131,28 @@ export function evaluate(p: Proposed, ctx: RuleContext): Verdict {
|
||||
for (const [what, re] of SECRET_PATTERNS) {
|
||||
if (newMatches(re, p.text, before).length) reasons.push({ code: "secret", message: `The note now contains what looks like ${what}` });
|
||||
}
|
||||
if (p.author === "agent") {
|
||||
const who = p.author === "sync" ? "A sync" : "An agent";
|
||||
if (isLimited(p.author)) {
|
||||
const allowed = new Set(settings.allowedEmails.map((e) => e.toLowerCase()));
|
||||
const emails = newMatches(EMAIL, p.text, before).filter((e) => !allowed.has(e.toLowerCase()));
|
||||
if (emails.length) reasons.push({ code: "contact", message: `An agent added an email address: ${emails.join(", ")}` });
|
||||
if (newMatches(PHONE, p.text, before).length) reasons.push({ code: "contact", message: "An agent added a phone number" });
|
||||
if (emails.length) reasons.push({ code: "contact", message: `${who} added an email address: ${emails.join(", ")}` });
|
||||
if (newMatches(PHONE, p.text, before).length) reasons.push({ code: "contact", message: `${who} added a phone number` });
|
||||
}
|
||||
|
||||
// 6. Agents stay inside their lane.
|
||||
if (p.author === "agent") {
|
||||
if (live && live.author !== "agent" && live.data.agentEdits !== true) {
|
||||
reasons.push({ code: "agent-edit", message: "An agent changed a note you wrote, which needs your review" });
|
||||
if (isLimited(p.author)) {
|
||||
// A sync may change notes it wrote itself; anything else needs agentEdits.
|
||||
if (live && live.author !== p.author && live.data.agentEdits !== true) {
|
||||
reasons.push({ code: "agent-edit", message: `${who} changed a note you wrote, which needs your review` });
|
||||
}
|
||||
const publishing = data.publish === true && !(live?.publish ?? false);
|
||||
if (publishing && !settings.agentAutoPublish.includes(p.checked.type)) {
|
||||
reasons.push({ code: "agent-publish", message: `An agent wants to publish a ${p.checked.type}, which needs your approval` });
|
||||
reasons.push({ code: "agent-publish", message: `${who} wants to publish a ${p.checked.type}, which needs your approval` });
|
||||
}
|
||||
if (ctx.recentAgentChanges >= settings.agentBatchLimit) {
|
||||
reasons.push({
|
||||
code: "batch",
|
||||
message: `More than ${settings.agentBatchLimit} agent changes in ${settings.agentBatchWindowMinutes} minutes`,
|
||||
message: `More than ${settings.agentBatchLimit} ${p.author} changes in ${settings.agentBatchWindowMinutes} minutes`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
*/
|
||||
import { basename } from "node:path";
|
||||
import { checkNote, checkVault, formatNote, notePath, outgoingLinks, slugify, typeDef, type CheckedNote } from "@gk/schema";
|
||||
import { defaultRules, evaluate, type RuleSettings } from "./rules.ts";
|
||||
import { defaultRules, evaluate, isLimited, type RuleSettings } from "./rules.ts";
|
||||
import type { Author, HeldChange, HeldReason, StoredNote, Store } from "./store.ts";
|
||||
import { hashText, isNotePath, type Vault } from "./vault.ts";
|
||||
|
||||
@@ -264,7 +264,7 @@ export class ContentService {
|
||||
current,
|
||||
nameTakenBy,
|
||||
slugTakenBy,
|
||||
recentAgentChanges: author === "agent" ? this.store.changesSince("agent", window) : 0,
|
||||
recentAgentChanges: isLimited(author) ? this.store.changesSince(author, window) : 0,
|
||||
settings: this.rules,
|
||||
},
|
||||
);
|
||||
|
||||
@@ -8,9 +8,12 @@
|
||||
*/
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import { mkdirSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
import { dirname, join } from "node:path";
|
||||
import { Submissions } from "./submissions.ts";
|
||||
import { SyncCache } from "./sync/cache.ts";
|
||||
|
||||
export type Author = "greg" | "agent" | "admin";
|
||||
/** Who made a change. `sync` is a sync job writing through ContentService.apply; it runs the agent rules. */
|
||||
export type Author = "greg" | "agent" | "admin" | "sync";
|
||||
|
||||
export interface StoredNote {
|
||||
path: string;
|
||||
@@ -176,12 +179,18 @@ const SYNC_RUNS_KEPT = 50;
|
||||
|
||||
export class Store {
|
||||
readonly db: DatabaseSync;
|
||||
readonly submissions: Submissions;
|
||||
/** What sync jobs last fetched per note; the notes API shows it as `synced`. */
|
||||
readonly syncCache: SyncCache;
|
||||
|
||||
constructor(file: string) {
|
||||
if (file !== ":memory:") mkdirSync(dirname(file), { recursive: true });
|
||||
this.db = new DatabaseSync(file);
|
||||
this.db.exec("pragma journal_mode = wal; pragma foreign_keys = on;");
|
||||
this.db.exec(SCHEMA);
|
||||
// Saved copies of the site's forms; attachments sit beside content.db.
|
||||
this.syncCache = new SyncCache(this.db);
|
||||
this.submissions = new Submissions(this.db, file === ":memory:" ? null : join(dirname(file), "submissions"));
|
||||
}
|
||||
|
||||
close() {
|
||||
|
||||
@@ -0,0 +1,376 @@
|
||||
/**
|
||||
* Saved copies of what visitors send through the site's forms: inquiries and
|
||||
* store order requests. Greg's decision is email plus a saved copy, so a lost
|
||||
* email never loses a request. The rows live in content.db (backed up with
|
||||
* the rest of it); an attachment is a file under DATA_DIR/submissions.
|
||||
*
|
||||
* Who may do what:
|
||||
* - `intake` (the site) files a submission and reports, once, how its email
|
||||
* went, using the receipt it got back. It can read nothing, not even what
|
||||
* it just filed.
|
||||
* - `admin` lists, reads and marks them handled, and downloads attachments.
|
||||
* - Every other role gets 403 here.
|
||||
*/
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { createReadStream, mkdirSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import type { IncomingMessage, ServerResponse } from "node:http";
|
||||
import { join } from "node:path";
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { z } from "zod";
|
||||
import type { Role } from "./config.ts";
|
||||
|
||||
export const SUBMISSIONS_SCHEMA = `
|
||||
create table if not exists submissions (
|
||||
id integer primary key autoincrement,
|
||||
kind text not null,
|
||||
type text not null,
|
||||
name text not null,
|
||||
email text not null,
|
||||
fields text not null,
|
||||
attachment_name text,
|
||||
attachment_file text,
|
||||
attachment_type text,
|
||||
attachment_size integer,
|
||||
created_at text not null,
|
||||
delivery text not null default 'pending',
|
||||
delivery_error text,
|
||||
delivery_at text,
|
||||
receipt text not null,
|
||||
handled integer not null default 0,
|
||||
handled_at text
|
||||
);
|
||||
create index if not exists submissions_inbox on submissions(handled, id);
|
||||
`;
|
||||
|
||||
export type Delivery = "pending" | "sent" | "failed" | "skipped";
|
||||
|
||||
export interface Submission {
|
||||
id: number;
|
||||
kind: "inquiry" | "order";
|
||||
/** The inquiry type (job, freelance, patiun) or the product line's slug. */
|
||||
type: string;
|
||||
name: string;
|
||||
email: string;
|
||||
/** What the visitor sent, in form order. */
|
||||
fields: { label: string; value: string }[];
|
||||
attachment: { name: string; type: string; size: number } | null;
|
||||
createdAt: string;
|
||||
delivery: Delivery;
|
||||
deliveryError: string | null;
|
||||
deliveryAt: string | null;
|
||||
handled: boolean;
|
||||
handledAt: string | null;
|
||||
}
|
||||
|
||||
/** File types an attachment may have, by extension, with the type it is served as. */
|
||||
export const ATTACHMENT_TYPES: Record<string, string> = {
|
||||
png: "image/png",
|
||||
jpg: "image/jpeg",
|
||||
jpeg: "image/jpeg",
|
||||
gif: "image/gif",
|
||||
webp: "image/webp",
|
||||
avif: "image/avif",
|
||||
heic: "image/heic",
|
||||
pdf: "application/pdf",
|
||||
txt: "text/plain; charset=utf-8",
|
||||
csv: "text/csv; charset=utf-8",
|
||||
dek: "text/plain; charset=utf-8",
|
||||
dec: "text/plain; charset=utf-8",
|
||||
};
|
||||
|
||||
export const DEFAULT_MAX_BYTES = 10 * 1024 * 1024;
|
||||
|
||||
export class SubmissionError extends Error {
|
||||
constructor(readonly status: number, message: string) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
/** A display name with nothing that could act as a path or a header break. */
|
||||
export function safeName(name: string): string {
|
||||
const base = name.split(/[\\/]/).pop() ?? "";
|
||||
const clean = base
|
||||
.normalize("NFKC")
|
||||
.replace(/[^\p{L}\p{N}._ -]+/gu, "_")
|
||||
.replace(/^[.\s]+/, "")
|
||||
.slice(-120)
|
||||
.trim();
|
||||
return clean || "attachment";
|
||||
}
|
||||
|
||||
export function extensionOf(name: string): string | null {
|
||||
const m = /\.([A-Za-z0-9]{1,8})$/.exec(name);
|
||||
const ext = m?.[1]?.toLowerCase();
|
||||
return ext && ATTACHMENT_TYPES[ext] ? ext : null;
|
||||
}
|
||||
|
||||
const fieldSchema = z.object({ label: z.string().trim().min(1).max(80), value: z.string().max(20_000) });
|
||||
|
||||
export const newSubmissionSchema = z.object({
|
||||
kind: z.enum(["inquiry", "order"]),
|
||||
type: z.string().trim().min(1).max(80),
|
||||
name: z.string().trim().min(1).max(120),
|
||||
email: z.email().max(200),
|
||||
fields: z.array(fieldSchema).max(40).default([]),
|
||||
attachment: z
|
||||
.object({ name: z.string().min(1).max(300), data: z.string().min(1) })
|
||||
.optional(),
|
||||
});
|
||||
|
||||
export type NewSubmission = z.input<typeof newSubmissionSchema>;
|
||||
|
||||
const deliverySchema = z.object({
|
||||
receipt: z.string().min(1).max(100).optional(),
|
||||
status: z.enum(["sent", "failed", "skipped"]),
|
||||
error: z.string().max(2000).optional(),
|
||||
});
|
||||
|
||||
type Row = Record<string, any>;
|
||||
|
||||
const toSubmission = (r: Row): Submission => ({
|
||||
id: r.id,
|
||||
kind: r.kind,
|
||||
type: r.type,
|
||||
name: r.name,
|
||||
email: r.email,
|
||||
fields: JSON.parse(r.fields),
|
||||
attachment: r.attachment_file ? { name: r.attachment_name, type: r.attachment_type, size: r.attachment_size } : null,
|
||||
createdAt: r.created_at,
|
||||
delivery: r.delivery,
|
||||
deliveryError: r.delivery_error,
|
||||
deliveryAt: r.delivery_at,
|
||||
handled: !!r.handled,
|
||||
handledAt: r.handled_at,
|
||||
});
|
||||
|
||||
export class Submissions {
|
||||
/**
|
||||
* @param dir where attachment files go; null keeps no files (an in-memory
|
||||
* store), and a submission with an attachment is then turned down.
|
||||
*/
|
||||
constructor(
|
||||
private readonly db: DatabaseSync,
|
||||
readonly dir: string | null,
|
||||
private readonly now: () => Date = () => new Date(),
|
||||
) {
|
||||
db.exec(SUBMISSIONS_SCHEMA);
|
||||
}
|
||||
|
||||
/** File a submission. Returns its id and the receipt for reporting delivery. */
|
||||
create(input: NewSubmission, opts: { maxBytes?: number } = {}): { id: number; receipt: string } {
|
||||
const parsed = newSubmissionSchema.safeParse(input);
|
||||
if (!parsed.success) throw new SubmissionError(400, z.prettifyError(parsed.error));
|
||||
const s = parsed.data;
|
||||
const maxBytes = opts.maxBytes ?? DEFAULT_MAX_BYTES;
|
||||
|
||||
let file: { name: string; type: string; bytes: Buffer; ext: string } | null = null;
|
||||
if (s.attachment) {
|
||||
if (!this.dir) throw new SubmissionError(400, "This store keeps no attachments");
|
||||
const ext = extensionOf(s.attachment.name);
|
||||
if (!ext) throw new SubmissionError(415, `Attachments must be one of: ${Object.keys(ATTACHMENT_TYPES).join(", ")}`);
|
||||
// Base64 is 4 characters per 3 bytes: refuse before decoding anything too large.
|
||||
if (s.attachment.data.length > Math.ceil(maxBytes / 3) * 4 + 4) throw new SubmissionError(413, "The attachment is too large");
|
||||
if (!/^[A-Za-z0-9+/]*={0,2}$/.test(s.attachment.data)) throw new SubmissionError(400, "The attachment is not base64");
|
||||
const bytes = Buffer.from(s.attachment.data, "base64");
|
||||
if (bytes.length > maxBytes) throw new SubmissionError(413, "The attachment is too large");
|
||||
if (bytes.length === 0) throw new SubmissionError(400, "The attachment is empty");
|
||||
file = { name: safeName(s.attachment.name), type: ATTACHMENT_TYPES[ext]!, bytes, ext };
|
||||
}
|
||||
|
||||
const receipt = randomBytes(24).toString("base64url");
|
||||
const at = this.now().toISOString();
|
||||
const r = this.db
|
||||
.prepare(
|
||||
`insert into submissions (kind, type, name, email, fields, created_at, receipt)
|
||||
values (?, ?, ?, ?, ?, ?, ?)`,
|
||||
)
|
||||
.run(s.kind, s.type, s.name, s.email, JSON.stringify(s.fields), at, receipt);
|
||||
const id = Number(r.lastInsertRowid);
|
||||
|
||||
if (file) {
|
||||
// The stored name is ours alone: the id, random bytes and a known extension.
|
||||
const stored = `${id}-${randomBytes(8).toString("hex")}.${file.ext}`;
|
||||
try {
|
||||
mkdirSync(this.dir!, { recursive: true });
|
||||
writeFileSync(join(this.dir!, stored), file.bytes, { flag: "wx" });
|
||||
} catch (err) {
|
||||
this.db.prepare("delete from submissions where id = ?").run(id);
|
||||
throw err;
|
||||
}
|
||||
this.db
|
||||
.prepare("update submissions set attachment_name = ?, attachment_file = ?, attachment_type = ?, attachment_size = ? where id = ?")
|
||||
.run(file.name, stored, file.type, file.bytes.length, id);
|
||||
}
|
||||
return { id, receipt };
|
||||
}
|
||||
|
||||
/**
|
||||
* Record how the email went. With a receipt (the site), only once and only
|
||||
* for the submission the receipt belongs to.
|
||||
*/
|
||||
setDelivery(id: number, d: { status: Exclude<Delivery, "pending">; error?: string; receipt?: string }, opts: { needReceipt: boolean }) {
|
||||
const row = this.db.prepare("select receipt, delivery from submissions where id = ?").get(id) as Row | undefined;
|
||||
if (opts.needReceipt) {
|
||||
if (!row || !d.receipt || row.receipt !== d.receipt) throw new SubmissionError(403, "That receipt does not match");
|
||||
if (row.delivery !== "pending") throw new SubmissionError(409, "Delivery is already recorded");
|
||||
} else if (!row) throw new SubmissionError(404, "No such submission");
|
||||
this.db
|
||||
.prepare("update submissions set delivery = ?, delivery_error = ?, delivery_at = ? where id = ?")
|
||||
.run(d.status, d.error ?? null, this.now().toISOString(), id);
|
||||
}
|
||||
|
||||
/** Unhandled first, then newest first. */
|
||||
list(opts: { handled?: boolean; limit?: number; offset?: number } = {}): Submission[] {
|
||||
const where = opts.handled === undefined ? "" : `where handled = ${opts.handled ? 1 : 0}`;
|
||||
return (
|
||||
this.db
|
||||
.prepare(`select * from submissions ${where} order by handled asc, id desc limit ? offset ?`)
|
||||
.all(opts.limit ?? 200, opts.offset ?? 0) as Row[]
|
||||
).map(toSubmission);
|
||||
}
|
||||
|
||||
get(id: number): Submission | null {
|
||||
const r = this.db.prepare("select * from submissions where id = ?").get(id) as Row | undefined;
|
||||
return r ? toSubmission(r) : null;
|
||||
}
|
||||
|
||||
counts(): { total: number; unhandled: number; failed: number } {
|
||||
const r = this.db
|
||||
.prepare("select count(*) total, coalesce(sum(handled = 0), 0) unhandled, coalesce(sum(delivery = 'failed' and handled = 0), 0) failed from submissions")
|
||||
.get() as Row;
|
||||
return { total: r.total, unhandled: r.unhandled, failed: r.failed };
|
||||
}
|
||||
|
||||
setHandled(id: number, handled: boolean): Submission {
|
||||
if (!this.get(id)) throw new SubmissionError(404, "No such submission");
|
||||
this.db.prepare("update submissions set handled = ?, handled_at = ? where id = ?").run(handled ? 1 : 0, handled ? this.now().toISOString() : null, id);
|
||||
return this.get(id)!;
|
||||
}
|
||||
|
||||
/** The attachment's file on disk, with its display name and type. */
|
||||
attachment(id: number): { file: string; name: string; type: string; size: number } | null {
|
||||
const r = this.db.prepare("select attachment_file, attachment_name, attachment_type from submissions where id = ?").get(id) as Row | undefined;
|
||||
if (!r?.attachment_file || !this.dir) return null;
|
||||
const file = join(this.dir, r.attachment_file as string);
|
||||
try {
|
||||
return { file, name: r.attachment_name, type: r.attachment_type, size: statSync(file).size };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** For tests: drop the attachment folder. */
|
||||
removeFiles() {
|
||||
if (this.dir) rmSync(this.dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------ HTTP
|
||||
|
||||
const json = (res: ServerResponse, status: number, body: unknown) => {
|
||||
res.writeHead(status, { "content-type": "application/json; charset=utf-8", "cache-control": "no-store" });
|
||||
res.end(JSON.stringify(body));
|
||||
};
|
||||
|
||||
async function readJson(req: IncomingMessage, limit: number): Promise<unknown> {
|
||||
let size = 0;
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of req) {
|
||||
size += (chunk as Buffer).length;
|
||||
if (size > limit) throw new SubmissionError(413, "The request is too large");
|
||||
chunks.push(chunk as Buffer);
|
||||
}
|
||||
const text = Buffer.concat(chunks).toString("utf8");
|
||||
if (!text) return undefined;
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch {
|
||||
throw new SubmissionError(400, "The body is not valid JSON");
|
||||
}
|
||||
}
|
||||
|
||||
/** RFC 6266 filename for a download, ASCII fallback plus the UTF-8 name. */
|
||||
const disposition = (name: string) =>
|
||||
`attachment; filename="${name.replace(/[^\x20-\x7e]|["\\]/g, "_")}"; filename*=UTF-8''${encodeURIComponent(name)}`;
|
||||
|
||||
/**
|
||||
* Handles /api/submissions and everything under it. Returns false when the
|
||||
* path is not a submissions route, so the main router carries on.
|
||||
*/
|
||||
export async function handleSubmissions(
|
||||
req: IncomingMessage,
|
||||
res: ServerResponse,
|
||||
ctx: { path: string; method: string; role: Role; url: URL; submissions: Submissions; maxBytes?: number },
|
||||
): Promise<boolean> {
|
||||
const { path, method, role, submissions } = ctx;
|
||||
if (path !== "/api/submissions" && !path.startsWith("/api/submissions/")) return false;
|
||||
const maxBytes = ctx.maxBytes ?? DEFAULT_MAX_BYTES;
|
||||
|
||||
try {
|
||||
const isIntake = role === "intake";
|
||||
const isAdmin = role === "admin";
|
||||
const forbid = () => {
|
||||
throw new SubmissionError(403, `The ${role} token cannot do this`);
|
||||
};
|
||||
|
||||
if (path === "/api/submissions" && method === "POST") {
|
||||
if (!isIntake && !isAdmin) forbid();
|
||||
const body = await readJson(req, Math.ceil(maxBytes / 3) * 4 + 256_000);
|
||||
const out = submissions.create(body as NewSubmission, { maxBytes });
|
||||
return (json(res, 201, out), true);
|
||||
}
|
||||
|
||||
const delivery = path.match(/^\/api\/submissions\/(\d+)\/delivery$/);
|
||||
if (delivery && method === "POST") {
|
||||
if (!isIntake && !isAdmin) forbid();
|
||||
const body = deliverySchema.safeParse(await readJson(req, 64_000));
|
||||
if (!body.success) throw new SubmissionError(400, z.prettifyError(body.error));
|
||||
submissions.setDelivery(Number(delivery[1]), body.data, { needReceipt: isIntake });
|
||||
return (json(res, 200, { ok: true }), true);
|
||||
}
|
||||
|
||||
// Everything else reads or changes saved copies: admin only.
|
||||
if (!isAdmin) forbid();
|
||||
|
||||
if (path === "/api/submissions" && method === "GET") {
|
||||
const h = ctx.url.searchParams.get("handled");
|
||||
const limit = Math.min(Number(ctx.url.searchParams.get("limit") ?? 200) || 200, 1000);
|
||||
return (json(res, 200, { counts: submissions.counts(), submissions: submissions.list({ handled: h === null ? undefined : h === "true", limit }) }), true);
|
||||
}
|
||||
|
||||
const one = path.match(/^\/api\/submissions\/(\d+)$/);
|
||||
if (one && method === "GET") {
|
||||
const s = submissions.get(Number(one[1]));
|
||||
if (!s) throw new SubmissionError(404, "No such submission");
|
||||
return (json(res, 200, s), true);
|
||||
}
|
||||
|
||||
const handled = path.match(/^\/api\/submissions\/(\d+)\/handled$/);
|
||||
if (handled && method === "POST") {
|
||||
const body = z.object({ handled: z.boolean() }).safeParse(await readJson(req, 10_000));
|
||||
if (!body.success) throw new SubmissionError(400, z.prettifyError(body.error));
|
||||
return (json(res, 200, submissions.setHandled(Number(handled[1]), body.data.handled)), true);
|
||||
}
|
||||
|
||||
const file = path.match(/^\/api\/submissions\/(\d+)\/attachment$/);
|
||||
if (file && method === "GET") {
|
||||
const a = submissions.attachment(Number(file[1]));
|
||||
if (!a) throw new SubmissionError(404, "No attachment");
|
||||
res.writeHead(200, {
|
||||
"content-type": a.type,
|
||||
"content-length": a.size,
|
||||
"content-disposition": disposition(a.name),
|
||||
"x-content-type-options": "nosniff",
|
||||
"content-security-policy": "default-src 'none'; sandbox",
|
||||
"cache-control": "no-store",
|
||||
});
|
||||
createReadStream(a.file).pipe(res);
|
||||
return true;
|
||||
}
|
||||
|
||||
throw new SubmissionError(404, "Not found");
|
||||
} catch (err) {
|
||||
if (err instanceof SubmissionError) return (json(res, err.status, { error: err.message }), true);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
/**
|
||||
* Library suggestions: a series Greg moved to Reading in PersonalReader that
|
||||
* has no Library item yet. The Suwayomi sync job keeps the list in its cache
|
||||
* (entry `_suggestions`); this module serves it to the admin page, leaves
|
||||
* out series that now have a Library item, and remembers dismissals in a
|
||||
* small table, so a dismissed series stays dismissed across runs.
|
||||
*
|
||||
* Adding one is not done here: the admin page creates the Library item with
|
||||
* the normal write API (`POST /api/changes`), so every rule applies.
|
||||
*
|
||||
* - `GET /api/suggestions` (editor, admin, agent): `[{ key, anilistId, title, medium, cover? }]`
|
||||
* - `POST /api/suggestions/<key>/dismiss` (editor, admin)
|
||||
*/
|
||||
import type { IncomingMessage, ServerResponse } from "node:http";
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { can, type Role } from "./config.ts";
|
||||
import type { Store } from "./store.ts";
|
||||
import { SUGGESTIONS_KEY } from "./sync/jobs/reader.ts";
|
||||
import type { Suggestion } from "./sync/jobs/suwayomi.ts";
|
||||
|
||||
/** Jobs whose cache may hold suggestions. */
|
||||
const SOURCES = ["suwayomi"];
|
||||
|
||||
export interface SuggestionView extends Suggestion {
|
||||
/** Stable id for dismissing: `anilist-<id>`. */
|
||||
key: string;
|
||||
}
|
||||
|
||||
const tables = new WeakMap<DatabaseSync, true>();
|
||||
function ready(db: DatabaseSync) {
|
||||
if (!tables.has(db)) {
|
||||
db.exec(`create table if not exists suggestion_dismissals (key text primary key, at text not null);`);
|
||||
tables.set(db, true);
|
||||
}
|
||||
return db;
|
||||
}
|
||||
|
||||
export function listSuggestions(store: Store): SuggestionView[] {
|
||||
const db = ready(store.db);
|
||||
const dismissed = new Set((db.prepare("select key from suggestion_dismissals").all() as { key: string }[]).map((r) => r.key));
|
||||
const linked = new Set(store.list({ type: "library-item" }).map((n) => n.data.anilistId).filter((id) => typeof id === "number"));
|
||||
const out = new Map<string, SuggestionView>();
|
||||
for (const job of SOURCES) {
|
||||
const items = (store.syncCache.get(job, SUGGESTIONS_KEY)?.values.items as Suggestion[] | undefined) ?? [];
|
||||
for (const s of items) {
|
||||
if (typeof s?.anilistId !== "number" || linked.has(s.anilistId)) continue;
|
||||
const key = `anilist-${s.anilistId}`;
|
||||
if (!dismissed.has(key) && !out.has(key)) out.set(key, { ...s, key });
|
||||
}
|
||||
}
|
||||
return [...out.values()].sort((a, b) => a.title.localeCompare(b.title));
|
||||
}
|
||||
|
||||
export function dismissSuggestion(store: Store, key: string, at = new Date().toISOString()) {
|
||||
ready(store.db).prepare("insert into suggestion_dismissals (key, at) values (?, ?) on conflict(key) do nothing").run(key, at);
|
||||
}
|
||||
|
||||
export async function handleSuggestions(
|
||||
_req: IncomingMessage,
|
||||
res: ServerResponse,
|
||||
o: { path: string; method: string; role: Role; store: Store },
|
||||
): Promise<boolean> {
|
||||
const send = (status: number, body: unknown) => {
|
||||
res.writeHead(status, { "content-type": "application/json; charset=utf-8", "cache-control": "no-store" });
|
||||
res.end(JSON.stringify(body));
|
||||
return true;
|
||||
};
|
||||
if (o.path === "/api/suggestions" && o.method === "GET") {
|
||||
if (!can.readDrafts(o.role)) return send(403, { error: `The ${o.role} token cannot do this` });
|
||||
return send(200, listSuggestions(o.store));
|
||||
}
|
||||
const m = o.path.match(/^\/api\/suggestions\/(anilist-\d{1,10})\/dismiss$/);
|
||||
if (m && o.method === "POST") {
|
||||
if (!can.review(o.role)) return send(403, { error: `The ${o.role} token cannot do this` });
|
||||
dismissSuggestion(o.store, m[1]!);
|
||||
return send(200, { ok: true });
|
||||
}
|
||||
return false;
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
/**
|
||||
* The sync cache: what each job last fetched for each note, as JSON, with
|
||||
* when it was fetched. The site reads it through the notes API (`synced`),
|
||||
* so a source being down never takes a page down: the last good values stay
|
||||
* until a later run replaces them.
|
||||
*
|
||||
* It lives in content.db but can be rebuilt by running the jobs again.
|
||||
*/
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import type { CacheEntry, SyncedValues } from "./types.ts";
|
||||
|
||||
type Row = { job: string; path: string; vals: string; fetched_at: string };
|
||||
|
||||
export class SyncCache {
|
||||
constructor(private readonly db: DatabaseSync) {
|
||||
db.exec(`create table if not exists sync_cache (
|
||||
job text not null,
|
||||
path text not null,
|
||||
vals text not null,
|
||||
fetched_at text not null,
|
||||
primary key (job, path)
|
||||
);`);
|
||||
}
|
||||
|
||||
get(job: string, path: string): CacheEntry | null {
|
||||
const r = this.db.prepare("select * from sync_cache where job = ? and path = ?").get(job, path) as Row | undefined;
|
||||
return r ? { values: JSON.parse(r.vals), fetchedAt: r.fetched_at } : null;
|
||||
}
|
||||
|
||||
set(job: string, path: string, values: SyncedValues, fetchedAt: string) {
|
||||
this.db
|
||||
.prepare(
|
||||
`insert into sync_cache (job, path, vals, fetched_at) values (?, ?, ?, ?)
|
||||
on conflict(job, path) do update set vals = excluded.vals, fetched_at = excluded.fetched_at`,
|
||||
)
|
||||
.run(job, path, JSON.stringify(values), fetchedAt);
|
||||
}
|
||||
|
||||
/** Drop a job's rows for notes outside `paths`. */
|
||||
retain(job: string, paths: Iterable<string>) {
|
||||
const keep = new Set(paths);
|
||||
const del = this.db.prepare("delete from sync_cache where job = ? and path = ?");
|
||||
for (const r of this.db.prepare("select path from sync_cache where job = ?").all(job) as { path: string }[]) {
|
||||
if (!keep.has(r.path)) del.run(job, r.path);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Every note's synced values, each job's merged in job-name order. Jobs
|
||||
* should not set the same key for one note; if two do, the later name wins.
|
||||
*/
|
||||
all(): Map<string, SyncedValues> {
|
||||
const out = new Map<string, SyncedValues>();
|
||||
for (const r of this.db.prepare("select * from sync_cache order by job").all() as Row[]) {
|
||||
out.set(r.path, { ...out.get(r.path), ...JSON.parse(r.vals) });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** One note's synced values, merged across jobs. */
|
||||
forPath(path: string): SyncedValues {
|
||||
let out: SyncedValues = {};
|
||||
for (const r of this.db.prepare("select * from sync_cache where path = ? order by job").all(path) as Row[]) {
|
||||
out = { ...out, ...JSON.parse(r.vals) };
|
||||
}
|
||||
return out;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
/**
|
||||
* The sync jobs this service can run, and which of them are enabled. To add
|
||||
* a source, write a `SyncJob` (see types.ts and docs/content-service.md,
|
||||
* "Sync jobs") and add it here behind the settings it needs.
|
||||
*/
|
||||
import { archidektJob } from "./jobs/archidekt.ts";
|
||||
import { bggJob } from "./jobs/bgg.ts";
|
||||
import { coversJob } from "./jobs/covers.ts";
|
||||
import { homelabJobs } from "./jobs/homelab.ts";
|
||||
import type { SyncSettings } from "./settings.ts";
|
||||
import type { SyncJob } from "./types.ts";
|
||||
|
||||
export { Scheduler } from "./scheduler.ts";
|
||||
export { SyncCache } from "./cache.ts";
|
||||
export { syncSettings, type SyncSettings } from "./settings.ts";
|
||||
export type { SyncJob, SyncContext, SyncResult, SyncedValues } from "./types.ts";
|
||||
|
||||
/** Public sources need nothing but their name in SYNC_JOBS. */
|
||||
export function enabledJobs(s: SyncSettings): SyncJob[] {
|
||||
const common = (name: string) => ({ everyMinutes: s.minutes[name], healthcheckUrl: s.healthchecks[name] });
|
||||
const jobs: SyncJob[] = [];
|
||||
if (s.enabled.has("archidekt")) jobs.push(archidektJob(common("archidekt")));
|
||||
if (s.enabled.has("bgg")) jobs.push(bggJob({ ...common("bgg"), token: s.bggToken }));
|
||||
if (s.enabled.has("covers")) jobs.push(coversJob({ ...common("covers"), tmdbApiKey: s.tmdbApiKey }));
|
||||
if (s.homelab) jobs.push(...homelabJobs(s.homelab, common));
|
||||
return jobs;
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
/**
|
||||
* Archidekt: for every deck note with an `archidektId`, the deck's commander
|
||||
* art, color identity, a short list summary and its address. Hourly.
|
||||
*
|
||||
* API: GET https://archidekt.com/api/decks/<id>/ (public JSON, no key). The
|
||||
* legacy site used the owner's deck list (/api/decks/v3/?ownerUsername=);
|
||||
* the single-deck endpoint adds the cards, so commanders and color identity
|
||||
* come from the list itself.
|
||||
*/
|
||||
import type { SyncJob, SyncedValues } from "../types.ts";
|
||||
|
||||
export const ARCHIDEKT_API = "https://archidekt.com/api/decks";
|
||||
const COLOR_ORDER = ["W", "U", "B", "R", "G"] as const;
|
||||
const COLOR_WORDS: Record<string, string> = { white: "W", blue: "U", black: "B", red: "R", green: "G" };
|
||||
|
||||
/** Archidekt deckFormat ids to labels, as the legacy site had them (legacy/src/lib/archidekt.ts). */
|
||||
const FORMATS: Record<number, string> = {
|
||||
1: "Casual",
|
||||
2: "Modern",
|
||||
3: "Commander",
|
||||
5: "Standard",
|
||||
7: "Legacy",
|
||||
10: "Pauper",
|
||||
14: "Historic",
|
||||
16: "Pioneer",
|
||||
37: "Oathbreaker",
|
||||
45: "Brawl",
|
||||
};
|
||||
|
||||
interface RawCard {
|
||||
quantity?: number;
|
||||
categories?: string[] | null;
|
||||
card?: {
|
||||
uid?: string;
|
||||
oracleCard?: { name?: string; colorIdentity?: string[] };
|
||||
};
|
||||
}
|
||||
export interface RawDeck {
|
||||
id: number;
|
||||
name?: string;
|
||||
deckFormat?: number;
|
||||
featured?: string;
|
||||
customFeatured?: string;
|
||||
private?: boolean;
|
||||
categories?: { name: string; includedInDeck?: boolean; isPremier?: boolean }[];
|
||||
cards?: RawCard[];
|
||||
}
|
||||
|
||||
export interface DeckValues extends SyncedValues {
|
||||
deckUrl: string;
|
||||
/** Art crop of the (first) commander, else the deck's featured image. */
|
||||
commanderArt?: string;
|
||||
/** Color identity as WUBRG letters, in that order. Empty means colorless. */
|
||||
colors: string[];
|
||||
commanders: string[];
|
||||
/** Cards in the deck, not counting the maybeboard or other excluded categories. */
|
||||
deckSize: number;
|
||||
format?: string;
|
||||
}
|
||||
|
||||
export const deckAddress = (id: number) => `https://archidekt.com/decks/${id}`;
|
||||
|
||||
/** Scryfall's art crop for a card by its Scryfall id (Archidekt's `uid`). */
|
||||
export const scryfallArt = (uid: string) => `https://cards.scryfall.io/art_crop/front/${uid[0]}/${uid[1]}/${uid}.jpg`;
|
||||
|
||||
const https = (url?: string) => (url && /^https?:\/\//.test(url) ? url.replace(/^http:/, "https:") : undefined);
|
||||
|
||||
const colorLetter = (c: string) => COLOR_WORDS[c.toLowerCase()] ?? (COLOR_ORDER.includes(c.toUpperCase() as "W") ? c.toUpperCase() : undefined);
|
||||
|
||||
/** Turn one Archidekt deck response into the values the site shows. */
|
||||
export function parseDeck(raw: RawDeck): DeckValues {
|
||||
const excluded = new Set((raw.categories ?? []).filter((c) => c.includedInDeck === false).map((c) => c.name));
|
||||
const premier = new Set((raw.categories ?? []).filter((c) => c.isPremier).map((c) => c.name));
|
||||
premier.add("Commander");
|
||||
const inDeck = (raw.cards ?? []).filter((c) => !(c.categories ?? []).some((cat) => excluded.has(cat)));
|
||||
const commanderCards = inDeck.filter((c) => (c.categories ?? []).some((cat) => premier.has(cat)));
|
||||
const identitySource = commanderCards.length ? commanderCards : inDeck;
|
||||
const letters = new Set(identitySource.flatMap((c) => (c.card?.oracleCard?.colorIdentity ?? []).map(colorLetter)));
|
||||
const uid = commanderCards.find((c) => c.card?.uid)?.card?.uid;
|
||||
|
||||
const values: DeckValues = {
|
||||
deckUrl: deckAddress(raw.id),
|
||||
colors: COLOR_ORDER.filter((l) => letters.has(l)),
|
||||
commanders: commanderCards.map((c) => c.card?.oracleCard?.name).filter((n): n is string => !!n),
|
||||
deckSize: inDeck.reduce((n, c) => n + (c.quantity ?? 1), 0),
|
||||
};
|
||||
const art = uid ? scryfallArt(uid) : https(raw.customFeatured) ?? https(raw.featured);
|
||||
if (art) values.commanderArt = art;
|
||||
const format = raw.deckFormat !== undefined ? FORMATS[raw.deckFormat] : undefined;
|
||||
if (format) values.format = format;
|
||||
return values;
|
||||
}
|
||||
|
||||
export function archidektJob(o: { everyMinutes?: number; healthcheckUrl?: string; delayMs?: number } = {}): SyncJob {
|
||||
return {
|
||||
name: "archidekt",
|
||||
everyMinutes: o.everyMinutes ?? 60,
|
||||
healthcheckUrl: o.healthcheckUrl,
|
||||
async run(ctx) {
|
||||
const decks = ctx.notes("deck").filter((n) => typeof n.data.archidektId === "number");
|
||||
let synced = 0;
|
||||
let missing = 0;
|
||||
let failed = 0;
|
||||
const errors: string[] = [];
|
||||
for (const [i, note] of decks.entries()) {
|
||||
if (ctx.signal.aborted) throw ctx.signal.reason;
|
||||
if (i > 0) await ctx.sleep(o.delayMs ?? 1000);
|
||||
const id = note.data.archidektId as number;
|
||||
try {
|
||||
const res = await ctx.fetch(`${ARCHIDEKT_API}/${id}/`, { headers: { accept: "application/json" } });
|
||||
if (res.status === 404 || res.status === 403) {
|
||||
missing++;
|
||||
ctx.log(`${note.path}: deck ${id} is private or gone (${res.status})`);
|
||||
continue;
|
||||
}
|
||||
if (!res.ok) throw new Error(`Archidekt answered ${res.status}`);
|
||||
ctx.set(note.path, parseDeck((await res.json()) as RawDeck));
|
||||
synced++;
|
||||
} catch (err) {
|
||||
if (ctx.signal.aborted) throw ctx.signal.reason;
|
||||
failed++;
|
||||
errors.push(`${id}: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
ctx.retain(decks.map((d) => d.path));
|
||||
return {
|
||||
ok: failed === 0,
|
||||
message: failed ? `${failed} of ${decks.length} decks failed: ${errors.slice(0, 3).join("; ")}` : `${synced} decks synced`,
|
||||
counts: { decks: decks.length, synced, missing, failed },
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
/**
|
||||
* Bambuddy: filament colors in stock, for the Store's Filament options.
|
||||
* Hourly. Read only.
|
||||
*
|
||||
* UNVERIFIED: Bambuddy's API is not documented anywhere this project has, and
|
||||
* no request has been made to a real Bambuddy. The endpoint path is a setting
|
||||
* (BAMBUDDY_SPOOLS_PATH) and everything that depends on the response's shape
|
||||
* is in `mapBambuddySpools` below, so fixing it against the real API is a
|
||||
* change to one function. Check it against a live Bambuddy before relying on
|
||||
* it (docs/content-service.md, "Sync jobs").
|
||||
*/
|
||||
import type { SyncJob } from "../types.ts";
|
||||
import { mergeOptions, normalizeHex, storeOnLines, type StockOption } from "./stock.ts";
|
||||
|
||||
export const BAMBUDDY_DEFAULT_PATH = "/api/v1/inventory/spools";
|
||||
|
||||
export interface BambuddyOptions {
|
||||
url: string;
|
||||
/** Path of the spool or filament inventory endpoint. */
|
||||
path?: string;
|
||||
/** Sent as `Authorization: Bearer`, or in `authHeader` when that is set. */
|
||||
apiKey?: string;
|
||||
authHeader?: string;
|
||||
/** Option group the colors feed. Default Filament. */
|
||||
group?: string;
|
||||
/** Only these materials (like PLA, PETG), when set. */
|
||||
materials?: string[];
|
||||
/** A spool with less filament than this, in grams, counts as out. Default 50. */
|
||||
minGrams?: number;
|
||||
everyMinutes?: number;
|
||||
healthcheckUrl?: string;
|
||||
}
|
||||
|
||||
export interface Spool {
|
||||
name: string;
|
||||
hex?: string;
|
||||
material?: string;
|
||||
/** Grams left, when the source says. */
|
||||
remaining?: number;
|
||||
}
|
||||
|
||||
const pick = (o: Record<string, unknown>, keys: string[]) => {
|
||||
for (const k of keys) {
|
||||
const v = o[k];
|
||||
if (v !== undefined && v !== null && v !== "") return v;
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
|
||||
/**
|
||||
* THE ONE PLACE that knows Bambuddy's response shape (a guess; see above).
|
||||
* Accepts a list, or an object holding one under items, spools, data or
|
||||
* results, and reads each spool's color name, hex, material and grams left
|
||||
* under the common spellings. Archived or empty-named entries are skipped.
|
||||
*/
|
||||
export function mapBambuddySpools(json: unknown): Spool[] {
|
||||
const root = json as Record<string, unknown> | unknown[];
|
||||
const list = Array.isArray(root)
|
||||
? root
|
||||
: (["items", "spools", "data", "results"].map((k) => (root as Record<string, unknown>)?.[k]).find(Array.isArray) as unknown[] | undefined);
|
||||
if (!list) throw new Error("Bambuddy answered with no list of spools");
|
||||
const out: Spool[] = [];
|
||||
for (const raw of list) {
|
||||
if (!raw || typeof raw !== "object") continue;
|
||||
const o = raw as Record<string, unknown>;
|
||||
if (o.archived === true || o.deleted === true) continue;
|
||||
const name = pick(o, ["color_name", "colorName", "color", "name"]);
|
||||
if (typeof name !== "string" || !name.trim()) continue;
|
||||
const s: Spool = { name: name.trim() };
|
||||
const hex = normalizeHex(pick(o, ["color_hex", "colorHex", "hex", "rgba", "color_code"]));
|
||||
if (hex) s.hex = hex;
|
||||
const material = pick(o, ["material", "filament_type", "type"]);
|
||||
if (typeof material === "string") s.material = material.trim();
|
||||
const grams = Number(pick(o, ["remaining_weight", "remainingWeight", "weight_remaining", "remaining", "remaining_grams"]));
|
||||
if (Number.isFinite(grams)) s.remaining = grams;
|
||||
out.push(s);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Spools as Filament options: in stock when some spool of that color has enough left. */
|
||||
export function filamentOptions(spools: Spool[], o: Pick<BambuddyOptions, "materials" | "minGrams"> = {}): StockOption[] {
|
||||
const materials = o.materials?.length ? new Set(o.materials.map((m) => m.toLowerCase())) : null;
|
||||
const min = o.minGrams ?? 50;
|
||||
return mergeOptions(
|
||||
spools
|
||||
.filter((s) => !materials || (s.material && materials.has(s.material.toLowerCase())))
|
||||
.map((s) => ({ name: s.name, ...(s.hex ? { hex: s.hex } : {}), inStock: s.remaining === undefined || s.remaining >= min })),
|
||||
);
|
||||
}
|
||||
|
||||
export function bambuddyJob(o: BambuddyOptions): SyncJob {
|
||||
const group = o.group ?? "Filament";
|
||||
return {
|
||||
name: "bambuddy",
|
||||
everyMinutes: o.everyMinutes ?? 60,
|
||||
healthcheckUrl: o.healthcheckUrl,
|
||||
async run(ctx) {
|
||||
const headers: Record<string, string> = { accept: "application/json" };
|
||||
if (o.apiKey) headers[o.authHeader?.toLowerCase() || "authorization"] = o.authHeader ? o.apiKey : `Bearer ${o.apiKey}`;
|
||||
const url = `${o.url.replace(/\/$/, "")}/${(o.path ?? BAMBUDDY_DEFAULT_PATH).replace(/^\//, "")}`;
|
||||
const res = await ctx.fetch(url, { headers });
|
||||
if (res.status === 401 || res.status === 403) throw new Error(`Bambuddy refused the request (${res.status}); check BAMBUDDY_API_KEY`);
|
||||
if (!res.ok) throw new Error(`Bambuddy answered ${res.status}; check BAMBUDDY_SPOOLS_PATH`);
|
||||
const options = filamentOptions(mapBambuddySpools(await res.json()), o);
|
||||
const lines = storeOnLines(ctx, "filamentStock", [{ group, options }]);
|
||||
const inStock = options.filter((x) => x.inStock).length;
|
||||
return { message: `${inStock} of ${options.length} colors in stock, on ${lines} lines`, counts: { colors: options.length, inStock, lines } };
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
/**
|
||||
* BoardGameGeek: for every board-game note, its player count, play time and
|
||||
* box art, from the game's `bggId`. Daily.
|
||||
*
|
||||
* API: XML API2 `thing`, GET https://boardgamegeek.com/xmlapi2/thing?id=1,2,3
|
||||
* Up to 20 ids per request, with a pause between requests. BGG may answer
|
||||
* 202 (request queued: ask again shortly) or 429 (slow down). BGG now asks
|
||||
* API users to register an application; when BGG_TOKEN is set it is sent as
|
||||
* `Authorization: Bearer <token>`.
|
||||
*/
|
||||
import type { SyncJob, SyncedValues } from "../types.ts";
|
||||
|
||||
export const BGG_API = "https://boardgamegeek.com/xmlapi2";
|
||||
const BATCH = 20;
|
||||
|
||||
export interface GameValues extends SyncedValues {
|
||||
minPlayers?: number;
|
||||
maxPlayers?: number;
|
||||
/** BGG's playing time in minutes, with its range when given. */
|
||||
playTime?: number;
|
||||
minPlayTime?: number;
|
||||
maxPlayTime?: number;
|
||||
/** BGG's box art. */
|
||||
cover?: string;
|
||||
}
|
||||
|
||||
const decode = (s: string) =>
|
||||
s
|
||||
.replace(/&#(\d+);/g, (_, n) => String.fromCharCode(Number(n)))
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/&/g, "&");
|
||||
|
||||
const attr = (tag: string, name: string) => {
|
||||
const m = new RegExp(`\\b${name}="([^"]*)"`).exec(tag);
|
||||
return m ? decode(m[1]!) : undefined;
|
||||
};
|
||||
|
||||
/** `<name value="3"/>` as a positive number. */
|
||||
const valueOf = (xml: string, el: string) => {
|
||||
const m = new RegExp(`<${el}\\b[^>]*>`).exec(xml);
|
||||
const n = m ? Number(attr(m[0], "value")) : NaN;
|
||||
return Number.isFinite(n) && n > 0 ? n : undefined;
|
||||
};
|
||||
|
||||
/** Parse a `thing` response into values keyed by BGG id. */
|
||||
export function parseThings(xml: string): Map<number, GameValues> {
|
||||
const out = new Map<number, GameValues>();
|
||||
for (const m of xml.matchAll(/<item\b([^>]*)>([\s\S]*?)<\/item>/g)) {
|
||||
const id = Number(attr(m[1]!, "id"));
|
||||
if (!Number.isInteger(id)) continue;
|
||||
const body = m[2]!;
|
||||
const v: GameValues = {};
|
||||
const set = (k: keyof GameValues, n: number | undefined) => {
|
||||
if (n !== undefined) v[k] = n;
|
||||
};
|
||||
set("minPlayers", valueOf(body, "minplayers"));
|
||||
set("maxPlayers", valueOf(body, "maxplayers"));
|
||||
set("playTime", valueOf(body, "playingtime"));
|
||||
set("minPlayTime", valueOf(body, "minplaytime"));
|
||||
set("maxPlayTime", valueOf(body, "maxplaytime"));
|
||||
const image = /<image>\s*([^<]+?)\s*<\/image>/.exec(body)?.[1];
|
||||
if (image) {
|
||||
const url = decode(image).replace(/^\/\//, "https://").replace(/^http:/, "https:");
|
||||
if (url.startsWith("https://")) v.cover = url;
|
||||
}
|
||||
out.set(id, v);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function bggJob(o: { everyMinutes?: number; healthcheckUrl?: string; token?: string; delayMs?: number } = {}): SyncJob {
|
||||
const delay = o.delayMs ?? 5000;
|
||||
return {
|
||||
name: "bgg",
|
||||
everyMinutes: o.everyMinutes ?? 24 * 60,
|
||||
healthcheckUrl: o.healthcheckUrl,
|
||||
timeoutMs: 30 * 60_000,
|
||||
async run(ctx) {
|
||||
const games = ctx.notes("board-game").filter((n) => typeof n.data.bggId === "number");
|
||||
const byId = new Map<number, string[]>();
|
||||
for (const g of games) byId.set(g.data.bggId as number, [...(byId.get(g.data.bggId as number) ?? []), g.path]);
|
||||
const ids = [...byId.keys()];
|
||||
const headers: Record<string, string> = { accept: "application/xml" };
|
||||
if (o.token) headers.authorization = `Bearer ${o.token}`;
|
||||
|
||||
let synced = 0;
|
||||
let failed = 0;
|
||||
const errors: string[] = [];
|
||||
for (let i = 0; i < ids.length; i += BATCH) {
|
||||
if (i > 0) await ctx.sleep(delay);
|
||||
if (ctx.signal.aborted) throw ctx.signal.reason;
|
||||
const batch = ids.slice(i, i + BATCH);
|
||||
const url = `${BGG_API}/thing?id=${batch.join(",")}&type=boardgame,boardgameexpansion`;
|
||||
let xml: string | null = null;
|
||||
try {
|
||||
for (let attempt = 0; attempt < 4 && xml === null; attempt++) {
|
||||
const res = await ctx.fetch(url, { headers });
|
||||
if (res.status === 202) {
|
||||
await ctx.sleep(delay * (attempt + 1));
|
||||
continue;
|
||||
}
|
||||
if (res.status === 429) throw new Error("BoardGameGeek asked us to slow down (429)");
|
||||
if (res.status === 401 || res.status === 403) throw new Error(`BoardGameGeek refused the request (${res.status}); set BGG_TOKEN`);
|
||||
if (!res.ok) throw new Error(`BoardGameGeek answered ${res.status}`);
|
||||
xml = await res.text();
|
||||
}
|
||||
if (xml === null) throw new Error("BoardGameGeek kept the request queued");
|
||||
} catch (err) {
|
||||
if (ctx.signal.aborted) throw ctx.signal.reason;
|
||||
failed += batch.length;
|
||||
errors.push(err instanceof Error ? err.message : String(err));
|
||||
if (/429|refused/.test(errors.at(-1)!)) break;
|
||||
continue;
|
||||
}
|
||||
const found = parseThings(xml);
|
||||
for (const id of batch) {
|
||||
const v = found.get(id);
|
||||
if (!v) {
|
||||
failed++;
|
||||
errors.push(`no game ${id}`);
|
||||
continue;
|
||||
}
|
||||
for (const path of byId.get(id) ?? []) ctx.set(path, v);
|
||||
synced++;
|
||||
}
|
||||
}
|
||||
ctx.retain(games.map((g) => g.path));
|
||||
const unsynced = ids.length - synced;
|
||||
return {
|
||||
ok: unsynced === 0,
|
||||
message: unsynced ? `${unsynced} of ${ids.length} games not synced: ${[...new Set(errors)].slice(0, 3).join("; ")}` : `${synced} games synced`,
|
||||
counts: { games: ids.length, synced, failed: unsynced },
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
/**
|
||||
* Cover art for Library items that have no `cover` of their own:
|
||||
*
|
||||
* - books: Open Library search (no key);
|
||||
* - films and TV: TMDB search (needs TMDB_API_KEY; skipped without it);
|
||||
* - anime, manga, manhwa, light novels and webnovels: AniList (no key),
|
||||
* by the note's `anilistId` when it has one, else by title.
|
||||
*
|
||||
* Video games have no source yet. Only the image address is stored; nothing
|
||||
* is downloaded. The job looks every 15 minutes, so a new item gets its
|
||||
* cover soon after it appears, and it re-checks old answers slowly: an item
|
||||
* with no cover found is asked about again after a day, one with a cover
|
||||
* after 30 days, at most `perRun` items a run. This follows the legacy
|
||||
* scripts in legacy/scripts/fetch-{book-covers,tmdb-art,anime-art}.mjs.
|
||||
*/
|
||||
import type { StoredNote } from "../../store.ts";
|
||||
import type { SyncContext, SyncJob } from "../types.ts";
|
||||
|
||||
export const OPEN_LIBRARY_SEARCH = "https://openlibrary.org/search.json";
|
||||
export const OPEN_LIBRARY_COVERS = "https://covers.openlibrary.org/b";
|
||||
export const TMDB_API = "https://api.themoviedb.org/3";
|
||||
export const TMDB_IMAGES = "https://image.tmdb.org/t/p/w500";
|
||||
export const ANILIST_API = "https://graphql.anilist.co";
|
||||
|
||||
const DAY = 24 * 60 * 60_000;
|
||||
const RETRY_FOUND_MS = 30 * DAY;
|
||||
const RETRY_MISSING_MS = DAY;
|
||||
|
||||
type Source = "openlibrary" | "tmdb" | "anilist";
|
||||
|
||||
export function sourceFor(medium: string): Source | null {
|
||||
if (medium === "book") return "openlibrary";
|
||||
if (medium === "movie" || medium === "TV") return "tmdb";
|
||||
if (["anime", "manga", "manhwa", "light novel", "webnovel"].includes(medium)) return "anilist";
|
||||
return null;
|
||||
}
|
||||
|
||||
class RateLimited extends Error {}
|
||||
|
||||
// ---------------------------------------------------------------- parsing
|
||||
|
||||
/** Open Library search.json: the first result with a cover. */
|
||||
export function parseOpenLibrary(json: unknown): string | null {
|
||||
const docs = ((json as { docs?: { cover_i?: number; cover_edition_key?: string }[] })?.docs ?? []);
|
||||
const withId = docs.find((d) => typeof d.cover_i === "number" && d.cover_i > 0);
|
||||
if (withId) return `${OPEN_LIBRARY_COVERS}/id/${withId.cover_i}-L.jpg`;
|
||||
const withEdition = docs.find((d) => d.cover_edition_key);
|
||||
return withEdition ? `${OPEN_LIBRARY_COVERS}/olid/${withEdition.cover_edition_key}-L.jpg` : null;
|
||||
}
|
||||
|
||||
/** TMDB search/movie or search/tv: the first result with a poster. */
|
||||
export function parseTmdb(json: unknown): string | null {
|
||||
const results = (json as { results?: { poster_path?: string | null }[] })?.results ?? [];
|
||||
const hit = results.find((r) => r.poster_path);
|
||||
return hit ? `${TMDB_IMAGES}${hit.poster_path}` : null;
|
||||
}
|
||||
|
||||
/** AniList Media query: the largest cover. */
|
||||
export function parseAniList(json: unknown): string | null {
|
||||
const media = (json as { data?: { Media?: { coverImage?: { extraLarge?: string; large?: string } } | null } })?.data?.Media;
|
||||
const url = media?.coverImage?.extraLarge ?? media?.coverImage?.large;
|
||||
return url && url.startsWith("https://") ? url : null;
|
||||
}
|
||||
|
||||
export const ANILIST_QUERY = `query ($id: Int, $search: String, $type: MediaType, $format: MediaFormat) {
|
||||
Media(id: $id, search: $search, type: $type, format: $format) {
|
||||
id
|
||||
title { english romaji }
|
||||
coverImage { extraLarge large }
|
||||
}
|
||||
}`;
|
||||
|
||||
// ---------------------------------------------------------------- lookups
|
||||
|
||||
async function json(ctx: SyncContext, url: string, init: RequestInit = {}) {
|
||||
const res = await ctx.fetch(url, { ...init, headers: { accept: "application/json", ...(init.headers as Record<string, string>) } });
|
||||
if (res.status === 429) throw new RateLimited("rate limited (429)");
|
||||
if (res.status === 404) return null;
|
||||
if (!res.ok) throw new Error(`answered ${res.status}`);
|
||||
return res.json();
|
||||
}
|
||||
|
||||
export async function lookup(ctx: SyncContext, note: StoredNote, tmdbKey?: string): Promise<string | null> {
|
||||
const d = note.data as { medium: string; creator?: string; year?: number; anilistId?: number };
|
||||
const title = note.title;
|
||||
switch (sourceFor(d.medium)) {
|
||||
case "openlibrary": {
|
||||
const p = new URLSearchParams({ title, limit: "5", fields: "title,cover_i,cover_edition_key" });
|
||||
if (d.creator) p.set("author", d.creator);
|
||||
return parseOpenLibrary(await json(ctx, `${OPEN_LIBRARY_SEARCH}?${p}`));
|
||||
}
|
||||
case "tmdb": {
|
||||
if (!tmdbKey) return null;
|
||||
const kind = d.medium === "movie" ? "movie" : "tv";
|
||||
const p = new URLSearchParams({ query: title, include_adult: "false" });
|
||||
if (d.year) p.set(kind === "movie" ? "year" : "first_air_date_year", String(d.year));
|
||||
// A v4 read access token (a JWT) goes in the header; a v3 key in the query.
|
||||
const bearer = tmdbKey.startsWith("eyJ");
|
||||
if (!bearer) p.set("api_key", tmdbKey);
|
||||
return parseTmdb(await json(ctx, `${TMDB_API}/search/${kind}?${p}`, bearer ? { headers: { authorization: `Bearer ${tmdbKey}` } } : {}));
|
||||
}
|
||||
case "anilist": {
|
||||
const variables: Record<string, unknown> = { type: d.medium === "anime" ? "ANIME" : "MANGA" };
|
||||
if (d.anilistId) variables.id = d.anilistId;
|
||||
else {
|
||||
variables.search = title;
|
||||
if (d.medium === "light novel" || d.medium === "webnovel") variables.format = "NOVEL";
|
||||
}
|
||||
return parseAniList(
|
||||
await json(ctx, ANILIST_API, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ query: ANILIST_QUERY, variables }),
|
||||
}),
|
||||
);
|
||||
}
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- the job
|
||||
|
||||
export function coversJob(
|
||||
o: { everyMinutes?: number; healthcheckUrl?: string; tmdbApiKey?: string; perRun?: number; delayMs?: number } = {},
|
||||
): SyncJob {
|
||||
return {
|
||||
name: "covers",
|
||||
everyMinutes: o.everyMinutes ?? 15,
|
||||
healthcheckUrl: o.healthcheckUrl,
|
||||
async run(ctx) {
|
||||
const now = ctx.now().getTime();
|
||||
const items = ctx.notes("library-item").filter((n) => !n.data.cover && sourceFor(String(n.data.medium)) !== null);
|
||||
const usable = items.filter((n) => sourceFor(String(n.data.medium)) !== "tmdb" || o.tmdbApiKey);
|
||||
// New items first, then the oldest answers that are due again.
|
||||
const due = usable
|
||||
.map((n) => ({ n, c: ctx.cached(n.path) }))
|
||||
.filter(({ c }) => !c || now - Date.parse(c.fetchedAt) >= (c.values.cover ? RETRY_FOUND_MS : RETRY_MISSING_MS))
|
||||
.sort((a, b) => (a.c ? Date.parse(a.c.fetchedAt) : 0) - (b.c ? Date.parse(b.c.fetchedAt) : 0))
|
||||
.slice(0, o.perRun ?? 20);
|
||||
|
||||
let found = 0;
|
||||
let missing = 0;
|
||||
let failed = 0;
|
||||
const errors: string[] = [];
|
||||
for (const [i, { n, c }] of due.entries()) {
|
||||
if (i > 0) await ctx.sleep(o.delayMs ?? 1000);
|
||||
if (ctx.signal.aborted) throw ctx.signal.reason;
|
||||
try {
|
||||
const cover = await lookup(ctx, n, o.tmdbApiKey);
|
||||
if (cover) {
|
||||
ctx.set(n.path, { cover });
|
||||
found++;
|
||||
} else {
|
||||
// Nothing found: keep a cover found earlier, else remember the miss.
|
||||
ctx.set(n.path, c?.values.cover ? c.values : {});
|
||||
missing++;
|
||||
}
|
||||
} catch (err) {
|
||||
if (ctx.signal.aborted) throw ctx.signal.reason;
|
||||
failed++;
|
||||
errors.push(`${n.title}: ${err instanceof Error ? err.message : String(err)}`);
|
||||
if (err instanceof RateLimited) break;
|
||||
}
|
||||
}
|
||||
ctx.retain(items.map((n) => n.path));
|
||||
return {
|
||||
ok: failed === 0,
|
||||
message: failed
|
||||
? `${failed} lookups failed: ${errors.slice(0, 3).join("; ")}`
|
||||
: `${due.length} checked: ${found} covers found, ${missing} not found`,
|
||||
counts: { checked: due.length, found, missing, failed, waiting: usable.length - due.length },
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
/**
|
||||
* Homebox: leather and paper materials in stock, for the Store's Leather and
|
||||
* Paper options. Hourly. Read only.
|
||||
*
|
||||
* API (Homebox's REST API, v1):
|
||||
* - `POST /api/v1/users/login` with username and password gives a token
|
||||
* (already prefixed `Bearer `), sent as `Authorization`.
|
||||
* - `GET /api/v1/labels` and `GET /api/v1/locations` name the ids to filter on.
|
||||
* - `GET /api/v1/items?labels=<id>&page=&pageSize=` (or `locations=<id>`)
|
||||
* lists the items, each with name, quantity and description.
|
||||
*
|
||||
* Which items count is a setting: HOMEBOX_LABELS (like `leather,paper`) and
|
||||
* HOMEBOX_LOCATIONS, each entry `name` or `name=Group`. Without `=Group` the
|
||||
* option group is the name capitalized (leather: Leather). An item is in
|
||||
* stock when its quantity is above zero; a `#rrggbb` in its description
|
||||
* becomes the swatch color.
|
||||
*
|
||||
* Written from Homebox's API documentation; not checked against a live
|
||||
* server from here (see docs/content-service.md).
|
||||
*/
|
||||
import type { SyncContext, SyncJob } from "../types.ts";
|
||||
import { mergeOptions, normalizeHex, storeOnLines, type StockGroup, type StockOption } from "./stock.ts";
|
||||
|
||||
export interface HomeboxFilter {
|
||||
/** Label or location name in Homebox, matched case-insensitively. */
|
||||
name: string;
|
||||
/** Store option group it feeds. */
|
||||
group: string;
|
||||
}
|
||||
|
||||
export interface HomeboxOptions {
|
||||
url: string;
|
||||
user: string;
|
||||
pass: string;
|
||||
labels: HomeboxFilter[];
|
||||
locations?: HomeboxFilter[];
|
||||
everyMinutes?: number;
|
||||
healthcheckUrl?: string;
|
||||
}
|
||||
|
||||
/** `leather,paper=Paper goods` as filters. */
|
||||
export function parseFilters(raw: string | undefined): HomeboxFilter[] {
|
||||
return (raw ?? "")
|
||||
.split(",")
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean)
|
||||
.map((s) => {
|
||||
const [name, group] = s.split("=").map((x) => x.trim());
|
||||
return { name: name!, group: group || name!.charAt(0).toUpperCase() + name!.slice(1) };
|
||||
});
|
||||
}
|
||||
|
||||
export interface HomeboxItem {
|
||||
name: string;
|
||||
quantity: number;
|
||||
description?: string;
|
||||
}
|
||||
|
||||
/** `GET /api/v1/items`: `{ items, page, pageSize, total }`. */
|
||||
export function parseItems(json: unknown): { items: HomeboxItem[]; total: number } {
|
||||
const r = json as { items?: { name?: string; quantity?: number; description?: string; archived?: boolean }[]; total?: number };
|
||||
if (!Array.isArray(r?.items)) throw new Error("Homebox answered with no list of items");
|
||||
const items = r.items
|
||||
.filter((i) => i && typeof i.name === "string" && i.name.trim() && i.archived !== true)
|
||||
.map((i) => ({ name: i.name!.trim(), quantity: Number(i.quantity ?? 0) || 0, ...(i.description ? { description: i.description } : {}) }));
|
||||
return { items, total: Number(r.total ?? items.length) };
|
||||
}
|
||||
|
||||
/** `GET /api/v1/labels` or `/locations`: `[{ id, name }]` to a name to id map. */
|
||||
export function parseNamed(json: unknown): Map<string, string> {
|
||||
const list = Array.isArray(json) ? json : ((json as { items?: unknown[] })?.items ?? []);
|
||||
const out = new Map<string, string>();
|
||||
for (const x of list as { id?: string; name?: string }[]) if (x?.id && x.name) out.set(x.name.trim().toLowerCase(), x.id);
|
||||
return out;
|
||||
}
|
||||
|
||||
export function materialOptions(items: HomeboxItem[]): StockOption[] {
|
||||
return mergeOptions(
|
||||
items.map((i) => {
|
||||
const hex = normalizeHex(/#[0-9a-f]{6}\b/i.exec(i.description ?? "")?.[0]);
|
||||
return { name: i.name, ...(hex ? { hex } : {}), inStock: i.quantity > 0 };
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async function call(ctx: SyncContext, url: string, init: RequestInit = {}) {
|
||||
const res = await ctx.fetch(url, init);
|
||||
if (res.status === 401 || res.status === 403) throw new Error(`Homebox refused the request (${res.status}); check HOMEBOX_USER and HOMEBOX_PASS`);
|
||||
if (!res.ok) throw new Error(`Homebox answered ${res.status}`);
|
||||
return res.json() as Promise<unknown>;
|
||||
}
|
||||
|
||||
export function homeboxJob(o: HomeboxOptions): SyncJob {
|
||||
const api = (path: string) => `${o.url.replace(/\/$/, "")}/api/v1${path}`;
|
||||
return {
|
||||
name: "homebox",
|
||||
everyMinutes: o.everyMinutes ?? 60,
|
||||
healthcheckUrl: o.healthcheckUrl,
|
||||
async run(ctx) {
|
||||
const login = (await call(ctx, api("/users/login"), {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", accept: "application/json" },
|
||||
body: JSON.stringify({ username: o.user, password: o.pass, stayLoggedIn: false }),
|
||||
})) as { token?: string };
|
||||
if (!login?.token) throw new Error("Homebox login gave no token");
|
||||
const headers = { accept: "application/json", authorization: /^Bearer /i.test(login.token) ? login.token : `Bearer ${login.token}` };
|
||||
|
||||
const filters: { kind: "labels" | "locations"; f: HomeboxFilter }[] = [
|
||||
...o.labels.map((f) => ({ kind: "labels" as const, f })),
|
||||
...(o.locations ?? []).map((f) => ({ kind: "locations" as const, f })),
|
||||
];
|
||||
const ids = { labels: new Map<string, string>(), locations: new Map<string, string>() };
|
||||
for (const kind of ["labels", "locations"] as const) {
|
||||
if (filters.some((x) => x.kind === kind)) ids[kind] = parseNamed(await call(ctx, api(`/${kind}`), { headers }));
|
||||
}
|
||||
|
||||
const byGroup = new Map<string, HomeboxItem[]>();
|
||||
const unknown: string[] = [];
|
||||
for (const { kind, f } of filters) {
|
||||
const id = ids[kind].get(f.name.toLowerCase());
|
||||
if (!id) {
|
||||
unknown.push(f.name);
|
||||
continue;
|
||||
}
|
||||
const items = byGroup.get(f.group) ?? [];
|
||||
for (let page = 1; page <= 20; page++) {
|
||||
const p = new URLSearchParams({ [kind]: id, page: String(page), pageSize: "100" });
|
||||
const r = parseItems(await call(ctx, api(`/items?${p}`), { headers }));
|
||||
items.push(...r.items);
|
||||
if (r.items.length < 100 || page * 100 >= r.total) break;
|
||||
}
|
||||
byGroup.set(f.group, items);
|
||||
}
|
||||
// A filter naming nothing in Homebox is a setup mistake: fail rather than mark everything out.
|
||||
if (unknown.length) throw new Error(`No Homebox label or location named ${unknown.join(", ")}`);
|
||||
|
||||
const groups: StockGroup[] = [...byGroup].map(([group, items]) => ({ group, options: materialOptions(items) }));
|
||||
const lines = storeOnLines(ctx, "materialStock", groups);
|
||||
const counts: Record<string, number> = { lines };
|
||||
for (const g of groups) counts[g.group.toLowerCase()] = g.options.filter((x) => x.inStock).length;
|
||||
return { message: `${groups.map((g) => `${g.group}: ${g.options.filter((x) => x.inStock).length} in stock`).join(", ")}; on ${lines} lines`, counts };
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
/**
|
||||
* Settings and jobs for the homelab sources on Cosmos: Suwayomi and Kavita
|
||||
* (PersonalReader), Bambuddy and Homebox (Store stock). Each is enabled by
|
||||
* its own URL and credentials alone, not by SYNC_JOBS. All are read only
|
||||
* against their source.
|
||||
*/
|
||||
import type { SyncJob } from "../types.ts";
|
||||
import { bambuddyJob, type BambuddyOptions } from "./bambuddy.ts";
|
||||
import { homeboxJob, parseFilters, type HomeboxOptions } from "./homebox.ts";
|
||||
import { kavitaJob, type KavitaOptions } from "./kavita.ts";
|
||||
import { suwayomiJob, type SuwayomiAuth, type SuwayomiOptions } from "./suwayomi.ts";
|
||||
|
||||
type Base<T> = Omit<T, "everyMinutes" | "healthcheckUrl">;
|
||||
|
||||
export interface HomelabSettings {
|
||||
suwayomi?: Base<SuwayomiOptions>;
|
||||
kavita?: Base<KavitaOptions>;
|
||||
bambuddy?: Base<BambuddyOptions>;
|
||||
homebox?: Base<HomeboxOptions>;
|
||||
}
|
||||
|
||||
const list = (v: string | undefined) => (v ?? "").split(",").map((s) => s.trim()).filter(Boolean);
|
||||
|
||||
export function homelabSettings(env: NodeJS.ProcessEnv): HomelabSettings {
|
||||
const s: HomelabSettings = {};
|
||||
if (env.SUWAYOMI_URL) {
|
||||
const auth = env.SUWAYOMI_AUTH?.trim().toLowerCase().replace("basic_auth", "basic");
|
||||
if (auth && !["ui_login", "basic", "none"].includes(auth)) throw new Error(`SUWAYOMI_AUTH: "${auth}" is not ui_login, basic or none`);
|
||||
s.suwayomi = {
|
||||
url: env.SUWAYOMI_URL,
|
||||
user: env.SUWAYOMI_USER || undefined,
|
||||
pass: env.SUWAYOMI_PASS || undefined,
|
||||
auth: (auth as SuwayomiAuth) || undefined,
|
||||
};
|
||||
}
|
||||
if (env.KAVITA_URL && env.KAVITA_API_KEY) s.kavita = { url: env.KAVITA_URL, apiKey: env.KAVITA_API_KEY };
|
||||
if (env.BAMBUDDY_URL) {
|
||||
s.bambuddy = {
|
||||
url: env.BAMBUDDY_URL,
|
||||
path: env.BAMBUDDY_SPOOLS_PATH || undefined,
|
||||
apiKey: env.BAMBUDDY_API_KEY || undefined,
|
||||
authHeader: env.BAMBUDDY_AUTH_HEADER || undefined,
|
||||
group: env.BAMBUDDY_GROUP || undefined,
|
||||
materials: list(env.BAMBUDDY_MATERIALS),
|
||||
minGrams: env.BAMBUDDY_MIN_GRAMS ? Number(env.BAMBUDDY_MIN_GRAMS) : undefined,
|
||||
};
|
||||
}
|
||||
if (env.HOMEBOX_URL && env.HOMEBOX_USER && env.HOMEBOX_PASS) {
|
||||
s.homebox = {
|
||||
url: env.HOMEBOX_URL,
|
||||
user: env.HOMEBOX_USER,
|
||||
pass: env.HOMEBOX_PASS,
|
||||
labels: parseFilters(env.HOMEBOX_LABELS ?? "leather,paper"),
|
||||
locations: parseFilters(env.HOMEBOX_LOCATIONS),
|
||||
};
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
export function homelabJobs(s: HomelabSettings, common: (name: string) => { everyMinutes?: number; healthcheckUrl?: string }): SyncJob[] {
|
||||
const jobs: SyncJob[] = [];
|
||||
if (s.suwayomi) jobs.push(suwayomiJob({ ...s.suwayomi, ...common("suwayomi") }));
|
||||
if (s.kavita) jobs.push(kavitaJob({ ...s.kavita, ...common("kavita") }));
|
||||
if (s.bambuddy) jobs.push(bambuddyJob({ ...s.bambuddy, ...common("bambuddy") }));
|
||||
if (s.homebox) jobs.push(homeboxJob({ ...s.homebox, ...common("homebox") }));
|
||||
return jobs;
|
||||
}
|
||||
@@ -0,0 +1,238 @@
|
||||
/**
|
||||
* Kavita (PersonalReader's light novels and books): reading progress for
|
||||
* Library items linked to a Kavita series. Daily. Read only against Kavita.
|
||||
*
|
||||
* A Library item is linked only explicitly, never by title:
|
||||
* - `kavitaSeriesId` on the note names the Kavita series; or
|
||||
* - a book, light novel or webnovel note's `anilistId` equals the AniList id
|
||||
* in a Kavita series' metadata (an anilist.co address in its web links).
|
||||
*
|
||||
* Auth: the API key is exchanged for a token at
|
||||
* `POST /api/Plugin/authenticate?apiKey=…&pluginName=…`, sent as
|
||||
* `Authorization: Bearer`. Series come from `POST /api/Series/all-v2` (only
|
||||
* when an AniList match is needed) or `GET /api/Series/<id>`, metadata from
|
||||
* `GET /api/Series/metadata?seriesId=`, the Want to Read list from
|
||||
* `POST /api/want-to-read/v2`.
|
||||
*
|
||||
* State: fully read is finished (or caught up while the series is still
|
||||
* ongoing), partly read is now, unread and on Want to Read is up next;
|
||||
* anything else leaves the note alone. Cached per note: `bookProgress`
|
||||
* (title, percent read, when). Nothing that links into Kavita.
|
||||
*
|
||||
* Written from Kavita's API documentation; not checked against a live server
|
||||
* from here (see docs/content-service.md).
|
||||
*/
|
||||
import type { StoredNote } from "../../store.ts";
|
||||
import type { SyncContext, SyncJob, SyncResult } from "../types.ts";
|
||||
import { errorText, isoTime, MEMO_KEY, StateProposer, targetState, type LibraryState } from "./reader.ts";
|
||||
|
||||
export interface KavitaOptions {
|
||||
url: string;
|
||||
apiKey: string;
|
||||
everyMinutes?: number;
|
||||
healthcheckUrl?: string;
|
||||
delayMs?: number;
|
||||
}
|
||||
|
||||
/** Media that may be linked to Kavita by AniList id. */
|
||||
export const KAVITA_MEDIA = ["book", "light novel", "webnovel"];
|
||||
|
||||
/** Kavita's PublicationStatus: OnGoing 0, Hiatus 1, Completed 2, Cancelled 3, Ended 4. */
|
||||
const DONE_STATUSES = new Set([2, 4]);
|
||||
|
||||
export interface KavitaSeries {
|
||||
id: number;
|
||||
name: string;
|
||||
pages: number;
|
||||
pagesRead: number;
|
||||
latestReadDate?: string;
|
||||
}
|
||||
|
||||
export interface KavitaMetadata {
|
||||
anilistId?: number;
|
||||
publicationStatus?: number;
|
||||
}
|
||||
|
||||
export function parseSeries(raw: unknown): KavitaSeries | null {
|
||||
const s = raw as { id?: number; name?: string; localizedName?: string; pages?: number; pagesRead?: number; latestReadDate?: string };
|
||||
if (typeof s?.id !== "number" || !s.name) return null;
|
||||
const out: KavitaSeries = { id: s.id, name: s.localizedName || s.name, pages: Math.max(0, Number(s.pages) || 0), pagesRead: Math.max(0, Number(s.pagesRead) || 0) };
|
||||
const at = isoTime(s.latestReadDate);
|
||||
if (at) out.latestReadDate = at;
|
||||
return out;
|
||||
}
|
||||
|
||||
const ANILIST_LINK = /anilist\.co\/(?:manga|anime)\/(\d+)/i;
|
||||
|
||||
/** AniList id from the series' web links (comma separated addresses), and its publication status. */
|
||||
export function parseMetadata(raw: unknown): KavitaMetadata {
|
||||
const m = raw as { webLinks?: string; publicationStatus?: number };
|
||||
const out: KavitaMetadata = {};
|
||||
const id = ANILIST_LINK.exec(m?.webLinks ?? "")?.[1];
|
||||
if (id) out.anilistId = Number(id);
|
||||
if (typeof m?.publicationStatus === "number") out.publicationStatus = m.publicationStatus;
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The Library state Kavita's progress means, or null to leave the note alone. */
|
||||
export function kavitaState(s: KavitaSeries, meta: KavitaMetadata, wantToRead: boolean): LibraryState | null {
|
||||
if (s.pages > 0 && s.pagesRead >= s.pages) return DONE_STATUSES.has(meta.publicationStatus ?? -1) ? "finished" : "caught up";
|
||||
if (s.pagesRead > 0) return "now";
|
||||
return wantToRead ? "up next" : null;
|
||||
}
|
||||
|
||||
export const percentRead = (s: KavitaSeries) => (s.pages > 0 ? Math.min(100, Math.floor((s.pagesRead / s.pages) * 100)) : 0);
|
||||
|
||||
// ---------------------------------------------------------------- requests
|
||||
|
||||
const FILTER = { statements: [], combination: 1, limitTo: 0, sortOptions: { sortField: 1, isAscending: true } };
|
||||
|
||||
class Kavita {
|
||||
private headers: Record<string, string> = {};
|
||||
constructor(private readonly ctx: SyncContext, private readonly o: KavitaOptions) {}
|
||||
|
||||
private url = (path: string) => `${this.o.url.replace(/\/$/, "")}${path}`;
|
||||
|
||||
async login() {
|
||||
const p = new URLSearchParams({ apiKey: this.o.apiKey, pluginName: "GregKilmer.com" });
|
||||
const res = await this.ctx.fetch(this.url(`/api/Plugin/authenticate?${p}`), { method: "POST", headers: { accept: "application/json" } });
|
||||
if (res.status === 401 || res.status === 403) throw new Error(`Kavita refused the API key (${res.status})`);
|
||||
if (!res.ok) throw new Error(`Kavita login answered ${res.status}`);
|
||||
const token = ((await res.json()) as { token?: string })?.token;
|
||||
if (!token) throw new Error("Kavita login gave no token");
|
||||
this.headers = { authorization: `Bearer ${token}` };
|
||||
}
|
||||
|
||||
async call(path: string, body?: unknown): Promise<unknown> {
|
||||
const res = await this.ctx.fetch(this.url(path), {
|
||||
method: body === undefined ? "GET" : "POST",
|
||||
headers: { accept: "application/json", ...(body === undefined ? {} : { "content-type": "application/json" }), ...this.headers },
|
||||
body: body === undefined ? undefined : JSON.stringify(body),
|
||||
});
|
||||
if (res.status === 404) return null;
|
||||
if (!res.ok) throw new Error(`Kavita answered ${res.status} for ${path.split("?")[0]}`);
|
||||
return res.json();
|
||||
}
|
||||
|
||||
async allSeries(): Promise<KavitaSeries[]> {
|
||||
const out: KavitaSeries[] = [];
|
||||
for (let page = 1; page <= 50; page++) {
|
||||
const list = (await this.call(`/api/Series/all-v2?PageNumber=${page}&PageSize=200`, FILTER)) as unknown[] | null;
|
||||
if (!Array.isArray(list) || !list.length) break;
|
||||
for (const r of list) {
|
||||
const s = parseSeries(r);
|
||||
if (s) out.push(s);
|
||||
}
|
||||
if (list.length < 200) break;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async series(id: number) {
|
||||
return parseSeries(await this.call(`/api/Series/${id}`));
|
||||
}
|
||||
|
||||
async metadata(id: number) {
|
||||
return parseMetadata(await this.call(`/api/Series/metadata?seriesId=${id}`));
|
||||
}
|
||||
|
||||
async wantToRead(): Promise<Set<number>> {
|
||||
try {
|
||||
const list = (await this.call("/api/want-to-read/v2?PageNumber=1&PageSize=1000", FILTER)) as { id?: number }[] | null;
|
||||
return new Set((list ?? []).map((s) => s.id).filter((id): id is number => typeof id === "number"));
|
||||
} catch (err) {
|
||||
if (this.ctx.signal.aborted) throw this.ctx.signal.reason;
|
||||
this.ctx.log(`Want to Read list not read: ${errorText(err)}`);
|
||||
return new Set();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- the job
|
||||
|
||||
export function kavitaJob(o: KavitaOptions): SyncJob {
|
||||
const delay = o.delayMs ?? 250;
|
||||
return {
|
||||
name: "kavita",
|
||||
everyMinutes: o.everyMinutes ?? 24 * 60,
|
||||
healthcheckUrl: o.healthcheckUrl,
|
||||
timeoutMs: 30 * 60_000,
|
||||
async run(ctx): Promise<SyncResult> {
|
||||
const items = ctx.notes("library-item");
|
||||
const byKavita = items.filter((n) => typeof n.data.kavitaSeriesId === "number");
|
||||
const byAniList = items.filter(
|
||||
(n) => typeof n.data.kavitaSeriesId !== "number" && typeof n.data.anilistId === "number" && KAVITA_MEDIA.includes(String(n.data.medium)),
|
||||
);
|
||||
if (!byKavita.length && !byAniList.length) {
|
||||
ctx.retain([MEMO_KEY]);
|
||||
return { message: "No Library item is linked to Kavita", counts: { linked: 0 } };
|
||||
}
|
||||
|
||||
const k = new Kavita(ctx, o);
|
||||
await k.login();
|
||||
|
||||
// Pairs of (note, Kavita series id), with the metadata already read.
|
||||
const pairs: { note: StoredNote; id: number }[] = byKavita.map((note) => ({ note, id: note.data.kavitaSeriesId as number }));
|
||||
const meta = new Map<number, KavitaMetadata>();
|
||||
const known = new Map<number, KavitaSeries>();
|
||||
if (byAniList.length) {
|
||||
const wanted = new Set(byAniList.map((n) => n.data.anilistId as number));
|
||||
for (const [i, s] of (await k.allSeries()).entries()) {
|
||||
if (i > 0) await ctx.sleep(delay);
|
||||
if (ctx.signal.aborted) throw ctx.signal.reason;
|
||||
const m = await k.metadata(s.id);
|
||||
if (!m.anilistId || !wanted.has(m.anilistId)) continue;
|
||||
meta.set(s.id, m);
|
||||
known.set(s.id, s);
|
||||
for (const note of byAniList.filter((n) => n.data.anilistId === m.anilistId)) pairs.push({ note, id: s.id });
|
||||
}
|
||||
}
|
||||
const wantToRead = await k.wantToRead();
|
||||
|
||||
const proposer = new StateProposer(ctx);
|
||||
const kept: string[] = [MEMO_KEY];
|
||||
let synced = 0;
|
||||
let missing = 0;
|
||||
let failed = 0;
|
||||
const errors: string[] = [];
|
||||
for (const { note, id } of pairs) {
|
||||
if (ctx.signal.aborted) throw ctx.signal.reason;
|
||||
kept.push(note.path);
|
||||
try {
|
||||
let s = known.get(id) ?? null;
|
||||
if (!s) {
|
||||
await ctx.sleep(delay);
|
||||
s = await k.series(id);
|
||||
}
|
||||
if (!s) {
|
||||
missing++;
|
||||
ctx.log(`${note.path}: no Kavita series ${id}`);
|
||||
continue;
|
||||
}
|
||||
const m = meta.get(id) ?? (await k.metadata(id));
|
||||
const progress: Record<string, unknown> = { title: s.name, percent: percentRead(s) };
|
||||
if (s.latestReadDate) progress.readAt = s.latestReadDate;
|
||||
ctx.set(note.path, { bookProgress: progress });
|
||||
synced++;
|
||||
proposer.propose(note, targetState(String(note.data.state), kavitaState(s, m, wantToRead.has(id))));
|
||||
} catch (err) {
|
||||
if (ctx.signal.aborted) throw ctx.signal.reason;
|
||||
failed++;
|
||||
errors.push(`${note.title}: ${errorText(err)}`);
|
||||
}
|
||||
}
|
||||
proposer.save(kept);
|
||||
ctx.retain(kept);
|
||||
|
||||
const c = proposer.counts;
|
||||
const problems = [...errors, ...proposer.errors];
|
||||
return {
|
||||
ok: problems.length === 0,
|
||||
message: problems.length
|
||||
? `${problems.length} problems: ${problems.slice(0, 3).join("; ")}`
|
||||
: `${synced} series synced: ${c.accepted} updated, ${c.held} held for review`,
|
||||
counts: { linked: pairs.length, synced, missing, failed, accepted: c.accepted, held: c.held, waiting: c.waiting, rejected: c.rejected },
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
/**
|
||||
* What the two PersonalReader jobs (Suwayomi and Kavita) share: the state
|
||||
* mapping, proposing a state change through `ctx.apply` without nagging, and
|
||||
* the job-private cache entries.
|
||||
*
|
||||
* PersonalReader is private. A job updates a Library item only when the note
|
||||
* carries the series' id (`anilistId`, or `kavitaSeriesId` for Kavita), and it
|
||||
* stores nothing that names a source site, a scan group or a link into the
|
||||
* reader: only title, chapter or percent, when it was read, and an AniList
|
||||
* cover.
|
||||
*/
|
||||
import { libraryStates } from "@gk/schema";
|
||||
import type { StoredNote } from "../../store.ts";
|
||||
import type { SyncContext, SyncedValues } from "../types.ts";
|
||||
|
||||
export type LibraryState = (typeof libraryStates)[number];
|
||||
|
||||
/**
|
||||
* Job-private cache entries. They are keyed like note paths but are not notes
|
||||
* (no `.md`), so the notes API never shows them to the site.
|
||||
*/
|
||||
export const MEMO_KEY = "_memo";
|
||||
export const SUGGESTIONS_KEY = "_suggestions";
|
||||
|
||||
/** Suwayomi category to Library state. `null`: leave the note alone (Dropped). */
|
||||
const CATEGORY_STATES: Record<string, LibraryState | null> = {
|
||||
reading: "now",
|
||||
"plan to read": "up next",
|
||||
completed: "finished",
|
||||
dropped: null,
|
||||
};
|
||||
|
||||
/** The Library state a reader category maps to; undefined for a category the site does not know. */
|
||||
export function stateForCategory(category: string): LibraryState | null | undefined {
|
||||
const key = category.trim().toLowerCase().replace(/\s+/g, " ");
|
||||
return key in CATEGORY_STATES ? CATEGORY_STATES[key] : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* The state to propose for a note, or null to leave it. Two refinements keep
|
||||
* a distinction the reader cannot make: "caught up" (read to the latest of an
|
||||
* ongoing series) is not turned back into "now", and "finished" is not
|
||||
* turned into "caught up".
|
||||
*/
|
||||
export function targetState(current: string, mapped: LibraryState | null | undefined): LibraryState | null {
|
||||
if (!mapped || mapped === current) return null;
|
||||
if (mapped === "now" && current === "caught up") return null;
|
||||
if (mapped === "caught up" && current === "finished") return null;
|
||||
return mapped;
|
||||
}
|
||||
|
||||
export type ProposeResult = "accepted" | "held" | "waiting" | "rejected" | "skipped";
|
||||
|
||||
interface Memo {
|
||||
/** The state last proposed for each note path. */
|
||||
proposed: Record<string, string>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Proposes state changes, once each. A change Greg rejected is not proposed
|
||||
* again until the reader says something different, so the review queue is not
|
||||
* refilled every hour.
|
||||
*/
|
||||
export class StateProposer {
|
||||
private readonly memo: Memo;
|
||||
readonly counts = { accepted: 0, held: 0, waiting: 0, rejected: 0 };
|
||||
readonly errors: string[] = [];
|
||||
|
||||
constructor(private readonly ctx: SyncContext) {
|
||||
const m = ctx.cached(MEMO_KEY)?.values as Partial<Memo> | undefined;
|
||||
this.memo = { proposed: { ...(m?.proposed ?? {}) } };
|
||||
}
|
||||
|
||||
propose(note: StoredNote, target: LibraryState | null): ProposeResult {
|
||||
const path = note.path;
|
||||
if (!target) {
|
||||
delete this.memo.proposed[path];
|
||||
return "skipped";
|
||||
}
|
||||
if (this.memo.proposed[path] === target) return "skipped";
|
||||
const o = this.ctx.apply({ op: "update", path, fields: { state: target } });
|
||||
if (o.status === "accepted" || o.status === "unchanged") {
|
||||
this.counts.accepted++;
|
||||
this.memo.proposed[path] = target;
|
||||
return "accepted";
|
||||
}
|
||||
if (o.status === "held") {
|
||||
this.counts.held++;
|
||||
this.memo.proposed[path] = target;
|
||||
return "held";
|
||||
}
|
||||
// A change already waits for review: try again next run.
|
||||
if (o.reasons.some((r) => r.code === "waiting")) {
|
||||
this.counts.waiting++;
|
||||
return "waiting";
|
||||
}
|
||||
this.counts.rejected++;
|
||||
this.errors.push(`${note.title}: ${o.reasons.map((r) => r.message).join("; ")}`);
|
||||
return "rejected";
|
||||
}
|
||||
|
||||
/** Save the memo, forgetting notes no longer in scope. */
|
||||
save(paths: Iterable<string>) {
|
||||
const keep = new Set(paths);
|
||||
const proposed = Object.fromEntries(Object.entries(this.memo.proposed).filter(([p]) => keep.has(p)));
|
||||
this.ctx.set(MEMO_KEY, { proposed } satisfies SyncedValues);
|
||||
}
|
||||
}
|
||||
|
||||
/** Epoch seconds or milliseconds (or an ISO string) as ISO; undefined for zero or junk. */
|
||||
export function isoTime(v: unknown): string | undefined {
|
||||
if (v === null || v === undefined || v === "") return undefined;
|
||||
if (typeof v === "string" && !/^\d+$/.test(v)) {
|
||||
const t = Date.parse(v);
|
||||
// Kavita says 0001-01-01 for "never".
|
||||
return Number.isFinite(t) && t > 0 ? new Date(t).toISOString() : undefined;
|
||||
}
|
||||
const n = Number(v);
|
||||
if (!Number.isFinite(n) || n <= 0) return undefined;
|
||||
return new Date(n < 1e12 ? n * 1000 : n).toISOString();
|
||||
}
|
||||
|
||||
export const errorText = (err: unknown) => (err instanceof Error ? err.message : String(err));
|
||||
@@ -0,0 +1,64 @@
|
||||
/**
|
||||
* What the two Store stock jobs (Bambuddy filament, Homebox leather and
|
||||
* paper) share: the synced shape and how it lands on product lines.
|
||||
*
|
||||
* A job reads what is in stock and stores it on every product-line note that
|
||||
* has an option in a group the job feeds (Filament for Bambuddy; the groups
|
||||
* named in HOMEBOX_LABELS for Homebox). The site lays it under the note's own
|
||||
* options (services/site/src/lib/store.ts): a note's option always wins, a
|
||||
* note option that leaves `inStock` or `hex` out takes the synced value, and
|
||||
* in-stock colors the note does not list are added after the note's own.
|
||||
*/
|
||||
import type { SyncContext, SyncedValues } from "../types.ts";
|
||||
|
||||
export interface StockOption {
|
||||
name: string;
|
||||
hex?: string;
|
||||
inStock: boolean;
|
||||
}
|
||||
|
||||
export interface StockGroup {
|
||||
group: string;
|
||||
options: StockOption[];
|
||||
}
|
||||
|
||||
/** `#rrggbb` from `rrggbb`, `#RRGGBB` or `#rrggbbaa`; undefined otherwise. */
|
||||
export function normalizeHex(v: unknown): string | undefined {
|
||||
if (typeof v !== "string") return undefined;
|
||||
const m = /^#?([0-9a-f]{6})(?:[0-9a-f]{2})?$/i.exec(v.trim());
|
||||
return m ? `#${m[1]!.toLowerCase()}` : undefined;
|
||||
}
|
||||
|
||||
/** One entry per name (case-insensitive): in stock when any entry is, keeping the first hex seen. */
|
||||
export function mergeOptions(options: StockOption[]): StockOption[] {
|
||||
const out = new Map<string, StockOption>();
|
||||
for (const o of options) {
|
||||
const key = o.name.trim().toLowerCase();
|
||||
if (!key) continue;
|
||||
const prev = out.get(key);
|
||||
if (!prev) out.set(key, { ...o, name: o.name.trim() });
|
||||
else {
|
||||
prev.inStock ||= o.inStock;
|
||||
prev.hex ??= o.hex;
|
||||
}
|
||||
}
|
||||
return [...out.values()].sort((a, b) => Number(b.inStock) - Number(a.inStock) || a.name.localeCompare(b.name));
|
||||
}
|
||||
|
||||
/**
|
||||
* Store `groups` under `key` on every product line that uses one of them,
|
||||
* forgetting lines that no longer do. Returns how many lines were updated.
|
||||
*/
|
||||
export function storeOnLines(ctx: SyncContext, key: string, groups: StockGroup[]): number {
|
||||
const fed = new Map(groups.map((g) => [g.group.toLowerCase(), g]));
|
||||
const kept: string[] = [];
|
||||
for (const line of ctx.notes("product-line")) {
|
||||
const used = new Set(((line.data.options as { group?: string }[] | undefined) ?? []).map((o) => String(o.group ?? "").toLowerCase()));
|
||||
const mine = [...used].map((g) => fed.get(g)).filter((g): g is StockGroup => !!g);
|
||||
if (!mine.length) continue;
|
||||
ctx.set(line.path, { [key]: mine } satisfies SyncedValues);
|
||||
kept.push(line.path);
|
||||
}
|
||||
ctx.retain(kept);
|
||||
return kept.length;
|
||||
}
|
||||
@@ -0,0 +1,313 @@
|
||||
/**
|
||||
* Suwayomi (PersonalReader's manga and manhwa): reading progress for Library
|
||||
* items that carry the series' `anilistId`. Hourly. Read only against
|
||||
* Suwayomi.
|
||||
*
|
||||
* - Logs in with its own read-only account. With Suwayomi's `ui_login` auth
|
||||
* mode that is the GraphQL `login` mutation, whose access token is sent as
|
||||
* `Authorization: Bearer`; with `basic_auth` it is HTTP basic auth.
|
||||
* - Reads the library over GraphQL (`<SUWAYOMI_URL>/api/graphql`): each
|
||||
* category (Reading, Plan to read, Completed, Dropped) with its series, the
|
||||
* furthest chapter read and when, and the AniList tracker record, whose
|
||||
* remote id is the AniList id.
|
||||
* - Maps the category to the note's state (Reading: now, Plan to read: up
|
||||
* next, Completed: finished, Dropped: left alone) through `ctx.apply` as
|
||||
* `sync`; Greg's notes are held for review.
|
||||
* - Caches `chapterProgress` (title, chapter, when) and the AniList `cover`.
|
||||
* Never a source name, scan group, thumbnail or any Suwayomi address.
|
||||
* - A series in Reading with no Library item becomes a suggestion on the
|
||||
* admin page.
|
||||
*
|
||||
* Written from Suwayomi-Server's GraphQL schema; not checked against a live
|
||||
* server from here (see docs/content-service.md).
|
||||
*/
|
||||
import type { StoredNote } from "../../store.ts";
|
||||
import type { SyncContext, SyncJob } from "../types.ts";
|
||||
import { ANILIST_API } from "./covers.ts";
|
||||
import { errorText, isoTime, StateProposer, stateForCategory, SUGGESTIONS_KEY, MEMO_KEY, targetState } from "./reader.ts";
|
||||
|
||||
/** Suwayomi's (and Mihon's) tracker id for AniList. */
|
||||
export const ANILIST_TRACKER_ID = 2;
|
||||
|
||||
export type SuwayomiAuth = "ui_login" | "basic" | "none";
|
||||
|
||||
export interface SuwayomiOptions {
|
||||
url: string;
|
||||
user?: string;
|
||||
pass?: string;
|
||||
/** Default `ui_login` when a user is set, else `none`. */
|
||||
auth?: SuwayomiAuth;
|
||||
everyMinutes?: number;
|
||||
healthcheckUrl?: string;
|
||||
}
|
||||
|
||||
export const LOGIN_MUTATION = `mutation GkLogin($username: String!, $password: String!) {
|
||||
login(input: { username: $username, password: $password }) { accessToken }
|
||||
}`;
|
||||
|
||||
/** Only what the site needs: no ids, urls, thumbnails, sources or scanlators. */
|
||||
export const LIBRARY_QUERY = `query GkLibrary {
|
||||
categories {
|
||||
nodes {
|
||||
name
|
||||
mangas {
|
||||
nodes {
|
||||
title
|
||||
inLibrary
|
||||
latestReadChapter { chapterNumber lastReadAt }
|
||||
trackRecords { nodes { trackerId remoteId title lastChapterRead } }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}`;
|
||||
|
||||
/** Batched AniList lookup for the cover and what a suggestion needs. */
|
||||
export const ANILIST_BATCH_QUERY = `query ($ids: [Int]) {
|
||||
Page(perPage: 50) {
|
||||
media(id_in: $ids, type: MANGA) {
|
||||
id
|
||||
title { english romaji }
|
||||
format
|
||||
countryOfOrigin
|
||||
coverImage { extraLarge large }
|
||||
}
|
||||
}
|
||||
}`;
|
||||
|
||||
export interface Series {
|
||||
title: string;
|
||||
category: string;
|
||||
anilistId?: number;
|
||||
/** Furthest chapter read. */
|
||||
chapter?: number;
|
||||
readAt?: string;
|
||||
}
|
||||
|
||||
export interface ChapterProgress {
|
||||
title: string;
|
||||
/** Library state the category maps to, or the category itself for Dropped. */
|
||||
status: string;
|
||||
chapter?: number;
|
||||
readAt?: string;
|
||||
}
|
||||
|
||||
export interface AniListInfo {
|
||||
title?: string;
|
||||
medium: "manga" | "manhwa" | "light novel";
|
||||
cover?: string;
|
||||
}
|
||||
|
||||
const CATEGORY_ORDER = ["reading", "plan to read", "completed", "dropped"];
|
||||
|
||||
interface RawChapter {
|
||||
chapterNumber?: number | string | null;
|
||||
lastReadAt?: number | string | null;
|
||||
}
|
||||
interface RawManga {
|
||||
title?: string;
|
||||
inLibrary?: boolean;
|
||||
latestReadChapter?: RawChapter | null;
|
||||
trackRecords?: { nodes?: { trackerId?: number; remoteId?: number | string; title?: string; lastChapterRead?: number | null }[] } | null;
|
||||
}
|
||||
interface RawLibrary {
|
||||
data?: { categories?: { nodes?: { name?: string; mangas?: { nodes?: RawManga[] } | null }[] } | null } | null;
|
||||
errors?: { message?: string }[];
|
||||
}
|
||||
|
||||
const positive = (v: unknown) => {
|
||||
const n = Number(v);
|
||||
return Number.isFinite(n) && n > 0 ? n : undefined;
|
||||
};
|
||||
|
||||
/**
|
||||
* The library response as series. A series in several known categories takes
|
||||
* the first of Reading, Plan to read, Completed, Dropped; series only in
|
||||
* categories the site does not know are left out.
|
||||
*/
|
||||
export function parseLibrary(json: unknown): Series[] {
|
||||
const raw = json as RawLibrary;
|
||||
if (raw?.errors?.length) throw new Error(`Suwayomi: ${raw.errors.map((e) => e.message).join("; ")}`);
|
||||
const cats = raw?.data?.categories?.nodes;
|
||||
if (!Array.isArray(cats)) throw new Error("Suwayomi answered without a library");
|
||||
const best = new Map<string, Series & { rank: number }>();
|
||||
for (const cat of cats) {
|
||||
const name = (cat.name ?? "").trim();
|
||||
const rank = CATEGORY_ORDER.indexOf(name.toLowerCase().replace(/\s+/g, " "));
|
||||
if (rank < 0 || stateForCategory(name) === undefined) continue;
|
||||
for (const m of cat.mangas?.nodes ?? []) {
|
||||
if (m.inLibrary === false || !m.title) continue;
|
||||
const track = (m.trackRecords?.nodes ?? []).find((t) => t.trackerId === ANILIST_TRACKER_ID);
|
||||
const anilistId = positive(track?.remoteId);
|
||||
const s: Series & { rank: number } = { title: (track?.title || m.title).trim(), category: name, rank };
|
||||
if (anilistId && Number.isInteger(anilistId)) s.anilistId = anilistId;
|
||||
const chapter = positive(m.latestReadChapter?.chapterNumber) ?? positive(track?.lastChapterRead);
|
||||
if (chapter !== undefined) s.chapter = chapter;
|
||||
const readAt = isoTime(m.latestReadChapter?.lastReadAt);
|
||||
if (readAt) s.readAt = readAt;
|
||||
// Key by AniList id when there is one, else by title.
|
||||
const key = anilistId ? `a${anilistId}` : `t${m.title}`;
|
||||
const prev = best.get(key);
|
||||
if (!prev || rank < prev.rank) best.set(key, s);
|
||||
}
|
||||
}
|
||||
return [...best.values()].map(({ rank: _, ...s }) => s);
|
||||
}
|
||||
|
||||
/** AniList's batched Media answer, by id. */
|
||||
export function parseAniListBatch(json: unknown): Map<number, AniListInfo> {
|
||||
type M = { id: number; title?: { english?: string; romaji?: string }; format?: string; countryOfOrigin?: string; coverImage?: { extraLarge?: string; large?: string } };
|
||||
const media = (json as { data?: { Page?: { media?: M[] } } })?.data?.Page?.media ?? [];
|
||||
const out = new Map<number, AniListInfo>();
|
||||
for (const m of media) {
|
||||
if (typeof m?.id !== "number") continue;
|
||||
const cover = m.coverImage?.extraLarge ?? m.coverImage?.large;
|
||||
const info: AniListInfo = { medium: m.format === "NOVEL" ? "light novel" : m.countryOfOrigin === "KR" ? "manhwa" : "manga" };
|
||||
const title = m.title?.english || m.title?.romaji;
|
||||
if (title) info.title = title;
|
||||
if (cover?.startsWith("https://")) info.cover = cover;
|
||||
out.set(m.id, info);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- requests
|
||||
|
||||
const endpoint = (base: string) => `${base.replace(/\/$/, "")}/api/graphql`;
|
||||
|
||||
async function graphql(ctx: SyncContext, url: string, headers: Record<string, string>, query: string, variables?: Record<string, unknown>) {
|
||||
const res = await ctx.fetch(url, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", accept: "application/json", ...headers },
|
||||
body: JSON.stringify({ query, variables }),
|
||||
});
|
||||
if (res.status === 401 || res.status === 403) throw new Error(`Suwayomi refused the login (${res.status})`);
|
||||
if (!res.ok) throw new Error(`Suwayomi answered ${res.status}`);
|
||||
return res.json() as Promise<unknown>;
|
||||
}
|
||||
|
||||
/** The headers that authenticate one run. */
|
||||
export async function authHeaders(ctx: SyncContext, o: SuwayomiOptions): Promise<Record<string, string>> {
|
||||
const mode = o.auth ?? (o.user ? "ui_login" : "none");
|
||||
if (mode === "none") return {};
|
||||
if (!o.user || !o.pass) throw new Error("SUWAYOMI_USER and SUWAYOMI_PASS are required for this auth mode");
|
||||
if (mode === "basic") return { authorization: `Basic ${Buffer.from(`${o.user}:${o.pass}`).toString("base64")}` };
|
||||
const json = (await graphql(ctx, endpoint(o.url), {}, LOGIN_MUTATION, { username: o.user, password: o.pass })) as {
|
||||
data?: { login?: { accessToken?: string } };
|
||||
errors?: { message?: string }[];
|
||||
};
|
||||
const token = json?.data?.login?.accessToken;
|
||||
if (!token) throw new Error(`Suwayomi login failed${json?.errors?.length ? `: ${json.errors.map((e) => e.message).join("; ")}` : ""}`);
|
||||
return { authorization: `Bearer ${token}` };
|
||||
}
|
||||
|
||||
async function aniList(ctx: SyncContext, ids: number[]): Promise<Map<number, AniListInfo>> {
|
||||
const out = new Map<number, AniListInfo>();
|
||||
for (let i = 0; i < ids.length; i += 50) {
|
||||
if (i > 0) await ctx.sleep(1000);
|
||||
const res = await ctx.fetch(ANILIST_API, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", accept: "application/json" },
|
||||
body: JSON.stringify({ query: ANILIST_BATCH_QUERY, variables: { ids: ids.slice(i, i + 50) } }),
|
||||
});
|
||||
if (!res.ok) throw new Error(`AniList answered ${res.status}`);
|
||||
for (const [k, v] of parseAniListBatch(await res.json())) out.set(k, v);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- the job
|
||||
|
||||
export interface Suggestion {
|
||||
anilistId: number;
|
||||
title: string;
|
||||
medium: AniListInfo["medium"];
|
||||
cover?: string;
|
||||
}
|
||||
|
||||
export function suwayomiJob(o: SuwayomiOptions): SyncJob {
|
||||
return {
|
||||
name: "suwayomi",
|
||||
everyMinutes: o.everyMinutes ?? 60,
|
||||
healthcheckUrl: o.healthcheckUrl,
|
||||
async run(ctx) {
|
||||
const headers = await authHeaders(ctx, o);
|
||||
const series = parseLibrary(await graphql(ctx, endpoint(o.url), headers, LIBRARY_QUERY));
|
||||
|
||||
const items = ctx.notes("library-item");
|
||||
const byAniList = new Map<number, StoredNote[]>();
|
||||
for (const n of items) {
|
||||
const id = n.data.anilistId;
|
||||
if (typeof id === "number") byAniList.set(id, [...(byAniList.get(id) ?? []), n]);
|
||||
}
|
||||
|
||||
// AniList: covers for matched series, titles and media for suggestions.
|
||||
const tracked = series.filter((s) => s.anilistId);
|
||||
const wanted = tracked.filter((s) => byAniList.has(s.anilistId!) || stateForCategory(s.category) === "now").map((s) => s.anilistId!);
|
||||
let info = new Map<number, AniListInfo>();
|
||||
let anilistError: string | null = null;
|
||||
try {
|
||||
if (wanted.length) info = await aniList(ctx, [...new Set(wanted)]);
|
||||
} catch (err) {
|
||||
if (ctx.signal.aborted) throw ctx.signal.reason;
|
||||
anilistError = errorText(err);
|
||||
}
|
||||
|
||||
const proposer = new StateProposer(ctx);
|
||||
const kept: string[] = [MEMO_KEY, SUGGESTIONS_KEY];
|
||||
const suggestions: Suggestion[] = [];
|
||||
const oldSuggestions = (ctx.cached(SUGGESTIONS_KEY)?.values.items as Suggestion[] | undefined) ?? [];
|
||||
let matched = 0;
|
||||
|
||||
for (const s of series) {
|
||||
const mapped = stateForCategory(s.category);
|
||||
const notes = s.anilistId ? (byAniList.get(s.anilistId) ?? []) : [];
|
||||
if (!notes.length) {
|
||||
if (mapped === "now" && s.anilistId) {
|
||||
const a = info.get(s.anilistId);
|
||||
const old = oldSuggestions.find((x) => x.anilistId === s.anilistId);
|
||||
const sug: Suggestion = { anilistId: s.anilistId, title: a?.title ?? old?.title ?? s.title, medium: a?.medium ?? old?.medium ?? "manga" };
|
||||
const cover = a?.cover ?? old?.cover;
|
||||
if (cover) sug.cover = cover;
|
||||
suggestions.push(sug);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
for (const note of notes) {
|
||||
matched++;
|
||||
kept.push(note.path);
|
||||
const progress: Record<string, unknown> = { title: s.title, status: mapped ?? s.category.toLowerCase() };
|
||||
if (s.chapter !== undefined) progress.chapter = s.chapter;
|
||||
if (s.readAt) progress.readAt = s.readAt;
|
||||
// Keep the last good cover when AniList could not be asked.
|
||||
const cover = info.get(s.anilistId!)?.cover ?? (ctx.cached(note.path)?.values.cover as string | undefined);
|
||||
ctx.set(note.path, cover ? { chapterProgress: progress, cover } : { chapterProgress: progress });
|
||||
proposer.propose(note, targetState(String(note.data.state), mapped));
|
||||
}
|
||||
}
|
||||
|
||||
ctx.set(SUGGESTIONS_KEY, { items: suggestions });
|
||||
proposer.save(kept);
|
||||
ctx.retain(kept);
|
||||
|
||||
const c = proposer.counts;
|
||||
const problems = [...(anilistError ? [`AniList: ${anilistError}`] : []), ...proposer.errors];
|
||||
return {
|
||||
ok: problems.length === 0,
|
||||
message: problems.length
|
||||
? problems.slice(0, 3).join("; ")
|
||||
: `${matched} series synced: ${c.accepted} updated, ${c.held} held for review, ${suggestions.length} suggested`,
|
||||
counts: {
|
||||
series: series.length,
|
||||
matched,
|
||||
untracked: series.length - tracked.length,
|
||||
accepted: c.accepted,
|
||||
held: c.held,
|
||||
waiting: c.waiting,
|
||||
rejected: c.rejected,
|
||||
suggestions: suggestions.length,
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
/**
|
||||
* Runs each sync job on its interval: once shortly after start (staggered so
|
||||
* the jobs do not all hit the network together), then every `everyMinutes`.
|
||||
* A job never runs twice at once: a tick that finds it still running is
|
||||
* skipped. A run longer than its timeout is aborted and reported as failed.
|
||||
*
|
||||
* Every run is reported to sync_runs (shown on /api/syncs and the admin
|
||||
* Overview) and pinged to the job's Healthchecks URL when it has one.
|
||||
*/
|
||||
import type { ContentService } from "../service.ts";
|
||||
import type { SyncCache } from "./cache.ts";
|
||||
import type { SyncContext, SyncJob, SyncResult } from "./types.ts";
|
||||
|
||||
export const SYNC_USER_AGENT = "GregKilmer.com content sync (+https://gregkilmer.com)";
|
||||
const DEFAULT_TIMEOUT_MS = 10 * 60_000;
|
||||
|
||||
export interface SchedulerOptions {
|
||||
/** Delay between the jobs' first runs. Default 20 seconds. */
|
||||
staggerMs?: number;
|
||||
/** Injectable for tests. */
|
||||
fetch?: typeof fetch;
|
||||
now?: () => Date;
|
||||
log?: (message: string) => void;
|
||||
}
|
||||
|
||||
export interface RunReport {
|
||||
job: string;
|
||||
ok: boolean;
|
||||
message: string;
|
||||
counts: Record<string, number>;
|
||||
/** True when the run was skipped because the job was already running. */
|
||||
skipped?: boolean;
|
||||
}
|
||||
|
||||
export class Scheduler {
|
||||
readonly cache: SyncCache;
|
||||
private readonly running = new Map<string, Promise<RunReport>>();
|
||||
private readonly timers = new Set<NodeJS.Timeout>();
|
||||
private readonly stopping = new AbortController();
|
||||
private readonly fetchFn: typeof fetch;
|
||||
private readonly now: () => Date;
|
||||
private readonly log: (message: string) => void;
|
||||
|
||||
constructor(
|
||||
private readonly service: ContentService,
|
||||
readonly jobs: SyncJob[],
|
||||
private readonly o: SchedulerOptions = {},
|
||||
) {
|
||||
this.cache = service.store.syncCache;
|
||||
this.fetchFn = o.fetch ?? fetch;
|
||||
this.now = o.now ?? (() => new Date());
|
||||
this.log = o.log ?? ((m) => console.log(m));
|
||||
}
|
||||
|
||||
/** Run every job once, staggered, then on its interval. */
|
||||
start() {
|
||||
const stagger = this.o.staggerMs ?? 20_000;
|
||||
this.jobs.forEach((job, i) => {
|
||||
const loop = () => {
|
||||
void this.runNow(job.name);
|
||||
this.later(loop, job.everyMinutes * 60_000);
|
||||
};
|
||||
this.later(loop, i * stagger);
|
||||
});
|
||||
}
|
||||
|
||||
stop() {
|
||||
for (const t of this.timers) clearTimeout(t);
|
||||
this.timers.clear();
|
||||
this.stopping.abort();
|
||||
}
|
||||
|
||||
/** Run one job now, unless it is already running. */
|
||||
runNow(name: string): Promise<RunReport> {
|
||||
const job = this.jobs.find((j) => j.name === name);
|
||||
if (!job) return Promise.resolve({ job: name, ok: false, message: "No such job", counts: {} });
|
||||
if (this.running.has(name)) {
|
||||
return Promise.resolve({ job: name, ok: false, message: "Already running", counts: {}, skipped: true });
|
||||
}
|
||||
const run = this.execute(job);
|
||||
this.running.set(name, run);
|
||||
// Released only when the job's promise settles, even after a timeout, so
|
||||
// a job that ignores its abort signal still never overlaps itself.
|
||||
return run;
|
||||
}
|
||||
|
||||
isRunning(name: string) {
|
||||
return this.running.has(name);
|
||||
}
|
||||
|
||||
private later(fn: () => void, ms: number) {
|
||||
const t = setTimeout(() => {
|
||||
this.timers.delete(t);
|
||||
fn();
|
||||
}, ms);
|
||||
t.unref?.();
|
||||
this.timers.add(t);
|
||||
}
|
||||
|
||||
private async execute(job: SyncJob): Promise<RunReport> {
|
||||
const abort = new AbortController();
|
||||
const onStop = () => abort.abort(new Error("The service is stopping"));
|
||||
this.stopping.signal.addEventListener("abort", onStop);
|
||||
const timeoutMs = job.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
||||
this.ping(job, "/start");
|
||||
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
const work = job.run(this.context(job, abort.signal));
|
||||
const timedOut = new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => {
|
||||
const err = new Error(`Timed out after ${Math.round(timeoutMs / 1000)} seconds`);
|
||||
abort.abort(err);
|
||||
reject(err);
|
||||
}, timeoutMs);
|
||||
});
|
||||
|
||||
let report: RunReport;
|
||||
try {
|
||||
const result: SyncResult = await Promise.race([work, timedOut]);
|
||||
const ok = result.ok !== false;
|
||||
report = { job: job.name, ok, message: result.message ?? (ok ? "OK" : "Some items failed"), counts: result.counts ?? {} };
|
||||
} catch (err) {
|
||||
report = { job: job.name, ok: false, message: err instanceof Error ? err.message : String(err), counts: {} };
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
this.stopping.signal.removeEventListener("abort", onStop);
|
||||
}
|
||||
|
||||
this.service.store.reportSync({ ...report, message: report.message.slice(0, 2000), at: this.now().toISOString() });
|
||||
this.ping(job, report.ok ? "" : "/fail", report.message);
|
||||
if (!report.ok) this.log(`Sync ${job.name} failed: ${report.message}`);
|
||||
|
||||
// Hold the running flag until the job itself has finished.
|
||||
work.then(
|
||||
() => this.running.delete(job.name),
|
||||
() => this.running.delete(job.name),
|
||||
);
|
||||
return report;
|
||||
}
|
||||
|
||||
private context(job: SyncJob, signal: AbortSignal): SyncContext {
|
||||
const { service, cache } = this;
|
||||
return {
|
||||
signal,
|
||||
now: this.now,
|
||||
notes: (type) => service.store.list({ type }),
|
||||
cached: (path) => cache.get(job.name, path),
|
||||
set: (path, values) => cache.set(job.name, path, values, this.now().toISOString()),
|
||||
retain: (paths) => cache.retain(job.name, paths),
|
||||
apply: (change) => service.apply(change, "sync"),
|
||||
fetch: (url, init = {}) => {
|
||||
const { timeoutMs = 30_000, headers, ...rest } = init;
|
||||
return this.fetchFn(url, {
|
||||
...rest,
|
||||
headers: { "user-agent": SYNC_USER_AGENT, ...(headers as Record<string, string> | undefined) },
|
||||
signal: AbortSignal.any([signal, AbortSignal.timeout(timeoutMs)]),
|
||||
});
|
||||
},
|
||||
sleep: (ms) =>
|
||||
new Promise<void>((resolve) => {
|
||||
if (signal.aborted) return resolve();
|
||||
const t = setTimeout(done, ms);
|
||||
function done() {
|
||||
clearTimeout(t);
|
||||
signal.removeEventListener("abort", done);
|
||||
resolve();
|
||||
}
|
||||
signal.addEventListener("abort", done);
|
||||
}),
|
||||
log: (m) => this.log(`Sync ${job.name}: ${m}`),
|
||||
};
|
||||
}
|
||||
|
||||
/** Healthchecks: `<url>/start`, `<url>` on success, `<url>/fail` with the message. */
|
||||
private ping(job: SyncJob, suffix: string, body?: string) {
|
||||
if (!job.healthcheckUrl) return;
|
||||
const url = job.healthcheckUrl.replace(/\/$/, "") + suffix;
|
||||
this.fetchFn(url, { method: "POST", body: body ?? "", signal: AbortSignal.timeout(10_000) }).catch(() => {});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
/**
|
||||
* Sync settings from the environment. A job is enabled only when its config
|
||||
* is present: the public sources (no credentials) are named in SYNC_JOBS;
|
||||
* jobs that need a URL or a login are enabled by those variables alone.
|
||||
*/
|
||||
import { homelabSettings, type HomelabSettings } from "./jobs/homelab.ts";
|
||||
|
||||
export interface SyncSettings {
|
||||
/** Public jobs switched on in SYNC_JOBS, e.g. "archidekt,bgg,covers". */
|
||||
enabled: Set<string>;
|
||||
/** SYNC_<JOB>_MINUTES overrides a job's interval. */
|
||||
minutes: Record<string, number>;
|
||||
/** SYNC_<JOB>_HEALTHCHECK_URL. */
|
||||
healthchecks: Record<string, string>;
|
||||
bggToken?: string;
|
||||
tmdbApiKey?: string;
|
||||
/** Suwayomi, Kavita, Bambuddy and Homebox: enabled by their own URL and login. */
|
||||
homelab: HomelabSettings;
|
||||
}
|
||||
|
||||
const envKey = (job: string) => job.toUpperCase().replace(/[^A-Z0-9]/g, "_");
|
||||
|
||||
export function syncSettings(env: NodeJS.ProcessEnv = process.env): SyncSettings {
|
||||
const enabled = new Set((env.SYNC_JOBS ?? "").split(",").map((s) => s.trim().toLowerCase()).filter(Boolean));
|
||||
const minutes: Record<string, number> = {};
|
||||
const healthchecks: Record<string, string> = {};
|
||||
for (const [k, v] of Object.entries(env)) {
|
||||
if (!v) continue;
|
||||
const m = /^SYNC_([A-Z0-9_]+)_(MINUTES|HEALTHCHECK_URL)$/.exec(k);
|
||||
if (!m) continue;
|
||||
const job = m[1]!.toLowerCase().replace(/_/g, "-");
|
||||
if (m[2] === "MINUTES" && Number(v) > 0) minutes[job] = Number(v);
|
||||
if (m[2] === "HEALTHCHECK_URL") healthchecks[job] = v;
|
||||
}
|
||||
return {
|
||||
enabled,
|
||||
minutes,
|
||||
healthchecks,
|
||||
bggToken: env.BGG_TOKEN || undefined,
|
||||
tmdbApiKey: env.TMDB_API_KEY || undefined,
|
||||
homelab: homelabSettings(env),
|
||||
};
|
||||
}
|
||||
|
||||
/** The env var name for one job's setting, for docs and messages. */
|
||||
export const syncEnvName = (job: string, what: "MINUTES" | "HEALTHCHECK_URL") => `SYNC_${envKey(job)}_${what}`;
|
||||
@@ -0,0 +1,73 @@
|
||||
/**
|
||||
* The sync job interface. A job reads notes, asks one outside source about
|
||||
* them, and either stores what it learned in the sync cache (shown on the
|
||||
* site under the note's own values) or, for jobs that must change a note,
|
||||
* proposes the change through `ContentService.apply` as `sync`.
|
||||
*
|
||||
* A job never touches the vault, the store's tables or another job's cache
|
||||
* directly: everything goes through the context, so a sync can do no more
|
||||
* than an agent can. See docs/content-service.md, "Sync jobs".
|
||||
*/
|
||||
import type { Change, Outcome } from "../service.ts";
|
||||
import type { StoredNote } from "../store.ts";
|
||||
|
||||
/** Synced values for one note: plain JSON. */
|
||||
export type SyncedValues = Record<string, unknown>;
|
||||
|
||||
export interface CacheEntry {
|
||||
values: SyncedValues;
|
||||
/** When these values were fetched. */
|
||||
fetchedAt: string;
|
||||
}
|
||||
|
||||
export interface SyncContext {
|
||||
/** Aborted when the run times out or the service stops. Pass it to every request. */
|
||||
readonly signal: AbortSignal;
|
||||
/** The current time (injectable for tests). */
|
||||
now(): Date;
|
||||
/** Live notes of one type, published or not. Read only. */
|
||||
notes(type: string): StoredNote[];
|
||||
/** This job's cached values for a note, if any. */
|
||||
cached(path: string): CacheEntry | null;
|
||||
/**
|
||||
* Store values fetched for a note, replacing this job's previous values for
|
||||
* it. Only call this with values that were fetched successfully: whatever
|
||||
* is not set keeps its last good values.
|
||||
*/
|
||||
set(path: string, values: SyncedValues): void;
|
||||
/** Forget this job's values for every note not in `paths` (notes that left the job's scope). */
|
||||
retain(paths: Iterable<string>): void;
|
||||
/**
|
||||
* Propose a note change as `sync`. It runs the same rules as an agent's
|
||||
* change: contact details, secrets, text loss, notes the sync did not write
|
||||
* (unless `agentEdits: true`), publishing and the batch limit all hold it
|
||||
* for review. A sync cannot delete, move or rename a note.
|
||||
*/
|
||||
apply(change: Change): Outcome;
|
||||
/** fetch with the sync User-Agent, the run's abort signal and a per-request timeout. */
|
||||
fetch(url: string, init?: RequestInit & { timeoutMs?: number }): Promise<Response>;
|
||||
/** A polite pause between requests; ends early if the run is aborted. */
|
||||
sleep(ms: number): Promise<void>;
|
||||
log(message: string): void;
|
||||
}
|
||||
|
||||
export interface SyncResult {
|
||||
/** Shown on the admin Overview. */
|
||||
message?: string;
|
||||
counts?: Record<string, number>;
|
||||
/** Set false when the run finished but some items failed; the run is reported as failed. */
|
||||
ok?: boolean;
|
||||
}
|
||||
|
||||
export interface SyncJob {
|
||||
/** Lowercase id, as in sync_runs and the cache: `archidekt`, `bgg`, `covers`. */
|
||||
name: string;
|
||||
/** Minutes between runs. */
|
||||
everyMinutes: number;
|
||||
/** A run longer than this is aborted and reported as failed. Default 10 minutes. */
|
||||
timeoutMs?: number;
|
||||
/** Healthchecks ping URL: `/start` before, the URL on success, `/fail` on failure. */
|
||||
healthcheckUrl?: string;
|
||||
/** One run. Throwing marks the run failed; the cache keeps its last good values. */
|
||||
run(ctx: SyncContext): Promise<SyncResult>;
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
import { readdirSync } from "node:fs";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import { join } from "node:path";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { parseTokens } from "../src/config.ts";
|
||||
import { createHttpServer } from "../src/http.ts";
|
||||
import { extensionOf, safeName } from "../src/submissions.ts";
|
||||
import { book, rig, type Rig } from "./helpers.ts";
|
||||
|
||||
const T = { reader: "r".repeat(24), agent: "a".repeat(24), editor: "e".repeat(24), admin: "d".repeat(24), intake: "i".repeat(24) };
|
||||
const MAX = 1024;
|
||||
let r: Rig;
|
||||
let base: string;
|
||||
let server: ReturnType<typeof createHttpServer>;
|
||||
|
||||
beforeAll(async () => {
|
||||
r = rig();
|
||||
r.put(book("Mistborn"));
|
||||
r.service.scan();
|
||||
server = createHttpServer(r.service, {
|
||||
tokens: parseTokens(Object.entries(T).map(([k, v]) => `${k}:${v}`).join(",")),
|
||||
submissionMaxBytes: MAX,
|
||||
});
|
||||
await new Promise<void>((ok) => server.listen(0, "127.0.0.1", ok));
|
||||
base = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
|
||||
});
|
||||
afterAll(() => {
|
||||
server.close();
|
||||
r.done();
|
||||
});
|
||||
|
||||
const call = (method: string, path: string, token: string, body?: unknown) =>
|
||||
fetch(base + path, {
|
||||
method,
|
||||
headers: { authorization: `Bearer ${token}`, ...(body !== undefined ? { "content-type": "application/json" } : {}) },
|
||||
body: body !== undefined ? JSON.stringify(body) : undefined,
|
||||
});
|
||||
|
||||
const inquiry = (extra: Record<string, unknown> = {}) => ({
|
||||
kind: "inquiry",
|
||||
type: "job",
|
||||
name: "Jane Doe",
|
||||
email: "jane@example.com",
|
||||
fields: [{ label: "Details", value: "A role on the platform team." }],
|
||||
...extra,
|
||||
});
|
||||
|
||||
describe("submissions", () => {
|
||||
it("lets intake file a submission and nothing else", async () => {
|
||||
const res = await call("POST", "/api/submissions", T.intake, inquiry());
|
||||
expect(res.status).toBe(201);
|
||||
const { id, receipt } = (await res.json()) as { id: number; receipt: string };
|
||||
expect(id).toBeGreaterThan(0);
|
||||
expect(receipt.length).toBeGreaterThan(20);
|
||||
|
||||
for (const path of ["/api/submissions", `/api/submissions/${id}`, `/api/submissions/${id}/attachment`, "/api/notes", "/api/note?type=library-item&slug=mistborn", "/api/schema", "/api/redirects", "/api/status", "/api/held", "/attachments/x.png"]) {
|
||||
expect((await call("GET", path, T.intake)).status, path).toBe(403);
|
||||
}
|
||||
expect((await call("POST", `/api/submissions/${id}/handled`, T.intake, { handled: true })).status).toBe(403);
|
||||
expect((await call("POST", "/api/changes", T.intake, { op: "create", data: book("X") })).status).toBe(403);
|
||||
expect((await call("POST", "/api/syncs", T.intake, { job: "x", ok: true })).status).toBe(403);
|
||||
expect((await call("POST", "/mcp", T.intake, {})).status).toBe(403);
|
||||
});
|
||||
|
||||
it("keeps readers, agents and editors out of the saved copies", async () => {
|
||||
for (const t of [T.reader, T.agent, T.editor]) {
|
||||
expect((await call("GET", "/api/submissions", t)).status).toBe(403);
|
||||
expect((await call("POST", "/api/submissions", t, inquiry())).status).toBe(403);
|
||||
expect((await call("GET", "/api/submissions/1", t)).status).toBe(403);
|
||||
}
|
||||
expect((await call("GET", "/api/submissions", "nope".repeat(8))).status).toBe(401);
|
||||
});
|
||||
|
||||
it("records delivery once, only with the matching receipt", async () => {
|
||||
const { id, receipt } = (await (await call("POST", "/api/submissions", T.intake, inquiry())).json()) as { id: number; receipt: string };
|
||||
expect((await call("POST", `/api/submissions/${id}/delivery`, T.intake, { receipt: "wrong", status: "sent" })).status).toBe(403);
|
||||
expect((await call("POST", `/api/submissions/${id}/delivery`, T.intake, { status: "sent" })).status).toBe(403);
|
||||
expect((await call("POST", `/api/submissions/${id}/delivery`, T.intake, { receipt, status: "failed", error: "SMTP said no" })).status).toBe(200);
|
||||
expect((await call("POST", `/api/submissions/${id}/delivery`, T.intake, { receipt, status: "sent" })).status).toBe(409);
|
||||
const s = (await (await call("GET", `/api/submissions/${id}`, T.admin)).json()) as { delivery: string; deliveryError: string };
|
||||
expect(s.delivery).toBe("failed");
|
||||
expect(s.deliveryError).toBe("SMTP said no");
|
||||
expect(JSON.stringify(s)).not.toContain(receipt);
|
||||
});
|
||||
|
||||
it("lists unhandled first, newest first, and marks handled", async () => {
|
||||
const list = async () =>
|
||||
((await (await call("GET", "/api/submissions", T.admin)).json()) as { counts: { unhandled: number }; submissions: { id: number; handled: boolean }[] });
|
||||
const before = await list();
|
||||
const ids = before.submissions.map((s) => s.id);
|
||||
expect(ids).toEqual([...ids].sort((a, b) => b - a));
|
||||
const oldest = ids[ids.length - 1]!;
|
||||
const res = await call("POST", `/api/submissions/${oldest}/handled`, T.admin, { handled: true });
|
||||
expect(res.status).toBe(200);
|
||||
const after = await list();
|
||||
expect(after.submissions[after.submissions.length - 1]!.id).toBe(oldest);
|
||||
expect(after.submissions[after.submissions.length - 1]!.handled).toBe(true);
|
||||
expect(after.counts.unhandled).toBe(before.counts.unhandled - 1);
|
||||
expect((await call("POST", "/api/submissions/9999/handled", T.admin, { handled: true })).status).toBe(404);
|
||||
});
|
||||
|
||||
it("validates what it saves", async () => {
|
||||
expect((await call("POST", "/api/submissions", T.intake, inquiry({ email: "not an email" }))).status).toBe(400);
|
||||
expect((await call("POST", "/api/submissions", T.intake, inquiry({ kind: "spam" }))).status).toBe(400);
|
||||
expect((await call("POST", "/api/submissions", T.intake, inquiry({ name: "" }))).status).toBe(400);
|
||||
});
|
||||
|
||||
it("saves an attachment under a safe name and serves it as a download", async () => {
|
||||
const data = Buffer.from("1 Sol Ring\n1 Command Tower\n").toString("base64");
|
||||
const res = await call("POST", "/api/submissions", T.intake, inquiry({ attachment: { name: "../../etc/deck list.txt", data } }));
|
||||
expect(res.status).toBe(201);
|
||||
const { id } = (await res.json()) as { id: number };
|
||||
const files = readdirSync(join(r.dir, "submissions"));
|
||||
expect(files.some((f) => f.startsWith(`${id}-`) && f.endsWith(".txt"))).toBe(true);
|
||||
const s = (await (await call("GET", `/api/submissions/${id}`, T.admin)).json()) as { attachment: { name: string; size: number } };
|
||||
expect(s.attachment.name).toBe("deck list.txt");
|
||||
const dl = await call("GET", `/api/submissions/${id}/attachment`, T.admin);
|
||||
expect(dl.status).toBe(200);
|
||||
expect(dl.headers.get("content-disposition")).toContain("attachment;");
|
||||
expect(dl.headers.get("x-content-type-options")).toBe("nosniff");
|
||||
expect(await dl.text()).toBe("1 Sol Ring\n1 Command Tower\n");
|
||||
});
|
||||
|
||||
it("turns down attachments that are too large, of an unknown type, or not base64", async () => {
|
||||
const big = Buffer.alloc(MAX + 1, 1).toString("base64");
|
||||
const count = async () => ((await (await call("GET", "/api/submissions", T.admin)).json()) as { counts: { total: number } }).counts.total;
|
||||
const n = await count();
|
||||
expect((await call("POST", "/api/submissions", T.intake, inquiry({ attachment: { name: "a.png", data: big } }))).status).toBe(413);
|
||||
expect((await call("POST", "/api/submissions", T.intake, inquiry({ attachment: { name: "a.exe", data: "AAAA" } }))).status).toBe(415);
|
||||
expect((await call("POST", "/api/submissions", T.intake, inquiry({ attachment: { name: "a.html", data: "AAAA" } }))).status).toBe(415);
|
||||
expect((await call("POST", "/api/submissions", T.intake, inquiry({ attachment: { name: "a.png", data: "<script>" } }))).status).toBe(400);
|
||||
expect(await count()).toBe(n);
|
||||
});
|
||||
});
|
||||
|
||||
describe("attachment names", () => {
|
||||
it("strips paths and odd characters", () => {
|
||||
expect(safeName("C:\\Users\\x\\photo.png")).toBe("photo.png");
|
||||
expect(safeName("..")).toBe("attachment");
|
||||
expect(safeName("a\r\nb.txt")).toBe("a_b.txt");
|
||||
expect(extensionOf("deck.DEK")).toBe("dek");
|
||||
expect(extensionOf("x.svg")).toBeNull();
|
||||
expect(extensionOf("noext")).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,22 @@
|
||||
# Sync test fixtures
|
||||
|
||||
Small sample payloads written by hand from each API's documented response
|
||||
shape, for the parsing tests. They are **not** live responses: no request was
|
||||
made to Archidekt, BoardGameGeek, Open Library, TMDB, AniList, Suwayomi,
|
||||
Kavita, Bambuddy or Homebox to make them,
|
||||
and ids, names and image addresses are illustrative.
|
||||
|
||||
| File | Shape of |
|
||||
| --- | --- |
|
||||
| `archidekt-deck.sample.json` | `GET https://archidekt.com/api/decks/<id>/` |
|
||||
| `bgg-thing.sample.xml` | `GET https://boardgamegeek.com/xmlapi2/thing?id=…` |
|
||||
| `openlibrary-search.sample.json` | `GET https://openlibrary.org/search.json?title=…` |
|
||||
| `tmdb-search.sample.json` | `GET https://api.themoviedb.org/3/search/movie?query=…` |
|
||||
| `anilist-media.sample.json` | `POST https://graphql.anilist.co` with the `Media` query |
|
||||
| `suwayomi-library.sample.json` | `POST <SUWAYOMI_URL>/api/graphql` with `LIBRARY_QUERY` (Suwayomi-Server's GraphQL schema) |
|
||||
| `anilist-page.sample.json` | `POST https://graphql.anilist.co` with the batched `Page { media(id_in:) }` query |
|
||||
| `kavita-series.sample.json` | `POST <KAVITA_URL>/api/Series/all-v2` (SeriesDto list) |
|
||||
| `kavita-metadata.sample.json` | `GET <KAVITA_URL>/api/Series/metadata?seriesId=`, one answer per series id |
|
||||
| `bambuddy-spools.sample.json` | **A guess.** Bambuddy's API is not documented anywhere this project has; this only exercises `mapBambuddySpools`' tolerant reading. |
|
||||
| `homebox-labels.sample.json` | `GET <HOMEBOX_URL>/api/v1/labels` |
|
||||
| `homebox-items.sample.json` | `GET <HOMEBOX_URL>/api/v1/items?labels=…` |
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"_sample": "Hand-written from AniList's GraphQL Media response shape; not a live response.",
|
||||
"data": {
|
||||
"Media": {
|
||||
"id": 30013,
|
||||
"title": { "english": "One Piece", "romaji": "ONE PIECE" },
|
||||
"coverImage": {
|
||||
"extraLarge": "https://s4.anilist.co/file/anilistcdn/media/manga/cover/large/sample-xl.jpg",
|
||||
"large": "https://s4.anilist.co/file/anilistcdn/media/manga/cover/medium/sample-l.jpg"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"data": {
|
||||
"Page": {
|
||||
"media": [
|
||||
{ "id": 100001, "title": { "english": "Sample Tower Climber", "romaji": "Sample Tou" }, "format": "MANGA", "countryOfOrigin": "KR",
|
||||
"coverImage": { "extraLarge": "https://s4.anilist.co/file/anilistcdn/media/manga/cover/large/sample-100001.jpg", "large": null } },
|
||||
{ "id": 100003, "title": { "english": null, "romaji": "Sample Owari" }, "format": "MANGA", "countryOfOrigin": "JP",
|
||||
"coverImage": { "extraLarge": null, "large": "https://s4.anilist.co/file/anilistcdn/media/manga/cover/medium/sample-100003.jpg" } },
|
||||
{ "id": 100004, "title": { "english": "New Sample Series", "romaji": "Shin Sample" }, "format": "MANGA", "countryOfOrigin": "KR",
|
||||
"coverImage": { "extraLarge": "https://s4.anilist.co/file/anilistcdn/media/manga/cover/large/sample-100004.jpg" } },
|
||||
{ "id": 100005, "title": { "english": "Sample Dropped" }, "format": "MANGA", "countryOfOrigin": "JP", "coverImage": {} }
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"_sample": "Hand-written from Archidekt's deck response shape; not a live response.",
|
||||
"id": 1234567,
|
||||
"name": "Sample Atraxa Superfriends",
|
||||
"deckFormat": 3,
|
||||
"featured": "https://storage.googleapis.com/topdekt-user/images/sample-featured.jpg",
|
||||
"customFeatured": "",
|
||||
"private": false,
|
||||
"categories": [
|
||||
{ "id": 1, "name": "Commander", "isPremier": true, "includedInDeck": true, "includedInPrice": true },
|
||||
{ "id": 2, "name": "Ramp", "isPremier": false, "includedInDeck": true, "includedInPrice": true },
|
||||
{ "id": 3, "name": "Maybeboard", "isPremier": false, "includedInDeck": false, "includedInPrice": false }
|
||||
],
|
||||
"cards": [
|
||||
{
|
||||
"quantity": 1,
|
||||
"categories": ["Commander"],
|
||||
"card": {
|
||||
"uid": "d0d33d52-3d28-4635-b985-51e126289259",
|
||||
"oracleCard": { "name": "Atraxa, Praetors' Voice", "colorIdentity": ["White", "Blue", "Black", "Green"] }
|
||||
}
|
||||
},
|
||||
{
|
||||
"quantity": 1,
|
||||
"categories": ["Ramp"],
|
||||
"card": { "uid": "aaaa1111-0000-0000-0000-000000000000", "oracleCard": { "name": "Sol Ring", "colorIdentity": [] } }
|
||||
},
|
||||
{
|
||||
"quantity": 34,
|
||||
"categories": ["Land"],
|
||||
"card": { "uid": "bbbb2222-0000-0000-0000-000000000000", "oracleCard": { "name": "Forest", "colorIdentity": ["Green"] } }
|
||||
},
|
||||
{
|
||||
"quantity": 1,
|
||||
"categories": ["Maybeboard"],
|
||||
"card": { "uid": "cccc3333-0000-0000-0000-000000000000", "oracleCard": { "name": "Lightning Bolt", "colorIdentity": ["Red"] } }
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"_comment": "GUESSED SHAPE. Bambuddy's API is undocumented here; this only exercises mapBambuddySpools' tolerant reading.",
|
||||
"items": [
|
||||
{ "id": 1, "material": "PLA", "color_name": "Jade White", "color_hex": "F5F5DCFF", "remaining_weight": 640 },
|
||||
{ "id": 2, "material": "PLA", "color_name": "Jade White", "color_hex": "F5F5DC", "remaining_weight": 20 },
|
||||
{ "id": 3, "material": "PLA", "colorName": "Bambu Green", "colorHex": "#00AE42", "remainingWeight": 15 },
|
||||
{ "id": 4, "material": "PETG", "color": "Translucent Orange", "hex": "#FF6A13", "remaining": 900 },
|
||||
{ "id": 5, "material": "PLA", "color_name": "Old Spool", "archived": true, "remaining_weight": 500 },
|
||||
{ "id": 6, "material": "PLA", "color_name": "Charcoal", "color_hex": "#2b2b2b" }
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- Hand-written from BoardGameGeek's XML API2 "thing" response shape; not a live response. -->
|
||||
<items termsofuse="https://boardgamegeek.com/xmlapi/termsofuse">
|
||||
<item type="boardgame" id="266192">
|
||||
<thumbnail>https://cf.geekdo-images.com/sample__thumb/img/wingspan.jpg</thumbnail>
|
||||
<image>https://cf.geekdo-images.com/sample__original/img/wingspan.jpg</image>
|
||||
<name type="primary" sortindex="1" value="Wingspan" />
|
||||
<description>Attract a beautiful and diverse collection of birds &amp; more.</description>
|
||||
<yearpublished value="2019" />
|
||||
<minplayers value="1" />
|
||||
<maxplayers value="5" />
|
||||
<playingtime value="70" />
|
||||
<minplaytime value="40" />
|
||||
<maxplaytime value="70" />
|
||||
<minage value="10" />
|
||||
</item>
|
||||
<item type="boardgame" id="13">
|
||||
<image>http://cf.geekdo-images.com/sample__original/img/catan.jpg?a=1&b=2</image>
|
||||
<name type="primary" sortindex="1" value="CATAN" />
|
||||
<minplayers value="3" />
|
||||
<maxplayers value="4" />
|
||||
<playingtime value="120" />
|
||||
<minplaytime value="60" />
|
||||
<maxplaytime value="120" />
|
||||
</item>
|
||||
<item type="boardgame" id="99">
|
||||
<name type="primary" sortindex="1" value="No Data Yet" />
|
||||
<minplayers value="0" />
|
||||
<maxplayers value="0" />
|
||||
<playingtime value="0" />
|
||||
</item>
|
||||
</items>
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"page": 1,
|
||||
"pageSize": 100,
|
||||
"total": 3,
|
||||
"items": [
|
||||
{ "id": "a1", "name": "Veg tan, Walnut", "quantity": 2, "description": "Swatch #5c3a21, 3-4 oz", "archived": false },
|
||||
{ "id": "a2", "name": "Chrome tan, Black", "quantity": 0, "description": "", "archived": false },
|
||||
{ "id": "a3", "name": "Retired hide", "quantity": 4, "description": "", "archived": true }
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
[
|
||||
{ "id": "6f1c0a52-0000-4000-8000-000000000001", "name": "Leather", "description": "" },
|
||||
{ "id": "6f1c0a52-0000-4000-8000-000000000002", "name": "Paper", "description": "" },
|
||||
{ "id": "6f1c0a52-0000-4000-8000-000000000003", "name": "Tools", "description": "" }
|
||||
]
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"11": { "seriesId": 11, "publicationStatus": 0, "webLinks": "https://www.novelupdates.com/series/sample/,https://anilist.co/manga/200001/Sample-Light-Novel/" },
|
||||
"12": { "seriesId": 12, "publicationStatus": 0, "webLinks": "" },
|
||||
"13": { "seriesId": 13, "publicationStatus": 2, "webLinks": "https://anilist.co/manga/200003" }
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
[
|
||||
{ "id": 11, "name": "Sample Light Novel", "localizedName": "", "pages": 400, "pagesRead": 400, "latestReadDate": "2026-10-02T21:15:00", "libraryId": 2 },
|
||||
{ "id": 12, "name": "Sample Ongoing Novel", "localizedName": "Sample Ongoing Novel (EN)", "pages": 300, "pagesRead": 120, "latestReadDate": "2026-10-09T08:00:00", "libraryId": 2 },
|
||||
{ "id": 13, "name": "Sample Unread Novel", "pages": 250, "pagesRead": 0, "latestReadDate": "0001-01-01T00:00:00", "libraryId": 2 }
|
||||
]
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"_sample": "Hand-written from Open Library's search.json response shape; not a live response.",
|
||||
"numFound": 2,
|
||||
"docs": [
|
||||
{ "title": "The Way of Kings (study guide)" },
|
||||
{ "title": "The Way of Kings", "cover_i": 8221255, "cover_edition_key": "OL24395542M" }
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"data": {
|
||||
"categories": {
|
||||
"nodes": [
|
||||
{ "name": "Default", "mangas": { "nodes": [
|
||||
{ "title": "Uncategorised Series", "inLibrary": true, "latestReadChapter": null, "trackRecords": { "nodes": [] } }
|
||||
] } },
|
||||
{ "name": "Reading", "mangas": { "nodes": [
|
||||
{ "title": "Sample Tower Climber (Source Title)", "inLibrary": true,
|
||||
"latestReadChapter": { "chapterNumber": 142, "lastReadAt": "1759900000" },
|
||||
"trackRecords": { "nodes": [
|
||||
{ "trackerId": 1, "remoteId": "555", "title": "Sample Tower Climber MAL", "lastChapterRead": 140 },
|
||||
{ "trackerId": 2, "remoteId": "100001", "title": "Sample Tower Climber", "lastChapterRead": 142 }
|
||||
] } },
|
||||
{ "title": "New Sample Series", "inLibrary": true,
|
||||
"latestReadChapter": { "chapterNumber": 3, "lastReadAt": "1759990000" },
|
||||
"trackRecords": { "nodes": [ { "trackerId": 2, "remoteId": "100004", "title": "New Sample Series", "lastChapterRead": 3 } ] } },
|
||||
{ "title": "Untracked Sample", "inLibrary": true,
|
||||
"latestReadChapter": { "chapterNumber": 7, "lastReadAt": "0" },
|
||||
"trackRecords": { "nodes": [] } }
|
||||
] } },
|
||||
{ "name": "Plan to read", "mangas": { "nodes": [
|
||||
{ "title": "Sample Later", "inLibrary": true, "latestReadChapter": null,
|
||||
"trackRecords": { "nodes": [ { "trackerId": 2, "remoteId": "100002", "title": "Sample Later", "lastChapterRead": 0 } ] } }
|
||||
] } },
|
||||
{ "name": "Completed", "mangas": { "nodes": [
|
||||
{ "title": "Sample Tower Climber (Source Title)", "inLibrary": true,
|
||||
"latestReadChapter": { "chapterNumber": 142, "lastReadAt": "1759900000" },
|
||||
"trackRecords": { "nodes": [ { "trackerId": 2, "remoteId": "100001", "title": "Sample Tower Climber", "lastChapterRead": 142 } ] } },
|
||||
{ "title": "Sample Finished", "inLibrary": true,
|
||||
"latestReadChapter": { "chapterNumber": 88.5, "lastReadAt": "1750000000000" },
|
||||
"trackRecords": { "nodes": [ { "trackerId": 2, "remoteId": "100003", "title": "Sample Finished", "lastChapterRead": 88 } ] } }
|
||||
] } },
|
||||
{ "name": "Dropped", "mangas": { "nodes": [
|
||||
{ "title": "Sample Dropped", "inLibrary": true,
|
||||
"latestReadChapter": { "chapterNumber": 12, "lastReadAt": "1740000000" },
|
||||
"trackRecords": { "nodes": [ { "trackerId": 2, "remoteId": "100005", "title": "Sample Dropped", "lastChapterRead": 12 } ] } }
|
||||
] } }
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"_sample": "Hand-written from TMDB's search/movie response shape; not a live response.",
|
||||
"page": 1,
|
||||
"results": [
|
||||
{ "id": 1, "title": "Arrival (short)", "poster_path": null },
|
||||
{ "id": 329865, "title": "Arrival", "release_date": "2016-11-10", "poster_path": "/x2FJsf1ElAgr63Y3PNPtJrcmpoe.jpg" }
|
||||
],
|
||||
"total_results": 2
|
||||
}
|
||||
@@ -0,0 +1,410 @@
|
||||
/**
|
||||
* The homelab jobs (Suwayomi, Kavita, Bambuddy, Homebox) and Library
|
||||
* suggestions, against the hand-written sample payloads in ./fixtures. Those
|
||||
* are documented shapes (Bambuddy's is a guess), never live responses; see
|
||||
* fixtures/README.md.
|
||||
*/
|
||||
import { readFileSync } from "node:fs";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import { parseTokens } from "../../src/config.ts";
|
||||
import { createHttpServer } from "../../src/http.ts";
|
||||
import { listSuggestions } from "../../src/suggestions.ts";
|
||||
import { bambuddyJob, filamentOptions, mapBambuddySpools } from "../../src/sync/jobs/bambuddy.ts";
|
||||
import { homelabSettings } from "../../src/sync/jobs/homelab.ts";
|
||||
import { homeboxJob, materialOptions, parseFilters, parseItems } from "../../src/sync/jobs/homebox.ts";
|
||||
import { kavitaJob, kavitaState, parseMetadata, parseSeries } from "../../src/sync/jobs/kavita.ts";
|
||||
import { stateForCategory, targetState } from "../../src/sync/jobs/reader.ts";
|
||||
import { ANILIST_TRACKER_ID, parseAniListBatch, parseLibrary, suwayomiJob } from "../../src/sync/jobs/suwayomi.ts";
|
||||
import { enabledJobs, syncSettings } from "../../src/sync/index.ts";
|
||||
import { Scheduler } from "../../src/sync/scheduler.ts";
|
||||
import type { SyncJob } from "../../src/sync/types.ts";
|
||||
import { rig, type Rig } from "../helpers.ts";
|
||||
|
||||
const fixture = (name: string) => JSON.parse(readFileSync(join(import.meta.dirname, "fixtures", name), "utf8"));
|
||||
const ok = (body: unknown, status = 200) => new Response(JSON.stringify(body), { status });
|
||||
|
||||
type Call = { url: string; init?: RequestInit };
|
||||
function fakeFetch(answer: (url: string, init?: RequestInit) => Response) {
|
||||
const calls: Call[] = [];
|
||||
const fn = (async (input: string | URL | Request, init?: RequestInit) => {
|
||||
calls.push({ url: String(input), init });
|
||||
return answer(String(input), init);
|
||||
}) as typeof fetch;
|
||||
return { fn, calls };
|
||||
}
|
||||
const headerOf = (c: Call, k: string) => (c.init?.headers as Record<string, string> | undefined)?.[k];
|
||||
const bodyOf = (c: Call) => JSON.parse(String(c.init?.body ?? "{}"));
|
||||
|
||||
let r: Rig;
|
||||
beforeEach(() => {
|
||||
r = rig();
|
||||
});
|
||||
afterEach(() => r.done());
|
||||
|
||||
const scheduler = (job: SyncJob, fetchFn: typeof fetch) =>
|
||||
new Scheduler(r.service, [job], { fetch: fetchFn, now: () => new Date(r.clock.t), log: () => {} });
|
||||
|
||||
const manga = (title: string, extra: Record<string, unknown> = {}) => ({
|
||||
type: "library-item",
|
||||
title,
|
||||
medium: "manhwa",
|
||||
state: "up next",
|
||||
publish: true,
|
||||
...extra,
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------- state mapping
|
||||
|
||||
describe("PersonalReader state mapping", () => {
|
||||
it("maps categories: Reading now, Plan to read up next, Completed finished, Dropped left alone", () => {
|
||||
expect(stateForCategory("Reading")).toBe("now");
|
||||
expect(stateForCategory(" plan to Read ")).toBe("up next");
|
||||
expect(stateForCategory("Completed")).toBe("finished");
|
||||
expect(stateForCategory("Dropped")).toBeNull();
|
||||
expect(stateForCategory("Default")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("proposes only a different state, and keeps caught up and finished", () => {
|
||||
expect(targetState("up next", "now")).toBe("now");
|
||||
expect(targetState("now", "now")).toBeNull();
|
||||
expect(targetState("now", null)).toBeNull();
|
||||
expect(targetState("caught up", "now")).toBeNull();
|
||||
expect(targetState("finished", "caught up")).toBeNull();
|
||||
expect(targetState("dropped", "finished")).toBe("finished");
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------- Suwayomi
|
||||
|
||||
describe("Suwayomi", () => {
|
||||
it("parses the library: AniList tracker id, furthest chapter, when, best category", () => {
|
||||
const s = parseLibrary(fixture("suwayomi-library.sample.json"));
|
||||
const by = (t: string) => s.find((x) => x.title === t);
|
||||
// In Reading and Completed: Reading wins. The AniList record's title is used, not the source's.
|
||||
expect(by("Sample Tower Climber")).toEqual({ title: "Sample Tower Climber", category: "Reading", anilistId: 100001, chapter: 142, readAt: "2025-10-08T05:06:40.000Z" });
|
||||
expect(by("Sample Finished")).toMatchObject({ category: "Completed", anilistId: 100003, chapter: 88.5, readAt: new Date(1750000000000).toISOString() });
|
||||
expect(by("Untracked Sample")).toEqual({ title: "Untracked Sample", category: "Reading", chapter: 7 });
|
||||
expect(by("Uncategorised Series")).toBeUndefined();
|
||||
expect(s).toHaveLength(6);
|
||||
expect(ANILIST_TRACKER_ID).toBe(2);
|
||||
expect(() => parseLibrary({ errors: [{ message: "Unauthorized" }] })).toThrow(/Unauthorized/);
|
||||
});
|
||||
|
||||
it("parses AniList's batch: cover, title and medium", () => {
|
||||
const m = parseAniListBatch(fixture("anilist-page.sample.json"));
|
||||
expect(m.get(100001)).toEqual({ title: "Sample Tower Climber", medium: "manhwa", cover: "https://s4.anilist.co/file/anilistcdn/media/manga/cover/large/sample-100001.jpg" });
|
||||
expect(m.get(100003)).toMatchObject({ title: "Sample Owari", medium: "manga" });
|
||||
expect(m.get(100005)?.cover).toBeUndefined();
|
||||
});
|
||||
|
||||
function suwayomiFetch() {
|
||||
return fakeFetch((url, init) => {
|
||||
if (url === "https://graphql.anilist.co") return ok(fixture("anilist-page.sample.json"));
|
||||
const q = String(JSON.parse(String(init?.body)).query);
|
||||
if (q.includes("login")) return ok({ data: { login: { accessToken: "jwt-access" } } });
|
||||
return ok(fixture("suwayomi-library.sample.json"));
|
||||
});
|
||||
}
|
||||
|
||||
it("logs in (ui_login), caches progress and cover, writes states as sync, holds Greg's notes, suggests", async () => {
|
||||
// A note the sync wrote earlier (so it may change it) and notes Greg wrote.
|
||||
const created = r.service.apply({ op: "create", data: manga("Sample Tower Climber", { anilistId: 100001, publish: true }) }, "sync");
|
||||
expect(created.status).toBe("accepted");
|
||||
const later = r.put(manga("Sample Later", { anilistId: 100002, state: "now" }));
|
||||
const finished = r.put(manga("Sample Finished", { medium: "manga", anilistId: 100003, state: "now" }));
|
||||
const dropped = r.put(manga("Sample Dropped", { medium: "manga", anilistId: 100005, state: "now" }));
|
||||
r.service.scan();
|
||||
|
||||
const f = suwayomiFetch();
|
||||
const job = suwayomiJob({ url: "http://suwayomi:4567/", user: "site-reader", pass: "pw" });
|
||||
const report = await scheduler(job, f.fn).runNow("suwayomi");
|
||||
expect(report).toMatchObject({ ok: true, counts: { series: 6, matched: 4, untracked: 1, accepted: 1, held: 2, suggestions: 1 } });
|
||||
|
||||
// Login, then the library with the access token.
|
||||
expect(f.calls[0]!.url).toBe("http://suwayomi:4567/api/graphql");
|
||||
expect(bodyOf(f.calls[0]!).variables).toEqual({ username: "site-reader", password: "pw" });
|
||||
expect(headerOf(f.calls[1]!, "authorization")).toBe("Bearer jwt-access");
|
||||
|
||||
// The sync's own note changed; Greg's are held for review; Dropped is untouched.
|
||||
const tower = created.status === "accepted" ? created.path : "";
|
||||
expect(r.store.get(tower)!.data).toMatchObject({ state: "now", updatedBy: "sync" });
|
||||
expect(r.store.get(later)!.data.state).toBe("now");
|
||||
expect(r.store.pendingFor(later)?.reasons.map((x) => x.code)).toContain("agent-edit");
|
||||
expect(r.store.pendingFor(finished)?.text).toContain("state: finished");
|
||||
expect(r.store.pendingFor(dropped)).toBeNull();
|
||||
|
||||
// Cached for the site: title, chapter, status, when, AniList cover. Nothing else.
|
||||
const synced = r.store.syncCache.forPath(tower);
|
||||
expect(synced).toEqual({
|
||||
chapterProgress: { title: "Sample Tower Climber", status: "now", chapter: 142, readAt: "2025-10-08T05:06:40.000Z" },
|
||||
cover: "https://s4.anilist.co/file/anilistcdn/media/manga/cover/large/sample-100001.jpg",
|
||||
});
|
||||
const everything = JSON.stringify([...r.store.syncCache.all()]);
|
||||
expect(everything).not.toMatch(/suwayomi|Source Title|4567|thumbnail|scanlator/i);
|
||||
|
||||
// The new Reading series is a suggestion; the untracked one cannot be (no AniList id).
|
||||
expect(listSuggestions(r.store)).toEqual([
|
||||
{
|
||||
key: "anilist-100004",
|
||||
anilistId: 100004,
|
||||
title: "New Sample Series",
|
||||
medium: "manhwa",
|
||||
cover: "https://s4.anilist.co/file/anilistcdn/media/manga/cover/large/sample-100004.jpg",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("proposes a held change once: not again while it waits, nor after Greg rejects it", async () => {
|
||||
const later = r.put(manga("Sample Later", { anilistId: 100002, state: "now" }));
|
||||
r.service.scan();
|
||||
const s = scheduler(suwayomiJob({ url: "http://suwayomi:4567", auth: "none" }), suwayomiFetch().fn);
|
||||
expect((await s.runNow("suwayomi")).counts).toMatchObject({ held: 1 });
|
||||
expect((await s.runNow("suwayomi")).counts).toMatchObject({ held: 0, waiting: 0, accepted: 0 });
|
||||
r.service.rejectHeld(r.store.pendingFor(later)!.id);
|
||||
expect((await s.runNow("suwayomi")).counts).toMatchObject({ held: 0 });
|
||||
expect(r.store.pendingFor(later)).toBeNull();
|
||||
expect(r.store.get(later)!.data.state).toBe("now");
|
||||
});
|
||||
|
||||
it("keeps the last cover when AniList is down, and reports the run as failed", async () => {
|
||||
r.service.apply({ op: "create", data: manga("Sample Tower Climber", { anilistId: 100001, state: "now" }) }, "sync");
|
||||
const good = suwayomiFetch();
|
||||
await scheduler(suwayomiJob({ url: "http://s", auth: "none" }), good.fn).runNow("suwayomi");
|
||||
const down = fakeFetch((url) => (url.includes("anilist") ? ok({}, 503) : ok(fixture("suwayomi-library.sample.json"))));
|
||||
const report = await scheduler(suwayomiJob({ url: "http://s", auth: "none" }), down.fn).runNow("suwayomi");
|
||||
expect(report).toMatchObject({ ok: false, message: expect.stringContaining("AniList answered 503") });
|
||||
const [path] = [...r.store.syncCache.all().keys()].filter((p) => p.endsWith(".md"));
|
||||
expect(r.store.syncCache.forPath(path!).cover).toMatch(/sample-100001/);
|
||||
});
|
||||
|
||||
it("uses basic auth when set, and fails the run on a refused login", async () => {
|
||||
const f = fakeFetch(() => ok(fixture("suwayomi-library.sample.json")));
|
||||
await scheduler(suwayomiJob({ url: "http://s", user: "u", pass: "p", auth: "basic" }), f.fn).runNow("suwayomi");
|
||||
expect(headerOf(f.calls[0]!, "authorization")).toBe(`Basic ${Buffer.from("u:p").toString("base64")}`);
|
||||
const no = fakeFetch(() => ok({ data: { login: null }, errors: [{ message: "Invalid credentials" }] }));
|
||||
const report = await scheduler(suwayomiJob({ url: "http://s", user: "u", pass: "bad" }), no.fn).runNow("suwayomi");
|
||||
expect(report).toMatchObject({ ok: false, message: expect.stringContaining("Invalid credentials") });
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------- suggestions
|
||||
|
||||
describe("Library suggestions", () => {
|
||||
it("are listed, dismissed for good, and leave once a Library item has the AniList id", async () => {
|
||||
await scheduler(suwayomiJob({ url: "http://s", auth: "none" }), fakeFetch((url) =>
|
||||
ok(url.includes("anilist") ? fixture("anilist-page.sample.json") : fixture("suwayomi-library.sample.json")),
|
||||
).fn).runNow("suwayomi");
|
||||
// Every Reading series with an AniList id and no note: 100001 and 100004.
|
||||
const ADMIN = "a".repeat(24);
|
||||
const READER = "r".repeat(24);
|
||||
const server = createHttpServer(r.service, { tokens: parseTokens(`admin:${ADMIN},reader:${READER}`) });
|
||||
await new Promise<void>((done) => server.listen(0, "127.0.0.1", done));
|
||||
const base = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
|
||||
const get = (token = ADMIN) => fetch(`${base}/api/suggestions`, { headers: { authorization: `Bearer ${token}` } });
|
||||
try {
|
||||
expect((await get(READER)).status).toBe(403);
|
||||
const list = (await (await get()).json()) as { key: string; anilistId: number }[];
|
||||
// By title: New Sample Series, Sample Tower Climber.
|
||||
expect(list.map((s) => s.key)).toEqual(["anilist-100004", "anilist-100001"]);
|
||||
|
||||
// Dismiss one.
|
||||
const dis = await fetch(`${base}/api/suggestions/anilist-100004/dismiss`, { method: "POST", headers: { authorization: `Bearer ${ADMIN}` } });
|
||||
expect(dis.status).toBe(200);
|
||||
// Add the other through the normal write path.
|
||||
const add = await fetch(`${base}/api/changes`, {
|
||||
method: "POST",
|
||||
headers: { authorization: `Bearer ${ADMIN}`, "content-type": "application/json" },
|
||||
body: JSON.stringify({ op: "create", data: { type: "library-item", title: "Sample Tower Climber", medium: "manhwa", anilistId: 100001, state: "now", publish: true } }),
|
||||
});
|
||||
expect(((await add.json()) as { status: string }).status).toBe("accepted");
|
||||
expect(await (await get()).json()).toEqual([]);
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------- Kavita
|
||||
|
||||
describe("Kavita", () => {
|
||||
const series = fixture("kavita-series.sample.json") as unknown[];
|
||||
const metadata = fixture("kavita-metadata.sample.json") as Record<string, unknown>;
|
||||
|
||||
it("parses series and metadata; AniList only from an explicit anilist.co link", () => {
|
||||
expect(parseSeries(series[1])).toEqual({ id: 12, name: "Sample Ongoing Novel (EN)", pages: 300, pagesRead: 120, latestReadDate: new Date("2026-10-09T08:00:00").toISOString() });
|
||||
expect(parseSeries(series[2])).not.toHaveProperty("latestReadDate");
|
||||
expect(parseMetadata(metadata["11"])).toEqual({ anilistId: 200001, publicationStatus: 0 });
|
||||
expect(parseMetadata(metadata["12"])).toEqual({ publicationStatus: 0 });
|
||||
});
|
||||
|
||||
it("maps progress to a state", () => {
|
||||
const s = (pages: number, pagesRead: number) => ({ id: 1, name: "x", pages, pagesRead });
|
||||
expect(kavitaState(s(10, 10), { publicationStatus: 2 }, false)).toBe("finished");
|
||||
expect(kavitaState(s(10, 10), { publicationStatus: 0 }, false)).toBe("caught up");
|
||||
expect(kavitaState(s(10, 3), {}, false)).toBe("now");
|
||||
expect(kavitaState(s(10, 0), {}, true)).toBe("up next");
|
||||
expect(kavitaState(s(10, 0), {}, false)).toBeNull();
|
||||
});
|
||||
|
||||
it("links by kavitaSeriesId or by AniList id in metadata, never by title", async () => {
|
||||
const byId = r.put({ type: "library-item", title: "Sample Ongoing Novel", medium: "light novel", state: "up next", kavitaSeriesId: 12, publish: true });
|
||||
const byAniList = r.service.apply(
|
||||
{ op: "create", data: { type: "library-item", title: "Sample Light Novel", medium: "light novel", state: "now", anilistId: 200001 } },
|
||||
"sync",
|
||||
);
|
||||
const sameTitle = r.put({ type: "library-item", title: "Sample Unread Novel", medium: "light novel", state: "now", publish: true });
|
||||
r.service.scan();
|
||||
|
||||
const f = fakeFetch((url) => {
|
||||
if (url.includes("/api/Plugin/authenticate")) return ok({ username: "greg", token: "kavita-jwt" });
|
||||
if (url.includes("/api/Series/all-v2")) return ok(series);
|
||||
if (url.includes("/api/Series/metadata")) return ok(metadata[new URL(url).searchParams.get("seriesId")!]);
|
||||
if (url.endsWith("/api/Series/12")) return ok(series[1]);
|
||||
if (url.includes("/api/want-to-read/v2")) return ok([series[2]]);
|
||||
return ok({}, 404);
|
||||
});
|
||||
const report = await scheduler(kavitaJob({ url: "http://kavita:5000", apiKey: "key-123", delayMs: 0 }), f.fn).runNow("kavita");
|
||||
expect(report).toMatchObject({ ok: true, counts: { linked: 2, synced: 2, accepted: 1, held: 1 } });
|
||||
expect(f.calls[0]!.url).toBe("http://kavita:5000/api/Plugin/authenticate?apiKey=key-123&pluginName=GregKilmer.com");
|
||||
expect(headerOf(f.calls[1]!, "authorization")).toBe("Bearer kavita-jwt");
|
||||
|
||||
// Read to the end of an ongoing series: caught up, written as sync.
|
||||
const lnPath = byAniList.status === "accepted" ? byAniList.path : "";
|
||||
expect(r.store.get(lnPath)!.data).toMatchObject({ state: "caught up", updatedBy: "sync" });
|
||||
expect(r.store.syncCache.forPath(lnPath)).toEqual({ bookProgress: { title: "Sample Light Novel", percent: 100, readAt: new Date("2026-10-02T21:15:00").toISOString() } });
|
||||
// Greg's note: partly read is now, held for review.
|
||||
expect(r.store.pendingFor(byId)?.text).toContain("state: now");
|
||||
// Same title, no link: untouched and nothing cached.
|
||||
expect(r.store.syncCache.forPath(sameTitle)).toEqual({});
|
||||
expect(r.store.pendingFor(sameTitle)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------- Bambuddy
|
||||
|
||||
describe("Bambuddy (unverified API shape)", () => {
|
||||
it("maps spools to colors: merged by name, enough grams is in stock, archived skipped", () => {
|
||||
const spools = mapBambuddySpools(fixture("bambuddy-spools.sample.json"));
|
||||
expect(spools.map((s) => s.name)).toEqual(["Jade White", "Jade White", "Bambu Green", "Translucent Orange", "Charcoal"]);
|
||||
expect(filamentOptions(spools)).toEqual([
|
||||
{ name: "Charcoal", hex: "#2b2b2b", inStock: true },
|
||||
{ name: "Jade White", hex: "#f5f5dc", inStock: true },
|
||||
{ name: "Translucent Orange", hex: "#ff6a13", inStock: true },
|
||||
{ name: "Bambu Green", hex: "#00ae42", inStock: false },
|
||||
]);
|
||||
expect(filamentOptions(spools, { materials: ["PETG"] }).map((o) => o.name)).toEqual(["Translucent Orange"]);
|
||||
expect(mapBambuddySpools([{ name: "Plain list" }])).toEqual([{ name: "Plain list" }]);
|
||||
expect(() => mapBambuddySpools({ nothing: true })).toThrow(/no list/);
|
||||
});
|
||||
|
||||
it("stores filamentStock on lines with a Filament option, at the configured path", async () => {
|
||||
const line = (title: string, group: string) => ({
|
||||
type: "product-line",
|
||||
title,
|
||||
category: "Tabletop",
|
||||
mode: "made to order",
|
||||
channel: "here",
|
||||
summary: "s",
|
||||
slots: { open: 1, total: 2 },
|
||||
priceTiers: [],
|
||||
options: [{ group, name: "Jade White", inStock: true }],
|
||||
publish: true,
|
||||
});
|
||||
const printed = r.put(line("Printed terrain", "Filament"));
|
||||
const cards = r.put(line("Cards", "Finish"));
|
||||
r.service.scan();
|
||||
const f = fakeFetch(() => ok(fixture("bambuddy-spools.sample.json")));
|
||||
const report = await scheduler(bambuddyJob({ url: "http://bambuddy:8000/", path: "/api/v1/spools", apiKey: "k" }), f.fn).runNow("bambuddy");
|
||||
expect(report).toMatchObject({ ok: true, counts: { colors: 4, inStock: 3, lines: 1 } });
|
||||
expect(f.calls[0]!.url).toBe("http://bambuddy:8000/api/v1/spools");
|
||||
expect(headerOf(f.calls[0]!, "authorization")).toBe("Bearer k");
|
||||
expect(r.store.syncCache.forPath(printed).filamentStock).toEqual([{ group: "Filament", options: expect.arrayContaining([{ name: "Jade White", hex: "#f5f5dc", inStock: true }]) }]);
|
||||
expect(r.store.syncCache.forPath(cards)).toEqual({});
|
||||
// Notes are never written.
|
||||
expect(r.store.get(printed)!.author).toBe("greg");
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------- Homebox
|
||||
|
||||
describe("Homebox", () => {
|
||||
it("parses filters, items and materials", () => {
|
||||
expect(parseFilters("leather, paper=Paper goods")).toEqual([
|
||||
{ name: "leather", group: "Leather" },
|
||||
{ name: "paper", group: "Paper goods" },
|
||||
]);
|
||||
const { items, total } = parseItems(fixture("homebox-items.sample.json"));
|
||||
expect(total).toBe(3);
|
||||
expect(materialOptions(items)).toEqual([
|
||||
{ name: "Veg tan, Walnut", hex: "#5c3a21", inStock: true },
|
||||
{ name: "Chrome tan, Black", inStock: false },
|
||||
]);
|
||||
});
|
||||
|
||||
it("logs in, filters by label and stores materialStock on Leather lines", async () => {
|
||||
const wallet = r.put({
|
||||
type: "product-line",
|
||||
title: "Wallet",
|
||||
category: "Leather",
|
||||
mode: "made to order",
|
||||
channel: "here",
|
||||
summary: "s",
|
||||
slots: { open: 1, total: 2 },
|
||||
priceTiers: [],
|
||||
options: [{ group: "Leather", name: "Chrome tan, Black" }],
|
||||
publish: true,
|
||||
});
|
||||
r.service.scan();
|
||||
const f = fakeFetch((url) => {
|
||||
if (url.endsWith("/api/v1/users/login")) return ok({ token: "Bearer hb-token", expiresAt: "2026-10-12T00:00:00Z" });
|
||||
if (url.endsWith("/api/v1/labels")) return ok(fixture("homebox-labels.sample.json"));
|
||||
if (url.includes("/api/v1/items?")) {
|
||||
return new URL(url).searchParams.get("labels")?.endsWith("01") ? ok(fixture("homebox-items.sample.json")) : ok({ items: [], total: 0 });
|
||||
}
|
||||
return ok({}, 404);
|
||||
});
|
||||
const report = await scheduler(homeboxJob({ url: "http://homebox:7745", user: "u", pass: "p", labels: parseFilters("leather,paper") }), f.fn).runNow("homebox");
|
||||
expect(report).toMatchObject({ ok: true, counts: { lines: 1, leather: 1, paper: 0 } });
|
||||
expect(headerOf(f.calls[2]!, "authorization")).toBe("Bearer hb-token");
|
||||
expect(r.store.syncCache.forPath(wallet).materialStock).toEqual([
|
||||
{ group: "Leather", options: [{ name: "Veg tan, Walnut", hex: "#5c3a21", inStock: true }, { name: "Chrome tan, Black", inStock: false }] },
|
||||
]);
|
||||
|
||||
const missing = await scheduler(homeboxJob({ url: "http://homebox:7745", user: "u", pass: "p", labels: parseFilters("vinyl") }), f.fn).runNow("homebox");
|
||||
expect(missing).toMatchObject({ ok: false, message: expect.stringContaining("vinyl") });
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------- settings
|
||||
|
||||
describe("homelab settings", () => {
|
||||
it("enable each job by its own URL and login, not by SYNC_JOBS", () => {
|
||||
expect(enabledJobs(syncSettings({})).map((j) => j.name)).toEqual([]);
|
||||
const env = {
|
||||
SUWAYOMI_URL: "http://suwayomi:4567",
|
||||
SUWAYOMI_USER: "u",
|
||||
SUWAYOMI_PASS: "p",
|
||||
KAVITA_URL: "http://kavita:5000",
|
||||
KAVITA_API_KEY: "k",
|
||||
BAMBUDDY_URL: "http://bambuddy:8000",
|
||||
HOMEBOX_URL: "http://homebox:7745",
|
||||
HOMEBOX_USER: "u",
|
||||
HOMEBOX_PASS: "p",
|
||||
SYNC_KAVITA_MINUTES: "720",
|
||||
SYNC_SUWAYOMI_HEALTHCHECK_URL: "https://hc/s",
|
||||
};
|
||||
const jobs = enabledJobs(syncSettings(env));
|
||||
expect(jobs.map((j) => [j.name, j.everyMinutes])).toEqual([
|
||||
["suwayomi", 60],
|
||||
["kavita", 720],
|
||||
["bambuddy", 60],
|
||||
["homebox", 60],
|
||||
]);
|
||||
expect(jobs[0]!.healthcheckUrl).toBe("https://hc/s");
|
||||
expect(homelabSettings(env).homebox?.labels).toEqual(parseFilters("leather,paper"));
|
||||
expect(homelabSettings({ KAVITA_URL: "http://k" }).kavita).toBeUndefined();
|
||||
expect(() => homelabSettings({ SUWAYOMI_URL: "http://s", SUWAYOMI_AUTH: "simple_login" })).toThrow(/SUWAYOMI_AUTH/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,204 @@
|
||||
import type { AddressInfo } from "node:net";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { parseTokens } from "../../src/config.ts";
|
||||
import { createHttpServer } from "../../src/http.ts";
|
||||
import { Scheduler } from "../../src/sync/scheduler.ts";
|
||||
import { syncSettings } from "../../src/sync/settings.ts";
|
||||
import { enabledJobs } from "../../src/sync/index.ts";
|
||||
import type { SyncContext, SyncJob, SyncResult } from "../../src/sync/types.ts";
|
||||
import { book, rig, type Rig } from "../helpers.ts";
|
||||
|
||||
let r: Rig;
|
||||
beforeEach(() => {
|
||||
r = rig();
|
||||
});
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
r.done();
|
||||
});
|
||||
|
||||
const pings: string[] = [];
|
||||
const fakeFetch = (async (url: string | URL | Request) => {
|
||||
pings.push(String(url));
|
||||
return new Response("OK");
|
||||
}) as typeof fetch;
|
||||
|
||||
const job = (name: string, run: (ctx: SyncContext) => Promise<SyncResult>, extra: Partial<SyncJob> = {}): SyncJob => ({
|
||||
name,
|
||||
everyMinutes: 60,
|
||||
run,
|
||||
...extra,
|
||||
});
|
||||
|
||||
const deferred = () => {
|
||||
let resolve!: (v: SyncResult) => void;
|
||||
const promise = new Promise<SyncResult>((ok) => (resolve = ok));
|
||||
return { promise, resolve };
|
||||
};
|
||||
|
||||
describe("scheduler", () => {
|
||||
it("reports each run to sync_runs and pings Healthchecks start, success and fail", async () => {
|
||||
pings.length = 0;
|
||||
let fail = false;
|
||||
const s = new Scheduler(
|
||||
r.service,
|
||||
[
|
||||
job("demo", async () => {
|
||||
if (fail) throw new Error("source down");
|
||||
return { message: "2 synced", counts: { synced: 2 } };
|
||||
}, { healthcheckUrl: "https://hc-ping.com/abc/" }),
|
||||
],
|
||||
{ fetch: fakeFetch, log: () => {} },
|
||||
);
|
||||
expect(await s.runNow("demo")).toMatchObject({ ok: true, message: "2 synced" });
|
||||
fail = true;
|
||||
expect(await s.runNow("demo")).toMatchObject({ ok: false, message: "source down" });
|
||||
expect(r.store.syncRuns("demo").map((x) => [x.ok, x.message])).toEqual([
|
||||
[false, "source down"],
|
||||
[true, "2 synced"],
|
||||
]);
|
||||
expect(r.store.syncJobs()[0]).toMatchObject({ job: "demo", ok: false, counts: {} });
|
||||
expect(pings).toEqual(["https://hc-ping.com/abc/start", "https://hc-ping.com/abc", "https://hc-ping.com/abc/start", "https://hc-ping.com/abc/fail"]);
|
||||
});
|
||||
|
||||
it("never runs one job twice at once", async () => {
|
||||
const d = deferred();
|
||||
let runs = 0;
|
||||
const s = new Scheduler(r.service, [job("slow", () => (runs++, d.promise))], { fetch: fakeFetch, log: () => {} });
|
||||
const first = s.runNow("slow");
|
||||
expect(await s.runNow("slow")).toMatchObject({ skipped: true });
|
||||
expect(runs).toBe(1);
|
||||
d.resolve({});
|
||||
await first;
|
||||
await Promise.resolve();
|
||||
expect(s.isRunning("slow")).toBe(false);
|
||||
});
|
||||
|
||||
it("aborts a run that passes its timeout and stays locked until the job really ends", async () => {
|
||||
vi.useFakeTimers();
|
||||
const d = deferred();
|
||||
let aborted = false;
|
||||
const s = new Scheduler(
|
||||
r.service,
|
||||
[
|
||||
job("stuck", (ctx) => {
|
||||
ctx.signal.addEventListener("abort", () => (aborted = true));
|
||||
return d.promise;
|
||||
}, { timeoutMs: 1000 }),
|
||||
],
|
||||
{ fetch: fakeFetch, log: () => {} },
|
||||
);
|
||||
const run = s.runNow("stuck");
|
||||
await vi.advanceTimersByTimeAsync(1001);
|
||||
expect(await run).toMatchObject({ ok: false, message: "Timed out after 1 seconds" });
|
||||
expect(aborted).toBe(true);
|
||||
expect(s.isRunning("stuck")).toBe(true);
|
||||
d.resolve({});
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(s.isRunning("stuck")).toBe(false);
|
||||
});
|
||||
|
||||
it("runs each job at start, staggered, then on its interval", async () => {
|
||||
vi.useFakeTimers();
|
||||
const ran: string[] = [];
|
||||
const mk = (name: string) => job(name, async () => (ran.push(name), {}), { everyMinutes: 10 });
|
||||
const s = new Scheduler(r.service, [mk("a"), mk("b")], { staggerMs: 1000, fetch: fakeFetch, log: () => {} });
|
||||
s.start();
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(ran).toEqual(["a"]);
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
expect(ran).toEqual(["a", "b"]);
|
||||
await vi.advanceTimersByTimeAsync(10 * 60_000);
|
||||
expect(ran).toEqual(["a", "b", "a", "b"]);
|
||||
s.stop();
|
||||
await vi.advanceTimersByTimeAsync(60 * 60_000);
|
||||
expect(ran).toHaveLength(4);
|
||||
});
|
||||
});
|
||||
|
||||
describe("sync cache", () => {
|
||||
it("keeps the last good values when a run fails, and shows them on the notes API", async () => {
|
||||
const path = r.put(book("Mistborn"));
|
||||
r.service.scan();
|
||||
let fail = false;
|
||||
const s = new Scheduler(
|
||||
r.service,
|
||||
[
|
||||
job("demo", async (ctx) => {
|
||||
if (fail) throw new Error("down");
|
||||
ctx.set(path, { cover: "https://example.com/a.jpg", pages: 541 });
|
||||
return {};
|
||||
}),
|
||||
],
|
||||
{ fetch: fakeFetch, log: () => {} },
|
||||
);
|
||||
await s.runNow("demo");
|
||||
const first = r.store.syncCache.get("demo", path);
|
||||
fail = true;
|
||||
await s.runNow("demo");
|
||||
expect(r.store.syncCache.get("demo", path)).toEqual(first);
|
||||
|
||||
// Values from several jobs merge per note.
|
||||
r.store.syncCache.set("other", path, { year: 2006 }, "2026-10-11T00:00:00Z");
|
||||
const token = "r".repeat(24);
|
||||
const server = createHttpServer(r.service, { tokens: parseTokens(`reader:${token}`) });
|
||||
await new Promise<void>((ok) => server.listen(0, "127.0.0.1", ok));
|
||||
const base = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
|
||||
const get = (p: string) => fetch(base + p, { headers: { authorization: `Bearer ${token}` } }).then((x) => x.json());
|
||||
try {
|
||||
const [listed] = (await get("/api/notes?full=true")) as { synced: unknown }[];
|
||||
expect(listed!.synced).toEqual({ cover: "https://example.com/a.jpg", pages: 541, year: 2006 });
|
||||
expect(((await get(`/api/note?path=${encodeURIComponent(path)}`)) as { synced: unknown }).synced).toMatchObject({ pages: 541 });
|
||||
expect(await get("/api/notes")).toEqual([expect.not.objectContaining({ synced: expect.anything() })]);
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("retain drops values for notes that left a job's scope", () => {
|
||||
r.store.syncCache.set("j", "a.md", { x: 1 }, "t");
|
||||
r.store.syncCache.set("j", "b.md", { x: 2 }, "t");
|
||||
r.store.syncCache.set("k", "b.md", { y: 3 }, "t");
|
||||
r.store.syncCache.retain("j", ["a.md"]);
|
||||
expect([...r.store.syncCache.all().entries()]).toEqual([
|
||||
["a.md", { x: 1 }],
|
||||
["b.md", { y: 3 }],
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("writing notes as sync", () => {
|
||||
it("runs the agent rules: a note Greg wrote is held unless it allows agent edits", async () => {
|
||||
const mine = r.put(book("Mine", { progress: "1/10" }));
|
||||
const open = r.put(book("Open", { progress: "1/10", agentEdits: true }));
|
||||
r.service.scan();
|
||||
const outcomes: string[] = [];
|
||||
const s = new Scheduler(
|
||||
r.service,
|
||||
[
|
||||
job("reader", async (ctx) => {
|
||||
for (const p of [mine, open]) outcomes.push(ctx.apply({ op: "update", path: p, fields: { progress: "2/10" } }).status);
|
||||
outcomes.push(ctx.apply({ op: "update", path: open, fields: { note: "Mail me at someone@example.com" } }).status);
|
||||
return {};
|
||||
}),
|
||||
],
|
||||
{ fetch: fakeFetch, log: () => {} },
|
||||
);
|
||||
await s.runNow("reader");
|
||||
expect(outcomes).toEqual(["held", "accepted", "held"]);
|
||||
expect(r.store.heldList()[0]!.reasons.map((x) => x.code)).toContain("contact");
|
||||
expect(r.store.get(open)!.data).toMatchObject({ progress: "2/10", updatedBy: "sync" });
|
||||
expect(r.store.get(open)!.author).toBe("sync");
|
||||
});
|
||||
});
|
||||
|
||||
describe("settings", () => {
|
||||
it("enables only the jobs whose config is present", () => {
|
||||
const s = syncSettings({ SYNC_JOBS: "bgg, covers", SYNC_BGG_MINUTES: "720", SYNC_COVERS_HEALTHCHECK_URL: "https://hc/x", BGG_TOKEN: "t" });
|
||||
expect(enabledJobs(s).map((j) => [j.name, j.everyMinutes, j.healthcheckUrl])).toEqual([
|
||||
["bgg", 720, undefined],
|
||||
["covers", 15, "https://hc/x"],
|
||||
]);
|
||||
expect(enabledJobs(syncSettings({}))).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,165 @@
|
||||
/**
|
||||
* Each source's parsing and its job, against the hand-written sample payloads
|
||||
* in ./fixtures (documented shapes, not live responses; see fixtures/README.md).
|
||||
*/
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import { archidektJob, parseDeck, scryfallArt, type RawDeck } from "../../src/sync/jobs/archidekt.ts";
|
||||
import { bggJob, parseThings } from "../../src/sync/jobs/bgg.ts";
|
||||
import { coversJob, parseAniList, parseOpenLibrary, parseTmdb } from "../../src/sync/jobs/covers.ts";
|
||||
import { Scheduler } from "../../src/sync/scheduler.ts";
|
||||
import type { SyncJob } from "../../src/sync/types.ts";
|
||||
import { book, rig, type Rig } from "../helpers.ts";
|
||||
|
||||
const fixture = (name: string) => readFileSync(join(import.meta.dirname, "fixtures", name), "utf8");
|
||||
const json = (name: string) => JSON.parse(fixture(name));
|
||||
|
||||
type Call = { url: string; init?: RequestInit };
|
||||
function fakeFetch(answer: (url: string, init?: RequestInit) => Response | Promise<Response>) {
|
||||
const calls: Call[] = [];
|
||||
const fn = (async (input: string | URL | Request, init?: RequestInit) => {
|
||||
const url = String(input);
|
||||
calls.push({ url, init });
|
||||
return answer(url, init);
|
||||
}) as typeof fetch;
|
||||
return { fn, calls };
|
||||
}
|
||||
const ok = (body: unknown, status = 200) =>
|
||||
new Response(typeof body === "string" ? body : JSON.stringify(body), { status });
|
||||
|
||||
describe("Archidekt", () => {
|
||||
it("parses commander art, color identity and a list summary", () => {
|
||||
const v = parseDeck(json("archidekt-deck.sample.json") as RawDeck);
|
||||
expect(v).toEqual({
|
||||
deckUrl: "https://archidekt.com/decks/1234567",
|
||||
commanderArt: scryfallArt("d0d33d52-3d28-4635-b985-51e126289259"),
|
||||
colors: ["W", "U", "B", "G"],
|
||||
commanders: ["Atraxa, Praetors' Voice"],
|
||||
deckSize: 36,
|
||||
format: "Commander",
|
||||
});
|
||||
expect(v.commanderArt).toBe("https://cards.scryfall.io/art_crop/front/d/0/d0d33d52-3d28-4635-b985-51e126289259.jpg");
|
||||
});
|
||||
|
||||
it("falls back to the featured image and the whole list's colors without a commander", () => {
|
||||
const v = parseDeck({
|
||||
id: 5,
|
||||
featured: "http://example.com/f.jpg",
|
||||
cards: [{ quantity: 4, categories: ["Main"], card: { oracleCard: { name: "Bolt", colorIdentity: ["R"] } } }],
|
||||
});
|
||||
expect(v).toMatchObject({ commanderArt: "https://example.com/f.jpg", colors: ["R"], commanders: [], deckSize: 4 });
|
||||
});
|
||||
});
|
||||
|
||||
describe("BoardGameGeek", () => {
|
||||
it("parses players, play time and box art per id", () => {
|
||||
const m = parseThings(fixture("bgg-thing.sample.xml"));
|
||||
expect(m.get(266192)).toEqual({
|
||||
minPlayers: 1,
|
||||
maxPlayers: 5,
|
||||
playTime: 70,
|
||||
minPlayTime: 40,
|
||||
maxPlayTime: 70,
|
||||
cover: "https://cf.geekdo-images.com/sample__original/img/wingspan.jpg",
|
||||
});
|
||||
expect(m.get(13)?.cover).toBe("https://cf.geekdo-images.com/sample__original/img/catan.jpg?a=1&b=2");
|
||||
// Zeros mean BGG has no value: left out, so nothing wrong shows.
|
||||
expect(m.get(99)).toEqual({});
|
||||
});
|
||||
});
|
||||
|
||||
describe("cover art", () => {
|
||||
it("parses Open Library, TMDB and AniList answers", () => {
|
||||
expect(parseOpenLibrary(json("openlibrary-search.sample.json"))).toBe("https://covers.openlibrary.org/b/id/8221255-L.jpg");
|
||||
expect(parseTmdb(json("tmdb-search.sample.json"))).toBe("https://image.tmdb.org/t/p/w500/x2FJsf1ElAgr63Y3PNPtJrcmpoe.jpg");
|
||||
expect(parseAniList(json("anilist-media.sample.json"))).toBe(
|
||||
"https://s4.anilist.co/file/anilistcdn/media/manga/cover/large/sample-xl.jpg",
|
||||
);
|
||||
expect(parseOpenLibrary({ docs: [] })).toBeNull();
|
||||
expect(parseAniList({ data: { Media: null } })).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------- the jobs, end to end
|
||||
|
||||
let r: Rig;
|
||||
beforeEach(() => {
|
||||
r = rig();
|
||||
});
|
||||
afterEach(() => r.done());
|
||||
|
||||
const scheduler = (job: SyncJob, fetchFn: typeof fetch) =>
|
||||
new Scheduler(r.service, [job], { fetch: fetchFn, now: () => new Date(r.clock.t), log: () => {} });
|
||||
|
||||
describe("jobs", () => {
|
||||
it("Archidekt syncs each deck with an id and forgets decks that lose it", async () => {
|
||||
r.put({ type: "card-game", title: "Magic", publish: true, stance: "Main event" });
|
||||
const atraxa = r.put({ type: "deck", title: "Atraxa", game: "[[Magic]]", archidektId: 1234567, publish: true });
|
||||
r.put({ type: "deck", title: "Paper only", game: "[[Magic]]", listUrl: "https://example.com/list", publish: true });
|
||||
r.service.scan();
|
||||
const f = fakeFetch(() => ok(fixture("archidekt-deck.sample.json")));
|
||||
const report = await scheduler(archidektJob({ delayMs: 0 }), f.fn).runNow("archidekt");
|
||||
expect(report).toMatchObject({ ok: true, counts: { decks: 1, synced: 1, failed: 0 } });
|
||||
expect(f.calls.map((c) => c.url)).toEqual(["https://archidekt.com/api/decks/1234567/"]);
|
||||
expect(r.store.syncCache.forPath(atraxa)).toMatchObject({ colors: ["W", "U", "B", "G"] });
|
||||
});
|
||||
|
||||
it("BoardGameGeek batches 20 ids a request, sends the token and waits out a 202", async () => {
|
||||
for (let i = 1; i <= 25; i++) r.put({ type: "board-game", title: `Game ${i}`, bggId: i === 1 ? 266192 : 1000 + i, blurb: "Fun.", publish: true });
|
||||
r.service.scan();
|
||||
let queued = true;
|
||||
const f = fakeFetch(() => {
|
||||
if (queued) {
|
||||
queued = false;
|
||||
return ok("", 202);
|
||||
}
|
||||
return ok(fixture("bgg-thing.sample.xml"));
|
||||
});
|
||||
const report = await scheduler(bggJob({ token: "app-token", delayMs: 0 }), f.fn).runNow("bgg");
|
||||
expect(f.calls).toHaveLength(3); // 202, then the first 20, then the last 5
|
||||
const ids = (c: Call) => new URL(c.url).searchParams.get("id")!.split(",");
|
||||
expect(f.calls[1]!.url).toMatch(/^https:\/\/boardgamegeek\.com\/xmlapi2\/thing\?id=/);
|
||||
expect([ids(f.calls[1]!).length, ids(f.calls[2]!).length]).toEqual([20, 5]);
|
||||
expect(new Set([...ids(f.calls[1]!), ...ids(f.calls[2]!)]).size).toBe(25);
|
||||
expect((f.calls[0]!.init?.headers as Record<string, string>).authorization).toBe("Bearer app-token");
|
||||
// Only Wingspan is in the sample answer; the rest count as not synced.
|
||||
expect(report).toMatchObject({ ok: false, counts: { games: 25, synced: 1, failed: 24 } });
|
||||
expect(r.store.syncCache.forPath("Shelf/Game 1.md")).toMatchObject({ minPlayers: 1, maxPlayers: 5, playTime: 70 });
|
||||
});
|
||||
|
||||
it("covers: picks the source by medium, skips TMDB without a key, keeps a found cover on a miss", async () => {
|
||||
const b = r.put(book("The Way of Kings", { creator: "Brandon Sanderson" }));
|
||||
const m = r.put(book("One Piece", { medium: "manga", anilistId: 30013 }));
|
||||
r.put(book("Arrival", { medium: "movie" }));
|
||||
r.put(book("Has its own", { cover: "https://example.com/own.jpg" }));
|
||||
r.service.scan();
|
||||
const f = fakeFetch((url) =>
|
||||
url.startsWith("https://openlibrary.org/") ? ok(fixture("openlibrary-search.sample.json")) : ok(fixture("anilist-media.sample.json")),
|
||||
);
|
||||
const s = scheduler(coversJob({ delayMs: 0 }), f.fn);
|
||||
expect(await s.runNow("covers")).toMatchObject({ ok: true, counts: { checked: 2, found: 2 } });
|
||||
expect(f.calls.map((c) => new URL(c.url).host).sort()).toEqual(["graphql.anilist.co", "openlibrary.org"]);
|
||||
expect(f.calls.find((c) => c.url.includes("openlibrary"))!.url).toContain("author=Brandon+Sanderson");
|
||||
expect(JSON.parse(String(f.calls.find((c) => c.url.includes("anilist"))!.init?.body)).variables).toEqual({ type: "MANGA", id: 30013 });
|
||||
expect(r.store.syncCache.forPath(b).cover).toBe("https://covers.openlibrary.org/b/id/8221255-L.jpg");
|
||||
|
||||
// Nothing is due again until its retry time.
|
||||
expect(await s.runNow("covers")).toMatchObject({ counts: { checked: 0 } });
|
||||
|
||||
// A month on, the sources find nothing: the earlier covers stay.
|
||||
r.clock.advance(31 * 24 * 60);
|
||||
const empty = fakeFetch(() => ok({ docs: [], data: { Media: null } }));
|
||||
await scheduler(coversJob({ delayMs: 0 }), empty.fn).runNow("covers");
|
||||
expect(r.store.syncCache.forPath(m).cover).toMatch(/anilist/);
|
||||
});
|
||||
|
||||
it("covers: TMDB with a key", async () => {
|
||||
const a = r.put(book("Arrival", { medium: "movie", year: 2016 }));
|
||||
r.service.scan();
|
||||
const f = fakeFetch(() => ok(fixture("tmdb-search.sample.json")));
|
||||
await scheduler(coversJob({ delayMs: 0, tmdbApiKey: "k".repeat(32) }), f.fn).runNow("covers");
|
||||
expect(f.calls[0]!.url).toMatch(/^https:\/\/api\.themoviedb\.org\/3\/search\/movie\?query=Arrival&include_adult=false&year=2016&api_key=k+$/);
|
||||
expect(r.store.syncCache.forPath(a).cover).toMatch(/image\.tmdb\.org/);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user