fix(pipeline): dedupe+P2002; add red maple-leaf cover system; replace dead CBC feeds (Google News CN / NatPost Canada); worker dead-letter

This commit is contained in:
2026-08-18 10:24:09 -04:00
parent 44d3220c4d
commit 08b9deb3d5
12 changed files with 211 additions and 61 deletions
+17 -1
View File
@@ -64,13 +64,29 @@ export async function synthesizePending(limit: number = 24) {
let ok = 0;
let failed = 0;
// Dead-letter: a row that is still `fetched` after 24h of continuous
// failure will never succeed on its own (poison content, oversized
// source, provider rejection). Mark it `failed` so it stops consuming
// a slot in every cron pass and out of the backlog. Manual re-queue by
// resetting status to `fetched` if the cause is later fixed.
const deadAfter = new Date(Date.now() - 24 * 60 * 60 * 1000);
for (const a of pending) {
try {
await synthesizeArticle(a.id);
ok += 1;
} catch (err) {
failed += 1;
console.error(`[synth] failed article ${a.slug}:`, (err as Error).message);
const dead = a.createdAt < deadAfter;
const why = (err as Error).message;
if (dead) {
await prisma.article.update({
where: { id: a.id },
data: { status: 'failed' },
});
console.error(`[synth] DEAD-LETTER ${a.slug} (failed >24h): ${why}`);
} else {
console.error(`[synth] failed article ${a.slug}: ${why}`);
}
}
}
return { total: pending.length, ok, failed };