Archived
fix(pipeline): dedupe+P2002; add red maple-leaf cover system; replace dead CBC feeds (Google News CN / NatPost Canada); worker dead-letter
This commit is contained in:
@@ -4,6 +4,7 @@ import { notFound } from 'next/navigation';
|
||||
import { getArticleFull, getRelatedArticles } from '@/lib/queries';
|
||||
import RelatedArticles from '@/components/RelatedArticles';
|
||||
import { InArticleAd, SidebarAd } from '@/components/ads/placements';
|
||||
import { categoryCover, resolveCoverImage } from '@/lib/cover';
|
||||
import { env } from '@/lib/env';
|
||||
import { formatFull } from '@/lib/format';
|
||||
import { safeParse } from '@/lib/llm/parse';
|
||||
@@ -56,15 +57,18 @@ export async function generateMetadata({
|
||||
modifiedTime: a.synthesizedAt?.toISOString(),
|
||||
section: sectionLabel,
|
||||
tags: a.tags ? safeParse<string[]>(a.tags).slice(0, 5) : undefined,
|
||||
images: a.image
|
||||
? [{ url: a.image, width: 1200, height: 630, alt: title }]
|
||||
: [{ url: '/og-image.png', width: 1200, height: 630, alt: 'MapleBrief' }],
|
||||
images: [{
|
||||
url: (a.image && a.image.trim()) || categoryCover(a.category),
|
||||
width: 1200,
|
||||
height: 630,
|
||||
alt: title,
|
||||
}],
|
||||
},
|
||||
twitter: {
|
||||
card: 'summary_large_image',
|
||||
title,
|
||||
description,
|
||||
images: a.image ? [a.image] : undefined,
|
||||
images: [(a.image && a.image.trim()) || categoryCover(a.category)],
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -102,14 +106,11 @@ export default async function ArticlePage({
|
||||
</h1>
|
||||
</header>
|
||||
|
||||
{a.image && (
|
||||
// eslint-disable-next-line @next/next/no-img-element
|
||||
<img
|
||||
src={a.image}
|
||||
alt={title}
|
||||
className="mt-6 aspect-[16/8] w-full rounded-2xl object-cover"
|
||||
/>
|
||||
)}
|
||||
<img
|
||||
src={resolveCoverImage(a.image, a.category)}
|
||||
alt={title}
|
||||
className="mt-6 aspect-[16/8] w-full rounded-2xl object-cover"
|
||||
/>
|
||||
|
||||
{takeaways.length > 0 && (
|
||||
<section className="mt-8 rounded-xl border-l-4 border-maple-600 bg-maple-50 p-5">
|
||||
|
||||
+7
-15
@@ -1,8 +1,8 @@
|
||||
import { getPublishedArticles, getTagList } from '@/lib/queries';
|
||||
import ArticleCard from '@/components/ArticleCard';
|
||||
import { resolveCoverImage } from '@/lib/cover';
|
||||
import { InFeedAd, SidebarAd } from '@/components/ads/placements';
|
||||
import Link from 'next/link';
|
||||
import { env } from '@/lib/env';
|
||||
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
@@ -33,20 +33,12 @@ export default async function HomePage({
|
||||
className="group mb-8 block overflow-hidden rounded-2xl border border-ink-200 bg-white shadow-sm"
|
||||
>
|
||||
<div className="relative aspect-[16/8] overflow-hidden bg-ink-100">
|
||||
{hero.image ? (
|
||||
// eslint-disable-next-line @next/next/no-img-element
|
||||
<img
|
||||
src={hero.image}
|
||||
alt=""
|
||||
className="h-full w-full object-cover transition duration-500 group-hover:scale-[1.02]"
|
||||
/>
|
||||
) : (
|
||||
<div className="flex h-full items-center justify-center bg-masthead-gradient">
|
||||
<span className="font-display text-2xl font-bold text-white/90">
|
||||
{env.siteName}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
{/* cover.ts falls back to a per-category brand cover when no photo was captured */}
|
||||
<img
|
||||
src={resolveCoverImage(hero.image, hero.category)}
|
||||
alt=""
|
||||
className="h-full w-full object-cover transition duration-500 group-hover:scale-[1.02]"
|
||||
/>
|
||||
<div className="absolute inset-x-0 bottom-0 bg-gradient-to-t from-ink-950/95 via-ink-950/70 to-transparent px-4 pb-4 pt-10 sm:px-5 sm:pb-5 sm:pt-16">
|
||||
<span className="mb-2 inline-block rounded bg-maple-600 px-2 py-0.5 text-[11px] font-bold uppercase tracking-wider text-white">
|
||||
Lead briefing
|
||||
|
||||
@@ -1,29 +1,24 @@
|
||||
import Link from 'next/link';
|
||||
import type { ArticleCard as Card } from '@/lib/queries';
|
||||
import { resolveCoverImage } from '@/lib/cover';
|
||||
import { formatRelativeTime } from '@/lib/format';
|
||||
|
||||
export default function ArticleCard({ card }: { card: Card }) {
|
||||
const href = `/article/${card.slug}`;
|
||||
const title = card.headline ?? card.title;
|
||||
const image = resolveCoverImage(card.image, card.category);
|
||||
return (
|
||||
<article className="group flex flex-col overflow-hidden rounded-xl border border-ink-200 bg-white shadow-sm transition hover:-translate-y-0.5 hover:shadow-md">
|
||||
<Link
|
||||
href={href}
|
||||
className="relative block aspect-[16/9] overflow-hidden bg-ink-100"
|
||||
>
|
||||
{card.image ? (
|
||||
// eslint-disable-next-line @next/next/no-img-element
|
||||
<img
|
||||
src={card.image}
|
||||
alt=""
|
||||
loading="lazy"
|
||||
className="h-full w-full object-cover transition duration-300 group-hover:scale-[1.03]"
|
||||
/>
|
||||
) : (
|
||||
<span className="flex h-full items-center justify-center font-display text-4xl text-ink-300">
|
||||
◆
|
||||
</span>
|
||||
)}
|
||||
<img
|
||||
src={image}
|
||||
alt=""
|
||||
loading="lazy"
|
||||
className="h-full w-full object-cover transition duration-300 group-hover:scale-[1.03]"
|
||||
/>
|
||||
</Link>
|
||||
<div className="flex flex-1 flex-col p-4">
|
||||
<h3 className="font-display text-lg font-bold leading-snug text-ink-900 group-hover:text-maple-700">
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import Link from 'next/link';
|
||||
import type { ArticleCard } from '@/lib/queries';
|
||||
import { resolveCoverImage } from '@/lib/cover';
|
||||
import { formatRelativeTime } from '@/lib/format';
|
||||
|
||||
export default function RelatedArticles({ items }: { items: ArticleCard[] }) {
|
||||
@@ -11,10 +12,12 @@ export default function RelatedArticles({ items }: { items: ArticleCard[] }) {
|
||||
{items.map((a) => (
|
||||
<li key={a.id} className="flex gap-3">
|
||||
<div className="h-16 w-24 flex-shrink-0 overflow-hidden rounded-md bg-ink-100">
|
||||
{a.image ? (
|
||||
// eslint-disable-next-line @next/next/no-img-element
|
||||
<img src={a.image} alt="" loading="lazy" className="h-full w-full object-cover" />
|
||||
) : null}
|
||||
<img
|
||||
src={resolveCoverImage(a.image, a.category)}
|
||||
alt=""
|
||||
loading="lazy"
|
||||
className="h-full w-full object-cover"
|
||||
/>
|
||||
</div>
|
||||
<div className="min-w-0">
|
||||
<Link
|
||||
|
||||
+14
-13
@@ -14,23 +14,24 @@ export interface FeedSeed {
|
||||
|
||||
export const STARTER_FEEDS: FeedSeed[] = [
|
||||
{
|
||||
// NOTE: these cbc-stats URLs are the official CBC feed paths from the
|
||||
// project spec, but as of 2026-08-15 CBC returns 404 for every public
|
||||
// RSS endpoint (repeatedly re-verified with multiple UAs). Keep them so
|
||||
// they light up if CBC restores the feeds; disable in `Feed.enabled`
|
||||
// or swap URLs in the DB if the 404s bother you (see README → Feeds).
|
||||
name: 'CBC News — Top Stories',
|
||||
url: 'https://www.cbc.ca/cbc-stats/rss/rss-topstories.xml',
|
||||
slug: 'cbc-top-stories',
|
||||
// Replacement for CBC News — Top Stories: cbc-stats returned 404 for
|
||||
// every public RSS path (re-verified from egress 2026-08-18). Google
|
||||
// News Canada hub gives a dense national top-stories stream (100 items).
|
||||
name: 'Google News — Canada',
|
||||
url: 'https://news.google.com/rss/search?q=canada%20when:48h&hl=en-CA&gl=CA&ceid=CA:en',
|
||||
slug: 'gn-canada',
|
||||
category: 'top-stories',
|
||||
siteName: 'CBC News',
|
||||
siteName: 'Google News',
|
||||
},
|
||||
{
|
||||
name: 'CBC News — Canada',
|
||||
url: 'https://www.cbc.ca/cbc-stats/rss/rss-canada.xml',
|
||||
slug: 'cbc-canada',
|
||||
// Replacement for CBC News — Canada (cbc-stats 404, re-verified
|
||||
// 2026-08-18). National Post Canada section — distinct from the
|
||||
// National Post News feed below; cross-feed GUIDs dedupe at ingest.
|
||||
name: 'National Post — Canada',
|
||||
url: 'https://nationalpost.com/category/canada/feed',
|
||||
slug: 'national-post-canada',
|
||||
category: 'canada',
|
||||
siteName: 'CBC News',
|
||||
siteName: 'National Post',
|
||||
},
|
||||
{
|
||||
// Live Arc outbound feed (verified 2026-08). The legacy
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
/**
|
||||
* Resolve the display image for an article.
|
||||
*
|
||||
* Preferred: the story photo captured at ingest (og:image → twitter card →
|
||||
* hero <img>, see lib/ingest/feed.ts). Last resort: a deterministic
|
||||
* per-category brand cover baked into /public/covers/ so that every card,
|
||||
* article page and social preview always shows a visual related to the
|
||||
* piece — never a bare glyph (2026-08-17 "every article has a photo").
|
||||
*/
|
||||
|
||||
/** Category slugs that have a generated cover asset. */
|
||||
const COVER_SLUGS = ['top-stories', 'canada', 'national'] as const;
|
||||
type CoverSlug = (typeof COVER_SLUGS)[number];
|
||||
|
||||
export function categoryCover(category?: string | null): string {
|
||||
const slug = (category ?? '')
|
||||
.toLowerCase()
|
||||
.trim()
|
||||
.replace(/[^a-z0-9]+/g, '-')
|
||||
.replace(/^-+|-+$/g, '');
|
||||
if ((COVER_SLUGS as readonly string[]).includes(slug)) {
|
||||
return `/covers/${slug}.svg`;
|
||||
}
|
||||
return '/covers/top-stories.svg';
|
||||
}
|
||||
|
||||
export function resolveCoverImage(
|
||||
image: string | null | undefined,
|
||||
category?: string | null,
|
||||
): string {
|
||||
return (image ?? '').trim() ? image! : categoryCover(category);
|
||||
}
|
||||
@@ -51,11 +51,15 @@ export async function ingestFeed(
|
||||
(i) => !i.publishedAt || i.publishedAt >= cutoff,
|
||||
);
|
||||
|
||||
// Build a set of keys already in the DB (bounded to recent window).
|
||||
// Build a set of keys already in the DB. Deliberately NOT scoped to the
|
||||
// publishedAt window: items saved without a pubDate (publishedAt=null)
|
||||
// would otherwise be invisible to every dedupe set and re-collide on the
|
||||
// @unique(guid) constraint on every pass (P2002 "item errors", fixed
|
||||
// 2026-08-18). Costs 3 scalar columns, so the full-history scan is cheap.
|
||||
const existing = await prisma.article.findMany({
|
||||
where: { publishedAt: { gte: cutoff }, guid: { not: null } },
|
||||
where: { guid: { not: null } },
|
||||
select: { guid: true, dedupKey: true, sourceUrl: true },
|
||||
take: 5000,
|
||||
take: 20000,
|
||||
});
|
||||
const seenGuid = new Set(existing.map((a) => a.guid).filter(Boolean) as string[]);
|
||||
const seenDedup = new Set(existing.map((a) => a.dedupKey));
|
||||
@@ -98,6 +102,7 @@ export async function ingestFeed(
|
||||
const sourceUrl = item.link;
|
||||
const canonicalUrl = `${env.siteUrl.replace(/\/$/, '')}/article/${slug}`;
|
||||
|
||||
try {
|
||||
await prisma.article.create({
|
||||
data: {
|
||||
feedId,
|
||||
@@ -117,6 +122,16 @@ export async function ingestFeed(
|
||||
status: 'fetched',
|
||||
},
|
||||
});
|
||||
} catch (err) {
|
||||
if (/P2002|Unique constraint failed/.test((err as Error).message)) {
|
||||
// A row claiming this guid/dedupKey/slug was written between the
|
||||
// pre-scan and the insert (concurrent pass or a pre-window row).
|
||||
// Count it as a duplicate, not a spurious item error.
|
||||
result.duplicates += 1;
|
||||
continue;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
result.newArticles += 1;
|
||||
} catch (err) {
|
||||
result.errors.push(`item "${item.title?.slice(0, 60)}": ${(err as Error).message}`);
|
||||
|
||||
@@ -64,13 +64,29 @@ export async function synthesizePending(limit: number = 24) {
|
||||
|
||||
let ok = 0;
|
||||
let failed = 0;
|
||||
// Dead-letter: a row that is still `fetched` after 24h of continuous
|
||||
// failure will never succeed on its own (poison content, oversized
|
||||
// source, provider rejection). Mark it `failed` so it stops consuming
|
||||
// a slot in every cron pass and out of the backlog. Manual re-queue by
|
||||
// resetting status to `fetched` if the cause is later fixed.
|
||||
const deadAfter = new Date(Date.now() - 24 * 60 * 60 * 1000);
|
||||
for (const a of pending) {
|
||||
try {
|
||||
await synthesizeArticle(a.id);
|
||||
ok += 1;
|
||||
} catch (err) {
|
||||
failed += 1;
|
||||
console.error(`[synth] failed article ${a.slug}:`, (err as Error).message);
|
||||
const dead = a.createdAt < deadAfter;
|
||||
const why = (err as Error).message;
|
||||
if (dead) {
|
||||
await prisma.article.update({
|
||||
where: { id: a.id },
|
||||
data: { status: 'failed' },
|
||||
});
|
||||
console.error(`[synth] DEAD-LETTER ${a.slug} (failed >24h): ${why}`);
|
||||
} else {
|
||||
console.error(`[synth] failed article ${a.slug}: ${why}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
return { total: pending.length, ok, failed };
|
||||
|
||||
Reference in New Issue
Block a user