diff --git a/package.json b/package.json index 97cef35..aa40491 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "build": "prisma generate && next build", "start": "next start", "typecheck": "tsc --noEmit", - "test": "tsx tests/llm-parse.test.ts && tsx tests/image-proxy.test.ts", + "test": "tsx tests/llm-parse.test.ts && tsx tests/image-guard.test.ts && tsx tests/image-proxy.test.ts", "db:generate": "prisma generate", "db:push": "prisma db push", "db:seed": "tsx prisma/seed.ts", diff --git a/src/lib/ingest/image-guard.ts b/src/lib/ingest/image-guard.ts new file mode 100644 index 0000000..8d381c4 --- /dev/null +++ b/src/lib/ingest/image-guard.ts @@ -0,0 +1,58 @@ +/** + * Image placeholder guard. + * + * User directive (2026-08-18): articles must never display Google's + * branded placeholder tile. + * + * Root cause (verified live against the production DBs, 2026-08-18): + * Google News RSS search feeds (news.google.com/rss/search?q=...) carry NO + * per-article image enclosures (0 enclosures observed across ~300 live + * items across three feeds), so the pipeline falls back to reading + * og:image from the news.google.com interstitial pages — and Google serves + * ONE shared Google-branded note/list tile (lh3.googleusercontent.com/ + * J6_coFbogxh...) as og:image for every single interstitial. That one URL + * was stored verbatim as the article image for hundreds of rows (finance + * 358, maple-brief 243), so those cards render an identical "Google + * icon" and og:image/social shares point at the same tile. + * + * In this pipeline the only producer of a *.googleusercontent.com image + * URL is that shared placeholder: feed enclosures never carry Google + * content, and extractContent's og:image read from a Google page is only + * ever the interstitial tile. Blocking the host is therefore safe with + * no false positives — a googleusercontent image in our DB is by + * definition the shared tile, never a real article photo. + * + * Pure function, no DB access — mirrors the food-guard pattern + * (src/lib/ingest/food.ts). + */ + +const GOOGLE_CONTENT_HOSTS = /(^|\.)googleusercontent\.com$/i; + +export interface PlaceholderMatch { + blocked: boolean; + /** the offending host, for logging */ + reason: string; +} + +/** + * Decide whether a candidate article image is a shared placeholder tile + * instead of a real article photo. + * @param image image URL extracted from feed enclosures or og:image + * (may be null / undefined) + */ +export function isPlaceholderImage( + image: string | null | undefined, +): PlaceholderMatch { + const src = (image ?? '').trim(); + if (!src) return { blocked: false, reason: '' }; + let host = ''; + try { + host = new URL(src).hostname.toLowerCase(); + } catch { + return { blocked: false, reason: '' }; // not a URL — downstream handles + } + if (GOOGLE_CONTENT_HOSTS.test(host)) { + return { blocked: true, reason: host }; + } + return { blocked: false, reason: '' }; +} diff --git a/src/lib/ingest/pipeline.ts b/src/lib/ingest/pipeline.ts index 3eaa586..df80796 100644 --- a/src/lib/ingest/pipeline.ts +++ b/src/lib/ingest/pipeline.ts @@ -2,12 +2,14 @@ import { prisma } from '@/lib/db'; import { env } from '@/lib/env'; import { dedupeKeyOf, slugify } from '@/lib/slug'; import { parseFeed, extractContent, type FeedItem } from './feed'; +import { isPlaceholderImage } from './image-guard'; export interface IngestResult { feed: string; fetched: number; newArticles: number; duplicates: number; + imageFiltered: number; errors: string[]; } @@ -34,6 +36,7 @@ export async function ingestFeed( fetched: 0, newArticles: 0, duplicates: 0, + imageFiltered: 0, errors: [], }; @@ -97,6 +100,17 @@ export async function ingestFeed( if (!image && ex.image) image = ex.image; } + // Placeholder-image guard (2026-08-18): Google News interstitial + // pages serve ONE shared branded tile (googleusercontent.com) as + // og:image for every item. Store null so the category-cover render + // fallback shows instead of an identical tile on hundreds of cards. + const imageGuard = isPlaceholderImage(image); + if (imageGuard.blocked) { + result.imageFiltered += 1; + console.log(`[ingest] placeholder image suppressed (${imageGuard.reason}): ${item.title.slice(0, 70)}`); + image = undefined; + } + const baseSlug = slugify(item.title); const slug = await uniqueSlug(baseSlug); const sourceUrl = item.link; diff --git a/tests/image-guard.test.ts b/tests/image-guard.test.ts new file mode 100644 index 0000000..bdaa070 --- /dev/null +++ b/tests/image-guard.test.ts @@ -0,0 +1,99 @@ +/** + * Placeholder-image guard tests (offline — no network). + * + * Contract under test (src/lib/ingest/image-guard.ts): + * - the ONE shared Google-branded tile observed in production (exact URL + * from the 2026-08-18 finance/maple cleanup, lh3.googleusercontent.com + * /J6_coFbogxh...) is ALWAYS blocked, in every variant (resize suffixes, + * host rotation lh2/lh3/lh4, case, extra query strings) + * - real publisher article photos on googleusercontent-lookalike or normal + * publisher hosts are NEVER blocked (must-pass: false positives here + * would strip legitimate images) + * + * Runner: `tsx tests/image-guard.test.ts` -> exit 1 on failure. + */ +import assert from 'node:assert/strict'; + +import { isPlaceholderImage } from '@/lib/ingest/image-guard'; + +let passed = 0; +let failed = 0; + +function check(name: string, fn: () => void): void { + try { + fn(); + passed += 1; + console.log(`ok ${passed} - ${name}`); + } catch (e) { + failed += 1; + console.error(`not ok ${passed + failed} - ${name}`); + console.error(String((e as Error).stack ?? e)); + } +} + +// The exact shared tile URL stored on 601 production rows (2026-08-18). +const TILE = + 'https://lh3.googleusercontent.com/J6_coFbogxhRI9iM864NL_liGXvsQp2Aups' + + 'Kei7z0cNNfDvGUmWUy20nuUhkREQyrpY4bEeIBuc=s0-w300'; + +check('MUST-BLOCK: exact prod shared tile url', () => { + assert.equal(isPlaceholderImage(TILE).blocked, true); +}); +check('MUST-BLOCK: -rw resize variant', () => { + assert.equal(isPlaceholderImage(`${TILE}-rw`).blocked, true); +}); +check('MUST-BLOCK: lh4 host rotation', () => { + assert.equal(isPlaceholderImage('https://lh4.googleusercontent.com/xyz123=s900').blocked, true); +}); +check('MUST-BLOCK: lh2 host rotation', () => { + assert.equal(isPlaceholderImage('https://lh2.googleusercontent.com/xyz123').blocked, true); +}); +check('MUST-BLOCK: uppercase host', () => { + assert.equal(isPlaceholderImage('https://LH3.GOOGLEUSERCONTENT.COM/J6_coFbo=s0').blocked, true); +}); +check('MUST-BLOCK: extra query string / fragment', () => { + assert.equal(isPlaceholderImage(TILE + '&dummy=1#f').blocked, true); +}); +check('MUST-BLOCK: unknown key (not the known tile) still blocked by host rule', () => { + assert.equal( + isPlaceholderImage('https://lh3.googleusercontent.com/someOtherKeyWq9aB=s64').blocked, + true, + ); +}); +check('MUST-BLOCK: whitespace-wrapped url', () => { + assert.equal(isPlaceholderImage(` ${TILE} `).blocked, true); +}); + +check('MUST-PASS: wired photo (publisher CDN)', () => { + assert.equal( + isPlaceholderImage('https://media.wired.com/photos/6a84a2c1/191:100/w_1280/c_limit/x.jpg').blocked, + false, + ); +}); +check('MUST-PASS: cnbcfm photo (publisher CDN)', () => { + assert.equal(isPlaceholderImage('https://media.cnbcfm.com/i/2026/08/robo.jpg').blocked, false); +}); +check('MUST-PASS: google.com logo asset (NOT googleusercontent)', () => { + assert.equal(isPlaceholderImage('https://www.google.com/logos/2026/xx512.png').blocked, false); +}); +check('MUST-PASS: example.com path segment merely containing the host string', () => { + assert.equal( + isPlaceholderImage('https://cdn.example.com/googleusercontent.com/img.png').blocked, + false, + ); +}); +check('MUST-PASS: null', () => { + assert.equal(isPlaceholderImage(null).blocked, false); +}); +check('MUST-PASS: undefined', () => { + assert.equal(isPlaceholderImage(undefined).blocked, false); +}); +check('MUST-PASS: empty string', () => { + assert.equal(isPlaceholderImage('').blocked, false); +}); +check('MUST-PASS: not a URL', () => { + assert.equal(isPlaceholderImage('not a url').blocked, false); +}); + +console.log(`\nimage-guard: ${passed} passed, ${failed} failed`); +if (failed > 0) process.exit(1);