ingest: suppress Google-news shared placeholder image (googleusercontent) -> null -> category cover fallback

Root cause: Google News RSS items carry no enclosures; pipeline fell back
to og:image from the news.google.com interstitial, which serves ONE shared
Google-branded tile (lh3.googleusercontent.com/J6_coFbo...) for every
article. 601 rows shared that URL (maple 243 / finance 358). New
image-guard blocks *.googleusercontent.com images at ingest; 16/16
regression tests; prod DBs migrated tile->NULL; images rebuilt
technews:10 finance:4 maple-brief:11 and deployed.
This commit is contained in:
2026-08-18 21:30:23 -04:00
parent 86a77b4907
commit 1e7e951c91
4 changed files with 172 additions and 1 deletions
+1 -1
View File
@@ -8,7 +8,7 @@
"build": "prisma generate && next build",
"start": "next start",
"typecheck": "tsc --noEmit",
"test": "tsx tests/llm-parse.test.ts && tsx tests/image-proxy.test.ts",
"test": "tsx tests/llm-parse.test.ts && tsx tests/image-guard.test.ts && tsx tests/image-proxy.test.ts",
"db:generate": "prisma generate",
"db:push": "prisma db push",
"db:seed": "tsx prisma/seed.ts",
+58
View File
@@ -0,0 +1,58 @@
/**
* Image placeholder guard.
*
* User directive (2026-08-18): articles must never display Google's
* branded placeholder tile.
*
* Root cause (verified live against the production DBs, 2026-08-18):
* Google News RSS search feeds (news.google.com/rss/search?q=...) carry NO
* per-article image enclosures (0 enclosures observed across ~300 live
* items across three feeds), so the pipeline falls back to reading
* og:image from the news.google.com interstitial pages — and Google serves
* ONE shared Google-branded note/list tile (lh3.googleusercontent.com/
* J6_coFbogxh...) as og:image for every single interstitial. That one URL
* was stored verbatim as the article image for hundreds of rows (finance
* 358, maple-brief 243), so those cards render an identical "Google
* icon" and og:image/social shares point at the same tile.
*
* In this pipeline the only producer of a *.googleusercontent.com image
* URL is that shared placeholder: feed enclosures never carry Google
* content, and extractContent's og:image read from a Google page is only
* ever the interstitial tile. Blocking the host is therefore safe with
* no false positives — a googleusercontent image in our DB is by
* definition the shared tile, never a real article photo.
*
* Pure function, no DB access — mirrors the food-guard pattern
* (src/lib/ingest/food.ts).
*/
const GOOGLE_CONTENT_HOSTS = /(^|\.)googleusercontent\.com$/i;
export interface PlaceholderMatch {
blocked: boolean;
/** the offending host, for logging */
reason: string;
}
/**
* Decide whether a candidate article image is a shared placeholder tile
* instead of a real article photo.
* @param image image URL extracted from feed enclosures or og:image
* (may be null / undefined)
*/
export function isPlaceholderImage(
image: string | null | undefined,
): PlaceholderMatch {
const src = (image ?? '').trim();
if (!src) return { blocked: false, reason: '' };
let host = '';
try {
host = new URL(src).hostname.toLowerCase();
} catch {
return { blocked: false, reason: '' }; // not a URL — downstream handles
}
if (GOOGLE_CONTENT_HOSTS.test(host)) {
return { blocked: true, reason: host };
}
return { blocked: false, reason: '' };
}
+14
View File
@@ -2,12 +2,14 @@ import { prisma } from '@/lib/db';
import { env } from '@/lib/env';
import { dedupeKeyOf, slugify } from '@/lib/slug';
import { parseFeed, extractContent, type FeedItem } from './feed';
import { isPlaceholderImage } from './image-guard';
export interface IngestResult {
feed: string;
fetched: number;
newArticles: number;
duplicates: number;
imageFiltered: number;
errors: string[];
}
@@ -34,6 +36,7 @@ export async function ingestFeed(
fetched: 0,
newArticles: 0,
duplicates: 0,
imageFiltered: 0,
errors: [],
};
@@ -97,6 +100,17 @@ export async function ingestFeed(
if (!image && ex.image) image = ex.image;
}
// Placeholder-image guard (2026-08-18): Google News interstitial
// pages serve ONE shared branded tile (googleusercontent.com) as
// og:image for every item. Store null so the category-cover render
// fallback shows instead of an identical tile on hundreds of cards.
const imageGuard = isPlaceholderImage(image);
if (imageGuard.blocked) {
result.imageFiltered += 1;
console.log(`[ingest] placeholder image suppressed (${imageGuard.reason}): ${item.title.slice(0, 70)}`);
image = undefined;
}
const baseSlug = slugify(item.title);
const slug = await uniqueSlug(baseSlug);
const sourceUrl = item.link;
+99
View File
@@ -0,0 +1,99 @@
/**
* Placeholder-image guard tests (offline — no network).
*
* Contract under test (src/lib/ingest/image-guard.ts):
* - the ONE shared Google-branded tile observed in production (exact URL
* from the 2026-08-18 finance/maple cleanup, lh3.googleusercontent.com
* /J6_coFbogxh...) is ALWAYS blocked, in every variant (resize suffixes,
* host rotation lh2/lh3/lh4, case, extra query strings)
* - real publisher article photos on googleusercontent-lookalike or normal
* publisher hosts are NEVER blocked (must-pass: false positives here
* would strip legitimate images)
*
* Runner: `tsx tests/image-guard.test.ts` -> exit 1 on failure.
*/
import assert from 'node:assert/strict';
import { isPlaceholderImage } from '@/lib/ingest/image-guard';
let passed = 0;
let failed = 0;
function check(name: string, fn: () => void): void {
try {
fn();
passed += 1;
console.log(`ok ${passed} - ${name}`);
} catch (e) {
failed += 1;
console.error(`not ok ${passed + failed} - ${name}`);
console.error(String((e as Error).stack ?? e));
}
}
// The exact shared tile URL stored on 601 production rows (2026-08-18).
const TILE =
'https://lh3.googleusercontent.com/J6_coFbogxhRI9iM864NL_liGXvsQp2Aups' +
'Kei7z0cNNfDvGUmWUy20nuUhkREQyrpY4bEeIBuc=s0-w300';
check('MUST-BLOCK: exact prod shared tile url', () => {
assert.equal(isPlaceholderImage(TILE).blocked, true);
});
check('MUST-BLOCK: -rw resize variant', () => {
assert.equal(isPlaceholderImage(`${TILE}-rw`).blocked, true);
});
check('MUST-BLOCK: lh4 host rotation', () => {
assert.equal(isPlaceholderImage('https://lh4.googleusercontent.com/xyz123=s900').blocked, true);
});
check('MUST-BLOCK: lh2 host rotation', () => {
assert.equal(isPlaceholderImage('https://lh2.googleusercontent.com/xyz123').blocked, true);
});
check('MUST-BLOCK: uppercase host', () => {
assert.equal(isPlaceholderImage('https://LH3.GOOGLEUSERCONTENT.COM/J6_coFbo=s0').blocked, true);
});
check('MUST-BLOCK: extra query string / fragment', () => {
assert.equal(isPlaceholderImage(TILE + '&dummy=1#f').blocked, true);
});
check('MUST-BLOCK: unknown key (not the known tile) still blocked by host rule', () => {
assert.equal(
isPlaceholderImage('https://lh3.googleusercontent.com/someOtherKeyWq9aB=s64').blocked,
true,
);
});
check('MUST-BLOCK: whitespace-wrapped url', () => {
assert.equal(isPlaceholderImage(` ${TILE} `).blocked, true);
});
check('MUST-PASS: wired photo (publisher CDN)', () => {
assert.equal(
isPlaceholderImage('https://media.wired.com/photos/6a84a2c1/191:100/w_1280/c_limit/x.jpg').blocked,
false,
);
});
check('MUST-PASS: cnbcfm photo (publisher CDN)', () => {
assert.equal(isPlaceholderImage('https://media.cnbcfm.com/i/2026/08/robo.jpg').blocked, false);
});
check('MUST-PASS: google.com logo asset (NOT googleusercontent)', () => {
assert.equal(isPlaceholderImage('https://www.google.com/logos/2026/xx512.png').blocked, false);
});
check('MUST-PASS: example.com path segment merely containing the host string', () => {
assert.equal(
isPlaceholderImage('https://cdn.example.com/googleusercontent.com/img.png').blocked,
false,
);
});
check('MUST-PASS: null', () => {
assert.equal(isPlaceholderImage(null).blocked, false);
});
check('MUST-PASS: undefined', () => {
assert.equal(isPlaceholderImage(undefined).blocked, false);
});
check('MUST-PASS: empty string', () => {
assert.equal(isPlaceholderImage('').blocked, false);
});
check('MUST-PASS: not a URL', () => {
assert.equal(isPlaceholderImage('not a url').blocked, false);
});
console.log(`\nimage-guard: ${passed} passed, ${failed} failed`);
if (failed > 0) process.exit(1);