Archived
ingest: suppress Google-news shared placeholder image (googleusercontent) -> null -> category cover fallback
Root cause: Google News RSS items carry no enclosures; pipeline fell back to og:image from the news.google.com interstitial, which serves ONE shared Google-branded tile (lh3.googleusercontent.com/J6_coFbo...) for every article. 601 rows shared that URL (maple 243 / finance 358). New image-guard blocks *.googleusercontent.com images at ingest; 16/16 regression tests; prod DBs migrated tile->NULL; images rebuilt technews:10 finance:4 maple-brief:11 and deployed.
This commit is contained in:
+1
-1
@@ -8,7 +8,7 @@
|
||||
"build": "prisma generate && next build",
|
||||
"start": "next start",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"test": "tsx tests/llm-parse.test.ts && tsx tests/image-proxy.test.ts",
|
||||
"test": "tsx tests/llm-parse.test.ts && tsx tests/image-guard.test.ts && tsx tests/image-proxy.test.ts",
|
||||
"db:generate": "prisma generate",
|
||||
"db:push": "prisma db push",
|
||||
"db:seed": "tsx prisma/seed.ts",
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
/**
|
||||
* Image placeholder guard.
|
||||
*
|
||||
* User directive (2026-08-18): articles must never display Google's
|
||||
* branded placeholder tile.
|
||||
*
|
||||
* Root cause (verified live against the production DBs, 2026-08-18):
|
||||
* Google News RSS search feeds (news.google.com/rss/search?q=...) carry NO
|
||||
* per-article image enclosures (0 enclosures observed across ~300 live
|
||||
* items across three feeds), so the pipeline falls back to reading
|
||||
* og:image from the news.google.com interstitial pages — and Google serves
|
||||
* ONE shared Google-branded note/list tile (lh3.googleusercontent.com/
|
||||
* J6_coFbogxh...) as og:image for every single interstitial. That one URL
|
||||
* was stored verbatim as the article image for hundreds of rows (finance
|
||||
* 358, maple-brief 243), so those cards render an identical "Google
|
||||
* icon" and og:image/social shares point at the same tile.
|
||||
*
|
||||
* In this pipeline the only producer of a *.googleusercontent.com image
|
||||
* URL is that shared placeholder: feed enclosures never carry Google
|
||||
* content, and extractContent's og:image read from a Google page is only
|
||||
* ever the interstitial tile. Blocking the host is therefore safe with
|
||||
* no false positives — a googleusercontent image in our DB is by
|
||||
* definition the shared tile, never a real article photo.
|
||||
*
|
||||
* Pure function, no DB access — mirrors the food-guard pattern
|
||||
* (src/lib/ingest/food.ts).
|
||||
*/
|
||||
|
||||
const GOOGLE_CONTENT_HOSTS = /(^|\.)googleusercontent\.com$/i;
|
||||
|
||||
export interface PlaceholderMatch {
|
||||
blocked: boolean;
|
||||
/** the offending host, for logging */
|
||||
reason: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Decide whether a candidate article image is a shared placeholder tile
|
||||
* instead of a real article photo.
|
||||
* @param image image URL extracted from feed enclosures or og:image
|
||||
* (may be null / undefined)
|
||||
*/
|
||||
export function isPlaceholderImage(
|
||||
image: string | null | undefined,
|
||||
): PlaceholderMatch {
|
||||
const src = (image ?? '').trim();
|
||||
if (!src) return { blocked: false, reason: '' };
|
||||
let host = '';
|
||||
try {
|
||||
host = new URL(src).hostname.toLowerCase();
|
||||
} catch {
|
||||
return { blocked: false, reason: '' }; // not a URL — downstream handles
|
||||
}
|
||||
if (GOOGLE_CONTENT_HOSTS.test(host)) {
|
||||
return { blocked: true, reason: host };
|
||||
}
|
||||
return { blocked: false, reason: '' };
|
||||
}
|
||||
@@ -2,12 +2,14 @@ import { prisma } from '@/lib/db';
|
||||
import { env } from '@/lib/env';
|
||||
import { dedupeKeyOf, slugify } from '@/lib/slug';
|
||||
import { parseFeed, extractContent, type FeedItem } from './feed';
|
||||
import { isPlaceholderImage } from './image-guard';
|
||||
|
||||
export interface IngestResult {
|
||||
feed: string;
|
||||
fetched: number;
|
||||
newArticles: number;
|
||||
duplicates: number;
|
||||
imageFiltered: number;
|
||||
errors: string[];
|
||||
}
|
||||
|
||||
@@ -34,6 +36,7 @@ export async function ingestFeed(
|
||||
fetched: 0,
|
||||
newArticles: 0,
|
||||
duplicates: 0,
|
||||
imageFiltered: 0,
|
||||
errors: [],
|
||||
};
|
||||
|
||||
@@ -97,6 +100,17 @@ export async function ingestFeed(
|
||||
if (!image && ex.image) image = ex.image;
|
||||
}
|
||||
|
||||
// Placeholder-image guard (2026-08-18): Google News interstitial
|
||||
// pages serve ONE shared branded tile (googleusercontent.com) as
|
||||
// og:image for every item. Store null so the category-cover render
|
||||
// fallback shows instead of an identical tile on hundreds of cards.
|
||||
const imageGuard = isPlaceholderImage(image);
|
||||
if (imageGuard.blocked) {
|
||||
result.imageFiltered += 1;
|
||||
console.log(`[ingest] placeholder image suppressed (${imageGuard.reason}): ${item.title.slice(0, 70)}`);
|
||||
image = undefined;
|
||||
}
|
||||
|
||||
const baseSlug = slugify(item.title);
|
||||
const slug = await uniqueSlug(baseSlug);
|
||||
const sourceUrl = item.link;
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
/**
|
||||
* Placeholder-image guard tests (offline — no network).
|
||||
*
|
||||
* Contract under test (src/lib/ingest/image-guard.ts):
|
||||
* - the ONE shared Google-branded tile observed in production (exact URL
|
||||
* from the 2026-08-18 finance/maple cleanup, lh3.googleusercontent.com
|
||||
* /J6_coFbogxh...) is ALWAYS blocked, in every variant (resize suffixes,
|
||||
* host rotation lh2/lh3/lh4, case, extra query strings)
|
||||
* - real publisher article photos on googleusercontent-lookalike or normal
|
||||
* publisher hosts are NEVER blocked (must-pass: false positives here
|
||||
* would strip legitimate images)
|
||||
*
|
||||
* Runner: `tsx tests/image-guard.test.ts` -> exit 1 on failure.
|
||||
*/
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
import { isPlaceholderImage } from '@/lib/ingest/image-guard';
|
||||
|
||||
let passed = 0;
|
||||
let failed = 0;
|
||||
|
||||
function check(name: string, fn: () => void): void {
|
||||
try {
|
||||
fn();
|
||||
passed += 1;
|
||||
console.log(`ok ${passed} - ${name}`);
|
||||
} catch (e) {
|
||||
failed += 1;
|
||||
console.error(`not ok ${passed + failed} - ${name}`);
|
||||
console.error(String((e as Error).stack ?? e));
|
||||
}
|
||||
}
|
||||
|
||||
// The exact shared tile URL stored on 601 production rows (2026-08-18).
|
||||
const TILE =
|
||||
'https://lh3.googleusercontent.com/J6_coFbogxhRI9iM864NL_liGXvsQp2Aups' +
|
||||
'Kei7z0cNNfDvGUmWUy20nuUhkREQyrpY4bEeIBuc=s0-w300';
|
||||
|
||||
check('MUST-BLOCK: exact prod shared tile url', () => {
|
||||
assert.equal(isPlaceholderImage(TILE).blocked, true);
|
||||
});
|
||||
check('MUST-BLOCK: -rw resize variant', () => {
|
||||
assert.equal(isPlaceholderImage(`${TILE}-rw`).blocked, true);
|
||||
});
|
||||
check('MUST-BLOCK: lh4 host rotation', () => {
|
||||
assert.equal(isPlaceholderImage('https://lh4.googleusercontent.com/xyz123=s900').blocked, true);
|
||||
});
|
||||
check('MUST-BLOCK: lh2 host rotation', () => {
|
||||
assert.equal(isPlaceholderImage('https://lh2.googleusercontent.com/xyz123').blocked, true);
|
||||
});
|
||||
check('MUST-BLOCK: uppercase host', () => {
|
||||
assert.equal(isPlaceholderImage('https://LH3.GOOGLEUSERCONTENT.COM/J6_coFbo=s0').blocked, true);
|
||||
});
|
||||
check('MUST-BLOCK: extra query string / fragment', () => {
|
||||
assert.equal(isPlaceholderImage(TILE + '&dummy=1#f').blocked, true);
|
||||
});
|
||||
check('MUST-BLOCK: unknown key (not the known tile) still blocked by host rule', () => {
|
||||
assert.equal(
|
||||
isPlaceholderImage('https://lh3.googleusercontent.com/someOtherKeyWq9aB=s64').blocked,
|
||||
true,
|
||||
);
|
||||
});
|
||||
check('MUST-BLOCK: whitespace-wrapped url', () => {
|
||||
assert.equal(isPlaceholderImage(` ${TILE} `).blocked, true);
|
||||
});
|
||||
|
||||
check('MUST-PASS: wired photo (publisher CDN)', () => {
|
||||
assert.equal(
|
||||
isPlaceholderImage('https://media.wired.com/photos/6a84a2c1/191:100/w_1280/c_limit/x.jpg').blocked,
|
||||
false,
|
||||
);
|
||||
});
|
||||
check('MUST-PASS: cnbcfm photo (publisher CDN)', () => {
|
||||
assert.equal(isPlaceholderImage('https://media.cnbcfm.com/i/2026/08/robo.jpg').blocked, false);
|
||||
});
|
||||
check('MUST-PASS: google.com logo asset (NOT googleusercontent)', () => {
|
||||
assert.equal(isPlaceholderImage('https://www.google.com/logos/2026/xx512.png').blocked, false);
|
||||
});
|
||||
check('MUST-PASS: example.com path segment merely containing the host string', () => {
|
||||
assert.equal(
|
||||
isPlaceholderImage('https://cdn.example.com/googleusercontent.com/img.png').blocked,
|
||||
false,
|
||||
);
|
||||
});
|
||||
check('MUST-PASS: null', () => {
|
||||
assert.equal(isPlaceholderImage(null).blocked, false);
|
||||
});
|
||||
check('MUST-PASS: undefined', () => {
|
||||
assert.equal(isPlaceholderImage(undefined).blocked, false);
|
||||
});
|
||||
check('MUST-PASS: empty string', () => {
|
||||
assert.equal(isPlaceholderImage('').blocked, false);
|
||||
});
|
||||
check('MUST-PASS: not a URL', () => {
|
||||
assert.equal(isPlaceholderImage('not a url').blocked, false);
|
||||
});
|
||||
|
||||
console.log(`\nimage-guard: ${passed} passed, ${failed} failed`);
|
||||
if (failed > 0) process.exit(1);
|
||||
Reference in New Issue
Block a user