Archived
ingest: suppress Google-news shared placeholder image (googleusercontent) -> null -> category cover fallback
Root cause: Google News RSS items carry no enclosures; pipeline fell back to og:image from the news.google.com interstitial, which serves ONE shared Google-branded tile (lh3.googleusercontent.com/J6_coFbo...) for every article. 601 rows shared that URL (maple 243 / finance 358). New image-guard blocks *.googleusercontent.com images at ingest; 16/16 regression tests; prod DBs migrated tile->NULL; images rebuilt technews:10 finance:4 maple-brief:11 and deployed.
This commit is contained in:
+1
-1
@@ -8,7 +8,7 @@
|
|||||||
"build": "prisma generate && next build",
|
"build": "prisma generate && next build",
|
||||||
"start": "next start",
|
"start": "next start",
|
||||||
"typecheck": "tsc --noEmit",
|
"typecheck": "tsc --noEmit",
|
||||||
"test": "tsx tests/llm-parse.test.ts && tsx tests/image-proxy.test.ts",
|
"test": "tsx tests/llm-parse.test.ts && tsx tests/image-guard.test.ts && tsx tests/image-proxy.test.ts",
|
||||||
"db:generate": "prisma generate",
|
"db:generate": "prisma generate",
|
||||||
"db:push": "prisma db push",
|
"db:push": "prisma db push",
|
||||||
"db:seed": "tsx prisma/seed.ts",
|
"db:seed": "tsx prisma/seed.ts",
|
||||||
|
|||||||
@@ -0,0 +1,58 @@
|
|||||||
|
/**
|
||||||
|
* Image placeholder guard.
|
||||||
|
*
|
||||||
|
* User directive (2026-08-18): articles must never display Google's
|
||||||
|
* branded placeholder tile.
|
||||||
|
*
|
||||||
|
* Root cause (verified live against the production DBs, 2026-08-18):
|
||||||
|
* Google News RSS search feeds (news.google.com/rss/search?q=...) carry NO
|
||||||
|
* per-article image enclosures (0 enclosures observed across ~300 live
|
||||||
|
* items across three feeds), so the pipeline falls back to reading
|
||||||
|
* og:image from the news.google.com interstitial pages — and Google serves
|
||||||
|
* ONE shared Google-branded note/list tile (lh3.googleusercontent.com/
|
||||||
|
* J6_coFbogxh...) as og:image for every single interstitial. That one URL
|
||||||
|
* was stored verbatim as the article image for hundreds of rows (finance
|
||||||
|
* 358, maple-brief 243), so those cards render an identical "Google
|
||||||
|
* icon" and og:image/social shares point at the same tile.
|
||||||
|
*
|
||||||
|
* In this pipeline the only producer of a *.googleusercontent.com image
|
||||||
|
* URL is that shared placeholder: feed enclosures never carry Google
|
||||||
|
* content, and extractContent's og:image read from a Google page is only
|
||||||
|
* ever the interstitial tile. Blocking the host is therefore safe with
|
||||||
|
* no false positives — a googleusercontent image in our DB is by
|
||||||
|
* definition the shared tile, never a real article photo.
|
||||||
|
*
|
||||||
|
* Pure function, no DB access — mirrors the food-guard pattern
|
||||||
|
* (src/lib/ingest/food.ts).
|
||||||
|
*/
|
||||||
|
|
||||||
|
const GOOGLE_CONTENT_HOSTS = /(^|\.)googleusercontent\.com$/i;
|
||||||
|
|
||||||
|
export interface PlaceholderMatch {
|
||||||
|
blocked: boolean;
|
||||||
|
/** the offending host, for logging */
|
||||||
|
reason: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Decide whether a candidate article image is a shared placeholder tile
|
||||||
|
* instead of a real article photo.
|
||||||
|
* @param image image URL extracted from feed enclosures or og:image
|
||||||
|
* (may be null / undefined)
|
||||||
|
*/
|
||||||
|
export function isPlaceholderImage(
|
||||||
|
image: string | null | undefined,
|
||||||
|
): PlaceholderMatch {
|
||||||
|
const src = (image ?? '').trim();
|
||||||
|
if (!src) return { blocked: false, reason: '' };
|
||||||
|
let host = '';
|
||||||
|
try {
|
||||||
|
host = new URL(src).hostname.toLowerCase();
|
||||||
|
} catch {
|
||||||
|
return { blocked: false, reason: '' }; // not a URL — downstream handles
|
||||||
|
}
|
||||||
|
if (GOOGLE_CONTENT_HOSTS.test(host)) {
|
||||||
|
return { blocked: true, reason: host };
|
||||||
|
}
|
||||||
|
return { blocked: false, reason: '' };
|
||||||
|
}
|
||||||
@@ -2,12 +2,14 @@ import { prisma } from '@/lib/db';
|
|||||||
import { env } from '@/lib/env';
|
import { env } from '@/lib/env';
|
||||||
import { dedupeKeyOf, slugify } from '@/lib/slug';
|
import { dedupeKeyOf, slugify } from '@/lib/slug';
|
||||||
import { parseFeed, extractContent, type FeedItem } from './feed';
|
import { parseFeed, extractContent, type FeedItem } from './feed';
|
||||||
|
import { isPlaceholderImage } from './image-guard';
|
||||||
|
|
||||||
export interface IngestResult {
|
export interface IngestResult {
|
||||||
feed: string;
|
feed: string;
|
||||||
fetched: number;
|
fetched: number;
|
||||||
newArticles: number;
|
newArticles: number;
|
||||||
duplicates: number;
|
duplicates: number;
|
||||||
|
imageFiltered: number;
|
||||||
errors: string[];
|
errors: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -34,6 +36,7 @@ export async function ingestFeed(
|
|||||||
fetched: 0,
|
fetched: 0,
|
||||||
newArticles: 0,
|
newArticles: 0,
|
||||||
duplicates: 0,
|
duplicates: 0,
|
||||||
|
imageFiltered: 0,
|
||||||
errors: [],
|
errors: [],
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -97,6 +100,17 @@ export async function ingestFeed(
|
|||||||
if (!image && ex.image) image = ex.image;
|
if (!image && ex.image) image = ex.image;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Placeholder-image guard (2026-08-18): Google News interstitial
|
||||||
|
// pages serve ONE shared branded tile (googleusercontent.com) as
|
||||||
|
// og:image for every item. Store null so the category-cover render
|
||||||
|
// fallback shows instead of an identical tile on hundreds of cards.
|
||||||
|
const imageGuard = isPlaceholderImage(image);
|
||||||
|
if (imageGuard.blocked) {
|
||||||
|
result.imageFiltered += 1;
|
||||||
|
console.log(`[ingest] placeholder image suppressed (${imageGuard.reason}): ${item.title.slice(0, 70)}`);
|
||||||
|
image = undefined;
|
||||||
|
}
|
||||||
|
|
||||||
const baseSlug = slugify(item.title);
|
const baseSlug = slugify(item.title);
|
||||||
const slug = await uniqueSlug(baseSlug);
|
const slug = await uniqueSlug(baseSlug);
|
||||||
const sourceUrl = item.link;
|
const sourceUrl = item.link;
|
||||||
|
|||||||
@@ -0,0 +1,99 @@
|
|||||||
|
/**
|
||||||
|
* Placeholder-image guard tests (offline — no network).
|
||||||
|
*
|
||||||
|
* Contract under test (src/lib/ingest/image-guard.ts):
|
||||||
|
* - the ONE shared Google-branded tile observed in production (exact URL
|
||||||
|
* from the 2026-08-18 finance/maple cleanup, lh3.googleusercontent.com
|
||||||
|
* /J6_coFbogxh...) is ALWAYS blocked, in every variant (resize suffixes,
|
||||||
|
* host rotation lh2/lh3/lh4, case, extra query strings)
|
||||||
|
* - real publisher article photos on googleusercontent-lookalike or normal
|
||||||
|
* publisher hosts are NEVER blocked (must-pass: false positives here
|
||||||
|
* would strip legitimate images)
|
||||||
|
*
|
||||||
|
* Runner: `tsx tests/image-guard.test.ts` -> exit 1 on failure.
|
||||||
|
*/
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
|
||||||
|
import { isPlaceholderImage } from '@/lib/ingest/image-guard';
|
||||||
|
|
||||||
|
let passed = 0;
|
||||||
|
let failed = 0;
|
||||||
|
|
||||||
|
function check(name: string, fn: () => void): void {
|
||||||
|
try {
|
||||||
|
fn();
|
||||||
|
passed += 1;
|
||||||
|
console.log(`ok ${passed} - ${name}`);
|
||||||
|
} catch (e) {
|
||||||
|
failed += 1;
|
||||||
|
console.error(`not ok ${passed + failed} - ${name}`);
|
||||||
|
console.error(String((e as Error).stack ?? e));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The exact shared tile URL stored on 601 production rows (2026-08-18).
|
||||||
|
const TILE =
|
||||||
|
'https://lh3.googleusercontent.com/J6_coFbogxhRI9iM864NL_liGXvsQp2Aups' +
|
||||||
|
'Kei7z0cNNfDvGUmWUy20nuUhkREQyrpY4bEeIBuc=s0-w300';
|
||||||
|
|
||||||
|
check('MUST-BLOCK: exact prod shared tile url', () => {
|
||||||
|
assert.equal(isPlaceholderImage(TILE).blocked, true);
|
||||||
|
});
|
||||||
|
check('MUST-BLOCK: -rw resize variant', () => {
|
||||||
|
assert.equal(isPlaceholderImage(`${TILE}-rw`).blocked, true);
|
||||||
|
});
|
||||||
|
check('MUST-BLOCK: lh4 host rotation', () => {
|
||||||
|
assert.equal(isPlaceholderImage('https://lh4.googleusercontent.com/xyz123=s900').blocked, true);
|
||||||
|
});
|
||||||
|
check('MUST-BLOCK: lh2 host rotation', () => {
|
||||||
|
assert.equal(isPlaceholderImage('https://lh2.googleusercontent.com/xyz123').blocked, true);
|
||||||
|
});
|
||||||
|
check('MUST-BLOCK: uppercase host', () => {
|
||||||
|
assert.equal(isPlaceholderImage('https://LH3.GOOGLEUSERCONTENT.COM/J6_coFbo=s0').blocked, true);
|
||||||
|
});
|
||||||
|
check('MUST-BLOCK: extra query string / fragment', () => {
|
||||||
|
assert.equal(isPlaceholderImage(TILE + '&dummy=1#f').blocked, true);
|
||||||
|
});
|
||||||
|
check('MUST-BLOCK: unknown key (not the known tile) still blocked by host rule', () => {
|
||||||
|
assert.equal(
|
||||||
|
isPlaceholderImage('https://lh3.googleusercontent.com/someOtherKeyWq9aB=s64').blocked,
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
check('MUST-BLOCK: whitespace-wrapped url', () => {
|
||||||
|
assert.equal(isPlaceholderImage(` ${TILE} `).blocked, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
check('MUST-PASS: wired photo (publisher CDN)', () => {
|
||||||
|
assert.equal(
|
||||||
|
isPlaceholderImage('https://media.wired.com/photos/6a84a2c1/191:100/w_1280/c_limit/x.jpg').blocked,
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
check('MUST-PASS: cnbcfm photo (publisher CDN)', () => {
|
||||||
|
assert.equal(isPlaceholderImage('https://media.cnbcfm.com/i/2026/08/robo.jpg').blocked, false);
|
||||||
|
});
|
||||||
|
check('MUST-PASS: google.com logo asset (NOT googleusercontent)', () => {
|
||||||
|
assert.equal(isPlaceholderImage('https://www.google.com/logos/2026/xx512.png').blocked, false);
|
||||||
|
});
|
||||||
|
check('MUST-PASS: example.com path segment merely containing the host string', () => {
|
||||||
|
assert.equal(
|
||||||
|
isPlaceholderImage('https://cdn.example.com/googleusercontent.com/img.png').blocked,
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
check('MUST-PASS: null', () => {
|
||||||
|
assert.equal(isPlaceholderImage(null).blocked, false);
|
||||||
|
});
|
||||||
|
check('MUST-PASS: undefined', () => {
|
||||||
|
assert.equal(isPlaceholderImage(undefined).blocked, false);
|
||||||
|
});
|
||||||
|
check('MUST-PASS: empty string', () => {
|
||||||
|
assert.equal(isPlaceholderImage('').blocked, false);
|
||||||
|
});
|
||||||
|
check('MUST-PASS: not a URL', () => {
|
||||||
|
assert.equal(isPlaceholderImage('not a url').blocked, false);
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log(`\nimage-guard: ${passed} passed, ${failed} failed`);
|
||||||
|
if (failed > 0) process.exit(1);
|
||||||
Reference in New Issue
Block a user