Archived
Compare commits
2
Commits
92373aa3c6
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1e7e951c91 | ||
|
|
86a77b4907 |
+1
-1
@@ -8,7 +8,7 @@
|
||||
"build": "prisma generate && next build",
|
||||
"start": "next start",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"test": "tsx tests/llm-parse.test.ts",
|
||||
"test": "tsx tests/llm-parse.test.ts && tsx tests/image-guard.test.ts && tsx tests/image-proxy.test.ts",
|
||||
"db:generate": "prisma generate",
|
||||
"db:push": "prisma db push",
|
||||
"db:seed": "tsx prisma/seed.ts",
|
||||
|
||||
@@ -4,7 +4,8 @@ import { notFound } from 'next/navigation';
|
||||
import { getArticleFull, getRelatedArticles } from '@/lib/queries';
|
||||
import RelatedArticles from '@/components/RelatedArticles';
|
||||
import { InArticleAd, SidebarAd } from '@/components/ads/placements';
|
||||
import { categoryCover, resolveCoverImage } from '@/lib/cover';
|
||||
import { categoryCover, ogImageUrl, resolveCoverImage } from '@/lib/cover';
|
||||
import { CoverImage } from '@/components/CoverImage';
|
||||
import { env } from '@/lib/env';
|
||||
import { formatFull } from '@/lib/format';
|
||||
import { safeParse } from '@/lib/llm/parse';
|
||||
@@ -58,7 +59,7 @@ export async function generateMetadata({
|
||||
section: sectionLabel,
|
||||
tags: a.tags ? safeParse<string[]>(a.tags).slice(0, 5) : undefined,
|
||||
images: [{
|
||||
url: (a.image && a.image.trim()) || categoryCover(a.category),
|
||||
url: ogImageUrl(a.image, a.category),
|
||||
width: 1200,
|
||||
height: 630,
|
||||
alt: title,
|
||||
@@ -68,7 +69,7 @@ export async function generateMetadata({
|
||||
card: 'summary_large_image',
|
||||
title,
|
||||
description,
|
||||
images: [(a.image && a.image.trim()) || categoryCover(a.category)],
|
||||
images: [ogImageUrl(a.image, a.category)],
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -106,9 +107,11 @@ export default async function ArticlePage({
|
||||
</h1>
|
||||
</header>
|
||||
|
||||
<img
|
||||
<CoverImage
|
||||
src={resolveCoverImage(a.image, a.category)}
|
||||
category={a.category}
|
||||
alt={title}
|
||||
loading="eager"
|
||||
className="mt-6 aspect-[16/8] w-full rounded-2xl object-cover"
|
||||
/>
|
||||
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
import { NextRequest } from 'next/server';
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
|
||||
import {
|
||||
MAX_PROXY_BYTES,
|
||||
MEDIA_DIR,
|
||||
cacheImage,
|
||||
fetchUpstreamImage,
|
||||
mediaKey,
|
||||
sniffImage,
|
||||
} from '@/lib/images';
|
||||
import { isSafeProxyTarget } from '@/lib/image-mapping';
|
||||
|
||||
export const dynamic = 'force-dynamic';
|
||||
const MAX_AGE_1Y = 'public, max-age=31536000, immutable';
|
||||
|
||||
function imageResponse(
|
||||
buf: Buffer,
|
||||
type: string,
|
||||
headers?: Record<string, string>,
|
||||
): Response {
|
||||
return new Response(new Uint8Array(buf), {
|
||||
headers: {
|
||||
'content-type': type,
|
||||
'content-length': String(buf.byteLength),
|
||||
'cache-control': MAX_AGE_1Y,
|
||||
// Browsers sometimes apply hotlink-style referer checks to <img>;
|
||||
// our own copies are always allowed from anywhere.
|
||||
'access-control-allow-origin': '*',
|
||||
vary: 'accept-encoding, referer',
|
||||
...headers,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Server-side image proxy + local cache.
|
||||
*
|
||||
* GET /images?url=<absolute https:// publisher image url>
|
||||
*
|
||||
* 1. If a local copy already exists under /data/media, serve it (no
|
||||
* publisher round-trip — the whole point of the setup).
|
||||
* 2. Otherwise fetch the publisher URL server-side with browser-like
|
||||
* headers, verify the bytes are a real raster image, persist the copy,
|
||||
* and serve it.
|
||||
*
|
||||
* Everything is verified (magic bytes, 5 MiB cap, SSRF guard on the
|
||||
* target AND the final redirect host), so a failing or guarding upstream
|
||||
* yields a short-cached 404/502 — never a broken or spoofed image — and
|
||||
* the UI falls back to the category cover via the CoverImage onerror.
|
||||
*/
|
||||
export async function GET(req: NextRequest): Promise<Response> {
|
||||
const url = req.nextUrl.searchParams.get('url') ?? '';
|
||||
const t0 = Date.now();
|
||||
|
||||
if (!url || !isSafeProxyTarget(url)) {
|
||||
return Response.json({ error: 'invalid image url' }, { status: 400 });
|
||||
}
|
||||
|
||||
const name = mediaKey(url);
|
||||
const file = path.join(MEDIA_DIR, name);
|
||||
|
||||
// 1) Local hit — serve the permanent copy.
|
||||
try {
|
||||
const buf = await readFile(file);
|
||||
const type = sniffImage(buf) ?? 'application/octet-stream';
|
||||
return imageResponse(buf, type, { 'x-image-cache': 'hit' });
|
||||
} catch {
|
||||
// miss → fall through to upstream fetch
|
||||
}
|
||||
|
||||
// 2) Upstream fetch (browser UA defeats referrer/hotlink guards).
|
||||
const got = await fetchUpstreamImage(url);
|
||||
if (!got.ok) {
|
||||
console.warn(
|
||||
`[images] upstream miss ${got.status} (${got.reason}) after ${Date.now() - t0}ms: ${url.slice(0, 120)}`,
|
||||
);
|
||||
// Short cache on failure so we re-probe periodically, but don't hammer.
|
||||
return Response.json(
|
||||
{ error: 'image unavailable' },
|
||||
{ status: got.status, headers: { 'cache-control': 'public, max-age=300' } },
|
||||
);
|
||||
}
|
||||
|
||||
// Await the write so a racing second request cannot double-fetch:
|
||||
// the first request is the only one that pays the upstream cost.
|
||||
await cacheImage(url, got.buf);
|
||||
|
||||
console.log(
|
||||
`[images] cached ${Math.round(got.buf.byteLength / 1024)} KiB from ${new URL(url).hostname} in ${Date.now() - t0}ms`,
|
||||
);
|
||||
return imageResponse(got.buf, got.type, { 'x-image-cache': 'miss' });
|
||||
}
|
||||
+4
-2
@@ -1,6 +1,7 @@
|
||||
import { getPublishedArticles, getTagList } from '@/lib/queries';
|
||||
import ArticleCard from '@/components/ArticleCard';
|
||||
import { resolveCoverImage } from '@/lib/cover';
|
||||
import { CoverImage } from '@/components/CoverImage';
|
||||
import { InFeedAd, SidebarAd } from '@/components/ads/placements';
|
||||
import Link from 'next/link';
|
||||
|
||||
@@ -33,10 +34,11 @@ export default async function HomePage({
|
||||
className="group mb-8 block overflow-hidden rounded-2xl border border-ink-200 bg-white shadow-sm"
|
||||
>
|
||||
<div className="relative aspect-[16/8] overflow-hidden bg-ink-100">
|
||||
{/* cover.ts falls back to a per-category brand cover when no photo was captured */}
|
||||
<img
|
||||
<CoverImage
|
||||
src={resolveCoverImage(hero.image, hero.category)}
|
||||
category={hero.category}
|
||||
alt=""
|
||||
loading="eager"
|
||||
className="h-full w-full object-cover transition duration-500 group-hover:scale-[1.02]"
|
||||
/>
|
||||
<div className="absolute inset-x-0 bottom-0 bg-gradient-to-t from-ink-950/95 via-ink-950/70 to-transparent px-4 pb-4 pt-10 sm:px-5 sm:pb-5 sm:pt-16">
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import Link from 'next/link';
|
||||
import type { ArticleCard as Card } from '@/lib/queries';
|
||||
import { resolveCoverImage } from '@/lib/cover';
|
||||
import { CoverImage } from '@/components/CoverImage';
|
||||
import { formatRelativeTime } from '@/lib/format';
|
||||
|
||||
export default function ArticleCard({ card }: { card: Card }) {
|
||||
@@ -13,10 +14,9 @@ export default function ArticleCard({ card }: { card: Card }) {
|
||||
href={href}
|
||||
className="relative block aspect-[16/9] overflow-hidden bg-ink-100"
|
||||
>
|
||||
<img
|
||||
<CoverImage
|
||||
src={image}
|
||||
alt=""
|
||||
loading="lazy"
|
||||
className="h-full w-full object-cover transition duration-300 group-hover:scale-[1.03]"
|
||||
/>
|
||||
</Link>
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
/**
|
||||
* CoverImage — the single <img> used for every article photo on the site.
|
||||
*
|
||||
* The `src` here is the result of cover.resolveCoverImage: either a local
|
||||
* category cover (/covers/<slug>.svg) or /images?url=... (the server-side
|
||||
* proxy). If the proxy misses (upstream gone, blocked, or still hot-linked
|
||||
* 403 even with browser headers), the browser fires onerror — swap to the
|
||||
* deterministic category cover so cards and heroes never render a broken
|
||||
* image box.
|
||||
*/
|
||||
'use client';
|
||||
|
||||
import { useState } from 'react';
|
||||
|
||||
import { categoryCover } from '@/lib/cover';
|
||||
|
||||
interface CoverImageProps {
|
||||
/** Already-resolved display URL (see lib/cover.ts). */
|
||||
src: string;
|
||||
alt?: string;
|
||||
category?: string | null;
|
||||
className?: string;
|
||||
loading?: 'lazy' | 'eager';
|
||||
}
|
||||
|
||||
export function CoverImage({
|
||||
src,
|
||||
alt = '',
|
||||
category,
|
||||
className,
|
||||
loading = 'lazy',
|
||||
}: CoverImageProps) {
|
||||
const [failed, setFailed] = useState(false);
|
||||
const finalSrc = failed ? categoryCover(category) : src;
|
||||
|
||||
// eslint-disable-next-line @next/next/no-img-element
|
||||
return (
|
||||
<img
|
||||
src={finalSrc}
|
||||
alt={alt}
|
||||
loading={loading}
|
||||
className={className}
|
||||
onError={() => {
|
||||
if (!failed) setFailed(true);
|
||||
}}
|
||||
/>
|
||||
);
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import Link from 'next/link';
|
||||
import type { ArticleCard } from '@/lib/queries';
|
||||
import { resolveCoverImage } from '@/lib/cover';
|
||||
import { CoverImage } from '@/components/CoverImage';
|
||||
import { formatRelativeTime } from '@/lib/format';
|
||||
|
||||
export default function RelatedArticles({ items }: { items: ArticleCard[] }) {
|
||||
@@ -12,10 +13,10 @@ export default function RelatedArticles({ items }: { items: ArticleCard[] }) {
|
||||
{items.map((a) => (
|
||||
<li key={a.id} className="flex gap-3">
|
||||
<div className="h-16 w-24 flex-shrink-0 overflow-hidden rounded-md bg-ink-100">
|
||||
<img
|
||||
<CoverImage
|
||||
src={resolveCoverImage(a.image, a.category)}
|
||||
category={a.category}
|
||||
alt=""
|
||||
loading="lazy"
|
||||
className="h-full w-full object-cover"
|
||||
/>
|
||||
</div>
|
||||
|
||||
+20
-1
@@ -8,6 +8,8 @@
|
||||
* piece — never a bare glyph (2026-08-17 "every article has a photo").
|
||||
*/
|
||||
|
||||
import { absSiteUrl, proxyImageUrl } from '@/lib/image-mapping';
|
||||
|
||||
/** Category slugs that have a generated cover asset. */
|
||||
const COVER_SLUGS = ['top-stories', 'canada', 'national'] as const;
|
||||
type CoverSlug = (typeof COVER_SLUGS)[number];
|
||||
@@ -24,9 +26,26 @@ export function categoryCover(category?: string | null): string {
|
||||
return '/covers/top-stories.svg';
|
||||
}
|
||||
|
||||
/**
|
||||
* Display-image URL for OG/Twitter tags. Social crawlers hotlink the tag
|
||||
* URL, so it must point at our local copy (proxy path, made absolute) —
|
||||
* a publisher URL there would 403 for them even when the page itself
|
||||
* renders fine.
|
||||
*/
|
||||
export function ogImageUrl(
|
||||
image: string | null | undefined,
|
||||
category?: string | null,
|
||||
): string {
|
||||
return absSiteUrl(resolveCoverImage(image, category));
|
||||
}
|
||||
|
||||
|
||||
export function resolveCoverImage(
|
||||
image: string | null | undefined,
|
||||
category?: string | null,
|
||||
): string {
|
||||
return (image ?? '').trim() ? image! : categoryCover(category);
|
||||
const stored = (image ?? '').trim();
|
||||
if (!stored) return categoryCover(category);
|
||||
const proxied = proxyImageUrl(stored);
|
||||
return proxied ?? stored;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
/**
|
||||
* Pure (node-free) URL-mapping helpers for the image proxy.
|
||||
*
|
||||
* Kept separate from lib/images.ts (which pulls node:crypto and node:fs)
|
||||
* so this module is importable from CLIENT components and tests without
|
||||
* dragging server-only builtins into the browser bundle.
|
||||
*
|
||||
* See app/images/route.ts for the server-side half of the pipeline.
|
||||
*/
|
||||
|
||||
import { env } from '@/lib/env';
|
||||
|
||||
/** Absolute origin like https://technews.krisforbes.ca (no trailing slash). */
|
||||
export function siteOrigin(): string {
|
||||
return (env.siteUrl || `http://localhost:${process.env.PORT ?? 3000}`).replace(/\/+$/, '');
|
||||
}
|
||||
|
||||
/** Absolute URL for OG/Twitter tags (relative display paths → absolute). */
|
||||
export function absSiteUrl(path: string): string {
|
||||
if (/^https?:\/\//i.test(path)) return path;
|
||||
return `${siteOrigin()}${path.startsWith('/') ? '' : '/'}${path}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* If the stored image URL is remote, return the locally-proxied display
|
||||
* path (`/images?url=...`); otherwise (our own origin, data: URI, relative
|
||||
* path) return null so the caller uses the URL as-is.
|
||||
*/
|
||||
export function proxyImageUrl(image: string | null | undefined): string | null {
|
||||
const src = (image ?? '').trim();
|
||||
if (!src) return null;
|
||||
if (!/^https?:\/\//i.test(src)) return null; // relative or data: — local
|
||||
try {
|
||||
const u = new URL(src);
|
||||
if (u.origin === siteOrigin()) return null; // our own domain — direct
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
return `/images?url=${encodeURIComponent(src)}`;
|
||||
}
|
||||
|
||||
const LOOPBACK_HOSTNAME =
|
||||
/^(localhost|127\.0\.0\.1|0\.0\.0\.0|\[::1\]|::1|\[::\]|0|)$/;
|
||||
const NO_PUBLIC_TLD = /\.(local|internal|home\.arpa|lan)$/i;
|
||||
|
||||
function isPrivateIPv4(h: string): boolean {
|
||||
return (
|
||||
/^(10\.|192\.168\.|169\.254\.|127\.|0\.)/.test(h) ||
|
||||
/^172\.(1[6-9]|2[0-9]|3[01])\./.test(h)
|
||||
);
|
||||
}
|
||||
|
||||
function isPrivateIPv6(h: string): boolean {
|
||||
const s = h.toLowerCase();
|
||||
if (s === '::' || s === '::1') return true; // unspecified / loopback
|
||||
if (/^fe[89ab]/.test(s)) return true; // fe80::/10 link-local
|
||||
if (/^f[cd]/.test(s)) return true; // fc00::/7 unique local
|
||||
if (s.startsWith('::ffff:')) {
|
||||
// IPv4-mapped: ::ffff:a.b.c.d or ::ffff:HHHH:HHHH — retest as IPv4.
|
||||
const tail = s.slice(7);
|
||||
let v4: string | null = null;
|
||||
if (/^\d{1,3}(\.\d{1,3}){3}$/.test(tail)) {
|
||||
v4 = tail;
|
||||
} else if (/^[0-9a-f]{4}:[0-9a-f]{4}$/.test(tail)) {
|
||||
const [a, b] = tail.split(':').map((x) => parseInt(x, 16));
|
||||
v4 = `${a >> 8}.${a & 0xff}.${b >> 8}.${b & 0xff}`;
|
||||
}
|
||||
return v4 === null ? true : isPrivateIPv4(v4);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** True for loopback / private / link-local hosts (IPv4, IPv6, mapped). */
|
||||
export function isPrivateHost(hostname: string): boolean {
|
||||
const h = hostname.toLowerCase().replace(/^\[|\]$/g, '');
|
||||
return (
|
||||
LOOPBACK_HOSTNAME.test(h) || isPrivateIPv4(h) || isPrivateIPv6(h)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* SSRF guard: only plain public http(s) hosts are proxyable. Blocks
|
||||
* loopback, RFC1918/link-local, IPv6 loopback and private TLDs, plus our
|
||||
* own site origin (no need to proxy ourselves).
|
||||
*/
|
||||
export function isSafeProxyTarget(raw: string): boolean {
|
||||
let u: URL;
|
||||
try {
|
||||
u = new URL(raw);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
if (u.protocol !== 'http:' && u.protocol !== 'https:') return false;
|
||||
const h = u.hostname.toLowerCase().replace(/^\[|\]$/g, '');
|
||||
if (!h || isPrivateHost(h)) return false;
|
||||
if (NO_PUBLIC_TLD.test(h)) return false;
|
||||
if (h === new URL(siteOrigin()).hostname.toLowerCase()) return false;
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
/**
|
||||
* Server-side image proxy helpers (see app/images/route.ts for the HTTP
|
||||
* side and lib/image-mapping.ts for the pure URL-mapping half).
|
||||
*
|
||||
* Articles store the publisher's own CDN URL in Article.image, and the UI
|
||||
* used to hotlink it directly. Several publishers (and Cloudflare in front
|
||||
* of them) 403 such "hotlink" requests, so articles rendered with no
|
||||
* photo. The UI now renders any remote image through our own
|
||||
* `GET /images?url=...` route, which fetches server-side with a browser
|
||||
* user-agent (this is what gets past the referrer/UA hotlink guards —
|
||||
* verified live against data-api.investing.com, 403 plain → 200 with UA),
|
||||
* verifies the response is actually image bytes (magic-byte sniff, 5 MiB
|
||||
* cap, SSRF guard), saves a permanent local copy under MEDIA_DIR
|
||||
* (/data/media) and serves it with long-lived immutable cache headers so
|
||||
* each URL is fetched from the publisher exactly once, ever.
|
||||
*
|
||||
* SERVER-ONLY module — node:crypto/node:fs. Client code must import
|
||||
* lib/image-mapping instead.
|
||||
*/
|
||||
|
||||
import { createHash } from 'node:crypto';
|
||||
import path from 'node:path';
|
||||
import { mkdir, rename, writeFile } from 'node:fs/promises';
|
||||
|
||||
import { isPrivateHost, siteOrigin } from '@/lib/image-mapping';
|
||||
|
||||
/** Where lazily-downloaded image copies live (volume-backed, app-writable). */
|
||||
export const MEDIA_DIR = process.env.MEDIA_DIR ?? '/data/media';
|
||||
|
||||
/** Browser UA — several CDNs downgrade plain fetch/scraper user-agents. */
|
||||
export const PROXY_USER_AGENT =
|
||||
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36';
|
||||
|
||||
/** Same accept header Chrome sends for <img> — gets the optimized format. */
|
||||
const PROXY_ACCEPT =
|
||||
'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8';
|
||||
|
||||
/** Hard cap for a proxied image body (5 MiB). */
|
||||
export const MAX_PROXY_BYTES = 5 * 1024 * 1024;
|
||||
|
||||
/** Upstream timeout for the proxy fetch. */
|
||||
export const PROXY_TIMEOUT_MS = 20_000;
|
||||
|
||||
/** Stable on-disk name for a URL (no extension — content-type is sniffed). */
|
||||
export function mediaKey(url: string): string {
|
||||
return createHash('sha256').update(url).digest('hex').slice(0, 40);
|
||||
}
|
||||
|
||||
/**
|
||||
* Magic-byte sniff → IANA Content-Type, or null when the buffer is not a
|
||||
* known raster image format. (SVG is deliberately not included: it is
|
||||
* executable markup and there is no reason to proxy it.) Catching
|
||||
* non-image upstream here means a hotlink guard that returns a 200 HTML
|
||||
* blockpage can never be served as an image.
|
||||
*/
|
||||
export function sniffImage(buf: Buffer): string | null {
|
||||
if (buf.length >= 3 && buf[0] === 0xff && buf[1] === 0xd8) return 'image/jpeg';
|
||||
if (
|
||||
buf.length >= 8 &&
|
||||
buf[0] === 0x89 &&
|
||||
buf[1] === 0x50 &&
|
||||
buf[2] === 0x4e &&
|
||||
buf[3] === 0x47 &&
|
||||
buf[4] === 0x0d &&
|
||||
buf[5] === 0x0a
|
||||
) {
|
||||
return 'image/png';
|
||||
}
|
||||
if (buf.length >= 6 && buf.toString('ascii', 0, 3) === 'GIF') return 'image/gif';
|
||||
if (
|
||||
buf.length >= 12 &&
|
||||
buf.toString('ascii', 0, 4) === 'RIFF' &&
|
||||
buf.toString('ascii', 8, 12) === 'WEBP'
|
||||
) {
|
||||
return 'image/webp';
|
||||
}
|
||||
if (buf.length >= 12 && buf.toString('ascii', 4, 8) === 'ftyp') {
|
||||
const brand = buf.toString('ascii', 8, 12).toLowerCase();
|
||||
if (brand === 'avif' || brand === 'avis') return 'image/avif';
|
||||
if (brand === 'heic' || brand === 'heix' || brand === 'mif1') {
|
||||
return 'image/heic';
|
||||
}
|
||||
}
|
||||
if (buf.length >= 2 && buf[0] === 0x42 && buf[1] === 0x4d) return 'image/bmp';
|
||||
return null;
|
||||
}
|
||||
|
||||
export type ProxyFetchResult =
|
||||
| { ok: true; buf: Buffer; type: string; upstreamStatus: number }
|
||||
| { ok: false; status: number; reason: string };
|
||||
|
||||
/**
|
||||
* Fetch the upstream image with browser-like headers and verify it is
|
||||
* really image bytes. Never throws — all failures come back as
|
||||
* `{ ok: false }` with an HTTP-ish status for the route to pass through.
|
||||
*/
|
||||
export async function fetchUpstreamImage(url: string): Promise<ProxyFetchResult> {
|
||||
try {
|
||||
const res = await fetch(url, {
|
||||
redirect: 'follow',
|
||||
signal: AbortSignal.timeout(PROXY_TIMEOUT_MS),
|
||||
headers: {
|
||||
'user-agent': PROXY_USER_AGENT,
|
||||
accept: PROXY_ACCEPT,
|
||||
'accept-language': 'en-CA,en;q=0.9',
|
||||
referer: `${siteOrigin()}/articles`,
|
||||
},
|
||||
});
|
||||
if (!res.ok) {
|
||||
return { ok: false, status: res.status >= 500 ? 502 : res.status, reason: `upstream ${res.status}` };
|
||||
}
|
||||
const buf = Buffer.from(await res.arrayBuffer());
|
||||
if (buf.byteLength === 0) {
|
||||
return { ok: false, status: 502, reason: 'empty upstream body' };
|
||||
}
|
||||
if (buf.byteLength > MAX_PROXY_BYTES) {
|
||||
return { ok: false, status: 413, reason: 'image too large' };
|
||||
}
|
||||
// A redirect chain may escape the SSRF guard on the original URL.
|
||||
const finalHost = new URL(res.url).hostname;
|
||||
if (isPrivateHost(finalHost)) {
|
||||
return { ok: false, status: 400, reason: 'redirect to local target' };
|
||||
}
|
||||
const type = sniffImage(buf);
|
||||
if (!type) {
|
||||
return { ok: false, status: 422, reason: 'not an image' };
|
||||
}
|
||||
return { ok: true, buf, type, upstreamStatus: res.status };
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
if (/timeout|aborted/i.test(msg)) {
|
||||
return { ok: false, status: 504, reason: 'upstream timeout' };
|
||||
}
|
||||
return { ok: false, status: 502, reason: msg };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Best-effort durable cache: write atomically (tmp + rename) so readers
|
||||
* never observe a partial file. Returns the final path on success.
|
||||
*/
|
||||
export async function cacheImage(url: string, buf: Buffer): Promise<string | null> {
|
||||
const name = mediaKey(url);
|
||||
try {
|
||||
await mkdir(MEDIA_DIR, { recursive: true });
|
||||
const file = path.join(MEDIA_DIR, name);
|
||||
const tmp = `${file}.tmp${process.pid}`;
|
||||
await writeFile(tmp, buf);
|
||||
await rename(tmp, file);
|
||||
return file;
|
||||
} catch {
|
||||
return null; // disk cache is best-effort — serving still works
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
/**
|
||||
* Image placeholder guard.
|
||||
*
|
||||
* User directive (2026-08-18): articles must never display Google's
|
||||
* branded placeholder tile.
|
||||
*
|
||||
* Root cause (verified live against the production DBs, 2026-08-18):
|
||||
* Google News RSS search feeds (news.google.com/rss/search?q=...) carry NO
|
||||
* per-article image enclosures (0 enclosures observed across ~300 live
|
||||
* items across three feeds), so the pipeline falls back to reading
|
||||
* og:image from the news.google.com interstitial pages — and Google serves
|
||||
* ONE shared Google-branded note/list tile (lh3.googleusercontent.com/
|
||||
* J6_coFbogxh...) as og:image for every single interstitial. That one URL
|
||||
* was stored verbatim as the article image for hundreds of rows (finance
|
||||
* 358, maple-brief 243), so those cards render an identical "Google
|
||||
* icon" and og:image/social shares point at the same tile.
|
||||
*
|
||||
* In this pipeline the only producer of a *.googleusercontent.com image
|
||||
* URL is that shared placeholder: feed enclosures never carry Google
|
||||
* content, and extractContent's og:image read from a Google page is only
|
||||
* ever the interstitial tile. Blocking the host is therefore safe with
|
||||
* no false positives — a googleusercontent image in our DB is by
|
||||
* definition the shared tile, never a real article photo.
|
||||
*
|
||||
* Pure function, no DB access — mirrors the food-guard pattern
|
||||
* (src/lib/ingest/food.ts).
|
||||
*/
|
||||
|
||||
const GOOGLE_CONTENT_HOSTS = /(^|\.)googleusercontent\.com$/i;
|
||||
|
||||
export interface PlaceholderMatch {
|
||||
blocked: boolean;
|
||||
/** the offending host, for logging */
|
||||
reason: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Decide whether a candidate article image is a shared placeholder tile
|
||||
* instead of a real article photo.
|
||||
* @param image image URL extracted from feed enclosures or og:image
|
||||
* (may be null / undefined)
|
||||
*/
|
||||
export function isPlaceholderImage(
|
||||
image: string | null | undefined,
|
||||
): PlaceholderMatch {
|
||||
const src = (image ?? '').trim();
|
||||
if (!src) return { blocked: false, reason: '' };
|
||||
let host = '';
|
||||
try {
|
||||
host = new URL(src).hostname.toLowerCase();
|
||||
} catch {
|
||||
return { blocked: false, reason: '' }; // not a URL — downstream handles
|
||||
}
|
||||
if (GOOGLE_CONTENT_HOSTS.test(host)) {
|
||||
return { blocked: true, reason: host };
|
||||
}
|
||||
return { blocked: false, reason: '' };
|
||||
}
|
||||
@@ -2,12 +2,14 @@ import { prisma } from '@/lib/db';
|
||||
import { env } from '@/lib/env';
|
||||
import { dedupeKeyOf, slugify } from '@/lib/slug';
|
||||
import { parseFeed, extractContent, type FeedItem } from './feed';
|
||||
import { isPlaceholderImage } from './image-guard';
|
||||
|
||||
export interface IngestResult {
|
||||
feed: string;
|
||||
fetched: number;
|
||||
newArticles: number;
|
||||
duplicates: number;
|
||||
imageFiltered: number;
|
||||
errors: string[];
|
||||
}
|
||||
|
||||
@@ -34,6 +36,7 @@ export async function ingestFeed(
|
||||
fetched: 0,
|
||||
newArticles: 0,
|
||||
duplicates: 0,
|
||||
imageFiltered: 0,
|
||||
errors: [],
|
||||
};
|
||||
|
||||
@@ -97,6 +100,17 @@ export async function ingestFeed(
|
||||
if (!image && ex.image) image = ex.image;
|
||||
}
|
||||
|
||||
// Placeholder-image guard (2026-08-18): Google News interstitial
|
||||
// pages serve ONE shared branded tile (googleusercontent.com) as
|
||||
// og:image for every item. Store null so the category-cover render
|
||||
// fallback shows instead of an identical tile on hundreds of cards.
|
||||
const imageGuard = isPlaceholderImage(image);
|
||||
if (imageGuard.blocked) {
|
||||
result.imageFiltered += 1;
|
||||
console.log(`[ingest] placeholder image suppressed (${imageGuard.reason}): ${item.title.slice(0, 70)}`);
|
||||
image = undefined;
|
||||
}
|
||||
|
||||
const baseSlug = slugify(item.title);
|
||||
const slug = await uniqueSlug(baseSlug);
|
||||
const sourceUrl = item.link;
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
/**
|
||||
* Placeholder-image guard tests (offline — no network).
|
||||
*
|
||||
* Contract under test (src/lib/ingest/image-guard.ts):
|
||||
* - the ONE shared Google-branded tile observed in production (exact URL
|
||||
* from the 2026-08-18 finance/maple cleanup, lh3.googleusercontent.com
|
||||
* /J6_coFbogxh...) is ALWAYS blocked, in every variant (resize suffixes,
|
||||
* host rotation lh2/lh3/lh4, case, extra query strings)
|
||||
* - real publisher article photos on googleusercontent-lookalike or normal
|
||||
* publisher hosts are NEVER blocked (must-pass: false positives here
|
||||
* would strip legitimate images)
|
||||
*
|
||||
* Runner: `tsx tests/image-guard.test.ts` -> exit 1 on failure.
|
||||
*/
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
import { isPlaceholderImage } from '@/lib/ingest/image-guard';
|
||||
|
||||
let passed = 0;
|
||||
let failed = 0;
|
||||
|
||||
function check(name: string, fn: () => void): void {
|
||||
try {
|
||||
fn();
|
||||
passed += 1;
|
||||
console.log(`ok ${passed} - ${name}`);
|
||||
} catch (e) {
|
||||
failed += 1;
|
||||
console.error(`not ok ${passed + failed} - ${name}`);
|
||||
console.error(String((e as Error).stack ?? e));
|
||||
}
|
||||
}
|
||||
|
||||
// The exact shared tile URL stored on 601 production rows (2026-08-18).
|
||||
const TILE =
|
||||
'https://lh3.googleusercontent.com/J6_coFbogxhRI9iM864NL_liGXvsQp2Aups' +
|
||||
'Kei7z0cNNfDvGUmWUy20nuUhkREQyrpY4bEeIBuc=s0-w300';
|
||||
|
||||
check('MUST-BLOCK: exact prod shared tile url', () => {
|
||||
assert.equal(isPlaceholderImage(TILE).blocked, true);
|
||||
});
|
||||
check('MUST-BLOCK: -rw resize variant', () => {
|
||||
assert.equal(isPlaceholderImage(`${TILE}-rw`).blocked, true);
|
||||
});
|
||||
check('MUST-BLOCK: lh4 host rotation', () => {
|
||||
assert.equal(isPlaceholderImage('https://lh4.googleusercontent.com/xyz123=s900').blocked, true);
|
||||
});
|
||||
check('MUST-BLOCK: lh2 host rotation', () => {
|
||||
assert.equal(isPlaceholderImage('https://lh2.googleusercontent.com/xyz123').blocked, true);
|
||||
});
|
||||
check('MUST-BLOCK: uppercase host', () => {
|
||||
assert.equal(isPlaceholderImage('https://LH3.GOOGLEUSERCONTENT.COM/J6_coFbo=s0').blocked, true);
|
||||
});
|
||||
check('MUST-BLOCK: extra query string / fragment', () => {
|
||||
assert.equal(isPlaceholderImage(TILE + '&dummy=1#f').blocked, true);
|
||||
});
|
||||
check('MUST-BLOCK: unknown key (not the known tile) still blocked by host rule', () => {
|
||||
assert.equal(
|
||||
isPlaceholderImage('https://lh3.googleusercontent.com/someOtherKeyWq9aB=s64').blocked,
|
||||
true,
|
||||
);
|
||||
});
|
||||
check('MUST-BLOCK: whitespace-wrapped url', () => {
|
||||
assert.equal(isPlaceholderImage(` ${TILE} `).blocked, true);
|
||||
});
|
||||
|
||||
check('MUST-PASS: wired photo (publisher CDN)', () => {
|
||||
assert.equal(
|
||||
isPlaceholderImage('https://media.wired.com/photos/6a84a2c1/191:100/w_1280/c_limit/x.jpg').blocked,
|
||||
false,
|
||||
);
|
||||
});
|
||||
check('MUST-PASS: cnbcfm photo (publisher CDN)', () => {
|
||||
assert.equal(isPlaceholderImage('https://media.cnbcfm.com/i/2026/08/robo.jpg').blocked, false);
|
||||
});
|
||||
check('MUST-PASS: google.com logo asset (NOT googleusercontent)', () => {
|
||||
assert.equal(isPlaceholderImage('https://www.google.com/logos/2026/xx512.png').blocked, false);
|
||||
});
|
||||
check('MUST-PASS: example.com path segment merely containing the host string', () => {
|
||||
assert.equal(
|
||||
isPlaceholderImage('https://cdn.example.com/googleusercontent.com/img.png').blocked,
|
||||
false,
|
||||
);
|
||||
});
|
||||
check('MUST-PASS: null', () => {
|
||||
assert.equal(isPlaceholderImage(null).blocked, false);
|
||||
});
|
||||
check('MUST-PASS: undefined', () => {
|
||||
assert.equal(isPlaceholderImage(undefined).blocked, false);
|
||||
});
|
||||
check('MUST-PASS: empty string', () => {
|
||||
assert.equal(isPlaceholderImage('').blocked, false);
|
||||
});
|
||||
check('MUST-PASS: not a URL', () => {
|
||||
assert.equal(isPlaceholderImage('not a url').blocked, false);
|
||||
});
|
||||
|
||||
console.log(`\nimage-guard: ${passed} passed, ${failed} failed`);
|
||||
if (failed > 0) process.exit(1);
|
||||
@@ -0,0 +1,143 @@
|
||||
/**
|
||||
* Image-proxy helper tests (offline — no network).
|
||||
*
|
||||
* Contract under test:
|
||||
* - sniffImage: magic-byte → Content-Type for jpeg/png/gif/webp/avif/heic/bmp,
|
||||
* null for HTML blockpages and junk (a hotlink guard returning 200 HTML
|
||||
* must never be served as an image).
|
||||
* - isSafeProxyTarget: public http(s) only — no loopback, RFC1918, link-local,
|
||||
* IPv6 private, or non-web schemes; our own site origin rejected.
|
||||
* - proxyImageUrl: remote http(s) → /images?url=... ; relative / data: /
|
||||
* our-own-origin URLs pass through untouched.
|
||||
*
|
||||
* Runner: `tsx tests/image-proxy.test.ts` → TAP-style, exit 1 on failure.
|
||||
*/
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
process.env.NEXT_PUBLIC_SITE_URL = 'https://technews.krisforbes.ca';
|
||||
|
||||
import {
|
||||
isSafeProxyTarget,
|
||||
proxyImageUrl,
|
||||
isPrivateHost,
|
||||
} from '@/lib/image-mapping';
|
||||
import { mediaKey, sniffImage } from '@/lib/images';
|
||||
|
||||
let passed = 0;
|
||||
let failed = 0;
|
||||
|
||||
function check(name: string, fn: () => void): void {
|
||||
try {
|
||||
fn();
|
||||
passed += 1;
|
||||
console.log(`ok ${passed + failed} ${name}`);
|
||||
} catch (err) {
|
||||
failed += 1;
|
||||
console.log(`not ok ${passed + failed} ${name}`);
|
||||
console.log(` ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
const b = (b64: string) => Buffer.from(b64, 'base64');
|
||||
// Minimal valid magic headers (payload irrelevant to the sniff).
|
||||
const JPEG = b(
|
||||
'/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoKBwYKDwMNDhgMEggRCwUNDAwTFBMSFBQUFxQVFRUUgAFMAAQHBgMCAwYHBgcKEA0HCAkKDw0NDhERCg0RHREKCA8VEg0RERoNDAwQGiYNDg8VIRUQNBMfISEYGRM0KhwjGhs0MioaIxwkIhgY',
|
||||
);
|
||||
const PNG = b('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACklEQVR4nGMYAQCABQAB6V282gAAAABJRU5ErkJggg==');
|
||||
const WEBP = b('UklGRlQAAABXRUJQVlA4IBoAAAAwAQCdASoBAAEAAUAmJaQAA3AA/vuUAAA=');
|
||||
const AVIF = Buffer.concat([Buffer.alloc(4), Buffer.from('ftypavif', 'ascii')]);
|
||||
const GIF = b('R0lGODlhAQABAIAAAP///wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw==');
|
||||
const HTML = Buffer.from('<!doctype html><html><body><h1>Access Denied</h1><p>The referenced entity does not exist</p>');
|
||||
|
||||
check('sniff: jpeg magic → image/jpeg', () => {
|
||||
assert.equal(sniffImage(JPEG), 'image/jpeg');
|
||||
});
|
||||
check('sniff: png magic → image/png', () => {
|
||||
assert.equal(sniffImage(PNG), 'image/png');
|
||||
});
|
||||
check('sniff: webp magic → image/webp', () => {
|
||||
assert.equal(sniffImage(WEBP), 'image/webp');
|
||||
});
|
||||
check('sniff: avif ftyp → image/avif', () => {
|
||||
assert.equal(sniffImage(AVIF), 'image/avif');
|
||||
});
|
||||
check('sniff: gif magic → image/gif', () => {
|
||||
assert.equal(sniffImage(GIF), 'image/gif');
|
||||
});
|
||||
check('sniff: HTML blockpage → null', () => {
|
||||
assert.equal(sniffImage(HTML), null);
|
||||
});
|
||||
check('sniff: short/garbage buffer → null', () => {
|
||||
assert.equal(sniffImage(Buffer.from('ab')), null);
|
||||
});
|
||||
|
||||
check('ssrf: public https ok', () => {
|
||||
assert.ok(isSafeProxyTarget('https://ichef.bbci.co.uk/ace/branded_news/x.jpg'));
|
||||
});
|
||||
check('ssrf: public http ok', () => {
|
||||
assert.ok(isSafeProxyTarget('http://example.com/a.webp'));
|
||||
});
|
||||
check('ssrf: localhost blocked', () => {
|
||||
assert.ok(!isSafeProxyTarget('http://localhost:3000/images?url=x'));
|
||||
});
|
||||
check('ssrf: 127/10/192.168/169.254 blocked', () => {
|
||||
assert.ok(!isSafeProxyTarget('http://127.0.0.1/a'));
|
||||
assert.ok(!isSafeProxyTarget('http://10.0.0.5/a'));
|
||||
assert.ok(!isSafeProxyTarget('http://192.168.1.2/a'));
|
||||
assert.ok(!isSafeProxyTarget('http://169.254.169.254/latest'));
|
||||
});
|
||||
check('ssrf: 172.16-31 blocked, 172.15/172.32 ok', () => {
|
||||
assert.ok(!isSafeProxyTarget('http://172.17.0.1:11434/'));
|
||||
assert.ok(!isSafeProxyTarget('http://172.16.0.9/'));
|
||||
assert.ok(!isSafeProxyTarget('http://172.31.9.9/'));
|
||||
assert.ok(isSafeProxyTarget('http://172.15.255.1/')); // below private range
|
||||
assert.ok(isSafeProxyTarget('http://172.32.0.1/')); // above private range
|
||||
});
|
||||
check('ssrf: ipv6 loopback/link-local/ULA blocked, mapped private blocked', () => {
|
||||
assert.ok(!isSafeProxyTarget('http://[::1]/a'));
|
||||
assert.ok(!isSafeProxyTarget('http://[fe80::1]/a'));
|
||||
assert.ok(!isSafeProxyTarget('http://[fd00::1]/a'));
|
||||
assert.ok(!isSafeProxyTarget('http://[::ffff:127.0.0.1]/a'));
|
||||
assert.ok(!isSafeProxyTarget('http://[::ffff:192.168.1.2]/a'));
|
||||
assert.ok(!isSafeProxyTarget('http://[::]/a'));
|
||||
});
|
||||
check('ssrf: non-web schemes blocked', () => {
|
||||
assert.ok(!isSafeProxyTarget('file:///etc/passwd'));
|
||||
assert.ok(!isSafeProxyTarget('gopher://example.com'));
|
||||
assert.ok(!isSafeProxyTarget('blob:https://x.com/abc'));
|
||||
});
|
||||
check('ssrf: our own origin rejected (no self-proxy)', () => {
|
||||
assert.ok(!isSafeProxyTarget('https://technews.krisforbes.ca/article/x'));
|
||||
});
|
||||
check('ssrf: unparseable url → false', () => {
|
||||
assert.ok(!isSafeProxyTarget('not a url'));
|
||||
});
|
||||
|
||||
check('proxy: remote url → /images?url=...', () => {
|
||||
const u = 'https://data-api.investing.com/trkd-images/abc.jpg';
|
||||
assert.equal(proxyImageUrl(u), `/images?url=${encodeURIComponent(u)}`);
|
||||
});
|
||||
check('proxy: relative path passes through', () => {
|
||||
assert.equal(proxyImageUrl('/covers/tech.svg'), null);
|
||||
});
|
||||
check('proxy: data URI passes through', () => {
|
||||
assert.equal(proxyImageUrl('data:image/png;base64,AAA='), null);
|
||||
});
|
||||
check('proxy: own-origin url passes through', () => {
|
||||
assert.equal(proxyImageUrl('https://technews.krisforbes.ca/logo.png'), null);
|
||||
});
|
||||
check('proxy: empty/blank → null', () => {
|
||||
assert.equal(proxyImageUrl(null), null);
|
||||
assert.equal(proxyImageUrl(' '), null);
|
||||
});
|
||||
check('key: stable + 40 hex chars', () => {
|
||||
const k1 = mediaKey('https://x.com/a.jpg');
|
||||
const k2 = mediaKey('https://x.com/a.jpg');
|
||||
assert.equal(k1, k2);
|
||||
assert.match(k1, /^[0-9a-f]{40}$/);
|
||||
assert.notEqual(k1, mediaKey('https://x.com/b.jpg'));
|
||||
});
|
||||
|
||||
const total = passed + failed;
|
||||
console.log(`\n# tests ${total}, pass ${passed}, fail ${failed}`);
|
||||
process.exit(failed === 0 ? 0 : 1);
|
||||
Reference in New Issue
Block a user