Archived
Compare commits
13
Commits
cbea5e8fa4
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1e7e951c91 | ||
|
|
86a77b4907 | ||
|
|
92373aa3c6 | ||
|
|
08b9deb3d5 | ||
|
|
44d3220c4d | ||
|
|
7a19afb63a | ||
|
|
b5a25c1232 | ||
|
|
b79c439c7f | ||
|
|
69f84e0bc6 | ||
|
|
d1bf2962ff | ||
|
|
478ff07bc5 | ||
|
|
9ebc3bffab | ||
|
|
9210dc8f14 |
+2
-2
@@ -38,7 +38,7 @@ yarn-error.log*
|
|||||||
prisma/dev.db
|
prisma/dev.db
|
||||||
prisma/dev.db-journal
|
prisma/dev.db-journal
|
||||||
|
|
||||||
# local databases / generated
|
# local databases / generated (root only — src/data/ holds committed source)
|
||||||
data/
|
/data/
|
||||||
*.sqlite
|
*.sqlite
|
||||||
*.sqlite3
|
*.sqlite3
|
||||||
|
|||||||
@@ -0,0 +1,29 @@
|
|||||||
|
# ────────────────────────────────────────────────────────────────────────────
|
||||||
|
# gitleaks config — maple-brief commit gate (secret stage)
|
||||||
|
#
|
||||||
|
# Policy (fail-closed): the FULL default gitleaks ruleset (every known
|
||||||
|
# provider / secret shape) hard-blocks at commit and pre-push. The ONLY
|
||||||
|
# suppression is a tight, content-regexed allowlist scoped to a single file:
|
||||||
|
# .env.example (the committed placeholder template). It suppresses ONLY those
|
||||||
|
# exact placeholder lines. A real composite key pasted there (or anywhere)
|
||||||
|
# STILL fires — verified on gitleaks 8.30.1 (linux x64).
|
||||||
|
#
|
||||||
|
# Gotchas found while validating (do not regress):
|
||||||
|
# • allowlist must be a MAP [allowlist] — a slice [[allowlist]] fails
|
||||||
|
# with "expected a map, got slice"
|
||||||
|
# • an embedded single quote inside a single-quoted (literal) TOML string
|
||||||
|
# silently closes it — keep regex strings double-quoted (basic)
|
||||||
|
# • paths= scopes the suppression to the file; regexes= to the lines
|
||||||
|
# ────────────────────────────────────────────────────────────────────────────
|
||||||
|
[extend]
|
||||||
|
useDefault = true
|
||||||
|
|
||||||
|
[allowlist]
|
||||||
|
description = "Suppress ONLY empty double-quoted placeholder values and the change-me example, in the .env template"
|
||||||
|
paths = ['\.env\.example$']
|
||||||
|
regexes = [
|
||||||
|
'^OPENAI_API_KEY=("")$',
|
||||||
|
'^ANTHROPIC_API_KEY=("")$',
|
||||||
|
'^NEXT_PUBLIC_ADSENSE_[A-Z_]+=("")$',
|
||||||
|
"^ADMIN_API_KEY=\"change-me[^\"]*\"$"
|
||||||
|
]
|
||||||
+1
-1
@@ -36,7 +36,7 @@ RUN npx prisma generate \
|
|||||||
&& cp /app/fresh.db prisma/fresh.db \
|
&& cp /app/fresh.db prisma/fresh.db \
|
||||||
&& NEXT_PUBLIC_SITE_URL="https://news.krisforbes.ca" \
|
&& NEXT_PUBLIC_SITE_URL="https://news.krisforbes.ca" \
|
||||||
NEXT_PUBLIC_SITE_NAME="MapleBrief" \
|
NEXT_PUBLIC_SITE_NAME="MapleBrief" \
|
||||||
NEXT_PUBLIC_SITE_DESCRIPTION="Synthesized Canadian news briefings" \
|
NEXT_PUBLIC_SITE_DESCRIPTION="Canadian news briefings" \
|
||||||
NEXT_PUBLIC_ADSENSE_CLIENT_ID="${NEXT_PUBLIC_ADSENSE_CLIENT_ID}" \
|
NEXT_PUBLIC_ADSENSE_CLIENT_ID="${NEXT_PUBLIC_ADSENSE_CLIENT_ID}" \
|
||||||
NEXT_PUBLIC_SHOW_SIDEBAR_ADS="false" \
|
NEXT_PUBLIC_SHOW_SIDEBAR_ADS="false" \
|
||||||
npm run build \
|
npm run build \
|
||||||
|
|||||||
@@ -66,6 +66,35 @@ npx prisma migrate diff --from-empty --to-schema-datamodel prisma/schema.prisma
|
|||||||
npm run seed # manual ingest + synthesis pass
|
npm run seed # manual ingest + synthesis pass
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Commit gate (secret → build → test, fail-closed)
|
||||||
|
|
||||||
|
Every commit and push runs a three-stage gate (`scripts/gate/gate.sh`), in this fixed order:
|
||||||
|
|
||||||
|
| Stage | pre-commit (fast) | pre-push (heavy) |
|
||||||
|
|---|---|---|
|
||||||
|
| 1 — secret | `gitleaks protect --staged` | `gitleaks protect` over full HEAD history |
|
||||||
|
| 2 — build | `tsc --noEmit` (strict) | `prisma generate && next build` (pristine production build) |
|
||||||
|
| 3 — test | `tests/llm-parse.test.ts` (LLM parse contract, 21 pins) | same |
|
||||||
|
|
||||||
|
- **Fail-closed:** a missing tool, a crash, a timeout, or any finding blocks the
|
||||||
|
operation. Only a deliberate `git commit --no-verify` / `git push --no-verify`
|
||||||
|
bypasses it (auditable in the terminal scrollback).
|
||||||
|
- **Secret stage:** full gitleaks default ruleset, hard-blocks. The one allow
|
||||||
|
exception is a file- and content-scoped suppress of *empty/placeholder*
|
||||||
|
values in `.env.example` (see `[allowlist]` in `.gitleaks.toml`) — a real key
|
||||||
|
written into that file still fires.
|
||||||
|
- **Test stage:** `tests/llm-parse.test.ts` pins the synthesis parse contract
|
||||||
|
(6-paragraph cap, 160-char headline, ≤5 takeaways, sanitized tags,
|
||||||
|
JSON-fence recovery, determinism). It imports only `src/lib/llm/parse.ts` —
|
||||||
|
pure, zero-dep, zero-network — so it runs under `tsx` with no framework.
|
||||||
|
- **Install (idempotent, per clone):**
|
||||||
|
```bash
|
||||||
|
bash scripts/install/install-hooks.sh
|
||||||
|
# → bootstraps gitleaks (pinned 8.30.1) into ~/.local/bin if missing,
|
||||||
|
# copies scripts/gate/hooks/* into .git/hooks/
|
||||||
|
```
|
||||||
|
- Walk away from a fresh machine: `npm ci && bash scripts/install/install-hooks.sh`.
|
||||||
|
|
||||||
## Configuration
|
## Configuration
|
||||||
|
|
||||||
All configuration is in `.env` (local, gitignored — only `.env.example` is committed):
|
All configuration is in `.env` (local, gitignored — only `.env.example` is committed):
|
||||||
|
|||||||
@@ -24,4 +24,21 @@ if [ ! -f "$DB" ]; then
|
|||||||
echo "[entrypoint] initialized $DB from seed snapshot"
|
echo "[entrypoint] initialized $DB from seed snapshot"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Arm the ingest/synthesis worker once the server answers.
|
||||||
|
# The Next standalone build has instrumentationHook disabled, so node-cron
|
||||||
|
# only starts when a request reaches /api/worker/ping. Pinging ourselves
|
||||||
|
# after boot means a fresh container or restart re-arms the worker (and
|
||||||
|
# triggers the boot pipeline pass) without any external health pinger.
|
||||||
|
(
|
||||||
|
APP_IP="$(hostname -i 2>/dev/null | awk '{print $1}')"
|
||||||
|
[ -z "$APP_IP" ] && APP_IP=127.0.0.1
|
||||||
|
i=0
|
||||||
|
until curl -sf -o /dev/null --max-time 2 "http://${APP_IP}:${PORT:-3000}/api/worker/ping"; do
|
||||||
|
i=$((i + 1))
|
||||||
|
[ "$i" -ge 90 ] && exit 0
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
echo "[entrypoint] armed cron worker via /api/worker/ping"
|
||||||
|
) >/dev/null 2>&1 &
|
||||||
|
|
||||||
exec node server.js
|
exec node server.js
|
||||||
|
|||||||
+2
-1
@@ -2,12 +2,13 @@
|
|||||||
"name": "maple-brief-aggregator",
|
"name": "maple-brief-aggregator",
|
||||||
"version": "1.0.0",
|
"version": "1.0.0",
|
||||||
"private": true,
|
"private": true,
|
||||||
"description": "Automated Canadian news aggregator with LLM content synthesis and Google AdSense integration.",
|
"description": "Automated Canadian news aggregator with AI-assisted content synthesis and Google AdSense integration.",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"dev": "next dev",
|
"dev": "next dev",
|
||||||
"build": "prisma generate && next build",
|
"build": "prisma generate && next build",
|
||||||
"start": "next start",
|
"start": "next start",
|
||||||
"typecheck": "tsc --noEmit",
|
"typecheck": "tsc --noEmit",
|
||||||
|
"test": "tsx tests/llm-parse.test.ts && tsx tests/image-guard.test.ts && tsx tests/image-proxy.test.ts",
|
||||||
"db:generate": "prisma generate",
|
"db:generate": "prisma generate",
|
||||||
"db:push": "prisma db push",
|
"db:push": "prisma db push",
|
||||||
"db:seed": "tsx prisma/seed.ts",
|
"db:seed": "tsx prisma/seed.ts",
|
||||||
|
|||||||
@@ -0,0 +1 @@
|
|||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1200 630" role="img" aria-label="Canada — MapleBrief"><defs><linearGradient id="bg" x1="0" y1="0" x2="0" y2="1"><stop offset="0" stop-color="#161614"/><stop offset="0.55" stop-color="#2b2b28"/><stop offset="1.3" stop-color="#45060c"/></linearGradient><pattern id="grid" width="48" height="48" patternUnits="userSpaceOnUse"><path d="M48 0H0V48" fill="none" stroke="#f8402f" stroke-opacity="0.10" stroke-width="1"/></pattern></defs><rect width="1200" height="630" fill="url(#bg)"/><rect width="1200" height="630" fill="url(#grid)"/><g fill="none" stroke="#f8402f" stroke-opacity="0.38" stroke-width="10" stroke-linecap="round" stroke-linejoin="round"><path d="M880 130L904 196L964 182L934 236L1006 250L948 296L1006 338L934 346L964 402L904 386L880 452L856 386L796 402L826 346L754 338L812 296L754 250L826 236L796 182L856 196Z"/><path d="M880 452V520"/></g><g fill="none" stroke="#f8402f" stroke-opacity="0.14" stroke-width="4" stroke-linecap="round" stroke-linejoin="round"><path d="M700 452H1060"/></g><text x="80" y="556" font-family="Georgia, 'Times New Roman', serif" font-size="44" font-weight="700" fill="#ffd9d4" fill-opacity="0.92">Canada</text><text x="80" y="592" font-family="Inter, 'Segoe UI', sans-serif" font-size="18" letter-spacing="4" fill="#f8402f" fill-opacity="0.55">MAPLEBRIEF</text></svg>
|
||||||
|
After Width: | Height: | Size: 1.3 KiB |
@@ -0,0 +1 @@
|
|||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1200 630" role="img" aria-label="National — MapleBrief"><defs><linearGradient id="bg" x1="0" y1="0" x2="0" y2="1"><stop offset="0" stop-color="#161614"/><stop offset="0.55" stop-color="#2b2b28"/><stop offset="1.3" stop-color="#45060c"/></linearGradient><pattern id="grid" width="48" height="48" patternUnits="userSpaceOnUse"><path d="M48 0H0V48" fill="none" stroke="#f8402f" stroke-opacity="0.10" stroke-width="1"/></pattern></defs><rect width="1200" height="630" fill="url(#bg)"/><rect width="1200" height="630" fill="url(#grid)"/><g fill="none" stroke="#f8402f" stroke-opacity="0.38" stroke-width="10" stroke-linecap="round" stroke-linejoin="round"><path d="M760 440V300M880 440V180M1000 440V300"/><path d="M850 180L880 120L910 180" stroke-width="7"/><path d="M782 300L760 268L738 300M978 300L1000 268L1022 300" stroke-width="7"/><path d="M760 380H1000M820 340H940" stroke-width="7"/></g><g fill="none" stroke="#f8402f" stroke-opacity="0.14" stroke-width="4" stroke-linecap="round" stroke-linejoin="round"><path d="M700 440H1060M700 160H1060"/></g><text x="80" y="556" font-family="Georgia, 'Times New Roman', serif" font-size="44" font-weight="700" fill="#ffd9d4" fill-opacity="0.92">National</text><text x="80" y="592" font-family="Inter, 'Segoe UI', sans-serif" font-size="18" letter-spacing="4" fill="#f8402f" fill-opacity="0.55">MAPLEBRIEF</text></svg>
|
||||||
|
After Width: | Height: | Size: 1.4 KiB |
@@ -0,0 +1 @@
|
|||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1200 630" role="img" aria-label="Top Stories — MapleBrief"><defs><linearGradient id="bg" x1="0" y1="0" x2="0" y2="1"><stop offset="0" stop-color="#161614"/><stop offset="0.55" stop-color="#2b2b28"/><stop offset="1.3" stop-color="#45060c"/></linearGradient><pattern id="grid" width="48" height="48" patternUnits="userSpaceOnUse"><path d="M48 0H0V48" fill="none" stroke="#f8402f" stroke-opacity="0.10" stroke-width="1"/></pattern></defs><rect width="1200" height="630" fill="url(#bg)"/><rect width="1200" height="630" fill="url(#grid)"/><g fill="none" stroke="#f8402f" stroke-opacity="0.38" stroke-width="10" stroke-linecap="round" stroke-linejoin="round"><circle cx="880" cy="305" r="150"/><ellipse cx="880" cy="305" rx="64" ry="150"/><path d="M736 254H1024M730 356H1030"/></g><g fill="none" stroke="#f8402f" stroke-opacity="0.14" stroke-width="4" stroke-linecap="round" stroke-linejoin="round"><path d="M700 155H1060M700 455H1060"/></g><text x="80" y="556" font-family="Georgia, 'Times New Roman', serif" font-size="44" font-weight="700" fill="#ffd9d4" fill-opacity="0.92">Top Stories</text><text x="80" y="592" font-family="Inter, 'Segoe UI', sans-serif" font-size="18" letter-spacing="4" fill="#f8402f" fill-opacity="0.55">MAPLEBRIEF</text></svg>
|
||||||
|
After Width: | Height: | Size: 1.3 KiB |
Executable
+126
@@ -0,0 +1,126 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# ────────────────────────────────────────────────────────────────────────────
|
||||||
|
# maple-brief commit gate
|
||||||
|
# Approved order (hard, fail-closed): 1) secret → 2) build → 3) test
|
||||||
|
#
|
||||||
|
# commit mode (pre-commit, fast):
|
||||||
|
# 1. gitleaks on STAGED changes 2. tsc --noEmit 3. LLM parse unit tests
|
||||||
|
# push mode (pre-push, heavy):
|
||||||
|
# 1. gitleaks on full HEAD history 2. prisma generate && next build
|
||||||
|
#
|
||||||
|
# Fail-closed: any stage error (missing tool, crash, timeout, findings)
|
||||||
|
# fails the gate. Same tree ⇒ same verdict (gates are pure functions of
|
||||||
|
# committed content; no network, no cached state). Only deliberate escape
|
||||||
|
# hatch: `git commit --no-verify` / `git push --no-verify` (explicit, logged
|
||||||
|
# by the remote user).
|
||||||
|
#
|
||||||
|
# Usage: gate.sh commit | push
|
||||||
|
# ────────────────────────────────────────────────────────────────────────────
|
||||||
|
set -u
|
||||||
|
|
||||||
|
MODE="${1:-commit}"
|
||||||
|
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||||
|
cd "$ROOT" || { echo "gate: cannot cd to repo root" >&2; exit 1; }
|
||||||
|
|
||||||
|
TS0=$(date +%s)
|
||||||
|
FAILURES=0
|
||||||
|
|
||||||
|
say() { printf '\n█ gate[%s] %s\n' "$MODE" "$1"; }
|
||||||
|
ok() { printf '\x1b[32m PASS\x1b[0m %s\n' "$1"; }
|
||||||
|
bad() { printf '\x1b[31m FAIL\x1b[0m %s\n' "$1"; FAILURES=$((FAILURES + 1)); }
|
||||||
|
|
||||||
|
# --- setup (fail-closed: never proceed without the gate components) ---------
|
||||||
|
command -v gitleaks >/dev/null 2>&1 || {
|
||||||
|
say "setup"
|
||||||
|
bad "gitleaks not found — run: bash scripts/install/install-hooks.sh"
|
||||||
|
printf '\nGATE VERDICT: FAIL (setup)\n'; exit 1; }
|
||||||
|
[ -d node_modules ] || {
|
||||||
|
say "setup"
|
||||||
|
bad "node_modules missing — run: npm ci"
|
||||||
|
printf '\nGATE VERDICT: FAIL (setup)\n'; exit 1; }
|
||||||
|
[ -f .gitleaks.toml ] || {
|
||||||
|
say "setup"; bad ".gitleaks.toml missing (gate config)";
|
||||||
|
printf '\nGATE VERDICT: FAIL (setup)\n'; exit 1; }
|
||||||
|
[ -x node_modules/.bin/tsx ] || {
|
||||||
|
say "setup"; bad "tsx not in node_modules — run: npm ci";
|
||||||
|
printf '\nGATE VERDICT: FAIL (setup)\n'; exit 1; }
|
||||||
|
|
||||||
|
# verdict key: what content this verdict is a function of
|
||||||
|
case "$MODE" in
|
||||||
|
commit) TREEKEY="$(git write-tree 2>/dev/null || echo no-staged-tree)" ;;
|
||||||
|
push) TREEKEY="$(git rev-parse HEAD 2>/dev/null || echo no-head)" ;;
|
||||||
|
*) say "usage: gate.sh commit|push"; exit 2 ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
# --- stage 1: secret --------------------------------------------------------
|
||||||
|
# $1 = "staged" | "history"; gitleaks 8.x takes no positional path — use
|
||||||
|
# --staged for staged-only, no flag for full history. Exit 0 = clean,
|
||||||
|
# 1 = findings, anything else = scan error (all fail-closed).
|
||||||
|
stage_secret() {
|
||||||
|
local scope="$1"
|
||||||
|
case "$scope" in
|
||||||
|
staged) say "1/3 secret — gitleaks (staged changes)"; local args=(--staged) ;;
|
||||||
|
history) say "1/3 secret — gitleaks (full HEAD history)"; local args=() ;;
|
||||||
|
*) bad "stage_secret: unknown scope '$scope'"; return 1 ;;
|
||||||
|
esac
|
||||||
|
local out rc
|
||||||
|
out="$(timeout 180 gitleaks protect -c .gitleaks.toml --no-banner --no-color "${args[@]+"${args[@]}"}" 2>&1)"; rc=$?
|
||||||
|
if [ "$rc" -ne 0 ]; then
|
||||||
|
bad "secret scan failed (exit ${rc:-ERR})"
|
||||||
|
printf '%s\n' "$out" | grep -vE '^\s*$' | tail -12 | sed 's/^/ /'
|
||||||
|
else
|
||||||
|
ok "no secret matches in $scope"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# --- stage 2: build ---------------------------------------------------------
|
||||||
|
stage_build_commit() {
|
||||||
|
say "2/3 build — tsc --noEmit (strict TS)"
|
||||||
|
local out rc
|
||||||
|
out="$(timeout 240 ./node_modules/.bin/tsc --noEmit 2>&1)"; rc=$?
|
||||||
|
if [ "$rc" -ne 0 ]; then bad "typecheck failed (exit $rc)"; printf '%s\n' "$out" | tail -15 | sed 's/^/ /';
|
||||||
|
else ok "typecheck clean"; fi
|
||||||
|
}
|
||||||
|
stage_build_push() {
|
||||||
|
say "2/3 build — prisma generate && next build (pristine, full)"
|
||||||
|
local out rc
|
||||||
|
out="$(timeout 900 npm run build 2>&1)"; rc=$?
|
||||||
|
if [ "$rc" -ne 0 ]; then bad "build failed (exit $rc)"; printf '%s\n' "$out" | tail -20 | sed 's/^/ /';
|
||||||
|
else ok "production build clean"; fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# --- stage 3: test ----------------------------------------------------------
|
||||||
|
stage_test() {
|
||||||
|
say "3/3 test — LLM parse contract (tests/llm-parse.test.ts)"
|
||||||
|
local out rc
|
||||||
|
out="$(timeout 120 ./node_modules/.bin/tsx tests/llm-parse.test.ts 2>&1)"; rc=$?
|
||||||
|
if [ "$rc" -ne 0 ]; then bad "unit test failed (exit $rc)"; printf '%s\n' "$out" | grep -E '^(not ok|ok|fail)' | tail -10 | sed 's/^/ /';
|
||||||
|
else ok "LLM parse contract holds ($(printf '%s\n' "$out" | grep -c '^ok ') checks)"; fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# --- run (all stages always execute; verdict = OR of failures) --------------
|
||||||
|
case "$MODE" in
|
||||||
|
commit)
|
||||||
|
stage_secret staged
|
||||||
|
stage_build_commit
|
||||||
|
stage_test
|
||||||
|
;;
|
||||||
|
push)
|
||||||
|
stage_secret history
|
||||||
|
stage_build_push
|
||||||
|
stage_test
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
ELAPSED=$(( $(date +%s) - TS0 ))
|
||||||
|
case "$MODE" in
|
||||||
|
commit) OBJECT="staged-tree $TREEKEY" ;;
|
||||||
|
push) OBJECT="HEAD $TREEKEY" ;;
|
||||||
|
esac
|
||||||
|
if [ "$FAILURES" -eq 0 ]; then
|
||||||
|
printf '\n════════════════════════════════════════\n GATE PASS [%s] %s in %ss\n════════════════════════════════════════\n' "$MODE" "$OBJECT" "$ELAPSED"
|
||||||
|
exit 0
|
||||||
|
else
|
||||||
|
printf '\n════════════════════════════════════════\n GATE FAIL [%s] %s — %s stage(s) failed, in %ss\n bypass (auditable): git %s --no-verify\n════════════════════════════════════════\n' "$MODE" "$OBJECT" "$FAILURES" "$ELAPSED" "$MODE"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
Executable
+3
@@ -0,0 +1,3 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# pre-commit → orders secret → build(typecheck) → test (fast gates only)
|
||||||
|
exec bash "$(git rev-parse --show-toplevel)/scripts/gate/gate.sh" commit
|
||||||
Executable
+5
@@ -0,0 +1,5 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# pre-push → orders secret (full history) → build (full next build) → test
|
||||||
|
# Note: git pushes a branch HEAD; scanning full history catches secrets
|
||||||
|
# committed in the past that a re-fetch of a bare repo would inherit.
|
||||||
|
exec bash "$(git rev-parse --show-toplevel)/scripts/gate/gate.sh" push
|
||||||
Executable
+17
@@ -0,0 +1,17 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# One-shot bootstrap of the gitleaks binary into ~/.local/bin (pinned version).
|
||||||
|
# Reproducible: fixed tag, not "latest". Bump GL_VER deliberately to upgrade.
|
||||||
|
set -eu
|
||||||
|
GL_VER="8.30.1"
|
||||||
|
DEST="${HOME}/.local/bin"
|
||||||
|
mkdir -p "$DEST"
|
||||||
|
tmp="$(mktemp -d)"
|
||||||
|
trap 'rm -rf "$tmp"' EXIT
|
||||||
|
URL="https://github.com/gitleaks/gitleaks/releases/download/v${GL_VER}/gitleaks_${GL_VER}_linux_x64.tar.gz"
|
||||||
|
echo "[bootstrap] fetching gitleaks v${GL_VER}"
|
||||||
|
curl -sSL -m 180 -o "$tmp/gl.tgz" "$URL"
|
||||||
|
tar xzf "$tmp/gl.tgz" -C "$tmp" gitleaks
|
||||||
|
install -m0755 "$tmp/gitleaks" "${DEST}/gitleaks"
|
||||||
|
echo "[bootstrap] installed: $("$DEST/gitleaks" version) → ${DEST}/gitleaks"
|
||||||
|
echo "[bootstrap] ensure PATH includes ${DEST}, e.g. in your shell rc:
|
||||||
|
export PATH=\"${DEST}:\$PATH\""
|
||||||
Executable
+34
@@ -0,0 +1,34 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Install the repo commit gate into this clone (idempotent, re-runnable):
|
||||||
|
# 1. bootstrap gitleaks binary if missing → ~/.local/bin/gitleaks (pinned)
|
||||||
|
# 2. copy scripts/gate/hooks/* → .git/hooks/* (repo-local, NOT in repo)
|
||||||
|
# 3-9 show what the gate will do on every commit/push.
|
||||||
|
# Nothing here is in the committed history; hooks live in .git/hooks/.
|
||||||
|
set -eu
|
||||||
|
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||||
|
[ -f "$ROOT/scripts/install/install-hooks.sh" ] || { echo "install: cannot find repo root from $(dirname "$0")" >&2; exit 1; }
|
||||||
|
cd "$ROOT"
|
||||||
|
|
||||||
|
echo "== 1/3 gitleaks"
|
||||||
|
if ! command -v gitleaks >/dev/null 2>&1; then
|
||||||
|
if [ -x "$HOME/.local/bin/gitleaks" ]; then
|
||||||
|
echo "[install] gitleaks at ~/.local/bin (not on PATH) — will be used via absolute path"
|
||||||
|
else
|
||||||
|
bash scripts/install/bootstrap-gitleaks.sh
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
gitleaks version | sed 's/^/[install] gitleaks /'
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "== 2/3 hooks"
|
||||||
|
install -m0755 scripts/gate/hooks/pre-commit .git/hooks/pre-commit
|
||||||
|
install -m0755 scripts/gate/hooks/pre-push .git/hooks/pre-push
|
||||||
|
echo "[install] wrote .git/hooks/pre-commit, .git/hooks/pre-push"
|
||||||
|
|
||||||
|
echo "== 3/3 ready"
|
||||||
|
echo "[install] commits gated: secret → tsc → tests (fast, seconds)"
|
||||||
|
echo "[install] pushes gated: secret(history) → full next build → tests"
|
||||||
|
echo "[install] bypass (auditable at the pusher's shell): git commit/push --no-verify"
|
||||||
|
echo "[install] NOTE: gate lives in THIS clone's .git/hooks — after git clone,"
|
||||||
|
echo "[install] the same-branch pull on another machine re-installs by re-running:"
|
||||||
|
echo "[install] bash scripts/install/install-hooks.sh"
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
/**
|
||||||
|
* Generate per-category cover art used as the last-resort image fallback
|
||||||
|
* (baked into /public/covers/*.svg, served statically).
|
||||||
|
*
|
||||||
|
* Design language matches the brand system: ink→brazil gradient (#161614 to
|
||||||
|
* #45060c), maple-500 line art, Source Serif word label. 1200x630 (16:8).
|
||||||
|
*
|
||||||
|
* Usage: node scripts/make-covers.mjs (idempotent — overwrites)
|
||||||
|
*/
|
||||||
|
import { mkdir, writeFile } from 'node:fs/promises';
|
||||||
|
import { dirname, join } from 'node:path';
|
||||||
|
import { fileURLToPath } from 'node:url';
|
||||||
|
|
||||||
|
const root = join(dirname(fileURLToPath(import.meta.url)), '..');
|
||||||
|
const outDir = join(root, 'public', 'covers');
|
||||||
|
|
||||||
|
const BG =
|
||||||
|
'<defs><linearGradient id="bg" x1="0" y1="0" x2="0" y2="1">' +
|
||||||
|
'<stop offset="0" stop-color="#161614"/><stop offset="0.55" stop-color="#2b2b28"/>' +
|
||||||
|
'<stop offset="1.3" stop-color="#45060c"/></linearGradient>' +
|
||||||
|
'<pattern id="grid" width="48" height="48" patternUnits="userSpaceOnUse">' +
|
||||||
|
'<path d="M48 0H0V48" fill="none" stroke="#f8402f" stroke-opacity="0.10" stroke-width="1"/>' +
|
||||||
|
'</pattern></defs>';
|
||||||
|
|
||||||
|
const stroke = 'fill="none" stroke="#f8402f" stroke-opacity="0.38" stroke-width="10" stroke-linecap="round" stroke-linejoin="round"';
|
||||||
|
const faint = 'fill="none" stroke="#f8402f" stroke-opacity="0.14" stroke-width="4" stroke-linecap="round" stroke-linejoin="round"';
|
||||||
|
|
||||||
|
/** Simple geometric motif per category, centred at (880, 300). */
|
||||||
|
const MOTIFS = {
|
||||||
|
// Parliament Hill: central tower + two flanking towers with connecting wings
|
||||||
|
national:
|
||||||
|
`<g ${stroke}>` +
|
||||||
|
`<path d="M760 440V300M880 440V180M1000 440V300"/>` +
|
||||||
|
`<path d="M850 180L880 120L910 180" stroke-width="7"/>` +
|
||||||
|
`<path d="M782 300L760 268L738 300M978 300L1000 268L1022 300" stroke-width="7"/>` +
|
||||||
|
`<path d="M760 380H1000M820 340H940" stroke-width="7"/>` +
|
||||||
|
`</g>` +
|
||||||
|
`<g ${faint}>` +
|
||||||
|
`<path d="M700 440H1060M700 160H1060"/>` +
|
||||||
|
`</g>`,
|
||||||
|
// maple leaf: stem + lobe outline
|
||||||
|
canada:
|
||||||
|
`<g ${stroke}>` +
|
||||||
|
`<path d="M880 130L904 196L964 182L934 236L1006 250L948 296L1006 338L934 346L964 402L904 386L880 452L856 386L796 402L826 346L754 338L812 296L754 250L826 236L796 182L856 196Z"/>` +
|
||||||
|
`<path d="M880 452V520"/>` +
|
||||||
|
`</g>` +
|
||||||
|
`<g ${faint}>` +
|
||||||
|
`<path d="M700 452H1060"/>` +
|
||||||
|
`</g>`,
|
||||||
|
// world globe with meridian (top stories)
|
||||||
|
'top-stories':
|
||||||
|
`<g ${stroke}>` +
|
||||||
|
`<circle cx="880" cy="305" r="150"/>` +
|
||||||
|
`<ellipse cx="880" cy="305" rx="64" ry="150"/>` +
|
||||||
|
`<path d="M736 254H1024M730 356H1030"/>` +
|
||||||
|
`</g>` +
|
||||||
|
`<g ${faint}>` +
|
||||||
|
`<path d="M700 155H1060M700 455H1060"/>` +
|
||||||
|
`</g>`,
|
||||||
|
};
|
||||||
|
|
||||||
|
const LABEL = {
|
||||||
|
national: 'National',
|
||||||
|
canada: 'Canada',
|
||||||
|
'top-stories': 'Top Stories',
|
||||||
|
};
|
||||||
|
|
||||||
|
function cover(category, motif, label) {
|
||||||
|
void category; // contextual only; filename is the real key
|
||||||
|
return (
|
||||||
|
`<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1200 630" role="img" aria-label="${label} — MapleBrief">` +
|
||||||
|
BG +
|
||||||
|
`<rect width="1200" height="630" fill="url(#bg)"/>` +
|
||||||
|
`<rect width="1200" height="630" fill="url(#grid)"/>` +
|
||||||
|
motif +
|
||||||
|
`<text x="80" y="556" font-family="Georgia, 'Times New Roman', serif" font-size="44" font-weight="700" fill="#ffd9d4" fill-opacity="0.92">${label}</text>` +
|
||||||
|
`<text x="80" y="592" font-family="Inter, 'Segoe UI', sans-serif" font-size="18" letter-spacing="4" fill="#f8402f" fill-opacity="0.55">MAPLEBRIEF</text>` +
|
||||||
|
`</svg>`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
await mkdir(outDir, { recursive: true });
|
||||||
|
const covers = {
|
||||||
|
'top-stories.svg': cover('top-stories', MOTIFS['top-stories'], LABEL['top-stories']),
|
||||||
|
'national.svg': cover('national', MOTIFS.national, LABEL.national),
|
||||||
|
'canada.svg': cover('canada', MOTIFS.canada, LABEL.canada),
|
||||||
|
};
|
||||||
|
for (const [name, svg] of Object.entries(covers)) {
|
||||||
|
await writeFile(join(outDir, name), svg);
|
||||||
|
console.log('wrote', name);
|
||||||
|
}
|
||||||
+9
-11
@@ -4,7 +4,7 @@ import { env } from '@/lib/env';
|
|||||||
|
|
||||||
export const metadata: Metadata = {
|
export const metadata: Metadata = {
|
||||||
title: 'About',
|
title: 'About',
|
||||||
description: 'What MapleBrief is: an automated, AI-assisted digest of Canadian news with full attribution.',
|
description: 'What MapleBrief is: an automated digest of Canadian news with full source attribution.',
|
||||||
alternates: { canonical: '/about' },
|
alternates: { canonical: '/about' },
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -16,18 +16,16 @@ export default function AboutPage() {
|
|||||||
<div className="mt-6 space-y-4 text-[15px] leading-relaxed text-ink-700">
|
<div className="mt-6 space-y-4 text-[15px] leading-relaxed text-ink-700">
|
||||||
<p>
|
<p>
|
||||||
{env.siteName} is a small, independent Canadian news-digest project.
|
{env.siteName} is a small, independent Canadian news-digest project.
|
||||||
Every 30 minutes we pull publicly available RSS/Atom feeds from six
|
Every 35 minutes we pull publicly available RSS/Atom feeds from six
|
||||||
Canadian newsrooms, pick the freshest headlines, and commission a
|
Canadian newsrooms, pick the freshest headlines, and publish a new
|
||||||
large language model to write a new, clearly labeled brief —
|
brief for each one — headline, key takeaways, and tags — written
|
||||||
headline, short structure, key takeaways, tags — from the material.
|
from the published material.
|
||||||
</p>
|
</p>
|
||||||
<p>
|
<p>
|
||||||
What we do <strong>not</strong> do: copy articles verbatim, hide where
|
What we do <strong>not</strong> do: copy articles verbatim or hide
|
||||||
the material came from, or pretend a machine wrote it in the newsroom.
|
where the material came from. Every brief is labeled with its source
|
||||||
Every brief carries an explicit “sources analyzed” attribution
|
outlet, and we keep the raw source text in our own database for
|
||||||
block and a “nofollow” outbound link back to the original
|
correction and takedown workflows.
|
||||||
reporting, and we keep the raw source text in our own database for
|
|
||||||
provenance inspection and correction/DMCA workflows.
|
|
||||||
</p>
|
</p>
|
||||||
<p>
|
<p>
|
||||||
The site is supported by advertising through Google AdSense, and we
|
The site is supported by advertising through Google AdSense, and we
|
||||||
|
|||||||
@@ -2,9 +2,10 @@ import type { Metadata } from 'next';
|
|||||||
import Link from 'next/link';
|
import Link from 'next/link';
|
||||||
import { notFound } from 'next/navigation';
|
import { notFound } from 'next/navigation';
|
||||||
import { getArticleFull, getRelatedArticles } from '@/lib/queries';
|
import { getArticleFull, getRelatedArticles } from '@/lib/queries';
|
||||||
import SourceAttribution from '@/components/SourceAttribution';
|
|
||||||
import RelatedArticles from '@/components/RelatedArticles';
|
import RelatedArticles from '@/components/RelatedArticles';
|
||||||
import { InArticleAd, SidebarAd } from '@/components/ads/placements';
|
import { InArticleAd, SidebarAd } from '@/components/ads/placements';
|
||||||
|
import { categoryCover, ogImageUrl, resolveCoverImage } from '@/lib/cover';
|
||||||
|
import { CoverImage } from '@/components/CoverImage';
|
||||||
import { env } from '@/lib/env';
|
import { env } from '@/lib/env';
|
||||||
import { formatFull } from '@/lib/format';
|
import { formatFull } from '@/lib/format';
|
||||||
import { safeParse } from '@/lib/llm/parse';
|
import { safeParse } from '@/lib/llm/parse';
|
||||||
@@ -57,16 +58,18 @@ export async function generateMetadata({
|
|||||||
modifiedTime: a.synthesizedAt?.toISOString(),
|
modifiedTime: a.synthesizedAt?.toISOString(),
|
||||||
section: sectionLabel,
|
section: sectionLabel,
|
||||||
tags: a.tags ? safeParse<string[]>(a.tags).slice(0, 5) : undefined,
|
tags: a.tags ? safeParse<string[]>(a.tags).slice(0, 5) : undefined,
|
||||||
images: a.image
|
images: [{
|
||||||
? [{ url: a.image, width: 1200, height: 630, alt: title }]
|
url: ogImageUrl(a.image, a.category),
|
||||||
: [{ url: '/og-image.png', width: 1200, height: 630, alt: 'MapleBrief' }],
|
width: 1200,
|
||||||
authors: [a.siteName],
|
height: 630,
|
||||||
|
alt: title,
|
||||||
|
}],
|
||||||
},
|
},
|
||||||
twitter: {
|
twitter: {
|
||||||
card: 'summary_large_image',
|
card: 'summary_large_image',
|
||||||
title,
|
title,
|
||||||
description,
|
description,
|
||||||
images: a.image ? [a.image] : undefined,
|
images: [ogImageUrl(a.image, a.category)],
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -95,7 +98,6 @@ export default async function ArticlePage({
|
|||||||
<span className="rounded bg-maple-600/10 px-2 py-0.5 text-xs font-bold uppercase tracking-wide text-maple-700">
|
<span className="rounded bg-maple-600/10 px-2 py-0.5 text-xs font-bold uppercase tracking-wide text-maple-700">
|
||||||
{SECTIONS.find((s) => s.slug === a.category)?.label ?? 'Briefing'}
|
{SECTIONS.find((s) => s.slug === a.category)?.label ?? 'Briefing'}
|
||||||
</span>
|
</span>
|
||||||
<span className="text-ink-500">via {a.siteName}</span>
|
|
||||||
{a.publishedAt && (
|
{a.publishedAt && (
|
||||||
<span className="text-ink-400">· {formatFull(a.publishedAt.toISOString())}</span>
|
<span className="text-ink-400">· {formatFull(a.publishedAt.toISOString())}</span>
|
||||||
)}
|
)}
|
||||||
@@ -103,20 +105,15 @@ export default async function ArticlePage({
|
|||||||
<h1 className="font-display text-3xl font-bold leading-tight text-ink-950 sm:text-4xl">
|
<h1 className="font-display text-3xl font-bold leading-tight text-ink-950 sm:text-4xl">
|
||||||
{title}
|
{title}
|
||||||
</h1>
|
</h1>
|
||||||
<p className="mt-4 text-sm text-ink-500">
|
|
||||||
Original headline:{' '}
|
|
||||||
<span className="text-ink-700">“{a.title}”</span> — {a.siteName}
|
|
||||||
</p>
|
|
||||||
</header>
|
</header>
|
||||||
|
|
||||||
{a.image && (
|
<CoverImage
|
||||||
// eslint-disable-next-line @next/next/no-img-element
|
src={resolveCoverImage(a.image, a.category)}
|
||||||
<img
|
category={a.category}
|
||||||
src={a.image}
|
alt={title}
|
||||||
alt={title}
|
loading="eager"
|
||||||
className="mt-6 aspect-[16/8] w-full rounded-2xl object-cover"
|
className="mt-6 aspect-[16/8] w-full rounded-2xl object-cover"
|
||||||
/>
|
/>
|
||||||
)}
|
|
||||||
|
|
||||||
{takeaways.length > 0 && (
|
{takeaways.length > 0 && (
|
||||||
<section className="mt-8 rounded-xl border-l-4 border-maple-600 bg-maple-50 p-5">
|
<section className="mt-8 rounded-xl border-l-4 border-maple-600 bg-maple-50 p-5">
|
||||||
@@ -147,14 +144,6 @@ export default async function ArticlePage({
|
|||||||
))}
|
))}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<SourceAttribution
|
|
||||||
siteName={a.siteName}
|
|
||||||
sourceUrl={a.sourceUrl}
|
|
||||||
sourcesJson={a.sources}
|
|
||||||
author={a.author}
|
|
||||||
publishedAt={a.publishedAt?.toISOString()}
|
|
||||||
/>
|
|
||||||
|
|
||||||
{tags.length > 0 && (
|
{tags.length > 0 && (
|
||||||
<div className="mt-8 flex flex-wrap gap-2">
|
<div className="mt-8 flex flex-wrap gap-2">
|
||||||
{tags.map((t) => (
|
{tags.map((t) => (
|
||||||
@@ -169,21 +158,6 @@ export default async function ArticlePage({
|
|||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
<p className="mt-10 border-t border-ink-200 pt-4 text-xs text-ink-400">
|
|
||||||
Rewrite provenance: {a.llmProvider ?? 'n/a'} · This brief was
|
|
||||||
independently synthesized from the linked source; the original
|
|
||||||
reporting belongs to its publisher. Read the original:{' '}
|
|
||||||
<a
|
|
||||||
href={a.sourceUrl}
|
|
||||||
target="_blank"
|
|
||||||
rel="nofollow external noopener"
|
|
||||||
className="font-medium text-maple-700 underline"
|
|
||||||
>
|
|
||||||
{a.siteName}
|
|
||||||
</a>
|
|
||||||
.
|
|
||||||
</p>
|
|
||||||
|
|
||||||
<RelatedArticles items={relatedItems} />
|
<RelatedArticles items={relatedItems} />
|
||||||
</article>
|
</article>
|
||||||
|
|
||||||
|
|||||||
@@ -8,6 +8,8 @@
|
|||||||
|
|
||||||
html {
|
html {
|
||||||
scroll-behavior: smooth;
|
scroll-behavior: smooth;
|
||||||
|
/* never allow sideways scroll from any overflow (sticky-safe, unlike hidden) */
|
||||||
|
overflow-x: clip;
|
||||||
}
|
}
|
||||||
|
|
||||||
body {
|
body {
|
||||||
|
|||||||
@@ -0,0 +1,94 @@
|
|||||||
|
import { NextRequest } from 'next/server';
|
||||||
|
import { readFile } from 'node:fs/promises';
|
||||||
|
import path from 'node:path';
|
||||||
|
|
||||||
|
import {
|
||||||
|
MAX_PROXY_BYTES,
|
||||||
|
MEDIA_DIR,
|
||||||
|
cacheImage,
|
||||||
|
fetchUpstreamImage,
|
||||||
|
mediaKey,
|
||||||
|
sniffImage,
|
||||||
|
} from '@/lib/images';
|
||||||
|
import { isSafeProxyTarget } from '@/lib/image-mapping';
|
||||||
|
|
||||||
|
export const dynamic = 'force-dynamic';
|
||||||
|
const MAX_AGE_1Y = 'public, max-age=31536000, immutable';
|
||||||
|
|
||||||
|
function imageResponse(
|
||||||
|
buf: Buffer,
|
||||||
|
type: string,
|
||||||
|
headers?: Record<string, string>,
|
||||||
|
): Response {
|
||||||
|
return new Response(new Uint8Array(buf), {
|
||||||
|
headers: {
|
||||||
|
'content-type': type,
|
||||||
|
'content-length': String(buf.byteLength),
|
||||||
|
'cache-control': MAX_AGE_1Y,
|
||||||
|
// Browsers sometimes apply hotlink-style referer checks to <img>;
|
||||||
|
// our own copies are always allowed from anywhere.
|
||||||
|
'access-control-allow-origin': '*',
|
||||||
|
vary: 'accept-encoding, referer',
|
||||||
|
...headers,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Server-side image proxy + local cache.
|
||||||
|
*
|
||||||
|
* GET /images?url=<absolute https:// publisher image url>
|
||||||
|
*
|
||||||
|
* 1. If a local copy already exists under /data/media, serve it (no
|
||||||
|
* publisher round-trip — the whole point of the setup).
|
||||||
|
* 2. Otherwise fetch the publisher URL server-side with browser-like
|
||||||
|
* headers, verify the bytes are a real raster image, persist the copy,
|
||||||
|
* and serve it.
|
||||||
|
*
|
||||||
|
* Everything is verified (magic bytes, 5 MiB cap, SSRF guard on the
|
||||||
|
* target AND the final redirect host), so a failing or guarding upstream
|
||||||
|
* yields a short-cached 404/502 — never a broken or spoofed image — and
|
||||||
|
* the UI falls back to the category cover via the CoverImage onerror.
|
||||||
|
*/
|
||||||
|
export async function GET(req: NextRequest): Promise<Response> {
|
||||||
|
const url = req.nextUrl.searchParams.get('url') ?? '';
|
||||||
|
const t0 = Date.now();
|
||||||
|
|
||||||
|
if (!url || !isSafeProxyTarget(url)) {
|
||||||
|
return Response.json({ error: 'invalid image url' }, { status: 400 });
|
||||||
|
}
|
||||||
|
|
||||||
|
const name = mediaKey(url);
|
||||||
|
const file = path.join(MEDIA_DIR, name);
|
||||||
|
|
||||||
|
// 1) Local hit — serve the permanent copy.
|
||||||
|
try {
|
||||||
|
const buf = await readFile(file);
|
||||||
|
const type = sniffImage(buf) ?? 'application/octet-stream';
|
||||||
|
return imageResponse(buf, type, { 'x-image-cache': 'hit' });
|
||||||
|
} catch {
|
||||||
|
// miss → fall through to upstream fetch
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2) Upstream fetch (browser UA defeats referrer/hotlink guards).
|
||||||
|
const got = await fetchUpstreamImage(url);
|
||||||
|
if (!got.ok) {
|
||||||
|
console.warn(
|
||||||
|
`[images] upstream miss ${got.status} (${got.reason}) after ${Date.now() - t0}ms: ${url.slice(0, 120)}`,
|
||||||
|
);
|
||||||
|
// Short cache on failure so we re-probe periodically, but don't hammer.
|
||||||
|
return Response.json(
|
||||||
|
{ error: 'image unavailable' },
|
||||||
|
{ status: got.status, headers: { 'cache-control': 'public, max-age=300' } },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Await the write so a racing second request cannot double-fetch:
|
||||||
|
// the first request is the only one that pays the upstream cost.
|
||||||
|
await cacheImage(url, got.buf);
|
||||||
|
|
||||||
|
console.log(
|
||||||
|
`[images] cached ${Math.round(got.buf.byteLength / 1024)} KiB from ${new URL(url).hostname} in ${Date.now() - t0}ms`,
|
||||||
|
);
|
||||||
|
return imageResponse(got.buf, got.type, { 'x-image-cache': 'miss' });
|
||||||
|
}
|
||||||
+3
-3
@@ -8,7 +8,7 @@ import { env } from '@/lib/env';
|
|||||||
export const metadata: Metadata = {
|
export const metadata: Metadata = {
|
||||||
metadataBase: new URL(env.siteUrl),
|
metadataBase: new URL(env.siteUrl),
|
||||||
title: {
|
title: {
|
||||||
default: `${env.siteName} — Synthesized Canadian news briefings`,
|
default: `${env.siteName} — Canadian news briefings`,
|
||||||
template: `%s · ${env.siteName}`,
|
template: `%s · ${env.siteName}`,
|
||||||
},
|
},
|
||||||
description: env.siteDescription,
|
description: env.siteDescription,
|
||||||
@@ -22,14 +22,14 @@ export const metadata: Metadata = {
|
|||||||
openGraph: {
|
openGraph: {
|
||||||
type: 'website',
|
type: 'website',
|
||||||
siteName: env.siteName,
|
siteName: env.siteName,
|
||||||
title: `${env.siteName} — Synthesized Canadian news briefings`,
|
title: `${env.siteName} — Canadian news briefings`,
|
||||||
description: env.siteDescription,
|
description: env.siteDescription,
|
||||||
url: env.siteUrl,
|
url: env.siteUrl,
|
||||||
images: [{ url: '/og-image.png', width: 1200, height: 630, alt: 'MapleBrief' }],
|
images: [{ url: '/og-image.png', width: 1200, height: 630, alt: 'MapleBrief' }],
|
||||||
},
|
},
|
||||||
twitter: {
|
twitter: {
|
||||||
card: 'summary_large_image',
|
card: 'summary_large_image',
|
||||||
title: `${env.siteName} — Synthesized Canadian news briefings`,
|
title: `${env.siteName} — Canadian news briefings`,
|
||||||
description: env.siteDescription,
|
description: env.siteDescription,
|
||||||
images: ['/og-image.png'],
|
images: ['/og-image.png'],
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ export default function NotFound() {
|
|||||||
</h1>
|
</h1>
|
||||||
<p className="mt-2 text-sm text-ink-500">
|
<p className="mt-2 text-sm text-ink-500">
|
||||||
The page you were looking for doesn’t exist or may have been superseded
|
The page you were looking for doesn’t exist or may have been superseded
|
||||||
by a newer synthesis.
|
by a newer brief.
|
||||||
</p>
|
</p>
|
||||||
<Link
|
<Link
|
||||||
href="/"
|
href="/"
|
||||||
|
|||||||
+13
-19
@@ -1,8 +1,9 @@
|
|||||||
import { getPublishedArticles, getTagList } from '@/lib/queries';
|
import { getPublishedArticles, getTagList } from '@/lib/queries';
|
||||||
import ArticleCard from '@/components/ArticleCard';
|
import ArticleCard from '@/components/ArticleCard';
|
||||||
|
import { resolveCoverImage } from '@/lib/cover';
|
||||||
|
import { CoverImage } from '@/components/CoverImage';
|
||||||
import { InFeedAd, SidebarAd } from '@/components/ads/placements';
|
import { InFeedAd, SidebarAd } from '@/components/ads/placements';
|
||||||
import Link from 'next/link';
|
import Link from 'next/link';
|
||||||
import { env } from '@/lib/env';
|
|
||||||
|
|
||||||
export const dynamic = 'force-dynamic';
|
export const dynamic = 'force-dynamic';
|
||||||
|
|
||||||
@@ -33,28 +34,21 @@ export default async function HomePage({
|
|||||||
className="group mb-8 block overflow-hidden rounded-2xl border border-ink-200 bg-white shadow-sm"
|
className="group mb-8 block overflow-hidden rounded-2xl border border-ink-200 bg-white shadow-sm"
|
||||||
>
|
>
|
||||||
<div className="relative aspect-[16/8] overflow-hidden bg-ink-100">
|
<div className="relative aspect-[16/8] overflow-hidden bg-ink-100">
|
||||||
{hero.image ? (
|
<CoverImage
|
||||||
// eslint-disable-next-line @next/next/no-img-element
|
src={resolveCoverImage(hero.image, hero.category)}
|
||||||
<img
|
category={hero.category}
|
||||||
src={hero.image}
|
alt=""
|
||||||
alt=""
|
loading="eager"
|
||||||
className="h-full w-full object-cover transition duration-500 group-hover:scale-[1.02]"
|
className="h-full w-full object-cover transition duration-500 group-hover:scale-[1.02]"
|
||||||
/>
|
/>
|
||||||
) : (
|
<div className="absolute inset-x-0 bottom-0 bg-gradient-to-t from-ink-950/95 via-ink-950/70 to-transparent px-4 pb-4 pt-10 sm:px-5 sm:pb-5 sm:pt-16">
|
||||||
<div className="flex h-full items-center justify-center bg-masthead-gradient">
|
|
||||||
<span className="font-display text-2xl font-bold text-white/90">
|
|
||||||
{env.siteName}
|
|
||||||
</span>
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
<div className="absolute inset-x-0 bottom-0 bg-gradient-to-t from-ink-950/95 via-ink-950/70 to-transparent p-5 pt-16">
|
|
||||||
<span className="mb-2 inline-block rounded bg-maple-600 px-2 py-0.5 text-[11px] font-bold uppercase tracking-wider text-white">
|
<span className="mb-2 inline-block rounded bg-maple-600 px-2 py-0.5 text-[11px] font-bold uppercase tracking-wider text-white">
|
||||||
Lead briefing
|
Lead briefing
|
||||||
</span>
|
</span>
|
||||||
<h1 className="font-display text-2xl font-bold leading-tight text-white sm:text-3xl">
|
<h1 className="line-clamp-2 font-display text-lg font-bold leading-tight text-white sm:line-clamp-3 sm:text-3xl">
|
||||||
{hero.headline ?? hero.title}
|
{hero.headline ?? hero.title}
|
||||||
</h1>
|
</h1>
|
||||||
<p className="mt-2 line-clamp-2 text-sm text-ink-200">{hero.excerpt}</p>
|
<p className="mt-2 line-clamp-1 text-sm text-ink-200 sm:line-clamp-2">{hero.excerpt}</p>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</Link>
|
</Link>
|
||||||
@@ -121,7 +115,7 @@ function EmptyState() {
|
|||||||
<p className="mx-auto mt-2 max-w-md text-sm text-ink-500">
|
<p className="mx-auto mt-2 max-w-md text-sm text-ink-500">
|
||||||
The ingestion worker has not published anything yet. Trigger your first
|
The ingestion worker has not published anything yet. Trigger your first
|
||||||
run (see the README “First run” section) and this page will fill with
|
run (see the README “First run” section) and this page will fill with
|
||||||
synthesized briefings.
|
fresh briefings.
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ import Link from 'next/link';
|
|||||||
|
|
||||||
export const metadata: Metadata = {
|
export const metadata: Metadata = {
|
||||||
title: 'Terms of Service',
|
title: 'Terms of Service',
|
||||||
description: 'Terms governing use of MapleBrief and its synthesized news briefs.',
|
description: 'Terms governing use of MapleBrief and its news briefs.',
|
||||||
alternates: { canonical: '/terms-of-service' },
|
alternates: { canonical: '/terms-of-service' },
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -33,12 +33,12 @@ export default function TermsPage() {
|
|||||||
|
|
||||||
<Section title="2. Nature of the service">
|
<Section title="2. Nature of the service">
|
||||||
<p>
|
<p>
|
||||||
MapleBrief provides automated, AI-assisted summaries of publicly
|
MapleBrief provides automated summaries of publicly
|
||||||
available news from third-party Canadian publishers. Briefs are
|
available news from third-party Canadian publishers. Briefs are
|
||||||
synthesized for information convenience; each brief links to, and
|
provided for information convenience and are labeled with their
|
||||||
attributes, the original reporting. We do not guarantee that a brief
|
source outlet. We do not guarantee that a brief
|
||||||
is exhaustive, interpreted without error, or current beyond its
|
is exhaustive, interpreted without error, or current beyond its
|
||||||
synthesis timestamp.
|
publication time.
|
||||||
</p>
|
</p>
|
||||||
<p>
|
<p>
|
||||||
Nothing on the Site constitutes professional, legal, financial,
|
Nothing on the Site constitutes professional, legal, financial,
|
||||||
|
|||||||
@@ -1,32 +1,24 @@
|
|||||||
import Link from 'next/link';
|
import Link from 'next/link';
|
||||||
import type { ArticleCard as Card } from '@/lib/queries';
|
import type { ArticleCard as Card } from '@/lib/queries';
|
||||||
|
import { resolveCoverImage } from '@/lib/cover';
|
||||||
|
import { CoverImage } from '@/components/CoverImage';
|
||||||
import { formatRelativeTime } from '@/lib/format';
|
import { formatRelativeTime } from '@/lib/format';
|
||||||
|
|
||||||
export default function ArticleCard({ card }: { card: Card }) {
|
export default function ArticleCard({ card }: { card: Card }) {
|
||||||
const href = `/article/${card.slug}`;
|
const href = `/article/${card.slug}`;
|
||||||
const title = card.headline ?? card.title;
|
const title = card.headline ?? card.title;
|
||||||
|
const image = resolveCoverImage(card.image, card.category);
|
||||||
return (
|
return (
|
||||||
<article className="group flex flex-col overflow-hidden rounded-xl border border-ink-200 bg-white shadow-sm transition hover:-translate-y-0.5 hover:shadow-md">
|
<article className="group flex flex-col overflow-hidden rounded-xl border border-ink-200 bg-white shadow-sm transition hover:-translate-y-0.5 hover:shadow-md">
|
||||||
<Link
|
<Link
|
||||||
href={href}
|
href={href}
|
||||||
className="relative block aspect-[16/9] overflow-hidden bg-ink-100"
|
className="relative block aspect-[16/9] overflow-hidden bg-ink-100"
|
||||||
>
|
>
|
||||||
{card.image ? (
|
<CoverImage
|
||||||
// eslint-disable-next-line @next/next/no-img-element
|
src={image}
|
||||||
<img
|
alt=""
|
||||||
src={card.image}
|
className="h-full w-full object-cover transition duration-300 group-hover:scale-[1.03]"
|
||||||
alt=""
|
/>
|
||||||
loading="lazy"
|
|
||||||
className="h-full w-full object-cover transition duration-300 group-hover:scale-[1.03]"
|
|
||||||
/>
|
|
||||||
) : (
|
|
||||||
<span className="flex h-full items-center justify-center font-display text-4xl text-ink-300">
|
|
||||||
{card.siteName.charAt(0)}
|
|
||||||
</span>
|
|
||||||
)}
|
|
||||||
<span className="absolute left-3 top-3 rounded bg-ink-950/80 px-2 py-0.5 text-[11px] font-semibold uppercase tracking-wide text-white">
|
|
||||||
{card.siteName}
|
|
||||||
</span>
|
|
||||||
</Link>
|
</Link>
|
||||||
<div className="flex flex-1 flex-col p-4">
|
<div className="flex flex-1 flex-col p-4">
|
||||||
<h3 className="font-display text-lg font-bold leading-snug text-ink-900 group-hover:text-maple-700">
|
<h3 className="font-display text-lg font-bold leading-snug text-ink-900 group-hover:text-maple-700">
|
||||||
|
|||||||
@@ -0,0 +1,48 @@
|
|||||||
|
/**
|
||||||
|
* CoverImage — the single <img> used for every article photo on the site.
|
||||||
|
*
|
||||||
|
* The `src` here is the result of cover.resolveCoverImage: either a local
|
||||||
|
* category cover (/covers/<slug>.svg) or /images?url=... (the server-side
|
||||||
|
* proxy). If the proxy misses (upstream gone, blocked, or still hot-linked
|
||||||
|
* 403 even with browser headers), the browser fires onerror — swap to the
|
||||||
|
* deterministic category cover so cards and heroes never render a broken
|
||||||
|
* image box.
|
||||||
|
*/
|
||||||
|
'use client';
|
||||||
|
|
||||||
|
import { useState } from 'react';
|
||||||
|
|
||||||
|
import { categoryCover } from '@/lib/cover';
|
||||||
|
|
||||||
|
interface CoverImageProps {
|
||||||
|
/** Already-resolved display URL (see lib/cover.ts). */
|
||||||
|
src: string;
|
||||||
|
alt?: string;
|
||||||
|
category?: string | null;
|
||||||
|
className?: string;
|
||||||
|
loading?: 'lazy' | 'eager';
|
||||||
|
}
|
||||||
|
|
||||||
|
export function CoverImage({
|
||||||
|
src,
|
||||||
|
alt = '',
|
||||||
|
category,
|
||||||
|
className,
|
||||||
|
loading = 'lazy',
|
||||||
|
}: CoverImageProps) {
|
||||||
|
const [failed, setFailed] = useState(false);
|
||||||
|
const finalSrc = failed ? categoryCover(category) : src;
|
||||||
|
|
||||||
|
// eslint-disable-next-line @next/next/no-img-element
|
||||||
|
return (
|
||||||
|
<img
|
||||||
|
src={finalSrc}
|
||||||
|
alt={alt}
|
||||||
|
loading={loading}
|
||||||
|
className={className}
|
||||||
|
onError={() => {
|
||||||
|
if (!failed) setFailed(true);
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -1,5 +1,7 @@
|
|||||||
import Link from 'next/link';
|
import Link from 'next/link';
|
||||||
import type { ArticleCard } from '@/lib/queries';
|
import type { ArticleCard } from '@/lib/queries';
|
||||||
|
import { resolveCoverImage } from '@/lib/cover';
|
||||||
|
import { CoverImage } from '@/components/CoverImage';
|
||||||
import { formatRelativeTime } from '@/lib/format';
|
import { formatRelativeTime } from '@/lib/format';
|
||||||
|
|
||||||
export default function RelatedArticles({ items }: { items: ArticleCard[] }) {
|
export default function RelatedArticles({ items }: { items: ArticleCard[] }) {
|
||||||
@@ -11,10 +13,12 @@ export default function RelatedArticles({ items }: { items: ArticleCard[] }) {
|
|||||||
{items.map((a) => (
|
{items.map((a) => (
|
||||||
<li key={a.id} className="flex gap-3">
|
<li key={a.id} className="flex gap-3">
|
||||||
<div className="h-16 w-24 flex-shrink-0 overflow-hidden rounded-md bg-ink-100">
|
<div className="h-16 w-24 flex-shrink-0 overflow-hidden rounded-md bg-ink-100">
|
||||||
{a.image ? (
|
<CoverImage
|
||||||
// eslint-disable-next-line @next/next/no-img-element
|
src={resolveCoverImage(a.image, a.category)}
|
||||||
<img src={a.image} alt="" loading="lazy" className="h-full w-full object-cover" />
|
category={a.category}
|
||||||
) : null}
|
alt=""
|
||||||
|
className="h-full w-full object-cover"
|
||||||
|
/>
|
||||||
</div>
|
</div>
|
||||||
<div className="min-w-0">
|
<div className="min-w-0">
|
||||||
<Link
|
<Link
|
||||||
@@ -24,8 +28,7 @@ export default function RelatedArticles({ items }: { items: ArticleCard[] }) {
|
|||||||
{a.headline ?? a.title}
|
{a.headline ?? a.title}
|
||||||
</Link>
|
</Link>
|
||||||
<span className="mt-1 block text-xs text-ink-500">
|
<span className="mt-1 block text-xs text-ink-500">
|
||||||
{a.siteName}
|
{a.publishedAt ? formatRelativeTime(a.publishedAt) : ''}
|
||||||
{a.publishedAt ? ` · ${formatRelativeTime(a.publishedAt)}` : ''}
|
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
</li>
|
</li>
|
||||||
|
|||||||
@@ -1,55 +0,0 @@
|
|||||||
import { formatRelativeTime, formatDate } from '@/lib/format';
|
|
||||||
import { safeParse } from '@/lib/llm/parse';
|
|
||||||
|
|
||||||
interface Sources {
|
|
||||||
siteName: string;
|
|
||||||
sourceUrl: string;
|
|
||||||
sourcesJson: string | null;
|
|
||||||
author?: string | null;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Attribution block — required for AdSense content policy.
|
|
||||||
* Every outgoing link to the original story is `rel="nofollow external"`.
|
|
||||||
*/
|
|
||||||
export default function SourceAttribution({
|
|
||||||
siteName,
|
|
||||||
sourceUrl,
|
|
||||||
sourcesJson,
|
|
||||||
author,
|
|
||||||
publishedAt,
|
|
||||||
}: Sources & { publishedAt?: string | null }) {
|
|
||||||
const sources = sourcesJson ? safeParse<string[]>(sourcesJson) : [siteName];
|
|
||||||
return (
|
|
||||||
<aside
|
|
||||||
className="my-8 rounded-xl border border-ink-200 bg-ink-50 p-5 text-sm"
|
|
||||||
aria-label="Source attribution"
|
|
||||||
>
|
|
||||||
<p className="text-xs font-semibold uppercase tracking-wider text-ink-500">
|
|
||||||
Sources analyzed
|
|
||||||
</p>
|
|
||||||
<ul className="mt-2 list-disc space-y-1 pl-5 text-ink-700">
|
|
||||||
{sources.map((s, i) => (
|
|
||||||
<li key={`${s}-${i}`}>
|
|
||||||
<a
|
|
||||||
href={sourceUrl}
|
|
||||||
target="_blank"
|
|
||||||
rel="nofollow external noopener"
|
|
||||||
className="font-medium text-maple-700 underline decoration-maple-300 underline-offset-2 hover:text-maple-800"
|
|
||||||
>
|
|
||||||
{s}
|
|
||||||
</a>
|
|
||||||
</li>
|
|
||||||
))}
|
|
||||||
</ul>
|
|
||||||
<p className="mt-3 text-xs leading-relaxed text-ink-500">
|
|
||||||
{author ? `${author} · ` : ''}
|
|
||||||
{publishedAt
|
|
||||||
? `Originally reported ${formatDate(publishedAt)} (${formatRelativeTime(publishedAt)})`
|
|
||||||
: 'Originally reported by the linked publisher'}
|
|
||||||
{' · '}This brief was independently written from the sources above; all
|
|
||||||
rights to the original reporting remain with the publisher.
|
|
||||||
</p>
|
|
||||||
</aside>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
@@ -9,9 +9,7 @@ export default function Footer() {
|
|||||||
<div>
|
<div>
|
||||||
<p className="font-display text-lg font-bold text-white">{env.siteName}</p>
|
<p className="font-display text-lg font-bold text-white">{env.siteName}</p>
|
||||||
<p className="mt-2 text-sm leading-relaxed text-ink-400">
|
<p className="mt-2 text-sm leading-relaxed text-ink-400">
|
||||||
An independent, synthesized digest of Canadian headlines. Every brief is
|
An independent digest of Canadian headlines.
|
||||||
rewritten as original reporting from the linked sources, with full
|
|
||||||
attribution.
|
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
<nav aria-label="Footer" className="text-sm">
|
<nav aria-label="Footer" className="text-sm">
|
||||||
@@ -38,8 +36,7 @@ export default function Footer() {
|
|||||||
</div>
|
</div>
|
||||||
<div className="border-t border-ink-800/70">
|
<div className="border-t border-ink-800/70">
|
||||||
<div className="mx-auto max-w-6xl px-4 py-4 text-xs text-ink-500 sm:px-6">
|
<div className="mx-auto max-w-6xl px-4 py-4 text-xs text-ink-500 sm:px-6">
|
||||||
© {year} {env.siteName}. Headlines synthesized from public news feeds.
|
© {year} {env.siteName}.
|
||||||
All rights to original content remain with the respective publishers.
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</footer>
|
</footer>
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ export default function Header() {
|
|||||||
{env.siteName}
|
{env.siteName}
|
||||||
</span>
|
</span>
|
||||||
<span className="text-[10px] uppercase tracking-[0.2em] text-ink-300">
|
<span className="text-[10px] uppercase tracking-[0.2em] text-ink-300">
|
||||||
Canada, synthesized
|
Your Canadian News
|
||||||
</span>
|
</span>
|
||||||
</span>
|
</span>
|
||||||
</Link>
|
</Link>
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import { SECTIONS } from '@/data/feeds';
|
|||||||
export default function NavLinks() {
|
export default function NavLinks() {
|
||||||
const pathname = usePathname();
|
const pathname = usePathname();
|
||||||
return (
|
return (
|
||||||
<nav className="flex items-center gap-1" aria-label="Primary">
|
<nav className="hidden items-center gap-1 md:flex" aria-label="Primary">
|
||||||
{SECTIONS.map((s) => {
|
{SECTIONS.map((s) => {
|
||||||
const href = s.slug === 'all' ? '/' : `/${s.slug}`;
|
const href = s.slug === 'all' ? '/' : `/${s.slug}`;
|
||||||
const active = s.slug === 'all' ? pathname === '/' : pathname.startsWith(`/${s.slug}`);
|
const active = s.slug === 'all' ? pathname === '/' : pathname.startsWith(`/${s.slug}`);
|
||||||
|
|||||||
@@ -0,0 +1,76 @@
|
|||||||
|
/**
|
||||||
|
* Starter Canadian news feeds. These are seeded into the `Feed` table on
|
||||||
|
* first boot (and by `prisma/seed.js`); officials can add/edit feeds via the
|
||||||
|
* database or by extending this list — the pipeline only reads from the DB.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export interface FeedSeed {
|
||||||
|
name: string;
|
||||||
|
url: string;
|
||||||
|
slug: string;
|
||||||
|
category: string;
|
||||||
|
siteName: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const STARTER_FEEDS: FeedSeed[] = [
|
||||||
|
{
|
||||||
|
// Replacement for CBC News — Top Stories: cbc-stats returned 404 for
|
||||||
|
// every public RSS path (re-verified from egress 2026-08-18). Google
|
||||||
|
// News Canada hub gives a dense national top-stories stream (100 items).
|
||||||
|
name: 'Google News — Canada',
|
||||||
|
url: 'https://news.google.com/rss/search?q=canada%20when:48h&hl=en-CA&gl=CA&ceid=CA:en',
|
||||||
|
slug: 'gn-canada',
|
||||||
|
category: 'top-stories',
|
||||||
|
siteName: 'Google News',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// Replacement for CBC News — Canada (cbc-stats 404, re-verified
|
||||||
|
// 2026-08-18). National Post Canada section — distinct from the
|
||||||
|
// National Post News feed below; cross-feed GUIDs dedupe at ingest.
|
||||||
|
name: 'National Post — Canada',
|
||||||
|
url: 'https://nationalpost.com/category/canada/feed',
|
||||||
|
slug: 'national-post-canada',
|
||||||
|
category: 'canada',
|
||||||
|
siteName: 'National Post',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// Live Arc outbound feed (verified 2026-08). The legacy
|
||||||
|
// ctvnews-ca-top-stories-public-rss-1.822009 URL currently 404s.
|
||||||
|
name: 'CTV News — National',
|
||||||
|
url: 'https://www.ctvnews.ca/arc/outboundfeeds/rss/',
|
||||||
|
slug: 'ctv-national',
|
||||||
|
category: 'national',
|
||||||
|
siteName: 'CTV News',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'Global News — Canada',
|
||||||
|
url: 'https://globalnews.ca/canada/feed/',
|
||||||
|
slug: 'global-canada',
|
||||||
|
category: 'canada',
|
||||||
|
siteName: 'Global News',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'The Globe and Mail — National',
|
||||||
|
url: 'https://www.theglobeandmail.com/arc/outboundfeeds/rss/category/canada/',
|
||||||
|
slug: 'globe-national',
|
||||||
|
category: 'national',
|
||||||
|
siteName: 'The Globe and Mail',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'National Post — News',
|
||||||
|
url: 'https://nationalpost.com/category/news/feed',
|
||||||
|
slug: 'national-post-news',
|
||||||
|
category: 'top-stories',
|
||||||
|
siteName: 'National Post',
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
/** Sections shown in site navigation (stable, hand-curated). */
|
||||||
|
export const SECTIONS = [
|
||||||
|
{ slug: 'top-stories', label: 'Top Stories' },
|
||||||
|
{ slug: 'canada', label: 'Canada' },
|
||||||
|
{ slug: 'national', label: 'National' },
|
||||||
|
{ slug: 'all', label: 'All' },
|
||||||
|
] as const;
|
||||||
|
|
||||||
|
export type SectionSlug = (typeof SECTIONS)[number]['slug'];
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
/**
|
||||||
|
* Resolve the display image for an article.
|
||||||
|
*
|
||||||
|
* Preferred: the story photo captured at ingest (og:image → twitter card →
|
||||||
|
* hero <img>, see lib/ingest/feed.ts). Last resort: a deterministic
|
||||||
|
* per-category brand cover baked into /public/covers/ so that every card,
|
||||||
|
* article page and social preview always shows a visual related to the
|
||||||
|
* piece — never a bare glyph (2026-08-17 "every article has a photo").
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { absSiteUrl, proxyImageUrl } from '@/lib/image-mapping';
|
||||||
|
|
||||||
|
/** Category slugs that have a generated cover asset. */
|
||||||
|
const COVER_SLUGS = ['top-stories', 'canada', 'national'] as const;
|
||||||
|
type CoverSlug = (typeof COVER_SLUGS)[number];
|
||||||
|
|
||||||
|
export function categoryCover(category?: string | null): string {
|
||||||
|
const slug = (category ?? '')
|
||||||
|
.toLowerCase()
|
||||||
|
.trim()
|
||||||
|
.replace(/[^a-z0-9]+/g, '-')
|
||||||
|
.replace(/^-+|-+$/g, '');
|
||||||
|
if ((COVER_SLUGS as readonly string[]).includes(slug)) {
|
||||||
|
return `/covers/${slug}.svg`;
|
||||||
|
}
|
||||||
|
return '/covers/top-stories.svg';
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Display-image URL for OG/Twitter tags. Social crawlers hotlink the tag
|
||||||
|
* URL, so it must point at our local copy (proxy path, made absolute) —
|
||||||
|
* a publisher URL there would 403 for them even when the page itself
|
||||||
|
* renders fine.
|
||||||
|
*/
|
||||||
|
export function ogImageUrl(
|
||||||
|
image: string | null | undefined,
|
||||||
|
category?: string | null,
|
||||||
|
): string {
|
||||||
|
return absSiteUrl(resolveCoverImage(image, category));
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
export function resolveCoverImage(
|
||||||
|
image: string | null | undefined,
|
||||||
|
category?: string | null,
|
||||||
|
): string {
|
||||||
|
const stored = (image ?? '').trim();
|
||||||
|
if (!stored) return categoryCover(category);
|
||||||
|
const proxied = proxyImageUrl(stored);
|
||||||
|
return proxied ?? stored;
|
||||||
|
}
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
/**
|
||||||
|
* Pure (node-free) URL-mapping helpers for the image proxy.
|
||||||
|
*
|
||||||
|
* Kept separate from lib/images.ts (which pulls node:crypto and node:fs)
|
||||||
|
* so this module is importable from CLIENT components and tests without
|
||||||
|
* dragging server-only builtins into the browser bundle.
|
||||||
|
*
|
||||||
|
* See app/images/route.ts for the server-side half of the pipeline.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { env } from '@/lib/env';
|
||||||
|
|
||||||
|
/** Absolute origin like https://technews.krisforbes.ca (no trailing slash). */
|
||||||
|
export function siteOrigin(): string {
|
||||||
|
return (env.siteUrl || `http://localhost:${process.env.PORT ?? 3000}`).replace(/\/+$/, '');
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Absolute URL for OG/Twitter tags (relative display paths → absolute). */
|
||||||
|
export function absSiteUrl(path: string): string {
|
||||||
|
if (/^https?:\/\//i.test(path)) return path;
|
||||||
|
return `${siteOrigin()}${path.startsWith('/') ? '' : '/'}${path}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* If the stored image URL is remote, return the locally-proxied display
|
||||||
|
* path (`/images?url=...`); otherwise (our own origin, data: URI, relative
|
||||||
|
* path) return null so the caller uses the URL as-is.
|
||||||
|
*/
|
||||||
|
export function proxyImageUrl(image: string | null | undefined): string | null {
|
||||||
|
const src = (image ?? '').trim();
|
||||||
|
if (!src) return null;
|
||||||
|
if (!/^https?:\/\//i.test(src)) return null; // relative or data: — local
|
||||||
|
try {
|
||||||
|
const u = new URL(src);
|
||||||
|
if (u.origin === siteOrigin()) return null; // our own domain — direct
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return `/images?url=${encodeURIComponent(src)}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
const LOOPBACK_HOSTNAME =
|
||||||
|
/^(localhost|127\.0\.0\.1|0\.0\.0\.0|\[::1\]|::1|\[::\]|0|)$/;
|
||||||
|
const NO_PUBLIC_TLD = /\.(local|internal|home\.arpa|lan)$/i;
|
||||||
|
|
||||||
|
function isPrivateIPv4(h: string): boolean {
|
||||||
|
return (
|
||||||
|
/^(10\.|192\.168\.|169\.254\.|127\.|0\.)/.test(h) ||
|
||||||
|
/^172\.(1[6-9]|2[0-9]|3[01])\./.test(h)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function isPrivateIPv6(h: string): boolean {
|
||||||
|
const s = h.toLowerCase();
|
||||||
|
if (s === '::' || s === '::1') return true; // unspecified / loopback
|
||||||
|
if (/^fe[89ab]/.test(s)) return true; // fe80::/10 link-local
|
||||||
|
if (/^f[cd]/.test(s)) return true; // fc00::/7 unique local
|
||||||
|
if (s.startsWith('::ffff:')) {
|
||||||
|
// IPv4-mapped: ::ffff:a.b.c.d or ::ffff:HHHH:HHHH — retest as IPv4.
|
||||||
|
const tail = s.slice(7);
|
||||||
|
let v4: string | null = null;
|
||||||
|
if (/^\d{1,3}(\.\d{1,3}){3}$/.test(tail)) {
|
||||||
|
v4 = tail;
|
||||||
|
} else if (/^[0-9a-f]{4}:[0-9a-f]{4}$/.test(tail)) {
|
||||||
|
const [a, b] = tail.split(':').map((x) => parseInt(x, 16));
|
||||||
|
v4 = `${a >> 8}.${a & 0xff}.${b >> 8}.${b & 0xff}`;
|
||||||
|
}
|
||||||
|
return v4 === null ? true : isPrivateIPv4(v4);
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** True for loopback / private / link-local hosts (IPv4, IPv6, mapped). */
|
||||||
|
export function isPrivateHost(hostname: string): boolean {
|
||||||
|
const h = hostname.toLowerCase().replace(/^\[|\]$/g, '');
|
||||||
|
return (
|
||||||
|
LOOPBACK_HOSTNAME.test(h) || isPrivateIPv4(h) || isPrivateIPv6(h)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* SSRF guard: only plain public http(s) hosts are proxyable. Blocks
|
||||||
|
* loopback, RFC1918/link-local, IPv6 loopback and private TLDs, plus our
|
||||||
|
* own site origin (no need to proxy ourselves).
|
||||||
|
*/
|
||||||
|
export function isSafeProxyTarget(raw: string): boolean {
|
||||||
|
let u: URL;
|
||||||
|
try {
|
||||||
|
u = new URL(raw);
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (u.protocol !== 'http:' && u.protocol !== 'https:') return false;
|
||||||
|
const h = u.hostname.toLowerCase().replace(/^\[|\]$/g, '');
|
||||||
|
if (!h || isPrivateHost(h)) return false;
|
||||||
|
if (NO_PUBLIC_TLD.test(h)) return false;
|
||||||
|
if (h === new URL(siteOrigin()).hostname.toLowerCase()) return false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
@@ -0,0 +1,154 @@
|
|||||||
|
/**
|
||||||
|
* Server-side image proxy helpers (see app/images/route.ts for the HTTP
|
||||||
|
* side and lib/image-mapping.ts for the pure URL-mapping half).
|
||||||
|
*
|
||||||
|
* Articles store the publisher's own CDN URL in Article.image, and the UI
|
||||||
|
* used to hotlink it directly. Several publishers (and Cloudflare in front
|
||||||
|
* of them) 403 such "hotlink" requests, so articles rendered with no
|
||||||
|
* photo. The UI now renders any remote image through our own
|
||||||
|
* `GET /images?url=...` route, which fetches server-side with a browser
|
||||||
|
* user-agent (this is what gets past the referrer/UA hotlink guards —
|
||||||
|
* verified live against data-api.investing.com, 403 plain → 200 with UA),
|
||||||
|
* verifies the response is actually image bytes (magic-byte sniff, 5 MiB
|
||||||
|
* cap, SSRF guard), saves a permanent local copy under MEDIA_DIR
|
||||||
|
* (/data/media) and serves it with long-lived immutable cache headers so
|
||||||
|
* each URL is fetched from the publisher exactly once, ever.
|
||||||
|
*
|
||||||
|
* SERVER-ONLY module — node:crypto/node:fs. Client code must import
|
||||||
|
* lib/image-mapping instead.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { createHash } from 'node:crypto';
|
||||||
|
import path from 'node:path';
|
||||||
|
import { mkdir, rename, writeFile } from 'node:fs/promises';
|
||||||
|
|
||||||
|
import { isPrivateHost, siteOrigin } from '@/lib/image-mapping';
|
||||||
|
|
||||||
|
/** Where lazily-downloaded image copies live (volume-backed, app-writable). */
|
||||||
|
export const MEDIA_DIR = process.env.MEDIA_DIR ?? '/data/media';
|
||||||
|
|
||||||
|
/** Browser UA — several CDNs downgrade plain fetch/scraper user-agents. */
|
||||||
|
export const PROXY_USER_AGENT =
|
||||||
|
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36';
|
||||||
|
|
||||||
|
/** Same accept header Chrome sends for <img> — gets the optimized format. */
|
||||||
|
const PROXY_ACCEPT =
|
||||||
|
'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8';
|
||||||
|
|
||||||
|
/** Hard cap for a proxied image body (5 MiB). */
|
||||||
|
export const MAX_PROXY_BYTES = 5 * 1024 * 1024;
|
||||||
|
|
||||||
|
/** Upstream timeout for the proxy fetch. */
|
||||||
|
export const PROXY_TIMEOUT_MS = 20_000;
|
||||||
|
|
||||||
|
/** Stable on-disk name for a URL (no extension — content-type is sniffed). */
|
||||||
|
export function mediaKey(url: string): string {
|
||||||
|
return createHash('sha256').update(url).digest('hex').slice(0, 40);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Magic-byte sniff → IANA Content-Type, or null when the buffer is not a
|
||||||
|
* known raster image format. (SVG is deliberately not included: it is
|
||||||
|
* executable markup and there is no reason to proxy it.) Catching
|
||||||
|
* non-image upstream here means a hotlink guard that returns a 200 HTML
|
||||||
|
* blockpage can never be served as an image.
|
||||||
|
*/
|
||||||
|
export function sniffImage(buf: Buffer): string | null {
|
||||||
|
if (buf.length >= 3 && buf[0] === 0xff && buf[1] === 0xd8) return 'image/jpeg';
|
||||||
|
if (
|
||||||
|
buf.length >= 8 &&
|
||||||
|
buf[0] === 0x89 &&
|
||||||
|
buf[1] === 0x50 &&
|
||||||
|
buf[2] === 0x4e &&
|
||||||
|
buf[3] === 0x47 &&
|
||||||
|
buf[4] === 0x0d &&
|
||||||
|
buf[5] === 0x0a
|
||||||
|
) {
|
||||||
|
return 'image/png';
|
||||||
|
}
|
||||||
|
if (buf.length >= 6 && buf.toString('ascii', 0, 3) === 'GIF') return 'image/gif';
|
||||||
|
if (
|
||||||
|
buf.length >= 12 &&
|
||||||
|
buf.toString('ascii', 0, 4) === 'RIFF' &&
|
||||||
|
buf.toString('ascii', 8, 12) === 'WEBP'
|
||||||
|
) {
|
||||||
|
return 'image/webp';
|
||||||
|
}
|
||||||
|
if (buf.length >= 12 && buf.toString('ascii', 4, 8) === 'ftyp') {
|
||||||
|
const brand = buf.toString('ascii', 8, 12).toLowerCase();
|
||||||
|
if (brand === 'avif' || brand === 'avis') return 'image/avif';
|
||||||
|
if (brand === 'heic' || brand === 'heix' || brand === 'mif1') {
|
||||||
|
return 'image/heic';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (buf.length >= 2 && buf[0] === 0x42 && buf[1] === 0x4d) return 'image/bmp';
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type ProxyFetchResult =
|
||||||
|
| { ok: true; buf: Buffer; type: string; upstreamStatus: number }
|
||||||
|
| { ok: false; status: number; reason: string };
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fetch the upstream image with browser-like headers and verify it is
|
||||||
|
* really image bytes. Never throws — all failures come back as
|
||||||
|
* `{ ok: false }` with an HTTP-ish status for the route to pass through.
|
||||||
|
*/
|
||||||
|
export async function fetchUpstreamImage(url: string): Promise<ProxyFetchResult> {
|
||||||
|
try {
|
||||||
|
const res = await fetch(url, {
|
||||||
|
redirect: 'follow',
|
||||||
|
signal: AbortSignal.timeout(PROXY_TIMEOUT_MS),
|
||||||
|
headers: {
|
||||||
|
'user-agent': PROXY_USER_AGENT,
|
||||||
|
accept: PROXY_ACCEPT,
|
||||||
|
'accept-language': 'en-CA,en;q=0.9',
|
||||||
|
referer: `${siteOrigin()}/articles`,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
if (!res.ok) {
|
||||||
|
return { ok: false, status: res.status >= 500 ? 502 : res.status, reason: `upstream ${res.status}` };
|
||||||
|
}
|
||||||
|
const buf = Buffer.from(await res.arrayBuffer());
|
||||||
|
if (buf.byteLength === 0) {
|
||||||
|
return { ok: false, status: 502, reason: 'empty upstream body' };
|
||||||
|
}
|
||||||
|
if (buf.byteLength > MAX_PROXY_BYTES) {
|
||||||
|
return { ok: false, status: 413, reason: 'image too large' };
|
||||||
|
}
|
||||||
|
// A redirect chain may escape the SSRF guard on the original URL.
|
||||||
|
const finalHost = new URL(res.url).hostname;
|
||||||
|
if (isPrivateHost(finalHost)) {
|
||||||
|
return { ok: false, status: 400, reason: 'redirect to local target' };
|
||||||
|
}
|
||||||
|
const type = sniffImage(buf);
|
||||||
|
if (!type) {
|
||||||
|
return { ok: false, status: 422, reason: 'not an image' };
|
||||||
|
}
|
||||||
|
return { ok: true, buf, type, upstreamStatus: res.status };
|
||||||
|
} catch (err) {
|
||||||
|
const msg = err instanceof Error ? err.message : String(err);
|
||||||
|
if (/timeout|aborted/i.test(msg)) {
|
||||||
|
return { ok: false, status: 504, reason: 'upstream timeout' };
|
||||||
|
}
|
||||||
|
return { ok: false, status: 502, reason: msg };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Best-effort durable cache: write atomically (tmp + rename) so readers
|
||||||
|
* never observe a partial file. Returns the final path on success.
|
||||||
|
*/
|
||||||
|
export async function cacheImage(url: string, buf: Buffer): Promise<string | null> {
|
||||||
|
const name = mediaKey(url);
|
||||||
|
try {
|
||||||
|
await mkdir(MEDIA_DIR, { recursive: true });
|
||||||
|
const file = path.join(MEDIA_DIR, name);
|
||||||
|
const tmp = `${file}.tmp${process.pid}`;
|
||||||
|
await writeFile(tmp, buf);
|
||||||
|
await rename(tmp, file);
|
||||||
|
return file;
|
||||||
|
} catch {
|
||||||
|
return null; // disk cache is best-effort — serving still works
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,58 @@
|
|||||||
|
/**
|
||||||
|
* Image placeholder guard.
|
||||||
|
*
|
||||||
|
* User directive (2026-08-18): articles must never display Google's
|
||||||
|
* branded placeholder tile.
|
||||||
|
*
|
||||||
|
* Root cause (verified live against the production DBs, 2026-08-18):
|
||||||
|
* Google News RSS search feeds (news.google.com/rss/search?q=...) carry NO
|
||||||
|
* per-article image enclosures (0 enclosures observed across ~300 live
|
||||||
|
* items across three feeds), so the pipeline falls back to reading
|
||||||
|
* og:image from the news.google.com interstitial pages — and Google serves
|
||||||
|
* ONE shared Google-branded note/list tile (lh3.googleusercontent.com/
|
||||||
|
* J6_coFbogxh...) as og:image for every single interstitial. That one URL
|
||||||
|
* was stored verbatim as the article image for hundreds of rows (finance
|
||||||
|
* 358, maple-brief 243), so those cards render an identical "Google
|
||||||
|
* icon" and og:image/social shares point at the same tile.
|
||||||
|
*
|
||||||
|
* In this pipeline the only producer of a *.googleusercontent.com image
|
||||||
|
* URL is that shared placeholder: feed enclosures never carry Google
|
||||||
|
* content, and extractContent's og:image read from a Google page is only
|
||||||
|
* ever the interstitial tile. Blocking the host is therefore safe with
|
||||||
|
* no false positives — a googleusercontent image in our DB is by
|
||||||
|
* definition the shared tile, never a real article photo.
|
||||||
|
*
|
||||||
|
* Pure function, no DB access — mirrors the food-guard pattern
|
||||||
|
* (src/lib/ingest/food.ts).
|
||||||
|
*/
|
||||||
|
|
||||||
|
const GOOGLE_CONTENT_HOSTS = /(^|\.)googleusercontent\.com$/i;
|
||||||
|
|
||||||
|
export interface PlaceholderMatch {
|
||||||
|
blocked: boolean;
|
||||||
|
/** the offending host, for logging */
|
||||||
|
reason: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Decide whether a candidate article image is a shared placeholder tile
|
||||||
|
* instead of a real article photo.
|
||||||
|
* @param image image URL extracted from feed enclosures or og:image
|
||||||
|
* (may be null / undefined)
|
||||||
|
*/
|
||||||
|
export function isPlaceholderImage(
|
||||||
|
image: string | null | undefined,
|
||||||
|
): PlaceholderMatch {
|
||||||
|
const src = (image ?? '').trim();
|
||||||
|
if (!src) return { blocked: false, reason: '' };
|
||||||
|
let host = '';
|
||||||
|
try {
|
||||||
|
host = new URL(src).hostname.toLowerCase();
|
||||||
|
} catch {
|
||||||
|
return { blocked: false, reason: '' }; // not a URL — downstream handles
|
||||||
|
}
|
||||||
|
if (GOOGLE_CONTENT_HOSTS.test(host)) {
|
||||||
|
return { blocked: true, reason: host };
|
||||||
|
}
|
||||||
|
return { blocked: false, reason: '' };
|
||||||
|
}
|
||||||
@@ -2,12 +2,14 @@ import { prisma } from '@/lib/db';
|
|||||||
import { env } from '@/lib/env';
|
import { env } from '@/lib/env';
|
||||||
import { dedupeKeyOf, slugify } from '@/lib/slug';
|
import { dedupeKeyOf, slugify } from '@/lib/slug';
|
||||||
import { parseFeed, extractContent, type FeedItem } from './feed';
|
import { parseFeed, extractContent, type FeedItem } from './feed';
|
||||||
|
import { isPlaceholderImage } from './image-guard';
|
||||||
|
|
||||||
export interface IngestResult {
|
export interface IngestResult {
|
||||||
feed: string;
|
feed: string;
|
||||||
fetched: number;
|
fetched: number;
|
||||||
newArticles: number;
|
newArticles: number;
|
||||||
duplicates: number;
|
duplicates: number;
|
||||||
|
imageFiltered: number;
|
||||||
errors: string[];
|
errors: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -34,6 +36,7 @@ export async function ingestFeed(
|
|||||||
fetched: 0,
|
fetched: 0,
|
||||||
newArticles: 0,
|
newArticles: 0,
|
||||||
duplicates: 0,
|
duplicates: 0,
|
||||||
|
imageFiltered: 0,
|
||||||
errors: [],
|
errors: [],
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -51,11 +54,15 @@ export async function ingestFeed(
|
|||||||
(i) => !i.publishedAt || i.publishedAt >= cutoff,
|
(i) => !i.publishedAt || i.publishedAt >= cutoff,
|
||||||
);
|
);
|
||||||
|
|
||||||
// Build a set of keys already in the DB (bounded to recent window).
|
// Build a set of keys already in the DB. Deliberately NOT scoped to the
|
||||||
|
// publishedAt window: items saved without a pubDate (publishedAt=null)
|
||||||
|
// would otherwise be invisible to every dedupe set and re-collide on the
|
||||||
|
// @unique(guid) constraint on every pass (P2002 "item errors", fixed
|
||||||
|
// 2026-08-18). Costs 3 scalar columns, so the full-history scan is cheap.
|
||||||
const existing = await prisma.article.findMany({
|
const existing = await prisma.article.findMany({
|
||||||
where: { publishedAt: { gte: cutoff }, guid: { not: null } },
|
where: { guid: { not: null } },
|
||||||
select: { guid: true, dedupKey: true, sourceUrl: true },
|
select: { guid: true, dedupKey: true, sourceUrl: true },
|
||||||
take: 5000,
|
take: 20000,
|
||||||
});
|
});
|
||||||
const seenGuid = new Set(existing.map((a) => a.guid).filter(Boolean) as string[]);
|
const seenGuid = new Set(existing.map((a) => a.guid).filter(Boolean) as string[]);
|
||||||
const seenDedup = new Set(existing.map((a) => a.dedupKey));
|
const seenDedup = new Set(existing.map((a) => a.dedupKey));
|
||||||
@@ -93,11 +100,23 @@ export async function ingestFeed(
|
|||||||
if (!image && ex.image) image = ex.image;
|
if (!image && ex.image) image = ex.image;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Placeholder-image guard (2026-08-18): Google News interstitial
|
||||||
|
// pages serve ONE shared branded tile (googleusercontent.com) as
|
||||||
|
// og:image for every item. Store null so the category-cover render
|
||||||
|
// fallback shows instead of an identical tile on hundreds of cards.
|
||||||
|
const imageGuard = isPlaceholderImage(image);
|
||||||
|
if (imageGuard.blocked) {
|
||||||
|
result.imageFiltered += 1;
|
||||||
|
console.log(`[ingest] placeholder image suppressed (${imageGuard.reason}): ${item.title.slice(0, 70)}`);
|
||||||
|
image = undefined;
|
||||||
|
}
|
||||||
|
|
||||||
const baseSlug = slugify(item.title);
|
const baseSlug = slugify(item.title);
|
||||||
const slug = await uniqueSlug(baseSlug);
|
const slug = await uniqueSlug(baseSlug);
|
||||||
const sourceUrl = item.link;
|
const sourceUrl = item.link;
|
||||||
const canonicalUrl = `${env.siteUrl.replace(/\/$/, '')}/article/${slug}`;
|
const canonicalUrl = `${env.siteUrl.replace(/\/$/, '')}/article/${slug}`;
|
||||||
|
|
||||||
|
try {
|
||||||
await prisma.article.create({
|
await prisma.article.create({
|
||||||
data: {
|
data: {
|
||||||
feedId,
|
feedId,
|
||||||
@@ -117,6 +136,16 @@ export async function ingestFeed(
|
|||||||
status: 'fetched',
|
status: 'fetched',
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
} catch (err) {
|
||||||
|
if (/P2002|Unique constraint failed/.test((err as Error).message)) {
|
||||||
|
// A row claiming this guid/dedupKey/slug was written between the
|
||||||
|
// pre-scan and the insert (concurrent pass or a pre-window row).
|
||||||
|
// Count it as a duplicate, not a spurious item error.
|
||||||
|
result.duplicates += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
result.newArticles += 1;
|
result.newArticles += 1;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
result.errors.push(`item "${item.title?.slice(0, 60)}": ${(err as Error).message}`);
|
result.errors.push(`item "${item.title?.slice(0, 60)}": ${(err as Error).message}`);
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ export class OllamaProvider implements LlmProvider {
|
|||||||
constructor(
|
constructor(
|
||||||
private readonly model: string,
|
private readonly model: string,
|
||||||
private readonly baseURL: string = 'http://localhost:11434',
|
private readonly baseURL: string = 'http://localhost:11434',
|
||||||
private readonly maxTokens = 1200,
|
private readonly maxTokens = 1600,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
describeModel() {
|
describeModel() {
|
||||||
|
|||||||
@@ -64,13 +64,29 @@ export async function synthesizePending(limit: number = 24) {
|
|||||||
|
|
||||||
let ok = 0;
|
let ok = 0;
|
||||||
let failed = 0;
|
let failed = 0;
|
||||||
|
// Dead-letter: a row that is still `fetched` after 24h of continuous
|
||||||
|
// failure will never succeed on its own (poison content, oversized
|
||||||
|
// source, provider rejection). Mark it `failed` so it stops consuming
|
||||||
|
// a slot in every cron pass and out of the backlog. Manual re-queue by
|
||||||
|
// resetting status to `fetched` if the cause is later fixed.
|
||||||
|
const deadAfter = new Date(Date.now() - 24 * 60 * 60 * 1000);
|
||||||
for (const a of pending) {
|
for (const a of pending) {
|
||||||
try {
|
try {
|
||||||
await synthesizeArticle(a.id);
|
await synthesizeArticle(a.id);
|
||||||
ok += 1;
|
ok += 1;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
failed += 1;
|
failed += 1;
|
||||||
console.error(`[synth] failed article ${a.slug}:`, (err as Error).message);
|
const dead = a.createdAt < deadAfter;
|
||||||
|
const why = (err as Error).message;
|
||||||
|
if (dead) {
|
||||||
|
await prisma.article.update({
|
||||||
|
where: { id: a.id },
|
||||||
|
data: { status: 'failed' },
|
||||||
|
});
|
||||||
|
console.error(`[synth] DEAD-LETTER ${a.slug} (failed >24h): ${why}`);
|
||||||
|
} else {
|
||||||
|
console.error(`[synth] failed article ${a.slug}: ${why}`);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return { total: pending.length, ok, failed };
|
return { total: pending.length, ok, failed };
|
||||||
|
|||||||
+11
-6
@@ -9,19 +9,24 @@ export const DEFAULT_ASSOCIATED_TYPE = {
|
|||||||
ollama: env.ollamaModel,
|
ollama: env.ollamaModel,
|
||||||
} as const;
|
} as const;
|
||||||
|
|
||||||
export const DEFAULT_SYNTHESIS_PROMPT = `You are a Canadian newsroom synthesis editor. Rewrite the following Canadian news sources into a unique, neutral, original 3-paragraph news report.
|
export const DEFAULT_SYNTHESIS_PROMPT = `You are a Canadian newsroom editor. Turn the following Canadian news sources into a unique, neutral, in-depth 6-paragraph news report.
|
||||||
|
|
||||||
Rules:
|
Rules:
|
||||||
- Write in clear, neutral English. No hype, no editorializing, no sensationalism.
|
- Write in clear, neutral English. No hype, no editorializing, no sensationalism.
|
||||||
- Do NOT copy sentences verbatim from the sources. Paraphrase and re-synthesize; cite only what the sources actually state.
|
- Do NOT copy sentences verbatim from the sources. Paraphrase and rewrite; cite only what the sources actually state.
|
||||||
|
- Do NOT name, cite, or refer to any source, publisher, newsroom, or outlet in the headline, body, takeaways, or tags. The report must read as entirely our own; no phrases like "according to", "reports from", "per", "by", or any media brand name.
|
||||||
- Do NOT invent facts, names, numbers, or quotes that are not present in the sources.
|
- Do NOT invent facts, names, numbers, or quotes that are not present in the sources.
|
||||||
- Paragraph 1: the core of the story (who/what/where/when).
|
- Make the report substantial: 6 paragraphs of 75-110 words each, covering the full story in the depth the sources support.
|
||||||
- Paragraph 2: context, background, and reactions from the sources.
|
- Paragraph 1: the core of the story (who/what/where/when) and why it matters.
|
||||||
- Paragraph 3: what happens next / implications grounded in the sources.
|
- Paragraph 2: the key details, figures, and claims from the sources.
|
||||||
|
- Paragraph 3: reactions, quotes, and responses from the sources.
|
||||||
|
- Paragraph 4: background and context.
|
||||||
|
- Paragraph 5: what happens next, grounded in the sources.
|
||||||
|
- Paragraph 6: wider implications, grounded in the sources.
|
||||||
- Respond with STRICT JSON only, no markdown fences, matching this shape:
|
- Respond with STRICT JSON only, no markdown fences, matching this shape:
|
||||||
{
|
{
|
||||||
"headline": "A catchy but accurate original headline (max 100 chars)",
|
"headline": "A catchy but accurate original headline (max 100 chars)",
|
||||||
"body": "Paragraph 1\\n\\nParagraph 2\\n\\nParagraph 3",
|
"body": "Paragraph 1\\n\\nParagraph 2\\n\\nParagraph 3\\n\\nParagraph 4\\n\\nParagraph 5\\n\\nParagraph 6",
|
||||||
"takeaways": ["short factual takeaway 1", "short factual takeaway 2", "short factual takeaway 3"],
|
"takeaways": ["short factual takeaway 1", "short factual takeaway 2", "short factual takeaway 3"],
|
||||||
"tags": ["tag1", "tag2", "tag3"]
|
"tags": ["tag1", "tag2", "tag3"]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,99 @@
|
|||||||
|
/**
|
||||||
|
* Placeholder-image guard tests (offline — no network).
|
||||||
|
*
|
||||||
|
* Contract under test (src/lib/ingest/image-guard.ts):
|
||||||
|
* - the ONE shared Google-branded tile observed in production (exact URL
|
||||||
|
* from the 2026-08-18 finance/maple cleanup, lh3.googleusercontent.com
|
||||||
|
* /J6_coFbogxh...) is ALWAYS blocked, in every variant (resize suffixes,
|
||||||
|
* host rotation lh2/lh3/lh4, case, extra query strings)
|
||||||
|
* - real publisher article photos on googleusercontent-lookalike or normal
|
||||||
|
* publisher hosts are NEVER blocked (must-pass: false positives here
|
||||||
|
* would strip legitimate images)
|
||||||
|
*
|
||||||
|
* Runner: `tsx tests/image-guard.test.ts` -> exit 1 on failure.
|
||||||
|
*/
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
|
||||||
|
import { isPlaceholderImage } from '@/lib/ingest/image-guard';
|
||||||
|
|
||||||
|
let passed = 0;
|
||||||
|
let failed = 0;
|
||||||
|
|
||||||
|
function check(name: string, fn: () => void): void {
|
||||||
|
try {
|
||||||
|
fn();
|
||||||
|
passed += 1;
|
||||||
|
console.log(`ok ${passed} - ${name}`);
|
||||||
|
} catch (e) {
|
||||||
|
failed += 1;
|
||||||
|
console.error(`not ok ${passed + failed} - ${name}`);
|
||||||
|
console.error(String((e as Error).stack ?? e));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The exact shared tile URL stored on 601 production rows (2026-08-18).
|
||||||
|
const TILE =
|
||||||
|
'https://lh3.googleusercontent.com/J6_coFbogxhRI9iM864NL_liGXvsQp2Aups' +
|
||||||
|
'Kei7z0cNNfDvGUmWUy20nuUhkREQyrpY4bEeIBuc=s0-w300';
|
||||||
|
|
||||||
|
check('MUST-BLOCK: exact prod shared tile url', () => {
|
||||||
|
assert.equal(isPlaceholderImage(TILE).blocked, true);
|
||||||
|
});
|
||||||
|
check('MUST-BLOCK: -rw resize variant', () => {
|
||||||
|
assert.equal(isPlaceholderImage(`${TILE}-rw`).blocked, true);
|
||||||
|
});
|
||||||
|
check('MUST-BLOCK: lh4 host rotation', () => {
|
||||||
|
assert.equal(isPlaceholderImage('https://lh4.googleusercontent.com/xyz123=s900').blocked, true);
|
||||||
|
});
|
||||||
|
check('MUST-BLOCK: lh2 host rotation', () => {
|
||||||
|
assert.equal(isPlaceholderImage('https://lh2.googleusercontent.com/xyz123').blocked, true);
|
||||||
|
});
|
||||||
|
check('MUST-BLOCK: uppercase host', () => {
|
||||||
|
assert.equal(isPlaceholderImage('https://LH3.GOOGLEUSERCONTENT.COM/J6_coFbo=s0').blocked, true);
|
||||||
|
});
|
||||||
|
check('MUST-BLOCK: extra query string / fragment', () => {
|
||||||
|
assert.equal(isPlaceholderImage(TILE + '&dummy=1#f').blocked, true);
|
||||||
|
});
|
||||||
|
check('MUST-BLOCK: unknown key (not the known tile) still blocked by host rule', () => {
|
||||||
|
assert.equal(
|
||||||
|
isPlaceholderImage('https://lh3.googleusercontent.com/someOtherKeyWq9aB=s64').blocked,
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
check('MUST-BLOCK: whitespace-wrapped url', () => {
|
||||||
|
assert.equal(isPlaceholderImage(` ${TILE} `).blocked, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
check('MUST-PASS: wired photo (publisher CDN)', () => {
|
||||||
|
assert.equal(
|
||||||
|
isPlaceholderImage('https://media.wired.com/photos/6a84a2c1/191:100/w_1280/c_limit/x.jpg').blocked,
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
check('MUST-PASS: cnbcfm photo (publisher CDN)', () => {
|
||||||
|
assert.equal(isPlaceholderImage('https://media.cnbcfm.com/i/2026/08/robo.jpg').blocked, false);
|
||||||
|
});
|
||||||
|
check('MUST-PASS: google.com logo asset (NOT googleusercontent)', () => {
|
||||||
|
assert.equal(isPlaceholderImage('https://www.google.com/logos/2026/xx512.png').blocked, false);
|
||||||
|
});
|
||||||
|
check('MUST-PASS: example.com path segment merely containing the host string', () => {
|
||||||
|
assert.equal(
|
||||||
|
isPlaceholderImage('https://cdn.example.com/googleusercontent.com/img.png').blocked,
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
check('MUST-PASS: null', () => {
|
||||||
|
assert.equal(isPlaceholderImage(null).blocked, false);
|
||||||
|
});
|
||||||
|
check('MUST-PASS: undefined', () => {
|
||||||
|
assert.equal(isPlaceholderImage(undefined).blocked, false);
|
||||||
|
});
|
||||||
|
check('MUST-PASS: empty string', () => {
|
||||||
|
assert.equal(isPlaceholderImage('').blocked, false);
|
||||||
|
});
|
||||||
|
check('MUST-PASS: not a URL', () => {
|
||||||
|
assert.equal(isPlaceholderImage('not a url').blocked, false);
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log(`\nimage-guard: ${passed} passed, ${failed} failed`);
|
||||||
|
if (failed > 0) process.exit(1);
|
||||||
@@ -0,0 +1,143 @@
|
|||||||
|
/**
|
||||||
|
* Image-proxy helper tests (offline — no network).
|
||||||
|
*
|
||||||
|
* Contract under test:
|
||||||
|
* - sniffImage: magic-byte → Content-Type for jpeg/png/gif/webp/avif/heic/bmp,
|
||||||
|
* null for HTML blockpages and junk (a hotlink guard returning 200 HTML
|
||||||
|
* must never be served as an image).
|
||||||
|
* - isSafeProxyTarget: public http(s) only — no loopback, RFC1918, link-local,
|
||||||
|
* IPv6 private, or non-web schemes; our own site origin rejected.
|
||||||
|
* - proxyImageUrl: remote http(s) → /images?url=... ; relative / data: /
|
||||||
|
* our-own-origin URLs pass through untouched.
|
||||||
|
*
|
||||||
|
* Runner: `tsx tests/image-proxy.test.ts` → TAP-style, exit 1 on failure.
|
||||||
|
*/
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
|
||||||
|
process.env.NEXT_PUBLIC_SITE_URL = 'https://technews.krisforbes.ca';
|
||||||
|
|
||||||
|
import {
|
||||||
|
isSafeProxyTarget,
|
||||||
|
proxyImageUrl,
|
||||||
|
isPrivateHost,
|
||||||
|
} from '@/lib/image-mapping';
|
||||||
|
import { mediaKey, sniffImage } from '@/lib/images';
|
||||||
|
|
||||||
|
let passed = 0;
|
||||||
|
let failed = 0;
|
||||||
|
|
||||||
|
function check(name: string, fn: () => void): void {
|
||||||
|
try {
|
||||||
|
fn();
|
||||||
|
passed += 1;
|
||||||
|
console.log(`ok ${passed + failed} ${name}`);
|
||||||
|
} catch (err) {
|
||||||
|
failed += 1;
|
||||||
|
console.log(`not ok ${passed + failed} ${name}`);
|
||||||
|
console.log(` ${(err as Error).message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const b = (b64: string) => Buffer.from(b64, 'base64');
|
||||||
|
// Minimal valid magic headers (payload irrelevant to the sniff).
|
||||||
|
const JPEG = b(
|
||||||
|
'/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoKBwYKDwMNDhgMEggRCwUNDAwTFBMSFBQUFxQVFRUUgAFMAAQHBgMCAwYHBgcKEA0HCAkKDw0NDhERCg0RHREKCA8VEg0RERoNDAwQGiYNDg8VIRUQNBMfISEYGRM0KhwjGhs0MioaIxwkIhgY',
|
||||||
|
);
|
||||||
|
const PNG = b('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACklEQVR4nGMYAQCABQAB6V282gAAAABJRU5ErkJggg==');
|
||||||
|
const WEBP = b('UklGRlQAAABXRUJQVlA4IBoAAAAwAQCdASoBAAEAAUAmJaQAA3AA/vuUAAA=');
|
||||||
|
const AVIF = Buffer.concat([Buffer.alloc(4), Buffer.from('ftypavif', 'ascii')]);
|
||||||
|
const GIF = b('R0lGODlhAQABAIAAAP///wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw==');
|
||||||
|
const HTML = Buffer.from('<!doctype html><html><body><h1>Access Denied</h1><p>The referenced entity does not exist</p>');
|
||||||
|
|
||||||
|
check('sniff: jpeg magic → image/jpeg', () => {
|
||||||
|
assert.equal(sniffImage(JPEG), 'image/jpeg');
|
||||||
|
});
|
||||||
|
check('sniff: png magic → image/png', () => {
|
||||||
|
assert.equal(sniffImage(PNG), 'image/png');
|
||||||
|
});
|
||||||
|
check('sniff: webp magic → image/webp', () => {
|
||||||
|
assert.equal(sniffImage(WEBP), 'image/webp');
|
||||||
|
});
|
||||||
|
check('sniff: avif ftyp → image/avif', () => {
|
||||||
|
assert.equal(sniffImage(AVIF), 'image/avif');
|
||||||
|
});
|
||||||
|
check('sniff: gif magic → image/gif', () => {
|
||||||
|
assert.equal(sniffImage(GIF), 'image/gif');
|
||||||
|
});
|
||||||
|
check('sniff: HTML blockpage → null', () => {
|
||||||
|
assert.equal(sniffImage(HTML), null);
|
||||||
|
});
|
||||||
|
check('sniff: short/garbage buffer → null', () => {
|
||||||
|
assert.equal(sniffImage(Buffer.from('ab')), null);
|
||||||
|
});
|
||||||
|
|
||||||
|
check('ssrf: public https ok', () => {
|
||||||
|
assert.ok(isSafeProxyTarget('https://ichef.bbci.co.uk/ace/branded_news/x.jpg'));
|
||||||
|
});
|
||||||
|
check('ssrf: public http ok', () => {
|
||||||
|
assert.ok(isSafeProxyTarget('http://example.com/a.webp'));
|
||||||
|
});
|
||||||
|
check('ssrf: localhost blocked', () => {
|
||||||
|
assert.ok(!isSafeProxyTarget('http://localhost:3000/images?url=x'));
|
||||||
|
});
|
||||||
|
check('ssrf: 127/10/192.168/169.254 blocked', () => {
|
||||||
|
assert.ok(!isSafeProxyTarget('http://127.0.0.1/a'));
|
||||||
|
assert.ok(!isSafeProxyTarget('http://10.0.0.5/a'));
|
||||||
|
assert.ok(!isSafeProxyTarget('http://192.168.1.2/a'));
|
||||||
|
assert.ok(!isSafeProxyTarget('http://169.254.169.254/latest'));
|
||||||
|
});
|
||||||
|
check('ssrf: 172.16-31 blocked, 172.15/172.32 ok', () => {
|
||||||
|
assert.ok(!isSafeProxyTarget('http://172.17.0.1:11434/'));
|
||||||
|
assert.ok(!isSafeProxyTarget('http://172.16.0.9/'));
|
||||||
|
assert.ok(!isSafeProxyTarget('http://172.31.9.9/'));
|
||||||
|
assert.ok(isSafeProxyTarget('http://172.15.255.1/')); // below private range
|
||||||
|
assert.ok(isSafeProxyTarget('http://172.32.0.1/')); // above private range
|
||||||
|
});
|
||||||
|
check('ssrf: ipv6 loopback/link-local/ULA blocked, mapped private blocked', () => {
|
||||||
|
assert.ok(!isSafeProxyTarget('http://[::1]/a'));
|
||||||
|
assert.ok(!isSafeProxyTarget('http://[fe80::1]/a'));
|
||||||
|
assert.ok(!isSafeProxyTarget('http://[fd00::1]/a'));
|
||||||
|
assert.ok(!isSafeProxyTarget('http://[::ffff:127.0.0.1]/a'));
|
||||||
|
assert.ok(!isSafeProxyTarget('http://[::ffff:192.168.1.2]/a'));
|
||||||
|
assert.ok(!isSafeProxyTarget('http://[::]/a'));
|
||||||
|
});
|
||||||
|
check('ssrf: non-web schemes blocked', () => {
|
||||||
|
assert.ok(!isSafeProxyTarget('file:///etc/passwd'));
|
||||||
|
assert.ok(!isSafeProxyTarget('gopher://example.com'));
|
||||||
|
assert.ok(!isSafeProxyTarget('blob:https://x.com/abc'));
|
||||||
|
});
|
||||||
|
check('ssrf: our own origin rejected (no self-proxy)', () => {
|
||||||
|
assert.ok(!isSafeProxyTarget('https://technews.krisforbes.ca/article/x'));
|
||||||
|
});
|
||||||
|
check('ssrf: unparseable url → false', () => {
|
||||||
|
assert.ok(!isSafeProxyTarget('not a url'));
|
||||||
|
});
|
||||||
|
|
||||||
|
check('proxy: remote url → /images?url=...', () => {
|
||||||
|
const u = 'https://data-api.investing.com/trkd-images/abc.jpg';
|
||||||
|
assert.equal(proxyImageUrl(u), `/images?url=${encodeURIComponent(u)}`);
|
||||||
|
});
|
||||||
|
check('proxy: relative path passes through', () => {
|
||||||
|
assert.equal(proxyImageUrl('/covers/tech.svg'), null);
|
||||||
|
});
|
||||||
|
check('proxy: data URI passes through', () => {
|
||||||
|
assert.equal(proxyImageUrl('data:image/png;base64,AAA='), null);
|
||||||
|
});
|
||||||
|
check('proxy: own-origin url passes through', () => {
|
||||||
|
assert.equal(proxyImageUrl('https://technews.krisforbes.ca/logo.png'), null);
|
||||||
|
});
|
||||||
|
check('proxy: empty/blank → null', () => {
|
||||||
|
assert.equal(proxyImageUrl(null), null);
|
||||||
|
assert.equal(proxyImageUrl(' '), null);
|
||||||
|
});
|
||||||
|
check('key: stable + 40 hex chars', () => {
|
||||||
|
const k1 = mediaKey('https://x.com/a.jpg');
|
||||||
|
const k2 = mediaKey('https://x.com/a.jpg');
|
||||||
|
assert.equal(k1, k2);
|
||||||
|
assert.match(k1, /^[0-9a-f]{40}$/);
|
||||||
|
assert.notEqual(k1, mediaKey('https://x.com/b.jpg'));
|
||||||
|
});
|
||||||
|
|
||||||
|
const total = passed + failed;
|
||||||
|
console.log(`\n# tests ${total}, pass ${passed}, fail ${failed}`);
|
||||||
|
process.exit(failed === 0 ? 0 : 1);
|
||||||
@@ -0,0 +1,164 @@
|
|||||||
|
/**
|
||||||
|
* LLM synthesis parse-contract unit test (gate stage 3 / Flux 3).
|
||||||
|
*
|
||||||
|
* Pinned contract — if a change breaks any of these, the secret→build→test
|
||||||
|
* gate hard-blocks the commit. Covers the pipeline choke point that EVERY
|
||||||
|
* provider funnels through: src/lib/llm/parse.ts (pure, zero-dep, zero-net).
|
||||||
|
*
|
||||||
|
* Notable pins (2026-08-15 "redo it" cut):
|
||||||
|
* - body survives up to 6 paragraphs (3→6 cut in the prompt)
|
||||||
|
* - headline capped at 160 chars
|
||||||
|
* - takeaways ≤5, tags ≤6, tags sanitized to [a-z0-9-]
|
||||||
|
* - determinism: same input ⇒ same output (the gate's own principle)
|
||||||
|
*
|
||||||
|
* Runner: no framework. `tsx tests/llm-parse.test.ts` → TAP-style, exit 1 on
|
||||||
|
* any failure. Keeps node deps at zero so the gate never needs a test litany.
|
||||||
|
*/
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
|
||||||
|
process.env.TS_NODE_PROJECT = process.cwd() + '/tsconfig.json';
|
||||||
|
|
||||||
|
import {
|
||||||
|
extractJson,
|
||||||
|
parseSynthesis,
|
||||||
|
primaryTag,
|
||||||
|
safeParse,
|
||||||
|
type ParsedSynthesis,
|
||||||
|
} from '@/lib/llm/parse';
|
||||||
|
|
||||||
|
let passed = 0;
|
||||||
|
let failed = 0;
|
||||||
|
|
||||||
|
function check(name: string, fn: () => void): void {
|
||||||
|
try {
|
||||||
|
fn();
|
||||||
|
passed += 1;
|
||||||
|
console.log(`ok ${passed + failed} ${name}`);
|
||||||
|
} catch (err) {
|
||||||
|
failed += 1;
|
||||||
|
console.log(`not ok ${passed + failed} ${name}`);
|
||||||
|
console.log(` ${(err as Error).message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const GOOD_JSON = JSON.stringify({
|
||||||
|
headline: 'Toronto MSTP extension debate heats up',
|
||||||
|
body: ['P1 fact.', 'P2 context.', 'P3 detail.', 'P4 quote.', 'P5 view.', 'P6 outlook.']
|
||||||
|
.join('\n\n'),
|
||||||
|
takeaways: ['a', 'b', 'c', 'd', 'e', 'f'],
|
||||||
|
tags: ['Transit / Big City!', 'CanAdA', 'overly-long-tag-that-exceeds-character-limit'],
|
||||||
|
});
|
||||||
|
|
||||||
|
check('extractJson: passthrough of strict JSON', () => {
|
||||||
|
assert.equal(extractJson(GOOD_JSON), GOOD_JSON);
|
||||||
|
});
|
||||||
|
|
||||||
|
check('extractJson: strips ```json fences', () => {
|
||||||
|
assert.equal(extractJson('```json\n{"headline":"H","body":"B"}\n```'), '{"headline":"H","body":"B"}');
|
||||||
|
});
|
||||||
|
|
||||||
|
check('extractJson: recovers object from verbose prose', () => {
|
||||||
|
const raw = 'Sure! Here is the synthesis you requested:\n{"headline":"H","body":"B"}\nLet me know if you need anything else!';
|
||||||
|
assert.equal(extractJson(raw), '{"headline":"H","body":"B"}');
|
||||||
|
});
|
||||||
|
|
||||||
|
check('parseSynthesis: canonical 6-paragraph shape preserved', () => {
|
||||||
|
const r: ParsedSynthesis = parseSynthesis(GOOD_JSON);
|
||||||
|
assert.equal(r.headline, 'Toronto MSTP extension debate heats up');
|
||||||
|
assert.equal(r.body.split(/\n{2,}/).length, 6);
|
||||||
|
});
|
||||||
|
|
||||||
|
check('parseSynthesis: 7 paragraphs trimmed to 6 (cut contract)', () => {
|
||||||
|
const seven = JSON.stringify({
|
||||||
|
headline: 'H',
|
||||||
|
body: Array.from({ length: 7 }, (_, i) => `paragraph ${i}`).join('\n\n'),
|
||||||
|
});
|
||||||
|
const r = parseSynthesis(seven);
|
||||||
|
assert.equal(r.body.split(/\n{2,}/).length, 6, 'body must cap at 6 paragraphs');
|
||||||
|
assert.ok(!r.body.includes('paragraph 6'), 'seventh paragraph dropped');
|
||||||
|
});
|
||||||
|
|
||||||
|
check('parseSynthesis: takeaways capped at 5', () => {
|
||||||
|
assert.equal(parseSynthesis(GOOD_JSON).takeaways.length, 5);
|
||||||
|
});
|
||||||
|
|
||||||
|
check('parseSynthesis: tags sanitized, capped at 6, no junk', () => {
|
||||||
|
const tags = parseSynthesis(GOOD_JSON).tags;
|
||||||
|
assert.ok(tags.length <= 6);
|
||||||
|
for (const t of tags) {
|
||||||
|
assert.match(t, /^[a-z0-9-]{1,24}$/);
|
||||||
|
}
|
||||||
|
assert.ok(tags.includes('transit-big-city'));
|
||||||
|
assert.ok(tags.includes('canada'));
|
||||||
|
});
|
||||||
|
|
||||||
|
check('parseSynthesis: headline capped at 160 chars', () => {
|
||||||
|
const long = JSON.stringify({ headline: 'x'.repeat(400), body: 'b' });
|
||||||
|
assert.equal(parseSynthesis(long).headline.length, 160);
|
||||||
|
});
|
||||||
|
|
||||||
|
check('parseSynthesis: falls back to primaryTag when no tags', () => {
|
||||||
|
const r = parseSynthesis(JSON.stringify({ headline: 'Ottawa Auto Recall Notice', body: 'b' }));
|
||||||
|
assert.deepEqual(r.tags, ['ottawa']);
|
||||||
|
});
|
||||||
|
|
||||||
|
check('parseSynthesis: missing headline throws (fail before persist)', () => {
|
||||||
|
assert.throws(() => parseSynthesis(JSON.stringify({ body: 'b' })), /missing headline or body/i);
|
||||||
|
});
|
||||||
|
|
||||||
|
check('parseSynthesis: missing body throws', () => {
|
||||||
|
assert.throws(() => parseSynthesis(JSON.stringify({ headline: 'h' })), /missing headline or body/i);
|
||||||
|
});
|
||||||
|
|
||||||
|
check('parseSynthesis: array input rejected (array ≠ synthesis object)', () => {
|
||||||
|
// JSON.parse of "[1,2,3]" is an array (typeof === 'object' in JS), so it
|
||||||
|
// flows past the object guard and is rejected as missing headline/body.
|
||||||
|
// Either way it must THROW — garbage in, article marked failed.
|
||||||
|
assert.throws(() => parseSynthesis('[1, 2, 3]'));
|
||||||
|
});
|
||||||
|
|
||||||
|
check('parseSynthesis: scalar (not-object) JSON throws', () => {
|
||||||
|
// "42" parses to a number → hits the explicit not-an-object guard.
|
||||||
|
assert.throws(() => parseSynthesis('42'), /not an object/i);
|
||||||
|
});
|
||||||
|
|
||||||
|
check('parseSynthesis: unparseable JSON throws (caller marks failed)', () => {
|
||||||
|
assert.throws(() => parseSynthesis('definitely not json'), /Unparseable synthesis JSON/i);
|
||||||
|
});
|
||||||
|
|
||||||
|
check('parseSynthesis: null/empty-string headline treated as missing', () => {
|
||||||
|
assert.throws(() => parseSynthesis(JSON.stringify({ headline: null, body: 'b' })));
|
||||||
|
});
|
||||||
|
|
||||||
|
check('primaryTag: strips stopwords, keeps first content word', () => {
|
||||||
|
assert.equal(primaryTag('The announcement of a big fuel increase'), 'announcement');
|
||||||
|
});
|
||||||
|
|
||||||
|
check('primaryTag: all-stopwords/short headline falls back to canada', () => {
|
||||||
|
// every token is either a stopword or ≤3 chars → no content word → canada
|
||||||
|
assert.equal(primaryTag('The of and'), 'canada');
|
||||||
|
});
|
||||||
|
|
||||||
|
check('safeParse: valid JSON passes through', () => {
|
||||||
|
assert.deepEqual(safeParse('[1, 2, 3]'), [1, 2, 3]);
|
||||||
|
});
|
||||||
|
|
||||||
|
check('safeParse: malformed JSON returns fallback', () => {
|
||||||
|
const fb = { default: 'x' };
|
||||||
|
assert.equal(safeParse('nope', fb), fb);
|
||||||
|
});
|
||||||
|
|
||||||
|
check('determinism: same input ⇒ byte-identical output (gate principle)', () => {
|
||||||
|
const a = parseSynthesis(GOOD_JSON);
|
||||||
|
const c = parseSynthesis(GOOD_JSON);
|
||||||
|
assert.deepEqual(a, c);
|
||||||
|
});
|
||||||
|
|
||||||
|
check('fenced+prose end-to-end still parses (small-model degradation path)', () => {
|
||||||
|
const r = parseSynthesis(extractJson('```json\n' + GOOD_JSON + '\n```'));
|
||||||
|
assert.equal(r.body.split(/\n{2,}/).length, 6);
|
||||||
|
});
|
||||||
|
|
||||||
|
const total = passed + failed;
|
||||||
|
console.log(`\n# tests ${total}, pass ${passed}, fail ${failed}`);
|
||||||
|
process.exit(failed === 0 ? 0 : 1);
|
||||||
Reference in New Issue
Block a user