/** * LLM synthesis parse-contract unit test (gate stage 3 / Flux 3). * * Pinned contract — if a change breaks any of these, the secret→build→test * gate hard-blocks the commit. Covers the pipeline choke point that EVERY * provider funnels through: src/lib/llm/parse.ts (pure, zero-dep, zero-net). * * Notable pins (2026-08-15 "redo it" cut): * - body survives up to 6 paragraphs (3→6 cut in the prompt) * - headline capped at 160 chars * - takeaways ≤5, tags ≤6, tags sanitized to [a-z0-9-] * - determinism: same input ⇒ same output (the gate's own principle) * * Runner: no framework. `tsx tests/llm-parse.test.ts` → TAP-style, exit 1 on * any failure. Keeps node deps at zero so the gate never needs a test litany. */ import assert from 'node:assert/strict'; process.env.TS_NODE_PROJECT = process.cwd() + '/tsconfig.json'; import { extractJson, parseSynthesis, primaryTag, safeParse, type ParsedSynthesis, } from '@/lib/llm/parse'; let passed = 0; let failed = 0; function check(name: string, fn: () => void): void { try { fn(); passed += 1; console.log(`ok ${passed + failed} ${name}`); } catch (err) { failed += 1; console.log(`not ok ${passed + failed} ${name}`); console.log(` ${(err as Error).message}`); } } const GOOD_JSON = JSON.stringify({ headline: 'Toronto MSTP extension debate heats up', body: ['P1 fact.', 'P2 context.', 'P3 detail.', 'P4 quote.', 'P5 view.', 'P6 outlook.'] .join('\n\n'), takeaways: ['a', 'b', 'c', 'd', 'e', 'f'], tags: ['Transit / Big City!', 'CanAdA', 'overly-long-tag-that-exceeds-character-limit'], }); check('extractJson: passthrough of strict JSON', () => { assert.equal(extractJson(GOOD_JSON), GOOD_JSON); }); check('extractJson: strips ```json fences', () => { assert.equal(extractJson('```json\n{"headline":"H","body":"B"}\n```'), '{"headline":"H","body":"B"}'); }); check('extractJson: recovers object from verbose prose', () => { const raw = 'Sure! Here is the synthesis you requested:\n{"headline":"H","body":"B"}\nLet me know if you need anything else!'; assert.equal(extractJson(raw), '{"headline":"H","body":"B"}'); }); check('parseSynthesis: canonical 6-paragraph shape preserved', () => { const r: ParsedSynthesis = parseSynthesis(GOOD_JSON); assert.equal(r.headline, 'Toronto MSTP extension debate heats up'); assert.equal(r.body.split(/\n{2,}/).length, 6); }); check('parseSynthesis: 7 paragraphs trimmed to 6 (cut contract)', () => { const seven = JSON.stringify({ headline: 'H', body: Array.from({ length: 7 }, (_, i) => `paragraph ${i}`).join('\n\n'), }); const r = parseSynthesis(seven); assert.equal(r.body.split(/\n{2,}/).length, 6, 'body must cap at 6 paragraphs'); assert.ok(!r.body.includes('paragraph 6'), 'seventh paragraph dropped'); }); check('parseSynthesis: takeaways capped at 5', () => { assert.equal(parseSynthesis(GOOD_JSON).takeaways.length, 5); }); check('parseSynthesis: tags sanitized, capped at 6, no junk', () => { const tags = parseSynthesis(GOOD_JSON).tags; assert.ok(tags.length <= 6); for (const t of tags) { assert.match(t, /^[a-z0-9-]{1,24}$/); } assert.ok(tags.includes('transit-big-city')); assert.ok(tags.includes('canada')); }); check('parseSynthesis: headline capped at 160 chars', () => { const long = JSON.stringify({ headline: 'x'.repeat(400), body: 'b' }); assert.equal(parseSynthesis(long).headline.length, 160); }); check('parseSynthesis: falls back to primaryTag when no tags', () => { const r = parseSynthesis(JSON.stringify({ headline: 'Ottawa Auto Recall Notice', body: 'b' })); assert.deepEqual(r.tags, ['ottawa']); }); check('parseSynthesis: missing headline throws (fail before persist)', () => { assert.throws(() => parseSynthesis(JSON.stringify({ body: 'b' })), /missing headline or body/i); }); check('parseSynthesis: missing body throws', () => { assert.throws(() => parseSynthesis(JSON.stringify({ headline: 'h' })), /missing headline or body/i); }); check('parseSynthesis: array input rejected (array ≠ synthesis object)', () => { // JSON.parse of "[1,2,3]" is an array (typeof === 'object' in JS), so it // flows past the object guard and is rejected as missing headline/body. // Either way it must THROW — garbage in, article marked failed. assert.throws(() => parseSynthesis('[1, 2, 3]')); }); check('parseSynthesis: scalar (not-object) JSON throws', () => { // "42" parses to a number → hits the explicit not-an-object guard. assert.throws(() => parseSynthesis('42'), /not an object/i); }); check('parseSynthesis: unparseable JSON throws (caller marks failed)', () => { assert.throws(() => parseSynthesis('definitely not json'), /Unparseable synthesis JSON/i); }); check('parseSynthesis: null/empty-string headline treated as missing', () => { assert.throws(() => parseSynthesis(JSON.stringify({ headline: null, body: 'b' }))); }); check('primaryTag: strips stopwords, keeps first content word', () => { assert.equal(primaryTag('The announcement of a big fuel increase'), 'announcement'); }); check('primaryTag: all-stopwords/short headline falls back to canada', () => { // every token is either a stopword or ≤3 chars → no content word → canada assert.equal(primaryTag('The of and'), 'canada'); }); check('safeParse: valid JSON passes through', () => { assert.deepEqual(safeParse('[1, 2, 3]'), [1, 2, 3]); }); check('safeParse: malformed JSON returns fallback', () => { const fb = { default: 'x' }; assert.equal(safeParse('nope', fb), fb); }); check('determinism: same input ⇒ byte-identical output (gate principle)', () => { const a = parseSynthesis(GOOD_JSON); const c = parseSynthesis(GOOD_JSON); assert.deepEqual(a, c); }); check('fenced+prose end-to-end still parses (small-model degradation path)', () => { const r = parseSynthesis(extractJson('```json\n' + GOOD_JSON + '\n```')); assert.equal(r.body.split(/\n{2,}/).length, 6); }); const total = passed + failed; console.log(`\n# tests ${total}, pass ${passed}, fail ${failed}`); process.exit(failed === 0 ? 0 : 1);