Archived
chore(gate): commit pre-push gate (secret -> build -> test, fail-closed)
- scripts/gate/gate.sh: 3-stage gate — 1) gitleaks, 2) build
(tsc on commit, full next build on push), 3) LLM parse contract
tests. Fail-closed: setup errors, timeouts, and findings all block.
- scripts/gate/hooks/{pre-commit,pre-push}: exec gate.sh commit|push.
- scripts/install/install-hooks.sh: idempotent installer (verifies repo
root, bootstraps pinned gitleaks 8.30.1 if absent, wires both hooks).
- scripts/install/bootstrap-gitleaks.sh: pinned per-user install,
x86_64/arm64, GitHub release download + SHA-less checksum pin.
- .gitleaks.toml: useDefault=true; single allowlist = .env.example
placeholder lines (secret= and change-me values) by path+regex.
Real secrets — even inside .env.example — still trip the gate
(empirically verified: OpenAI/AWS/Slack/GitHub tokens all caught).
- tests/llm-parse.test.ts: pins parse.ts contracts (strict 6-paragraph
body, headline/scalar/or array rejection, stopword rules, tag
fallback) — the choke point for LLM output parsing.
- package.json: 'test' script.
- README: 'Commit gate' section (install, stages, verified fail-closed
modes).
Verified before commit: clean tree PASSes all 3 stages; staged
realistic secret FAILs stage 1 (exit 1); broken type FAILs stage 2;
broken assertion FAILs stage 3; next build exit 0.
This commit is contained in:
@@ -0,0 +1,164 @@
|
||||
/**
|
||||
* LLM synthesis parse-contract unit test (gate stage 3 / Flux 3).
|
||||
*
|
||||
* Pinned contract — if a change breaks any of these, the secret→build→test
|
||||
* gate hard-blocks the commit. Covers the pipeline choke point that EVERY
|
||||
* provider funnels through: src/lib/llm/parse.ts (pure, zero-dep, zero-net).
|
||||
*
|
||||
* Notable pins (2026-08-15 "redo it" cut):
|
||||
* - body survives up to 6 paragraphs (3→6 cut in the prompt)
|
||||
* - headline capped at 160 chars
|
||||
* - takeaways ≤5, tags ≤6, tags sanitized to [a-z0-9-]
|
||||
* - determinism: same input ⇒ same output (the gate's own principle)
|
||||
*
|
||||
* Runner: no framework. `tsx tests/llm-parse.test.ts` → TAP-style, exit 1 on
|
||||
* any failure. Keeps node deps at zero so the gate never needs a test litany.
|
||||
*/
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
process.env.TS_NODE_PROJECT = process.cwd() + '/tsconfig.json';
|
||||
|
||||
import {
|
||||
extractJson,
|
||||
parseSynthesis,
|
||||
primaryTag,
|
||||
safeParse,
|
||||
type ParsedSynthesis,
|
||||
} from '@/lib/llm/parse';
|
||||
|
||||
let passed = 0;
|
||||
let failed = 0;
|
||||
|
||||
function check(name: string, fn: () => void): void {
|
||||
try {
|
||||
fn();
|
||||
passed += 1;
|
||||
console.log(`ok ${passed + failed} ${name}`);
|
||||
} catch (err) {
|
||||
failed += 1;
|
||||
console.log(`not ok ${passed + failed} ${name}`);
|
||||
console.log(` ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
const GOOD_JSON = JSON.stringify({
|
||||
headline: 'Toronto MSTP extension debate heats up',
|
||||
body: ['P1 fact.', 'P2 context.', 'P3 detail.', 'P4 quote.', 'P5 view.', 'P6 outlook.']
|
||||
.join('\n\n'),
|
||||
takeaways: ['a', 'b', 'c', 'd', 'e', 'f'],
|
||||
tags: ['Transit / Big City!', 'CanAdA', 'overly-long-tag-that-exceeds-character-limit'],
|
||||
});
|
||||
|
||||
check('extractJson: passthrough of strict JSON', () => {
|
||||
assert.equal(extractJson(GOOD_JSON), GOOD_JSON);
|
||||
});
|
||||
|
||||
check('extractJson: strips ```json fences', () => {
|
||||
assert.equal(extractJson('```json\n{"headline":"H","body":"B"}\n```'), '{"headline":"H","body":"B"}');
|
||||
});
|
||||
|
||||
check('extractJson: recovers object from verbose prose', () => {
|
||||
const raw = 'Sure! Here is the synthesis you requested:\n{"headline":"H","body":"B"}\nLet me know if you need anything else!';
|
||||
assert.equal(extractJson(raw), '{"headline":"H","body":"B"}');
|
||||
});
|
||||
|
||||
check('parseSynthesis: canonical 6-paragraph shape preserved', () => {
|
||||
const r: ParsedSynthesis = parseSynthesis(GOOD_JSON);
|
||||
assert.equal(r.headline, 'Toronto MSTP extension debate heats up');
|
||||
assert.equal(r.body.split(/\n{2,}/).length, 6);
|
||||
});
|
||||
|
||||
check('parseSynthesis: 7 paragraphs trimmed to 6 (cut contract)', () => {
|
||||
const seven = JSON.stringify({
|
||||
headline: 'H',
|
||||
body: Array.from({ length: 7 }, (_, i) => `paragraph ${i}`).join('\n\n'),
|
||||
});
|
||||
const r = parseSynthesis(seven);
|
||||
assert.equal(r.body.split(/\n{2,}/).length, 6, 'body must cap at 6 paragraphs');
|
||||
assert.ok(!r.body.includes('paragraph 6'), 'seventh paragraph dropped');
|
||||
});
|
||||
|
||||
check('parseSynthesis: takeaways capped at 5', () => {
|
||||
assert.equal(parseSynthesis(GOOD_JSON).takeaways.length, 5);
|
||||
});
|
||||
|
||||
check('parseSynthesis: tags sanitized, capped at 6, no junk', () => {
|
||||
const tags = parseSynthesis(GOOD_JSON).tags;
|
||||
assert.ok(tags.length <= 6);
|
||||
for (const t of tags) {
|
||||
assert.match(t, /^[a-z0-9-]{1,24}$/);
|
||||
}
|
||||
assert.ok(tags.includes('transit-big-city'));
|
||||
assert.ok(tags.includes('canada'));
|
||||
});
|
||||
|
||||
check('parseSynthesis: headline capped at 160 chars', () => {
|
||||
const long = JSON.stringify({ headline: 'x'.repeat(400), body: 'b' });
|
||||
assert.equal(parseSynthesis(long).headline.length, 160);
|
||||
});
|
||||
|
||||
check('parseSynthesis: falls back to primaryTag when no tags', () => {
|
||||
const r = parseSynthesis(JSON.stringify({ headline: 'Ottawa Auto Recall Notice', body: 'b' }));
|
||||
assert.deepEqual(r.tags, ['ottawa']);
|
||||
});
|
||||
|
||||
check('parseSynthesis: missing headline throws (fail before persist)', () => {
|
||||
assert.throws(() => parseSynthesis(JSON.stringify({ body: 'b' })), /missing headline or body/i);
|
||||
});
|
||||
|
||||
check('parseSynthesis: missing body throws', () => {
|
||||
assert.throws(() => parseSynthesis(JSON.stringify({ headline: 'h' })), /missing headline or body/i);
|
||||
});
|
||||
|
||||
check('parseSynthesis: array input rejected (array ≠ synthesis object)', () => {
|
||||
// JSON.parse of "[1,2,3]" is an array (typeof === 'object' in JS), so it
|
||||
// flows past the object guard and is rejected as missing headline/body.
|
||||
// Either way it must THROW — garbage in, article marked failed.
|
||||
assert.throws(() => parseSynthesis('[1, 2, 3]'));
|
||||
});
|
||||
|
||||
check('parseSynthesis: scalar (not-object) JSON throws', () => {
|
||||
// "42" parses to a number → hits the explicit not-an-object guard.
|
||||
assert.throws(() => parseSynthesis('42'), /not an object/i);
|
||||
});
|
||||
|
||||
check('parseSynthesis: unparseable JSON throws (caller marks failed)', () => {
|
||||
assert.throws(() => parseSynthesis('definitely not json'), /Unparseable synthesis JSON/i);
|
||||
});
|
||||
|
||||
check('parseSynthesis: null/empty-string headline treated as missing', () => {
|
||||
assert.throws(() => parseSynthesis(JSON.stringify({ headline: null, body: 'b' })));
|
||||
});
|
||||
|
||||
check('primaryTag: strips stopwords, keeps first content word', () => {
|
||||
assert.equal(primaryTag('The announcement of a big fuel increase'), 'announcement');
|
||||
});
|
||||
|
||||
check('primaryTag: all-stopwords/short headline falls back to canada', () => {
|
||||
// every token is either a stopword or ≤3 chars → no content word → canada
|
||||
assert.equal(primaryTag('The of and'), 'canada');
|
||||
});
|
||||
|
||||
check('safeParse: valid JSON passes through', () => {
|
||||
assert.deepEqual(safeParse('[1, 2, 3]'), [1, 2, 3]);
|
||||
});
|
||||
|
||||
check('safeParse: malformed JSON returns fallback', () => {
|
||||
const fb = { default: 'x' };
|
||||
assert.equal(safeParse('nope', fb), fb);
|
||||
});
|
||||
|
||||
check('determinism: same input ⇒ byte-identical output (gate principle)', () => {
|
||||
const a = parseSynthesis(GOOD_JSON);
|
||||
const c = parseSynthesis(GOOD_JSON);
|
||||
assert.deepEqual(a, c);
|
||||
});
|
||||
|
||||
check('fenced+prose end-to-end still parses (small-model degradation path)', () => {
|
||||
const r = parseSynthesis(extractJson('```json\n' + GOOD_JSON + '\n```'));
|
||||
assert.equal(r.body.split(/\n{2,}/).length, 6);
|
||||
});
|
||||
|
||||
const total = passed + failed;
|
||||
console.log(`\n# tests ${total}, pass ${passed}, fail ${failed}`);
|
||||
process.exit(failed === 0 ? 0 : 1);
|
||||
Reference in New Issue
Block a user