#!/usr/bin/env node /** * CI benchmark script for @arbiter/core — powered by @tenere/benchmark-lib * (possibilistic contours with valid alpha cuts). * * node --expose-gc scripts/benchmark.js compare against baseline * node --expose-gc scripts/benchmark.js --save save new baseline * node --expose-gc scripts/benchmark.js --json JSON output * node --expose-gc scripts/benchmark.js --baseline custom baseline path * * Exit codes: * 0 — all benchmarks pass, no high-severity regressions * 1 — high-severity regression found * 2 — crash (benchmark threw unexpectedly) */ import fs from 'fs' import { benchmark, createBaseline, detectRegressions, formatRegressions } from '@tenere/benchmark-lib' import { Arbiter } from '../src/core/Arbiter.js' import { PartialGraphContext } from '../src/core/PartialGraphContext.js' const SAVE = process.argv.includes('--save') const AS_JSON = process.argv.includes('--json') const BASELINE_PATH = (() => { const idx = process.argv.indexOf('--baseline') return idx >= 0 ? process.argv[idx + 1] : '.rigor-baseline.json' })() benchmark.config({ measurements: ['timing'], // Stop collecting once 90% confident (was 0.99 ≈ "confident" immediately, // so the loop stopped at the first check and the sub-ms checks were // measured from ~5 noisy samples). A tighter threshold forces the loop to // keep sampling until variance actually tightens. uncertaintyThreshold: 0.1, // Sample-count floor before any early stop (was 0). Sub-ms checks need a // real sample population for a stable mostPlausible / p95. minSamples: 200, // Headroom for noisy actions to run to (was 200, which was the effective // cap and doubled as the practical floor). maxSamples: 2000, overheadCompensation: true, // Per-sample GC injected a global.gc() between every iteration, which // dominated sub-ms timings and produced the wild run-to-run swings // (check[overlay-on-top] flagged -23%..-83% "faster" across runs of // unchanged code). GC is left to the runtime; the sample floor + tight // confidence threshold now stabilize the distribution instead. gcBetweenSamples: false, }) let CRASHED = 0 // ── Fixtures ────────────────────────────────────────────────────────── function buildEngine() { const a = new Arbiter() const groupCount = 40 for (let g = 0; g < groupCount; g++) a.addNode(`group:${g}`, 'group') for (let u = 0; u < 200; u++) { a.addNode(`user:${u}`, 'user') a.addNode(`doc:${u}`, 'doc') } a.setRelationConfig('owner', { type: 'direct' }) a.setRelationConfig('member_of', { type: 'direct' }) a.setRelationConfig('can_read', { type: 'union', rules: [{ kind: 'direct', relation: 'owner' }, { kind: 'tuple_to_userset', parentRelation: 'member_of', childRelation: 'owner' }] }) for (let u = 0; u < 200; u++) { a.addRelation(`user:${u}`, 'owner', `doc:${u}`, { possibility: 0.9 + (u % 10) / 100 }) a.addRelation(`user:${u}`, 'member_of', `group:${u % groupCount}`, { possibility: 0.7 }) } for (let g = 0; g < groupCount; g++) a.addRelation(`group:${g}`, 'owner', `doc:${g}`, { possibility: 1.0 }) return a } const engine = buildEngine() // ── Warmup ──────────────────────────────────────────────────────────── // Sub-ms checks measured without a warmup phase mix first-touch allocation, // lazy index construction, and JIT compilation into the sample population — // a bimodal distribution (check[direct-hit] ~4µs vs ~20µs) that flapped the // regression gate across runs of unchanged code. Run each hot path to a // steady state before any sampling. function warmupChecks(a, iterations = 20000) { const ctx = new PartialGraphContext(a, { relations: [{ src: 'user:1', relation: 'owner', dst: 'doc:1', possibility: 0.5 }] }) for (let i = 0; i < iterations; i++) { a.check('user:1', 'owner', 'doc:1') a.check('user:1', 'can_read', 'doc:1') a.check('user:5', 'owner', 'doc:1') a.check('user:1', 'can_read', 'doc:1', { includeMeta: true }) a.check('user:1', 'owner', 'doc:1', { partialGraphContext: ctx }) } } warmupChecks(engine) // Sub-ms checks are 4–20µs per call — below reliable single-call timing // resolution, so one GC tick or context switch inflates a sample (bimodal // distributions flapped the gate). Each sub-ms action measures a BATCH of // calls per sample; jitter amortizes across the batch and the relative // comparison against the baseline (which uses the same BATCH) stays exact. const BATCH = 100 const directBench = benchmark('check[direct-hit]', () => { for (let i = 0; i < BATCH; i++) engine.check('user:1', 'owner', 'doc:1') }) const unionBench = benchmark('check[union-ttu]', () => { for (let i = 0; i < BATCH; i++) engine.check('user:1', 'can_read', 'doc:1') }) const deniedBench = benchmark('check[denied-miss]', () => { for (let i = 0; i < BATCH; i++) engine.check('user:5', 'owner', 'doc:1') }) const metaBench = benchmark('check[include-meta]', () => { for (let i = 0; i < BATCH; i++) engine.check('user:1', 'can_read', 'doc:1', { includeMeta: true }) }) const overlayBench = benchmark('check[overlay-on-top]', () => { for (let i = 0; i < BATCH; i++) { const ctx = new PartialGraphContext(engine, { relations: [{ src: 'user:1', relation: 'owner', dst: 'doc:1', possibility: 0.5 }] }) engine.check('user:1', 'owner', 'doc:1', { partialGraphContext: ctx }) } }) const binaryBench = (() => { const snap = buildEngine() snap.enableCondensedSnapshot() for (let i = 0; i < 5000; i++) snap.check('user:1', 'owner', 'doc:1', { binary: true }) return benchmark('check[binary-direct]', () => { for (let i = 0; i < BATCH; i++) snap.check('user:1', 'owner', 'doc:1', { binary: true }) }) })() const snapshotBuildBench = (() => { const snap = buildEngine() snap.enableCondensedSnapshot() return benchmark('snapshot[build-binary]', () => { snap.toSnapshotBinary() }) })() const snapshotRestoreBench = (() => { const snap = buildEngine() snap.enableCondensedSnapshot() const buf = snap.toSnapshotBinary() return benchmark('snapshot[restore-binary]', () => { Arbiter.fromSnapshotBinary(buf) }) })() // ── Run ─────────────────────────────────────────────────────────────── function measure(handle) { try { return handle.contour('timing') } catch (e) { CRASHED++ console.error(`⚠ ${e.message}`) return null } } const results = {} const order = [directBench, unionBench, deniedBench, metaBench, overlayBench, binaryBench, snapshotBuildBench, snapshotRestoreBench] for (const handle of order) { const contour = measure(handle) if (!contour) continue const mp = contour.mostPlausible() results[handle.name] = { mostPlausible: mp.value, alphaCuts: contour.alphaCuts(), } } const current = { actions: results, campaign: null, failures: 0 } if (AS_JSON) { const out = { results: {}, regressions: [], improvements: [], crashed: CRASHED } for (const [name, r] of Object.entries(results)) { out.results[name] = { timing: { mp: r.mostPlausible, alphaCuts: r.alphaCuts } } } if (fs.existsSync(BASELINE_PATH) && !SAVE) { const baseline = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf8')) const regResult = detectRegressions(current, baseline) out.regressions = regResult.regressions out.improvements = regResult.improvements } console.log(JSON.stringify(out, null, 2)) } else { for (const [name, r] of Object.entries(results)) { const cuts = r.alphaCuts console.log(`${name}: mp=${r.mostPlausible.toFixed(4)}ms p95=[${cuts.p95.lower.toFixed(4)},${cuts.p95.upper.toFixed(4)}]`) } } if (SAVE) { const baseline = createBaseline([current]) fs.writeFileSync(BASELINE_PATH, JSON.stringify(baseline, null, 2), 'utf8') console.log(`baseline saved: ${BASELINE_PATH}`) } else if (fs.existsSync(BASELINE_PATH)) { const baseline = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf8')) const regResult = detectRegressions(current, baseline) if (!AS_JSON) console.log(formatRegressions(regResult, 'pretty')) // The lib flags a regression whenever current > baseline.p95/p99, which is // tighter than run-to-run noise at sub-ms scales (baseline alphaCuts capture // per-run sample spread, not cross-run variance). Require a minimum change // before treating a high-severity flag as a CI failure, so the gate fails on // real regressions (>10% past the p99) instead of ~3-5% noise. const MIN_HIGH_REGRESSION_PERCENT = Number(process.env.MIN_HIGH_REGRESSION_PERCENT ?? 10) const critical = regResult.regressions.filter( r => r.severity === 'high' && r.changePercent >= MIN_HIGH_REGRESSION_PERCENT ) if (critical.length > 0) { console.error(`❌ ${critical.length} critical regression(s): ${critical.map(r => r.name).join(', ')}`) process.exit(1) } } else if (!SAVE) { console.log('no baseline — first run, use --save') } if (CRASHED > 0) { console.error(`⚠ ${CRASHED} benchmark(s) crashed`) process.exit(SAVE ? 0 : 2) }