John Dvorak
ab0e569552
The CI benchmark gate was flagging wild run-to-run 'regressions'/'improvements' (-23%..-83% on unchanged code) because sub-ms checks were measured from ~5 samples with a global.gc() injected between every iteration: - uncertaintyThreshold 0.99 -> 0.1: the loop now keeps sampling until variance actually tightens instead of 'passing' at the first check. - minSamples 0 -> 200, maxSamples 200 -> 2000: real sample floor + headroom. - gcBetweenSamples true -> false: per-sample GC dominated sub-ms timings. - warmup phase: each hot path runs to steady state (JIT, lazy index, caches) before sampling, eliminating the bimodal ~4us vs ~20us distribution. - BATCH=100 for sub-ms checks: jitter amortizes across a batch per sample; the relative comparison stays exact because the baseline uses the same batch. - minimum-change threshold (MIN_HIGH_REGRESSION_PERCENT, default 10): a high-severity flag only fails CI when the change exceeds run-to-run noise. Result: within-run p95 spread is now ~5% instead of ~100x. Residual cross-run variance on loaded shared runners (this machine: load ~19) is environmental — the baseline alphaCuts capture per-run spread, not machine-load swings.
231 lines
9.2 KiB
JavaScript
231 lines
9.2 KiB
JavaScript
#!/usr/bin/env node
|
||
/**
|
||
* CI benchmark script for @arbiter/core — powered by @tenere/benchmark-lib
|
||
* (possibilistic contours with valid alpha cuts).
|
||
*
|
||
* node --expose-gc scripts/benchmark.js compare against baseline
|
||
* node --expose-gc scripts/benchmark.js --save save new baseline
|
||
* node --expose-gc scripts/benchmark.js --json JSON output
|
||
* node --expose-gc scripts/benchmark.js --baseline <path> custom baseline path
|
||
*
|
||
* Exit codes:
|
||
* 0 — all benchmarks pass, no high-severity regressions
|
||
* 1 — high-severity regression found
|
||
* 2 — crash (benchmark threw unexpectedly)
|
||
*/
|
||
|
||
import fs from 'fs'
|
||
import { benchmark, createBaseline, detectRegressions, formatRegressions } from '@tenere/benchmark-lib'
|
||
import { Arbiter } from '../src/core/Arbiter.js'
|
||
import { PartialGraphContext } from '../src/core/PartialGraphContext.js'
|
||
|
||
const SAVE = process.argv.includes('--save')
|
||
const AS_JSON = process.argv.includes('--json')
|
||
const BASELINE_PATH = (() => {
|
||
const idx = process.argv.indexOf('--baseline')
|
||
return idx >= 0 ? process.argv[idx + 1] : '.rigor-baseline.json'
|
||
})()
|
||
|
||
benchmark.config({
|
||
measurements: ['timing'],
|
||
// Stop collecting once 90% confident (was 0.99 ≈ "confident" immediately,
|
||
// so the loop stopped at the first check and the sub-ms checks were
|
||
// measured from ~5 noisy samples). A tighter threshold forces the loop to
|
||
// keep sampling until variance actually tightens.
|
||
uncertaintyThreshold: 0.1,
|
||
// Sample-count floor before any early stop (was 0). Sub-ms checks need a
|
||
// real sample population for a stable mostPlausible / p95.
|
||
minSamples: 200,
|
||
// Headroom for noisy actions to run to (was 200, which was the effective
|
||
// cap and doubled as the practical floor).
|
||
maxSamples: 2000,
|
||
overheadCompensation: true,
|
||
// Per-sample GC injected a global.gc() between every iteration, which
|
||
// dominated sub-ms timings and produced the wild run-to-run swings
|
||
// (check[overlay-on-top] flagged -23%..-83% "faster" across runs of
|
||
// unchanged code). GC is left to the runtime; the sample floor + tight
|
||
// confidence threshold now stabilize the distribution instead.
|
||
gcBetweenSamples: false,
|
||
})
|
||
|
||
let CRASHED = 0
|
||
|
||
// ── Fixtures ──────────────────────────────────────────────────────────
|
||
|
||
function buildEngine() {
|
||
const a = new Arbiter()
|
||
const groupCount = 40
|
||
for (let g = 0; g < groupCount; g++) a.addNode(`group:${g}`, 'group')
|
||
for (let u = 0; u < 200; u++) {
|
||
a.addNode(`user:${u}`, 'user')
|
||
a.addNode(`doc:${u}`, 'doc')
|
||
}
|
||
a.setRelationConfig('owner', { type: 'direct' })
|
||
a.setRelationConfig('member_of', { type: 'direct' })
|
||
a.setRelationConfig('can_read', {
|
||
type: 'union',
|
||
rules: [{ kind: 'direct', relation: 'owner' }, { kind: 'tuple_to_userset', parentRelation: 'member_of', childRelation: 'owner' }]
|
||
})
|
||
for (let u = 0; u < 200; u++) {
|
||
a.addRelation(`user:${u}`, 'owner', `doc:${u}`, { possibility: 0.9 + (u % 10) / 100 })
|
||
a.addRelation(`user:${u}`, 'member_of', `group:${u % groupCount}`, { possibility: 0.7 })
|
||
}
|
||
for (let g = 0; g < groupCount; g++) a.addRelation(`group:${g}`, 'owner', `doc:${g}`, { possibility: 1.0 })
|
||
return a
|
||
}
|
||
|
||
const engine = buildEngine()
|
||
|
||
// ── Warmup ────────────────────────────────────────────────────────────
|
||
// Sub-ms checks measured without a warmup phase mix first-touch allocation,
|
||
// lazy index construction, and JIT compilation into the sample population —
|
||
// a bimodal distribution (check[direct-hit] ~4µs vs ~20µs) that flapped the
|
||
// regression gate across runs of unchanged code. Run each hot path to a
|
||
// steady state before any sampling.
|
||
function warmupChecks(a, iterations = 20000) {
|
||
const ctx = new PartialGraphContext(a, {
|
||
relations: [{ src: 'user:1', relation: 'owner', dst: 'doc:1', possibility: 0.5 }]
|
||
})
|
||
for (let i = 0; i < iterations; i++) {
|
||
a.check('user:1', 'owner', 'doc:1')
|
||
a.check('user:1', 'can_read', 'doc:1')
|
||
a.check('user:5', 'owner', 'doc:1')
|
||
a.check('user:1', 'can_read', 'doc:1', { includeMeta: true })
|
||
a.check('user:1', 'owner', 'doc:1', { partialGraphContext: ctx })
|
||
}
|
||
}
|
||
warmupChecks(engine)
|
||
|
||
// Sub-ms checks are 4–20µs per call — below reliable single-call timing
|
||
// resolution, so one GC tick or context switch inflates a sample (bimodal
|
||
// distributions flapped the gate). Each sub-ms action measures a BATCH of
|
||
// calls per sample; jitter amortizes across the batch and the relative
|
||
// comparison against the baseline (which uses the same BATCH) stays exact.
|
||
const BATCH = 100
|
||
|
||
const directBench = benchmark('check[direct-hit]', () => {
|
||
for (let i = 0; i < BATCH; i++) engine.check('user:1', 'owner', 'doc:1')
|
||
})
|
||
|
||
const unionBench = benchmark('check[union-ttu]', () => {
|
||
for (let i = 0; i < BATCH; i++) engine.check('user:1', 'can_read', 'doc:1')
|
||
})
|
||
|
||
const deniedBench = benchmark('check[denied-miss]', () => {
|
||
for (let i = 0; i < BATCH; i++) engine.check('user:5', 'owner', 'doc:1')
|
||
})
|
||
|
||
const metaBench = benchmark('check[include-meta]', () => {
|
||
for (let i = 0; i < BATCH; i++) engine.check('user:1', 'can_read', 'doc:1', { includeMeta: true })
|
||
})
|
||
|
||
const overlayBench = benchmark('check[overlay-on-top]', () => {
|
||
for (let i = 0; i < BATCH; i++) {
|
||
const ctx = new PartialGraphContext(engine, {
|
||
relations: [{ src: 'user:1', relation: 'owner', dst: 'doc:1', possibility: 0.5 }]
|
||
})
|
||
engine.check('user:1', 'owner', 'doc:1', { partialGraphContext: ctx })
|
||
}
|
||
})
|
||
|
||
const binaryBench = (() => {
|
||
const snap = buildEngine()
|
||
snap.enableCondensedSnapshot()
|
||
for (let i = 0; i < 5000; i++) snap.check('user:1', 'owner', 'doc:1', { binary: true })
|
||
return benchmark('check[binary-direct]', () => {
|
||
for (let i = 0; i < BATCH; i++) snap.check('user:1', 'owner', 'doc:1', { binary: true })
|
||
})
|
||
})()
|
||
|
||
const snapshotBuildBench = (() => {
|
||
const snap = buildEngine()
|
||
snap.enableCondensedSnapshot()
|
||
return benchmark('snapshot[build-binary]', () => {
|
||
snap.toSnapshotBinary()
|
||
})
|
||
})()
|
||
|
||
const snapshotRestoreBench = (() => {
|
||
const snap = buildEngine()
|
||
snap.enableCondensedSnapshot()
|
||
const buf = snap.toSnapshotBinary()
|
||
return benchmark('snapshot[restore-binary]', () => {
|
||
Arbiter.fromSnapshotBinary(buf)
|
||
})
|
||
})()
|
||
|
||
// ── Run ───────────────────────────────────────────────────────────────
|
||
|
||
function measure(handle) {
|
||
try {
|
||
return handle.contour('timing')
|
||
} catch (e) {
|
||
CRASHED++
|
||
console.error(`⚠ ${e.message}`)
|
||
return null
|
||
}
|
||
}
|
||
|
||
const results = {}
|
||
const order = [directBench, unionBench, deniedBench, metaBench, overlayBench, binaryBench, snapshotBuildBench, snapshotRestoreBench]
|
||
for (const handle of order) {
|
||
const contour = measure(handle)
|
||
if (!contour) continue
|
||
const mp = contour.mostPlausible()
|
||
results[handle.name] = {
|
||
mostPlausible: mp.value,
|
||
alphaCuts: contour.alphaCuts(),
|
||
}
|
||
}
|
||
|
||
const current = { actions: results, campaign: null, failures: 0 }
|
||
|
||
if (AS_JSON) {
|
||
const out = { results: {}, regressions: [], improvements: [], crashed: CRASHED }
|
||
for (const [name, r] of Object.entries(results)) {
|
||
out.results[name] = { timing: { mp: r.mostPlausible, alphaCuts: r.alphaCuts } }
|
||
}
|
||
if (fs.existsSync(BASELINE_PATH) && !SAVE) {
|
||
const baseline = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf8'))
|
||
const regResult = detectRegressions(current, baseline)
|
||
out.regressions = regResult.regressions
|
||
out.improvements = regResult.improvements
|
||
}
|
||
console.log(JSON.stringify(out, null, 2))
|
||
} else {
|
||
for (const [name, r] of Object.entries(results)) {
|
||
const cuts = r.alphaCuts
|
||
console.log(`${name}: mp=${r.mostPlausible.toFixed(4)}ms p95=[${cuts.p95.lower.toFixed(4)},${cuts.p95.upper.toFixed(4)}]`)
|
||
}
|
||
}
|
||
|
||
if (SAVE) {
|
||
const baseline = createBaseline([current])
|
||
fs.writeFileSync(BASELINE_PATH, JSON.stringify(baseline, null, 2), 'utf8')
|
||
console.log(`baseline saved: ${BASELINE_PATH}`)
|
||
} else if (fs.existsSync(BASELINE_PATH)) {
|
||
const baseline = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf8'))
|
||
const regResult = detectRegressions(current, baseline)
|
||
if (!AS_JSON) console.log(formatRegressions(regResult, 'pretty'))
|
||
// The lib flags a regression whenever current > baseline.p95/p99, which is
|
||
// tighter than run-to-run noise at sub-ms scales (baseline alphaCuts capture
|
||
// per-run sample spread, not cross-run variance). Require a minimum change
|
||
// before treating a high-severity flag as a CI failure, so the gate fails on
|
||
// real regressions (>10% past the p99) instead of ~3-5% noise.
|
||
const MIN_HIGH_REGRESSION_PERCENT = Number(process.env.MIN_HIGH_REGRESSION_PERCENT ?? 10)
|
||
const critical = regResult.regressions.filter(
|
||
r => r.severity === 'high' && r.changePercent >= MIN_HIGH_REGRESSION_PERCENT
|
||
)
|
||
if (critical.length > 0) {
|
||
console.error(`❌ ${critical.length} critical regression(s): ${critical.map(r => r.name).join(', ')}`)
|
||
process.exit(1)
|
||
}
|
||
} else if (!SAVE) {
|
||
console.log('no baseline — first run, use --save')
|
||
}
|
||
|
||
if (CRASHED > 0) {
|
||
console.error(`⚠ ${CRASHED} benchmark(s) crashed`)
|
||
process.exit(SAVE ? 0 : 2)
|
||
}
|