Arbiter/core
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bench: stabilize sub-ms measurements — more samples, no per-sample GC, warmup, batching, regression threshold
CI / test (push) Successful in 5m53s
Details
CI / benchmark (push) Successful in 49s
Details
CI / publish (push) Has been skipped
Details

Browse Source 
The CI benchmark gate was flagging wild run-to-run 'regressions'/'improvements'
(-23%..-83% on unchanged code) because sub-ms checks were measured from ~5
samples with a global.gc() injected between every iteration:

- uncertaintyThreshold 0.99 -> 0.1: the loop now keeps sampling until variance
  actually tightens instead of 'passing' at the first check.
- minSamples 0 -> 200, maxSamples 200 -> 2000: real sample floor + headroom.
- gcBetweenSamples true -> false: per-sample GC dominated sub-ms timings.
- warmup phase: each hot path runs to steady state (JIT, lazy index, caches)
  before sampling, eliminating the bimodal ~4us vs ~20us distribution.
- BATCH=100 for sub-ms checks: jitter amortizes across a batch per sample; the
  relative comparison stays exact because the baseline uses the same batch.
- minimum-change threshold (MIN_HIGH_REGRESSION_PERCENT, default 10): a
  high-severity flag only fails CI when the change exceeds run-to-run noise.

Result: within-run p95 spread is now ~5% instead of ~100x. Residual cross-run
variance on loaded shared runners (this machine: load ~19) is environmental —
the baseline alphaCuts capture per-run spread, not machine-load swings.
This commit is contained in:
John Dvorak
2026-08-03 14:02:31 -07:00
parent ed34df4474
commit ab0e569552
2 changed files with 122 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.rigor-baseline.json
+57 -57
View File
@@ -1,140 +1,140 @@
{ {
"version": 1, "version": 1,
"generated": "2026-08-03T03:25:46.644Z", "generated": "2026-08-03T21:02:18.659Z",
"actions": { "actions": {
"check[direct-hit]": { "check[direct-hit]": {
"mostPlausible": 0.2641342114909732, "mostPlausible": 0.13270348756111364,
"alphaCuts": { "alphaCuts": {
"p50": { "p50": {
"lower": 0.11993100000017193, "lower": 0.13113648835948058,
"upper": 0.6825795028928235 "upper": 0.13428864360995807
}, },
"p95": { "p95": {
"lower": 0.11993100000017193, "lower": 0.12820153883971488,
"upper": 0.6825795028928235 "upper": 0.13736359391644548
}, },
"p99": { "p99": {
"lower": 0.11993100000017193, "lower": 0.12681899549493264,
"upper": 0.7088692690070402 "upper": 0.13886133499002185
} }
} }
}, },
"check[union-ttu]": { "check[union-ttu]": {
"mostPlausible": 0.5572143564747223, "mostPlausible": 0.13272407787199741,
"alphaCuts": { "alphaCuts": {
"p50": { "p50": {
"lower": 0.10743099999989908, "lower": 0.13111540538562938,
"upper": 0.5572148030094048 "upper": 0.13435208823425798
}, },
"p95": { "p95": {
"lower": 0.10743099999989908, "lower": 0.12810370624158607,
"upper": 0.5572148030094048 "upper": 0.13751164134564425
}, },
"p99": { "p99": {
"lower": 0.10743099999989908, "lower": 0.12668522950869812,
"upper": 0.5572148030094048 "upper": 0.1390508250938302
} }
} }
}, },
"check[denied-miss]": { "check[denied-miss]": {
"mostPlausible": 0.17001072644302176, "mostPlausible": 0.10464274362739566,
"alphaCuts": { "alphaCuts": {
"p50": { "p50": {
"lower": 0.10983099999997875, "lower": 0.1035911653567046,
"upper": 0.27403866354448264 "upper": 0.10570542262187044
}, },
"p95": { "p95": {
"lower": 0.10983099999997875, "lower": 0.10161518016378715,
"upper": 0.27403866354448264 "upper": 0.10776062041045117
}, },
"p99": { "p99": {
"lower": 0.10983099999997875, "lower": 0.10068232198106161,
"upper": 0.2792968083762517 "upper": 0.10875916306014438
} }
} }
}, },
"check[include-meta]": { "check[include-meta]": {
"mostPlausible": 0.4313421276490078, "mostPlausible": 0.2032086398120905,
"alphaCuts": { "alphaCuts": {
"p50": { "p50": {
"lower": 0.2319310000000246, "lower": 0.20130903880967277,
"upper": 0.9209354557840623 "upper": 0.20512602330365767
}, },
"p95": { "p95": {
"lower": 0.2319310000000246, "lower": 0.19773841496722822,
"upper": 0.9209354557840623 "upper": 0.20882979063960022
}, },
"p99": { "p99": {
"lower": 0.2319310000000246, "lower": 0.19605028516281864,
"upper": 0.9491636618587536 "upper": 0.21062828369506048
} }
} }
}, },
"check[overlay-on-top]": { "check[overlay-on-top]": {
"mostPlausible": 2.4209884301909703, "mostPlausible": 0.43801922339274096,
"alphaCuts": { "alphaCuts": {
"p50": { "p50": {
"lower": 2.4209884301909703, "lower": 0.4331486167515239,
"upper": 2.4209887680862785 "upper": 0.4429442096584253
}, },
"p95": { "p95": {
"lower": 2.4209884301909703, "lower": 0.4240166116932131,
"upper": 2.4209887680862785 "upper": 0.45248423243066815
}, },
"p99": { "p99": {
"lower": 2.4209884301909703, "lower": 0.4197093259995535,
"upper": 2.4209887680862785 "upper": 0.45712770847629536
} }
} }
}, },
"check[binary-direct]": { "check[binary-direct]": {
"mostPlausible": 0.24005599110530157, "mostPlausible": 0.12194744387843882,
"alphaCuts": { "alphaCuts": {
"p50": { "p50": {
"lower": 0.154731000000304, "lower": 0.12053468994850938,
"upper": 0.2756826494555559 "upper": 0.12337633544038318
}, },
"p95": { "p95": {
"lower": 0.154731000000304, "lower": 0.1178875990998377,
"upper": 0.41012851350897955 "upper": 0.1261469438761139
}, },
"p99": { "p99": {
"lower": 0.154731000000304, "lower": 0.11663982372031695,
"upper": 0.41896614930102694 "upper": 0.12749639398836587
} }
} }
}, },
"snapshot[build-binary]": { "snapshot[build-binary]": {
"mostPlausible": 7.446251599999954, "mostPlausible": 4.699470380992978,
"alphaCuts": { "alphaCuts": {
"p50": { "p50": {
"lower": 5.3102783288306075, "lower": 4.649083103245884,
"upper": 9.582224244110606 "upper": 4.750403850161681
}, },
"p95": { "p95": {
"lower": 5.3102783288306075, "lower": 4.554543327557406,
"upper": 9.582224244110606 "upper": 4.849008463493959
}, },
"p99": { "p99": {
"lower": 5.225227783447299, "lower": 4.509934735938718,
"upper": 9.667235344457687 "upper": 4.896973552928932
} }
} }
}, },
"snapshot[restore-binary]": { "snapshot[restore-binary]": {
"mostPlausible": 9.853481541731316, "mostPlausible": 5.015702105071855,
"alphaCuts": { "alphaCuts": {
"p50": { "p50": {
"lower": 7.845275634169648, "lower": 4.960977858992745,
"upper": 12.375747204692138 "upper": 5.071029889482844
}, },
"p95": { "p95": {
"lower": 4.7317310000000745, "lower": 4.858330254069017,
"upper": 21.54997911145655 "upper": 5.178172214368928
}, },
"p99": { "p99": {
"lower": 4.7317310000000745, "lower": 4.809905283270391,
"upper": 25.40483299999982 "upper": 5.2303028693638565
} }
} }
} }
scripts/benchmark.js
+65 -14
View File
@@ -28,11 +28,24 @@ const BASELINE_PATH = (() => {
benchmark.config({ benchmark.config({
measurements: ['timing'], measurements: ['timing'],
uncertaintyThreshold: 0.99, // Stop collecting once 90% confident (was 0.99 ≈ "confident" immediately,
minSamples: 0, // so the loop stopped at the first check and the sub-ms checks were
maxSamples: 200, // measured from ~5 noisy samples). A tighter threshold forces the loop to
// keep sampling until variance actually tightens.
uncertaintyThreshold: 0.1,
// Sample-count floor before any early stop (was 0). Sub-ms checks need a
// real sample population for a stable mostPlausible / p95.
minSamples: 200,
// Headroom for noisy actions to run to (was 200, which was the effective
// cap and doubled as the practical floor).
maxSamples: 2000,
overheadCompensation: true, overheadCompensation: true,
gcBetweenSamples: true, // Per-sample GC injected a global.gc() between every iteration, which
// dominated sub-ms timings and produced the wild run-to-run swings
// (check[overlay-on-top] flagged -23%..-83% "faster" across runs of
// unchanged code). GC is left to the runtime; the sample floor + tight
// confidence threshold now stabilize the distribution instead.
gcBetweenSamples: false,
}) })
let CRASHED = 0 let CRASHED = 0
@@ -63,34 +76,64 @@ function buildEngine() {
const engine = buildEngine() const engine = buildEngine()
// ── Warmup ────────────────────────────────────────────────────────────
// Sub-ms checks measured without a warmup phase mix first-touch allocation,
// lazy index construction, and JIT compilation into the sample population —
// a bimodal distribution (check[direct-hit] ~4µs vs ~20µs) that flapped the
// regression gate across runs of unchanged code. Run each hot path to a
// steady state before any sampling.
function warmupChecks(a, iterations = 20000) {
const ctx = new PartialGraphContext(a, {
relations: [{ src: 'user:1', relation: 'owner', dst: 'doc:1', possibility: 0.5 }]
})
for (let i = 0; i < iterations; i++) {
a.check('user:1', 'owner', 'doc:1')
a.check('user:1', 'can_read', 'doc:1')
a.check('user:5', 'owner', 'doc:1')
a.check('user:1', 'can_read', 'doc:1', { includeMeta: true })
a.check('user:1', 'owner', 'doc:1', { partialGraphContext: ctx })
}
}
warmupChecks(engine)
// Sub-ms checks are 4–20µs per call — below reliable single-call timing
// resolution, so one GC tick or context switch inflates a sample (bimodal
// distributions flapped the gate). Each sub-ms action measures a BATCH of
// calls per sample; jitter amortizes across the batch and the relative
// comparison against the baseline (which uses the same BATCH) stays exact.
const BATCH = 100
const directBench = benchmark('check[direct-hit]', () => { const directBench = benchmark('check[direct-hit]', () => {
engine.check('user:1', 'owner', 'doc:1') for (let i = 0; i < BATCH; i++) engine.check('user:1', 'owner', 'doc:1')
}) })
const unionBench = benchmark('check[union-ttu]', () => { const unionBench = benchmark('check[union-ttu]', () => {
engine.check('user:1', 'can_read', 'doc:1') for (let i = 0; i < BATCH; i++) engine.check('user:1', 'can_read', 'doc:1')
}) })
const deniedBench = benchmark('check[denied-miss]', () => { const deniedBench = benchmark('check[denied-miss]', () => {
engine.check('user:5', 'owner', 'doc:1') for (let i = 0; i < BATCH; i++) engine.check('user:5', 'owner', 'doc:1')
}) })
const metaBench = benchmark('check[include-meta]', () => { const metaBench = benchmark('check[include-meta]', () => {
engine.check('user:1', 'can_read', 'doc:1', { includeMeta: true }) for (let i = 0; i < BATCH; i++) engine.check('user:1', 'can_read', 'doc:1', { includeMeta: true })
}) })
const overlayBench = benchmark('check[overlay-on-top]', () => { const overlayBench = benchmark('check[overlay-on-top]', () => {
const ctx = new PartialGraphContext(engine, { for (let i = 0; i < BATCH; i++) {
relations: [{ src: 'user:1', relation: 'owner', dst: 'doc:1', possibility: 0.5 }] const ctx = new PartialGraphContext(engine, {
}) relations: [{ src: 'user:1', relation: 'owner', dst: 'doc:1', possibility: 0.5 }]
engine.check('user:1', 'owner', 'doc:1', { partialGraphContext: ctx }) })
engine.check('user:1', 'owner', 'doc:1', { partialGraphContext: ctx })
}
}) })
const binaryBench = (() => { const binaryBench = (() => {
const snap = buildEngine() const snap = buildEngine()
snap.enableCondensedSnapshot() snap.enableCondensedSnapshot()
for (let i = 0; i < 5000; i++) snap.check('user:1', 'owner', 'doc:1', { binary: true })
return benchmark('check[binary-direct]', () => { return benchmark('check[binary-direct]', () => {
snap.check('user:1', 'owner', 'doc:1', { binary: true }) for (let i = 0; i < BATCH; i++) snap.check('user:1', 'owner', 'doc:1', { binary: true })
}) })
})() })()
@@ -164,7 +207,15 @@ if (SAVE) {
const baseline = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf8')) const baseline = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf8'))
const regResult = detectRegressions(current, baseline) const regResult = detectRegressions(current, baseline)
if (!AS_JSON) console.log(formatRegressions(regResult, 'pretty')) if (!AS_JSON) console.log(formatRegressions(regResult, 'pretty'))
const critical = regResult.regressions.filter(r => r.severity === 'high') // The lib flags a regression whenever current > baseline.p95/p99, which is
// tighter than run-to-run noise at sub-ms scales (baseline alphaCuts capture
// per-run sample spread, not cross-run variance). Require a minimum change
// before treating a high-severity flag as a CI failure, so the gate fails on
// real regressions (>10% past the p99) instead of ~3-5% noise.
const MIN_HIGH_REGRESSION_PERCENT = Number(process.env.MIN_HIGH_REGRESSION_PERCENT ?? 10)
const critical = regResult.regressions.filter(
r => r.severity === 'high' && r.changePercent >= MIN_HIGH_REGRESSION_PERCENT
)
if (critical.length > 0) { if (critical.length > 0) {
console.error(`❌ ${critical.length} critical regression(s): ${critical.map(r => r.name).join(', ')}`) console.error(`❌ ${critical.length} critical regression(s): ${critical.map(r => r.name).join(', ')}`)
process.exit(1) process.exit(1)