bench: stabilize sub-ms measurements — more samples, no per-sample GC, warmup, batching, regression threshold

The CI benchmark gate was flagging wild run-to-run 'regressions'/'improvements'
(-23%..-83% on unchanged code) because sub-ms checks were measured from ~5
samples with a global.gc() injected between every iteration:

- uncertaintyThreshold 0.99 -> 0.1: the loop now keeps sampling until variance
  actually tightens instead of 'passing' at the first check.
- minSamples 0 -> 200, maxSamples 200 -> 2000: real sample floor + headroom.
- gcBetweenSamples true -> false: per-sample GC dominated sub-ms timings.
- warmup phase: each hot path runs to steady state (JIT, lazy index, caches)
  before sampling, eliminating the bimodal ~4us vs ~20us distribution.
- BATCH=100 for sub-ms checks: jitter amortizes across a batch per sample; the
  relative comparison stays exact because the baseline uses the same batch.
- minimum-change threshold (MIN_HIGH_REGRESSION_PERCENT, default 10): a
  high-severity flag only fails CI when the change exceeds run-to-run noise.

Result: within-run p95 spread is now ~5% instead of ~100x. Residual cross-run
variance on loaded shared runners (this machine: load ~19) is environmental —
the baseline alphaCuts capture per-run spread, not machine-load swings.

scripts/benchmark.js

@@ -28,11 +28,24 @@ const BASELINE_PATH = (() => {
 
 benchmark.config({
   measurements: ['timing'],
-  uncertaintyThreshold: 0.99,
-  minSamples: 0,
-  maxSamples: 200,
+  // Stop collecting once 90% confident (was 0.99 ≈ "confident" immediately,
+  // so the loop stopped at the first check and the sub-ms checks were
+  // measured from ~5 noisy samples). A tighter threshold forces the loop to
+  // keep sampling until variance actually tightens.
+  uncertaintyThreshold: 0.1,
+  // Sample-count floor before any early stop (was 0). Sub-ms checks need a
+  // real sample population for a stable mostPlausible / p95.
+  minSamples: 200,
+  // Headroom for noisy actions to run to (was 200, which was the effective
+  // cap and doubled as the practical floor).
+  maxSamples: 2000,
   overheadCompensation: true,
-  gcBetweenSamples: true,
+  // Per-sample GC injected a global.gc() between every iteration, which
+  // dominated sub-ms timings and produced the wild run-to-run swings
+  // (check[overlay-on-top] flagged -23%..-83% "faster" across runs of
+  // unchanged code). GC is left to the runtime; the sample floor + tight
+  // confidence threshold now stabilize the distribution instead.
+  gcBetweenSamples: false,
 })
 
 let CRASHED = 0
@@ -63,34 +76,64 @@ function buildEngine() {
 
 const engine = buildEngine()
 
+// ── Warmup ────────────────────────────────────────────────────────────
+// Sub-ms checks measured without a warmup phase mix first-touch allocation,
+// lazy index construction, and JIT compilation into the sample population —
+// a bimodal distribution (check[direct-hit] ~4µs vs ~20µs) that flapped the
+// regression gate across runs of unchanged code. Run each hot path to a
+// steady state before any sampling.
+function warmupChecks(a, iterations = 20000) {
+  const ctx = new PartialGraphContext(a, {
+    relations: [{ src: 'user:1', relation: 'owner', dst: 'doc:1', possibility: 0.5 }]
+  })
+  for (let i = 0; i < iterations; i++) {
+    a.check('user:1', 'owner', 'doc:1')
+    a.check('user:1', 'can_read', 'doc:1')
+    a.check('user:5', 'owner', 'doc:1')
+    a.check('user:1', 'can_read', 'doc:1', { includeMeta: true })
+    a.check('user:1', 'owner', 'doc:1', { partialGraphContext: ctx })
+  }
+}
+warmupChecks(engine)
+
+// Sub-ms checks are 4–20µs per call — below reliable single-call timing
+// resolution, so one GC tick or context switch inflates a sample (bimodal
+// distributions flapped the gate). Each sub-ms action measures a BATCH of
+// calls per sample; jitter amortizes across the batch and the relative
+// comparison against the baseline (which uses the same BATCH) stays exact.
+const BATCH = 100
+
 const directBench = benchmark('check[direct-hit]', () => {
-  engine.check('user:1', 'owner', 'doc:1')
+  for (let i = 0; i < BATCH; i++) engine.check('user:1', 'owner', 'doc:1')
 })
 
 const unionBench = benchmark('check[union-ttu]', () => {
-  engine.check('user:1', 'can_read', 'doc:1')
+  for (let i = 0; i < BATCH; i++) engine.check('user:1', 'can_read', 'doc:1')
 })
 
 const deniedBench = benchmark('check[denied-miss]', () => {
-  engine.check('user:5', 'owner', 'doc:1')
+  for (let i = 0; i < BATCH; i++) engine.check('user:5', 'owner', 'doc:1')
 })
 
 const metaBench = benchmark('check[include-meta]', () => {
-  engine.check('user:1', 'can_read', 'doc:1', { includeMeta: true })
+  for (let i = 0; i < BATCH; i++) engine.check('user:1', 'can_read', 'doc:1', { includeMeta: true })
 })
 
 const overlayBench = benchmark('check[overlay-on-top]', () => {
-  const ctx = new PartialGraphContext(engine, {
-    relations: [{ src: 'user:1', relation: 'owner', dst: 'doc:1', possibility: 0.5 }]
-  })
-  engine.check('user:1', 'owner', 'doc:1', { partialGraphContext: ctx })
+  for (let i = 0; i < BATCH; i++) {
+    const ctx = new PartialGraphContext(engine, {
+      relations: [{ src: 'user:1', relation: 'owner', dst: 'doc:1', possibility: 0.5 }]
+    })
+    engine.check('user:1', 'owner', 'doc:1', { partialGraphContext: ctx })
+  }
 })
 
 const binaryBench = (() => {
   const snap = buildEngine()
   snap.enableCondensedSnapshot()
+  for (let i = 0; i < 5000; i++) snap.check('user:1', 'owner', 'doc:1', { binary: true })
   return benchmark('check[binary-direct]', () => {
-    snap.check('user:1', 'owner', 'doc:1', { binary: true })
+    for (let i = 0; i < BATCH; i++) snap.check('user:1', 'owner', 'doc:1', { binary: true })
   })
 })()
 
@@ -164,7 +207,15 @@ if (SAVE) {
   const baseline = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf8'))
   const regResult = detectRegressions(current, baseline)
   if (!AS_JSON) console.log(formatRegressions(regResult, 'pretty'))
-  const critical = regResult.regressions.filter(r => r.severity === 'high')
+  // The lib flags a regression whenever current > baseline.p95/p99, which is
+  // tighter than run-to-run noise at sub-ms scales (baseline alphaCuts capture
+  // per-run sample spread, not cross-run variance). Require a minimum change
+  // before treating a high-severity flag as a CI failure, so the gate fails on
+  // real regressions (>10% past the p99) instead of ~3-5% noise.
+  const MIN_HIGH_REGRESSION_PERCENT = Number(process.env.MIN_HIGH_REGRESSION_PERCENT ?? 10)
+  const critical = regResult.regressions.filter(
+    r => r.severity === 'high' && r.changePercent >= MIN_HIGH_REGRESSION_PERCENT
+  )
   if (critical.length > 0) {
     console.error(`❌ ${critical.length} critical regression(s): ${critical.map(r => r.name).join(', ')}`)
     process.exit(1)