John Dvorak
ab0e569552
The CI benchmark gate was flagging wild run-to-run 'regressions'/'improvements' (-23%..-83% on unchanged code) because sub-ms checks were measured from ~5 samples with a global.gc() injected between every iteration: - uncertaintyThreshold 0.99 -> 0.1: the loop now keeps sampling until variance actually tightens instead of 'passing' at the first check. - minSamples 0 -> 200, maxSamples 200 -> 2000: real sample floor + headroom. - gcBetweenSamples true -> false: per-sample GC dominated sub-ms timings. - warmup phase: each hot path runs to steady state (JIT, lazy index, caches) before sampling, eliminating the bimodal ~4us vs ~20us distribution. - BATCH=100 for sub-ms checks: jitter amortizes across a batch per sample; the relative comparison stays exact because the baseline uses the same batch. - minimum-change threshold (MIN_HIGH_REGRESSION_PERCENT, default 10): a high-severity flag only fails CI when the change exceeds run-to-run noise. Result: within-run p95 spread is now ~5% instead of ~100x. Residual cross-run variance on loaded shared runners (this machine: load ~19) is environmental — the baseline alphaCuts capture per-run spread, not machine-load swings.
142 lines
3.4 KiB
JSON
142 lines
3.4 KiB
JSON
{
|
|
"version": 1,
|
|
"generated": "2026-08-03T21:02:18.659Z",
|
|
"actions": {
|
|
"check[direct-hit]": {
|
|
"mostPlausible": 0.13270348756111364,
|
|
"alphaCuts": {
|
|
"p50": {
|
|
"lower": 0.13113648835948058,
|
|
"upper": 0.13428864360995807
|
|
},
|
|
"p95": {
|
|
"lower": 0.12820153883971488,
|
|
"upper": 0.13736359391644548
|
|
},
|
|
"p99": {
|
|
"lower": 0.12681899549493264,
|
|
"upper": 0.13886133499002185
|
|
}
|
|
}
|
|
},
|
|
"check[union-ttu]": {
|
|
"mostPlausible": 0.13272407787199741,
|
|
"alphaCuts": {
|
|
"p50": {
|
|
"lower": 0.13111540538562938,
|
|
"upper": 0.13435208823425798
|
|
},
|
|
"p95": {
|
|
"lower": 0.12810370624158607,
|
|
"upper": 0.13751164134564425
|
|
},
|
|
"p99": {
|
|
"lower": 0.12668522950869812,
|
|
"upper": 0.1390508250938302
|
|
}
|
|
}
|
|
},
|
|
"check[denied-miss]": {
|
|
"mostPlausible": 0.10464274362739566,
|
|
"alphaCuts": {
|
|
"p50": {
|
|
"lower": 0.1035911653567046,
|
|
"upper": 0.10570542262187044
|
|
},
|
|
"p95": {
|
|
"lower": 0.10161518016378715,
|
|
"upper": 0.10776062041045117
|
|
},
|
|
"p99": {
|
|
"lower": 0.10068232198106161,
|
|
"upper": 0.10875916306014438
|
|
}
|
|
}
|
|
},
|
|
"check[include-meta]": {
|
|
"mostPlausible": 0.2032086398120905,
|
|
"alphaCuts": {
|
|
"p50": {
|
|
"lower": 0.20130903880967277,
|
|
"upper": 0.20512602330365767
|
|
},
|
|
"p95": {
|
|
"lower": 0.19773841496722822,
|
|
"upper": 0.20882979063960022
|
|
},
|
|
"p99": {
|
|
"lower": 0.19605028516281864,
|
|
"upper": 0.21062828369506048
|
|
}
|
|
}
|
|
},
|
|
"check[overlay-on-top]": {
|
|
"mostPlausible": 0.43801922339274096,
|
|
"alphaCuts": {
|
|
"p50": {
|
|
"lower": 0.4331486167515239,
|
|
"upper": 0.4429442096584253
|
|
},
|
|
"p95": {
|
|
"lower": 0.4240166116932131,
|
|
"upper": 0.45248423243066815
|
|
},
|
|
"p99": {
|
|
"lower": 0.4197093259995535,
|
|
"upper": 0.45712770847629536
|
|
}
|
|
}
|
|
},
|
|
"check[binary-direct]": {
|
|
"mostPlausible": 0.12194744387843882,
|
|
"alphaCuts": {
|
|
"p50": {
|
|
"lower": 0.12053468994850938,
|
|
"upper": 0.12337633544038318
|
|
},
|
|
"p95": {
|
|
"lower": 0.1178875990998377,
|
|
"upper": 0.1261469438761139
|
|
},
|
|
"p99": {
|
|
"lower": 0.11663982372031695,
|
|
"upper": 0.12749639398836587
|
|
}
|
|
}
|
|
},
|
|
"snapshot[build-binary]": {
|
|
"mostPlausible": 4.699470380992978,
|
|
"alphaCuts": {
|
|
"p50": {
|
|
"lower": 4.649083103245884,
|
|
"upper": 4.750403850161681
|
|
},
|
|
"p95": {
|
|
"lower": 4.554543327557406,
|
|
"upper": 4.849008463493959
|
|
},
|
|
"p99": {
|
|
"lower": 4.509934735938718,
|
|
"upper": 4.896973552928932
|
|
}
|
|
}
|
|
},
|
|
"snapshot[restore-binary]": {
|
|
"mostPlausible": 5.015702105071855,
|
|
"alphaCuts": {
|
|
"p50": {
|
|
"lower": 4.960977858992745,
|
|
"upper": 5.071029889482844
|
|
},
|
|
"p95": {
|
|
"lower": 4.858330254069017,
|
|
"upper": 5.178172214368928
|
|
},
|
|
"p99": {
|
|
"lower": 4.809905283270391,
|
|
"upper": 5.2303028693638565
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} |