John Dvorak
bd9c74fb0e
- ChainRule._expandRuleFromSrc now expands tuple_to_userset configs: from a source node the reachable set is the objects sharing an intermediate with the source (src ->computed-> intermediate ->tupleset-> object, direction aware, weakest-link combined). Lets a TTU evidence serve as an intermediate condition step in a chain. - scripts/benchmark.js: ratio-based self-calibration. Comparing each action's RATIO to a cheap reference action (default check[direct-hit]) cancels machine-load swings that scale all actions proportionally, so the gate only fails on code regressions that shift a single action's ratio. The reference is still checked absolutely with a loose bound. Verified: stable across runs, and a simulated union-ttu slowdown is caught (+76.8% ratio). Tests: chain-condition-step intermediate TTU expansion.
264 lines
11 KiB
JavaScript
264 lines
11 KiB
JavaScript
#!/usr/bin/env node
|
||
/**
|
||
* CI benchmark script for @arbiter/core — powered by @tenere/benchmark-lib
|
||
* (possibilistic contours with valid alpha cuts).
|
||
*
|
||
* node --expose-gc scripts/benchmark.js compare against baseline
|
||
* node --expose-gc scripts/benchmark.js --save save new baseline
|
||
* node --expose-gc scripts/benchmark.js --json JSON output
|
||
* node --expose-gc scripts/benchmark.js --baseline <path> custom baseline path
|
||
*
|
||
* Exit codes:
|
||
* 0 — all benchmarks pass, no high-severity regressions
|
||
* 1 — high-severity regression found
|
||
* 2 — crash (benchmark threw unexpectedly)
|
||
*/
|
||
|
||
import fs from 'fs'
|
||
import { benchmark, createBaseline, detectRegressions, formatRegressions } from '@tenere/benchmark-lib'
|
||
import { Arbiter } from '../src/core/Arbiter.js'
|
||
import { PartialGraphContext } from '../src/core/PartialGraphContext.js'
|
||
|
||
const SAVE = process.argv.includes('--save')
|
||
const AS_JSON = process.argv.includes('--json')
|
||
const BASELINE_PATH = (() => {
|
||
const idx = process.argv.indexOf('--baseline')
|
||
return idx >= 0 ? process.argv[idx + 1] : '.rigor-baseline.json'
|
||
})()
|
||
|
||
benchmark.config({
|
||
measurements: ['timing'],
|
||
// Stop collecting once 90% confident (was 0.99 ≈ "confident" immediately,
|
||
// so the loop stopped at the first check and the sub-ms checks were
|
||
// measured from ~5 noisy samples). A tighter threshold forces the loop to
|
||
// keep sampling until variance actually tightens.
|
||
uncertaintyThreshold: 0.1,
|
||
// Sample-count floor before any early stop (was 0). Sub-ms checks need a
|
||
// real sample population for a stable mostPlausible / p95.
|
||
minSamples: 200,
|
||
// Headroom for noisy actions to run to (was 200, which was the effective
|
||
// cap and doubled as the practical floor).
|
||
maxSamples: 2000,
|
||
overheadCompensation: true,
|
||
// Per-sample GC injected a global.gc() between every iteration, which
|
||
// dominated sub-ms timings and produced the wild run-to-run swings
|
||
// (check[overlay-on-top] flagged -23%..-83% "faster" across runs of
|
||
// unchanged code). GC is left to the runtime; the sample floor + tight
|
||
// confidence threshold now stabilize the distribution instead.
|
||
gcBetweenSamples: false,
|
||
})
|
||
|
||
let CRASHED = 0
|
||
|
||
// ── Fixtures ──────────────────────────────────────────────────────────
|
||
|
||
function buildEngine() {
|
||
const a = new Arbiter()
|
||
const groupCount = 40
|
||
for (let g = 0; g < groupCount; g++) a.addNode(`group:${g}`, 'group')
|
||
for (let u = 0; u < 200; u++) {
|
||
a.addNode(`user:${u}`, 'user')
|
||
a.addNode(`doc:${u}`, 'doc')
|
||
}
|
||
a.setRelationConfig('owner', { type: 'direct' })
|
||
a.setRelationConfig('member_of', { type: 'direct' })
|
||
a.setRelationConfig('can_read', {
|
||
type: 'union',
|
||
rules: [{ kind: 'direct', relation: 'owner' }, { kind: 'tuple_to_userset', parentRelation: 'member_of', childRelation: 'owner' }]
|
||
})
|
||
for (let u = 0; u < 200; u++) {
|
||
a.addRelation(`user:${u}`, 'owner', `doc:${u}`, { possibility: 0.9 + (u % 10) / 100 })
|
||
a.addRelation(`user:${u}`, 'member_of', `group:${u % groupCount}`, { possibility: 0.7 })
|
||
}
|
||
for (let g = 0; g < groupCount; g++) a.addRelation(`group:${g}`, 'owner', `doc:${g}`, { possibility: 1.0 })
|
||
return a
|
||
}
|
||
|
||
const engine = buildEngine()
|
||
|
||
// ── Warmup ────────────────────────────────────────────────────────────
|
||
// Sub-ms checks measured without a warmup phase mix first-touch allocation,
|
||
// lazy index construction, and JIT compilation into the sample population —
|
||
// a bimodal distribution (check[direct-hit] ~4µs vs ~20µs) that flapped the
|
||
// regression gate across runs of unchanged code. Run each hot path to a
|
||
// steady state before any sampling.
|
||
function warmupChecks(a, iterations = 20000) {
|
||
const ctx = new PartialGraphContext(a, {
|
||
relations: [{ src: 'user:1', relation: 'owner', dst: 'doc:1', possibility: 0.5 }]
|
||
})
|
||
for (let i = 0; i < iterations; i++) {
|
||
a.check('user:1', 'owner', 'doc:1')
|
||
a.check('user:1', 'can_read', 'doc:1')
|
||
a.check('user:5', 'owner', 'doc:1')
|
||
a.check('user:1', 'can_read', 'doc:1', { includeMeta: true })
|
||
a.check('user:1', 'owner', 'doc:1', { partialGraphContext: ctx })
|
||
}
|
||
}
|
||
warmupChecks(engine)
|
||
|
||
// Sub-ms checks are 4–20µs per call — below reliable single-call timing
|
||
// resolution, so one GC tick or context switch inflates a sample (bimodal
|
||
// distributions flapped the gate). Each sub-ms action measures a BATCH of
|
||
// calls per sample; jitter amortizes across the batch and the relative
|
||
// comparison against the baseline (which uses the same BATCH) stays exact.
|
||
const BATCH = 100
|
||
|
||
const directBench = benchmark('check[direct-hit]', () => {
|
||
for (let i = 0; i < BATCH; i++) engine.check('user:1', 'owner', 'doc:1')
|
||
})
|
||
|
||
const unionBench = benchmark('check[union-ttu]', () => {
|
||
for (let i = 0; i < BATCH; i++) engine.check('user:1', 'can_read', 'doc:1')
|
||
})
|
||
|
||
const deniedBench = benchmark('check[denied-miss]', () => {
|
||
for (let i = 0; i < BATCH; i++) engine.check('user:5', 'owner', 'doc:1')
|
||
})
|
||
|
||
const metaBench = benchmark('check[include-meta]', () => {
|
||
for (let i = 0; i < BATCH; i++) engine.check('user:1', 'can_read', 'doc:1', { includeMeta: true })
|
||
})
|
||
|
||
const overlayBench = benchmark('check[overlay-on-top]', () => {
|
||
for (let i = 0; i < BATCH; i++) {
|
||
const ctx = new PartialGraphContext(engine, {
|
||
relations: [{ src: 'user:1', relation: 'owner', dst: 'doc:1', possibility: 0.5 }]
|
||
})
|
||
engine.check('user:1', 'owner', 'doc:1', { partialGraphContext: ctx })
|
||
}
|
||
})
|
||
|
||
const binaryBench = (() => {
|
||
const snap = buildEngine()
|
||
snap.enableCondensedSnapshot()
|
||
for (let i = 0; i < 5000; i++) snap.check('user:1', 'owner', 'doc:1', { binary: true })
|
||
return benchmark('check[binary-direct]', () => {
|
||
for (let i = 0; i < BATCH; i++) snap.check('user:1', 'owner', 'doc:1', { binary: true })
|
||
})
|
||
})()
|
||
|
||
const snapshotBuildBench = (() => {
|
||
const snap = buildEngine()
|
||
snap.enableCondensedSnapshot()
|
||
return benchmark('snapshot[build-binary]', () => {
|
||
snap.toSnapshotBinary()
|
||
})
|
||
})()
|
||
|
||
const snapshotRestoreBench = (() => {
|
||
const snap = buildEngine()
|
||
snap.enableCondensedSnapshot()
|
||
const buf = snap.toSnapshotBinary()
|
||
return benchmark('snapshot[restore-binary]', () => {
|
||
Arbiter.fromSnapshotBinary(buf)
|
||
})
|
||
})()
|
||
|
||
// ── Run ───────────────────────────────────────────────────────────────
|
||
|
||
function measure(handle) {
|
||
try {
|
||
return handle.contour('timing')
|
||
} catch (e) {
|
||
CRASHED++
|
||
console.error(`⚠ ${e.message}`)
|
||
return null
|
||
}
|
||
}
|
||
|
||
const results = {}
|
||
const order = [directBench, unionBench, deniedBench, metaBench, overlayBench, binaryBench, snapshotBuildBench, snapshotRestoreBench]
|
||
for (const handle of order) {
|
||
const contour = measure(handle)
|
||
if (!contour) continue
|
||
const mp = contour.mostPlausible()
|
||
results[handle.name] = {
|
||
mostPlausible: mp.value,
|
||
alphaCuts: contour.alphaCuts(),
|
||
}
|
||
}
|
||
|
||
const current = { actions: results, campaign: null, failures: 0 }
|
||
|
||
if (AS_JSON) {
|
||
const out = { results: {}, regressions: [], improvements: [], crashed: CRASHED }
|
||
for (const [name, r] of Object.entries(results)) {
|
||
out.results[name] = { timing: { mp: r.mostPlausible, alphaCuts: r.alphaCuts } }
|
||
}
|
||
if (fs.existsSync(BASELINE_PATH) && !SAVE) {
|
||
const baseline = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf8'))
|
||
const regResult = detectRegressions(current, baseline)
|
||
out.regressions = regResult.regressions
|
||
out.improvements = regResult.improvements
|
||
}
|
||
console.log(JSON.stringify(out, null, 2))
|
||
} else {
|
||
for (const [name, r] of Object.entries(results)) {
|
||
const cuts = r.alphaCuts
|
||
console.log(`${name}: mp=${r.mostPlausible.toFixed(4)}ms p95=[${cuts.p95.lower.toFixed(4)},${cuts.p95.upper.toFixed(4)}]`)
|
||
}
|
||
}
|
||
|
||
if (SAVE) {
|
||
const baseline = createBaseline([current])
|
||
fs.writeFileSync(BASELINE_PATH, JSON.stringify(baseline, null, 2), 'utf8')
|
||
console.log(`baseline saved: ${BASELINE_PATH}`)
|
||
} else if (fs.existsSync(BASELINE_PATH)) {
|
||
const baseline = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf8'))
|
||
const regResult = detectRegressions(current, baseline)
|
||
if (!AS_JSON) console.log(formatRegressions(regResult, 'pretty'))
|
||
|
||
// ── Ratio-based self-calibration ─────────────────────────────────────
|
||
// Absolute timings swing with machine load (a shared runner at load 19
|
||
// shifted every action +15..+50%). Comparing each action's RATIO to a cheap
|
||
// reference action cancels the load: load scales all actions proportionally,
|
||
// while a code regression shifts only the affected action's ratio. The
|
||
// reference action itself is still gate-checked absolutely with a loose
|
||
// bound (a regression of the reference would otherwise mask every ratio).
|
||
const RATIO_REFERENCE = process.env.BENCH_RATIO_REFERENCE || 'check[direct-hit]'
|
||
const RATIO_PERCENT = Number(process.env.RATIO_REGRESSION_PERCENT ?? 15)
|
||
const REFERENCE_PERCENT = Number(process.env.REFERENCE_REGRESSION_PERCENT ?? 30)
|
||
|
||
const currentRef = current.actions?.[RATIO_REFERENCE]?.mostPlausible
|
||
const baselineRef = baseline.actions?.[RATIO_REFERENCE]?.mostPlausible
|
||
const ratioViolations = []
|
||
let referenceViolation = false
|
||
|
||
if (currentRef > 0 && baselineRef > 0) {
|
||
if (currentRef > baselineRef * (1 + REFERENCE_PERCENT / 100)) {
|
||
referenceViolation = true
|
||
ratioViolations.push(
|
||
`${RATIO_REFERENCE} (reference) baseline=${baselineRef.toFixed(4)} → current=${currentRef.toFixed(4)} (+${(((currentRef / baselineRef) - 1) * 100).toFixed(1)}%)`
|
||
)
|
||
}
|
||
for (const [name, cur] of Object.entries(current.actions)) {
|
||
if (name === RATIO_REFERENCE) continue
|
||
const base = baseline.actions?.[name]
|
||
if (!base || !(base.mostPlausible > 0)) continue
|
||
const curRatio = cur.mostPlausible / currentRef
|
||
const baseRatio = base.mostPlausible / baselineRef
|
||
if (curRatio > baseRatio * (1 + RATIO_PERCENT / 100)) {
|
||
ratioViolations.push(
|
||
`${name} ratio ${curRatio.toFixed(3)} → baseline ${baseRatio.toFixed(3)} (+${(((curRatio / baseRatio) - 1) * 100).toFixed(1)}%)`
|
||
)
|
||
}
|
||
}
|
||
}
|
||
|
||
if (ratioViolations.length > 0) {
|
||
console.error(`❌ ${ratioViolations.length} ratio regression(s):`)
|
||
for (const v of ratioViolations) console.error(` - ${v}`)
|
||
if (referenceViolation) {
|
||
console.error(' (the reference action regressed absolutely — ratios may be unreliable)')
|
||
}
|
||
process.exit(1)
|
||
}
|
||
} else if (!SAVE) {
|
||
console.log('no baseline — first run, use --save')
|
||
}
|
||
|
||
if (CRASHED > 0) {
|
||
console.error(`⚠ ${CRASHED} benchmark(s) crashed`)
|
||
process.exit(SAVE ? 0 : 2)
|
||
}
|