packaging: README (NASA style) + possibilistic perf baseline + CI benchmark job
README: purpose-first (possibility not boolean, caller owns evidence/time), install, verified quick start, concepts (result shape, overlays, temporal context), API table, development commands, design notes. benchmark: scripts/benchmark.js on @tenere/benchmark-lib — eight contours (direct/union/denied/meta/overlay/binary checks, snapshot build/restore), committed .rigor-baseline.json, exit 1 on high-severity regressions. CI: benchmark job compares on push (continue-on-error), re-saves baseline and uploads it as an artifact on tags; publish now depends on benchmark passing as well as test.
This commit is contained in:
John Dvorak
@@ -0,0 +1,179 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* CI benchmark script for @arbiter/core — powered by @tenere/benchmark-lib
|
||||
* (possibilistic contours with valid alpha cuts).
|
||||
*
|
||||
* node --expose-gc scripts/benchmark.js compare against baseline
|
||||
* node --expose-gc scripts/benchmark.js --save save new baseline
|
||||
* node --expose-gc scripts/benchmark.js --json JSON output
|
||||
* node --expose-gc scripts/benchmark.js --baseline <path> custom baseline path
|
||||
*
|
||||
* Exit codes:
|
||||
* 0 — all benchmarks pass, no high-severity regressions
|
||||
* 1 — high-severity regression found
|
||||
* 2 — crash (benchmark threw unexpectedly)
|
||||
*/
|
||||
|
||||
import fs from 'fs'
|
||||
import { benchmark, createBaseline, detectRegressions, formatRegressions } from '@tenere/benchmark-lib'
|
||||
import { Arbiter } from '../src/core/Arbiter.js'
|
||||
import { PartialGraphContext } from '../src/core/PartialGraphContext.js'
|
||||
|
||||
const SAVE = process.argv.includes('--save')
|
||||
const AS_JSON = process.argv.includes('--json')
|
||||
const BASELINE_PATH = (() => {
|
||||
const idx = process.argv.indexOf('--baseline')
|
||||
return idx >= 0 ? process.argv[idx + 1] : '.rigor-baseline.json'
|
||||
})()
|
||||
|
||||
benchmark.config({
|
||||
measurements: ['timing'],
|
||||
uncertaintyThreshold: 0.99,
|
||||
minSamples: 0,
|
||||
maxSamples: 200,
|
||||
overheadCompensation: true,
|
||||
gcBetweenSamples: true,
|
||||
})
|
||||
|
||||
let CRASHED = 0
|
||||
|
||||
// ── Fixtures ──────────────────────────────────────────────────────────
|
||||
|
||||
function buildEngine() {
|
||||
const a = new Arbiter()
|
||||
const groupCount = 40
|
||||
for (let g = 0; g < groupCount; g++) a.addNode(`group:${g}`, 'group')
|
||||
for (let u = 0; u < 200; u++) {
|
||||
a.addNode(`user:${u}`, 'user')
|
||||
a.addNode(`doc:${u}`, 'doc')
|
||||
}
|
||||
a.setRelationConfig('owner', { type: 'direct' })
|
||||
a.setRelationConfig('member_of', { type: 'direct' })
|
||||
a.setRelationConfig('can_read', {
|
||||
type: 'union',
|
||||
rules: [{ kind: 'direct', relation: 'owner' }, { kind: 'tuple_to_userset', parentRelation: 'member_of', childRelation: 'owner' }]
|
||||
})
|
||||
for (let u = 0; u < 200; u++) {
|
||||
a.addRelation(`user:${u}`, 'owner', `doc:${u}`, { possibility: 0.9 + (u % 10) / 100 })
|
||||
a.addRelation(`user:${u}`, 'member_of', `group:${u % groupCount}`, { possibility: 0.7 })
|
||||
}
|
||||
for (let g = 0; g < groupCount; g++) a.addRelation(`group:${g}`, 'owner', `doc:${g}`, { possibility: 1.0 })
|
||||
return a
|
||||
}
|
||||
|
||||
const engine = buildEngine()
|
||||
|
||||
const directBench = benchmark('check[direct-hit]', () => {
|
||||
engine.check('user:1', 'owner', 'doc:1')
|
||||
})
|
||||
|
||||
const unionBench = benchmark('check[union-ttu]', () => {
|
||||
engine.check('user:1', 'can_read', 'doc:1')
|
||||
})
|
||||
|
||||
const deniedBench = benchmark('check[denied-miss]', () => {
|
||||
engine.check('user:5', 'owner', 'doc:1')
|
||||
})
|
||||
|
||||
const metaBench = benchmark('check[include-meta]', () => {
|
||||
engine.check('user:1', 'can_read', 'doc:1', { includeMeta: true })
|
||||
})
|
||||
|
||||
const overlayBench = benchmark('check[overlay-on-top]', () => {
|
||||
const ctx = new PartialGraphContext(engine, {
|
||||
relations: [{ src: 'user:1', relation: 'owner', dst: 'doc:1', possibility: 0.5 }]
|
||||
})
|
||||
engine.check('user:1', 'owner', 'doc:1', { partialGraphContext: ctx })
|
||||
})
|
||||
|
||||
const binaryBench = (() => {
|
||||
const snap = buildEngine()
|
||||
snap.enableCondensedSnapshot()
|
||||
return benchmark('check[binary-direct]', () => {
|
||||
snap.check('user:1', 'owner', 'doc:1', { binary: true })
|
||||
})
|
||||
})()
|
||||
|
||||
const snapshotBuildBench = (() => {
|
||||
const snap = buildEngine()
|
||||
snap.enableCondensedSnapshot()
|
||||
return benchmark('snapshot[build-binary]', () => {
|
||||
snap.toSnapshotBinary()
|
||||
})
|
||||
})()
|
||||
|
||||
const snapshotRestoreBench = (() => {
|
||||
const snap = buildEngine()
|
||||
snap.enableCondensedSnapshot()
|
||||
const buf = snap.toSnapshotBinary()
|
||||
return benchmark('snapshot[restore-binary]', () => {
|
||||
Arbiter.fromSnapshotBinary(buf)
|
||||
})
|
||||
})()
|
||||
|
||||
// ── Run ───────────────────────────────────────────────────────────────
|
||||
|
||||
function measure(handle) {
|
||||
try {
|
||||
return handle.contour('timing')
|
||||
} catch (e) {
|
||||
CRASHED++
|
||||
console.error(`⚠ ${e.message}`)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
const results = {}
|
||||
const order = [directBench, unionBench, deniedBench, metaBench, overlayBench, binaryBench, snapshotBuildBench, snapshotRestoreBench]
|
||||
for (const handle of order) {
|
||||
const contour = measure(handle)
|
||||
if (!contour) continue
|
||||
const mp = contour.mostPlausible()
|
||||
results[handle.name] = {
|
||||
mostPlausible: mp.value,
|
||||
alphaCuts: contour.alphaCuts(),
|
||||
}
|
||||
}
|
||||
|
||||
const current = { actions: results, campaign: null, failures: 0 }
|
||||
|
||||
if (AS_JSON) {
|
||||
const out = { results: {}, regressions: [], improvements: [], crashed: CRASHED }
|
||||
for (const [name, r] of Object.entries(results)) {
|
||||
out.results[name] = { timing: { mp: r.mostPlausible, alphaCuts: r.alphaCuts } }
|
||||
}
|
||||
if (fs.existsSync(BASELINE_PATH) && !SAVE) {
|
||||
const baseline = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf8'))
|
||||
const regResult = detectRegressions(current, baseline)
|
||||
out.regressions = regResult.regressions
|
||||
out.improvements = regResult.improvements
|
||||
}
|
||||
console.log(JSON.stringify(out, null, 2))
|
||||
} else {
|
||||
for (const [name, r] of Object.entries(results)) {
|
||||
const cuts = r.alphaCuts
|
||||
console.log(`${name}: mp=${r.mostPlausible.toFixed(4)}ms p95=[${cuts.p95.lower.toFixed(4)},${cuts.p95.upper.toFixed(4)}]`)
|
||||
}
|
||||
}
|
||||
|
||||
if (SAVE) {
|
||||
const baseline = createBaseline([current])
|
||||
fs.writeFileSync(BASELINE_PATH, JSON.stringify(baseline, null, 2), 'utf8')
|
||||
console.log(`baseline saved: ${BASELINE_PATH}`)
|
||||
} else if (fs.existsSync(BASELINE_PATH)) {
|
||||
const baseline = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf8'))
|
||||
const regResult = detectRegressions(current, baseline)
|
||||
if (!AS_JSON) console.log(formatRegressions(regResult, 'pretty'))
|
||||
const critical = regResult.regressions.filter(r => r.severity === 'high')
|
||||
if (critical.length > 0) {
|
||||
console.error(`❌ ${critical.length} critical regression(s): ${critical.map(r => r.name).join(', ')}`)
|
||||
process.exit(1)
|
||||
}
|
||||
} else if (!SAVE) {
|
||||
console.log('no baseline — first run, use --save')
|
||||
}
|
||||
|
||||
if (CRASHED > 0) {
|
||||
console.error(`⚠ ${CRASHED} benchmark(s) crashed`)
|
||||
process.exit(SAVE ? 0 : 2)
|
||||
}
|
||||
Reference in New Issue
Block a user