core/scripts/benchmark.js

#!/usr/bin/env node
/**
 * CI benchmark script for @arbiter/core — powered by @tenere/benchmark-lib
 * (possibilistic contours with valid alpha cuts).
 *
 *   node --expose-gc scripts/benchmark.js             compare against baseline
 *   node --expose-gc scripts/benchmark.js --save       save new baseline
 *   node --expose-gc scripts/benchmark.js --json       JSON output
 *   node --expose-gc scripts/benchmark.js --baseline <path>   custom baseline path
 *
 * Exit codes:
 *   0 — all benchmarks pass, no high-severity regressions
 *   1 — high-severity regression found
 *   2 — crash (benchmark threw unexpectedly)
 */

import fs from 'fs'
import { benchmark, createBaseline, detectRegressions, formatRegressions } from '@tenere/benchmark-lib'
import { Arbiter } from '../src/core/Arbiter.js'
import { PartialGraphContext } from '../src/core/PartialGraphContext.js'

const SAVE    = process.argv.includes('--save')
const AS_JSON = process.argv.includes('--json')
const BASELINE_PATH = (() => {
  const idx = process.argv.indexOf('--baseline')
  return idx >= 0 ? process.argv[idx + 1] : '.rigor-baseline.json'
})()

benchmark.config({
  measurements: ['timing'],
  uncertaintyThreshold: 0.99,
  minSamples: 0,
  maxSamples: 200,
  overheadCompensation: true,
  gcBetweenSamples: true,
})

let CRASHED = 0

// ── Fixtures ──────────────────────────────────────────────────────────

function buildEngine() {
  const a = new Arbiter()
  const groupCount = 40
  for (let g = 0; g < groupCount; g++) a.addNode(`group:${g}`, 'group')
  for (let u = 0; u < 200; u++) {
    a.addNode(`user:${u}`, 'user')
    a.addNode(`doc:${u}`, 'doc')
  }
  a.setRelationConfig('owner', { type: 'direct' })
  a.setRelationConfig('member_of', { type: 'direct' })
  a.setRelationConfig('can_read', {
    type: 'union',
    rules: [{ kind: 'direct', relation: 'owner' }, { kind: 'tuple_to_userset', parentRelation: 'member_of', childRelation: 'owner' }]
  })
  for (let u = 0; u < 200; u++) {
    a.addRelation(`user:${u}`, 'owner', `doc:${u}`, { possibility: 0.9 + (u % 10) / 100 })
    a.addRelation(`user:${u}`, 'member_of', `group:${u % groupCount}`, { possibility: 0.7 })
  }
  for (let g = 0; g < groupCount; g++) a.addRelation(`group:${g}`, 'owner', `doc:${g}`, { possibility: 1.0 })
  return a
}

const engine = buildEngine()

const directBench = benchmark('check[direct-hit]', () => {
  engine.check('user:1', 'owner', 'doc:1')
})

const unionBench = benchmark('check[union-ttu]', () => {
  engine.check('user:1', 'can_read', 'doc:1')
})

const deniedBench = benchmark('check[denied-miss]', () => {
  engine.check('user:5', 'owner', 'doc:1')
})

const metaBench = benchmark('check[include-meta]', () => {
  engine.check('user:1', 'can_read', 'doc:1', { includeMeta: true })
})

const overlayBench = benchmark('check[overlay-on-top]', () => {
  const ctx = new PartialGraphContext(engine, {
    relations: [{ src: 'user:1', relation: 'owner', dst: 'doc:1', possibility: 0.5 }]
  })
  engine.check('user:1', 'owner', 'doc:1', { partialGraphContext: ctx })
})

const binaryBench = (() => {
  const snap = buildEngine()
  snap.enableCondensedSnapshot()
  return benchmark('check[binary-direct]', () => {
    snap.check('user:1', 'owner', 'doc:1', { binary: true })
  })
})()

const snapshotBuildBench = (() => {
  const snap = buildEngine()
  snap.enableCondensedSnapshot()
  return benchmark('snapshot[build-binary]', () => {
    snap.toSnapshotBinary()
  })
})()

const snapshotRestoreBench = (() => {
  const snap = buildEngine()
  snap.enableCondensedSnapshot()
  const buf = snap.toSnapshotBinary()
  return benchmark('snapshot[restore-binary]', () => {
    Arbiter.fromSnapshotBinary(buf)
  })
})()

// ── Run ───────────────────────────────────────────────────────────────

function measure(handle) {
  try {
    return handle.contour('timing')
  } catch (e) {
    CRASHED++
    console.error(`⚠ ${e.message}`)
    return null
  }
}

const results = {}
const order = [directBench, unionBench, deniedBench, metaBench, overlayBench, binaryBench, snapshotBuildBench, snapshotRestoreBench]
for (const handle of order) {
  const contour = measure(handle)
  if (!contour) continue
  const mp = contour.mostPlausible()
  results[handle.name] = {
    mostPlausible: mp.value,
    alphaCuts: contour.alphaCuts(),
  }
}

const current = { actions: results, campaign: null, failures: 0 }

if (AS_JSON) {
  const out = { results: {}, regressions: [], improvements: [], crashed: CRASHED }
  for (const [name, r] of Object.entries(results)) {
    out.results[name] = { timing: { mp: r.mostPlausible, alphaCuts: r.alphaCuts } }
  }
  if (fs.existsSync(BASELINE_PATH) && !SAVE) {
    const baseline = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf8'))
    const regResult = detectRegressions(current, baseline)
    out.regressions = regResult.regressions
    out.improvements = regResult.improvements
  }
  console.log(JSON.stringify(out, null, 2))
} else {
  for (const [name, r] of Object.entries(results)) {
    const cuts = r.alphaCuts
    console.log(`${name}: mp=${r.mostPlausible.toFixed(4)}ms p95=[${cuts.p95.lower.toFixed(4)},${cuts.p95.upper.toFixed(4)}]`)
  }
}

if (SAVE) {
  const baseline = createBaseline([current])
  fs.writeFileSync(BASELINE_PATH, JSON.stringify(baseline, null, 2), 'utf8')
  console.log(`baseline saved: ${BASELINE_PATH}`)
} else if (fs.existsSync(BASELINE_PATH)) {
  const baseline = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf8'))
  const regResult = detectRegressions(current, baseline)
  if (!AS_JSON) console.log(formatRegressions(regResult, 'pretty'))
  const critical = regResult.regressions.filter(r => r.severity === 'high')
  if (critical.length > 0) {
    console.error(`❌ ${critical.length} critical regression(s): ${critical.map(r => r.name).join(', ')}`)
    process.exit(1)
  }
} else if (!SAVE) {
  console.log('no baseline — first run, use --save')
}

if (CRASHED > 0) {
  console.error(`⚠ ${CRASHED} benchmark(s) crashed`)
  process.exit(SAVE ? 0 : 2)
}