core/scripts/benchmark.js

#!/usr/bin/env node
/**
 * CI benchmark script for @arbiter/core — powered by @tenere/benchmark-lib
 * (possibilistic contours with valid alpha cuts).
 *
 *   node --expose-gc scripts/benchmark.js             compare against baseline
 *   node --expose-gc scripts/benchmark.js --save       save new baseline
 *   node --expose-gc scripts/benchmark.js --json       JSON output
 *   node --expose-gc scripts/benchmark.js --baseline <path>   custom baseline path
 *
 * Exit codes:
 *   0 — all benchmarks pass, no high-severity regressions
 *   1 — high-severity regression found
 *   2 — crash (benchmark threw unexpectedly)
 */

import fs from 'fs'
import { benchmark, createBaseline, detectRegressions, formatRegressions } from '@tenere/benchmark-lib'
import { Arbiter } from '../src/core/Arbiter.js'
import { PartialGraphContext } from '../src/core/PartialGraphContext.js'

const SAVE    = process.argv.includes('--save')
const AS_JSON = process.argv.includes('--json')
const BASELINE_PATH = (() => {
  const idx = process.argv.indexOf('--baseline')
  return idx >= 0 ? process.argv[idx + 1] : '.rigor-baseline.json'
})()

benchmark.config({
  measurements: ['timing'],
  // Stop collecting once 90% confident (was 0.99 ≈ "confident" immediately,
  // so the loop stopped at the first check and the sub-ms checks were
  // measured from ~5 noisy samples). A tighter threshold forces the loop to
  // keep sampling until variance actually tightens.
  uncertaintyThreshold: 0.1,
  // Sample-count floor before any early stop (was 0). Sub-ms checks need a
  // real sample population for a stable mostPlausible / p95.
  minSamples: 200,
  // Headroom for noisy actions to run to (was 200, which was the effective
  // cap and doubled as the practical floor).
  maxSamples: 2000,
  overheadCompensation: true,
  // Per-sample GC injected a global.gc() between every iteration, which
  // dominated sub-ms timings and produced the wild run-to-run swings
  // (check[overlay-on-top] flagged -23%..-83% "faster" across runs of
  // unchanged code). GC is left to the runtime; the sample floor + tight
  // confidence threshold now stabilize the distribution instead.
  gcBetweenSamples: false,
})

let CRASHED = 0

// ── Fixtures ──────────────────────────────────────────────────────────

function buildEngine() {
  const a = new Arbiter()
  const groupCount = 40
  for (let g = 0; g < groupCount; g++) a.addNode(`group:${g}`, 'group')
  for (let u = 0; u < 200; u++) {
    a.addNode(`user:${u}`, 'user')
    a.addNode(`doc:${u}`, 'doc')
  }
  a.setRelationConfig('owner', { type: 'direct' })
  a.setRelationConfig('member_of', { type: 'direct' })
  a.setRelationConfig('can_read', {
    type: 'union',
    rules: [{ kind: 'direct', relation: 'owner' }, { kind: 'tuple_to_userset', parentRelation: 'member_of', childRelation: 'owner' }]
  })
  for (let u = 0; u < 200; u++) {
    a.addRelation(`user:${u}`, 'owner', `doc:${u}`, { possibility: 0.9 + (u % 10) / 100 })
    a.addRelation(`user:${u}`, 'member_of', `group:${u % groupCount}`, { possibility: 0.7 })
  }
  for (let g = 0; g < groupCount; g++) a.addRelation(`group:${g}`, 'owner', `doc:${g}`, { possibility: 1.0 })
  return a
}

const engine = buildEngine()

// ── Warmup ────────────────────────────────────────────────────────────
// Sub-ms checks measured without a warmup phase mix first-touch allocation,
// lazy index construction, and JIT compilation into the sample population —
// a bimodal distribution (check[direct-hit] ~4µs vs ~20µs) that flapped the
// regression gate across runs of unchanged code. Run each hot path to a
// steady state before any sampling.
function warmupChecks(a, iterations = 20000) {
  const ctx = new PartialGraphContext(a, {
    relations: [{ src: 'user:1', relation: 'owner', dst: 'doc:1', possibility: 0.5 }]
  })
  for (let i = 0; i < iterations; i++) {
    a.check('user:1', 'owner', 'doc:1')
    a.check('user:1', 'can_read', 'doc:1')
    a.check('user:5', 'owner', 'doc:1')
    a.check('user:1', 'can_read', 'doc:1', { includeMeta: true })
    a.check('user:1', 'owner', 'doc:1', { partialGraphContext: ctx })
  }
}
warmupChecks(engine)

// Sub-ms checks are 4–20µs per call — below reliable single-call timing
// resolution, so one GC tick or context switch inflates a sample (bimodal
// distributions flapped the gate). Each sub-ms action measures a BATCH of
// calls per sample; jitter amortizes across the batch and the relative
// comparison against the baseline (which uses the same BATCH) stays exact.
const BATCH = 100

const directBench = benchmark('check[direct-hit]', () => {
  for (let i = 0; i < BATCH; i++) engine.check('user:1', 'owner', 'doc:1')
})

const unionBench = benchmark('check[union-ttu]', () => {
  for (let i = 0; i < BATCH; i++) engine.check('user:1', 'can_read', 'doc:1')
})

const deniedBench = benchmark('check[denied-miss]', () => {
  for (let i = 0; i < BATCH; i++) engine.check('user:5', 'owner', 'doc:1')
})

const metaBench = benchmark('check[include-meta]', () => {
  for (let i = 0; i < BATCH; i++) engine.check('user:1', 'can_read', 'doc:1', { includeMeta: true })
})

const overlayBench = benchmark('check[overlay-on-top]', () => {
  for (let i = 0; i < BATCH; i++) {
    const ctx = new PartialGraphContext(engine, {
      relations: [{ src: 'user:1', relation: 'owner', dst: 'doc:1', possibility: 0.5 }]
    })
    engine.check('user:1', 'owner', 'doc:1', { partialGraphContext: ctx })
  }
})

const binaryBench = (() => {
  const snap = buildEngine()
  snap.enableCondensedSnapshot()
  for (let i = 0; i < 5000; i++) snap.check('user:1', 'owner', 'doc:1', { binary: true })
  return benchmark('check[binary-direct]', () => {
    for (let i = 0; i < BATCH; i++) snap.check('user:1', 'owner', 'doc:1', { binary: true })
  })
})()

const snapshotBuildBench = (() => {
  const snap = buildEngine()
  snap.enableCondensedSnapshot()
  return benchmark('snapshot[build-binary]', () => {
    snap.toSnapshotBinary()
  })
})()

const snapshotRestoreBench = (() => {
  const snap = buildEngine()
  snap.enableCondensedSnapshot()
  const buf = snap.toSnapshotBinary()
  return benchmark('snapshot[restore-binary]', () => {
    Arbiter.fromSnapshotBinary(buf)
  })
})()

// ── Run ───────────────────────────────────────────────────────────────

function measure(handle) {
  try {
    return handle.contour('timing')
  } catch (e) {
    CRASHED++
    console.error(`⚠ ${e.message}`)
    return null
  }
}

const results = {}
const order = [directBench, unionBench, deniedBench, metaBench, overlayBench, binaryBench, snapshotBuildBench, snapshotRestoreBench]
for (const handle of order) {
  const contour = measure(handle)
  if (!contour) continue
  const mp = contour.mostPlausible()
  results[handle.name] = {
    mostPlausible: mp.value,
    alphaCuts: contour.alphaCuts(),
  }
}

const current = { actions: results, campaign: null, failures: 0 }

if (AS_JSON) {
  const out = { results: {}, regressions: [], improvements: [], crashed: CRASHED }
  for (const [name, r] of Object.entries(results)) {
    out.results[name] = { timing: { mp: r.mostPlausible, alphaCuts: r.alphaCuts } }
  }
  if (fs.existsSync(BASELINE_PATH) && !SAVE) {
    const baseline = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf8'))
    const regResult = detectRegressions(current, baseline)
    out.regressions = regResult.regressions
    out.improvements = regResult.improvements
  }
  console.log(JSON.stringify(out, null, 2))
} else {
  for (const [name, r] of Object.entries(results)) {
    const cuts = r.alphaCuts
    console.log(`${name}: mp=${r.mostPlausible.toFixed(4)}ms p95=[${cuts.p95.lower.toFixed(4)},${cuts.p95.upper.toFixed(4)}]`)
  }
}

if (SAVE) {
  const baseline = createBaseline([current])
  fs.writeFileSync(BASELINE_PATH, JSON.stringify(baseline, null, 2), 'utf8')
  console.log(`baseline saved: ${BASELINE_PATH}`)
} else if (fs.existsSync(BASELINE_PATH)) {
  const baseline = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf8'))
  const regResult = detectRegressions(current, baseline)
  if (!AS_JSON) console.log(formatRegressions(regResult, 'pretty'))

  // ── Ratio-based self-calibration ─────────────────────────────────────
  // Absolute timings swing with machine load (a shared runner at load 19
  // shifted every action +15..+50%). Comparing each action's RATIO to a cheap
  // reference action cancels the load: load scales all actions proportionally,
  // while a code regression shifts only the affected action's ratio. The
  // reference action itself is still gate-checked absolutely with a loose
  // bound (a regression of the reference would otherwise mask every ratio).
  const RATIO_REFERENCE = process.env.BENCH_RATIO_REFERENCE || 'check[direct-hit]'
  const RATIO_PERCENT = Number(process.env.RATIO_REGRESSION_PERCENT ?? 15)
  const REFERENCE_PERCENT = Number(process.env.REFERENCE_REGRESSION_PERCENT ?? 30)

  const currentRef = current.actions?.[RATIO_REFERENCE]?.mostPlausible
  const baselineRef = baseline.actions?.[RATIO_REFERENCE]?.mostPlausible
  const ratioViolations = []
  let referenceViolation = false

  if (currentRef > 0 && baselineRef > 0) {
    if (currentRef > baselineRef * (1 + REFERENCE_PERCENT / 100)) {
      referenceViolation = true
      ratioViolations.push(
        `${RATIO_REFERENCE} (reference) baseline=${baselineRef.toFixed(4)} → current=${currentRef.toFixed(4)} (+${(((currentRef / baselineRef) - 1) * 100).toFixed(1)}%)`
      )
    }
    for (const [name, cur] of Object.entries(current.actions)) {
      if (name === RATIO_REFERENCE) continue
      const base = baseline.actions?.[name]
      if (!base || !(base.mostPlausible > 0)) continue
      const curRatio = cur.mostPlausible / currentRef
      const baseRatio = base.mostPlausible / baselineRef
      if (curRatio > baseRatio * (1 + RATIO_PERCENT / 100)) {
        ratioViolations.push(
          `${name} ratio ${curRatio.toFixed(3)} → baseline ${baseRatio.toFixed(3)} (+${(((curRatio / baseRatio) - 1) * 100).toFixed(1)}%)`
        )
      }
    }
  }

  if (ratioViolations.length > 0) {
    console.error(`❌ ${ratioViolations.length} ratio regression(s):`)
    for (const v of ratioViolations) console.error(`   - ${v}`)
    if (referenceViolation) {
      console.error('   (the reference action regressed absolutely — ratios may be unreliable)')
    }
    process.exit(1)
  }
} else if (!SAVE) {
  console.log('no baseline — first run, use --save')
}

if (CRASHED > 0) {
  console.error(`⚠ ${CRASHED} benchmark(s) crashed`)
  process.exit(SAVE ? 0 : 2)
}