2 Commits

Author SHA1 Message Date
John Dvorak bd9c74fb0e feat: tuple_to_userset intermediate reachability; ratio-based benchmark gate
CI / publish (push) Successful in 15s
CI / test (push) Successful in 5m7s
CI / benchmark (push) Successful in 53s
- ChainRule._expandRuleFromSrc now expands tuple_to_userset configs: from a
  source node the reachable set is the objects sharing an intermediate with
  the source (src ->computed-> intermediate ->tupleset-> object, direction
  aware, weakest-link combined). Lets a TTU evidence serve as an intermediate
  condition step in a chain.
- scripts/benchmark.js: ratio-based self-calibration. Comparing each action's
  RATIO to a cheap reference action (default check[direct-hit]) cancels
  machine-load swings that scale all actions proportionally, so the gate only
  fails on code regressions that shift a single action's ratio. The reference
  is still checked absolutely with a loose bound. Verified: stable across
  runs, and a simulated union-ttu slowdown is caught (+76.8% ratio).

Tests: chain-condition-step intermediate TTU expansion.
2026-08-03 15:43:48 -07:00
John Dvorak ab0e569552 bench: stabilize sub-ms measurements — more samples, no per-sample GC, warmup, batching, regression threshold
CI / test (push) Successful in 5m53s
CI / benchmark (push) Successful in 49s
CI / publish (push) Has been skipped
The CI benchmark gate was flagging wild run-to-run 'regressions'/'improvements'
(-23%..-83% on unchanged code) because sub-ms checks were measured from ~5
samples with a global.gc() injected between every iteration:

- uncertaintyThreshold 0.99 -> 0.1: the loop now keeps sampling until variance
  actually tightens instead of 'passing' at the first check.
- minSamples 0 -> 200, maxSamples 200 -> 2000: real sample floor + headroom.
- gcBetweenSamples true -> false: per-sample GC dominated sub-ms timings.
- warmup phase: each hot path runs to steady state (JIT, lazy index, caches)
  before sampling, eliminating the bimodal ~4us vs ~20us distribution.
- BATCH=100 for sub-ms checks: jitter amortizes across a batch per sample; the
  relative comparison stays exact because the baseline uses the same batch.
- minimum-change threshold (MIN_HIGH_REGRESSION_PERCENT, default 10): a
  high-severity flag only fails CI when the change exceeds run-to-run noise.

Result: within-run p95 spread is now ~5% instead of ~100x. Residual cross-run
variance on loaded shared runners (this machine: load ~19) is environmental —
the baseline alphaCuts capture per-run spread, not machine-load swings.
2026-08-03 14:02:31 -07:00
5 changed files with 223 additions and 74 deletions
+57 -57
View File
@@ -1,140 +1,140 @@
{
"version": 1,
"generated": "2026-08-03T03:25:46.644Z",
"generated": "2026-08-03T22:37:59.060Z",
"actions": {
"check[direct-hit]": {
"mostPlausible": 0.2641342114909732,
"mostPlausible": 0.10672237227527051,
"alphaCuts": {
"p50": {
"lower": 0.11993100000017193,
"upper": 0.6825795028928235
"lower": 0.105663874043709,
"upper": 0.10779178510155751
},
"p95": {
"lower": 0.11993100000017193,
"upper": 0.6825795028928235
"lower": 0.10367558838055704,
"upper": 0.10985914641251807
},
"p99": {
"lower": 0.11993100000017193,
"upper": 0.7088692690070402
"lower": 0.10273583797909441,
"upper": 0.11086333384764993
}
}
},
"check[union-ttu]": {
"mostPlausible": 0.5572143564747223,
"mostPlausible": 0.11249228961043199,
"alphaCuts": {
"p50": {
"lower": 0.10743099999989908,
"upper": 0.5572148030094048
"lower": 0.111604019277894,
"upper": 0.11338800600185026
},
"p95": {
"lower": 0.10743099999989908,
"upper": 0.5572148030094048
"lower": 0.10993048219120201,
"upper": 0.11511402952231622
},
"p99": {
"lower": 0.10743099999989908,
"upper": 0.5572148030094048
"lower": 0.1091378967617418,
"upper": 0.11594970606842417
}
}
},
"check[denied-miss]": {
"mostPlausible": 0.17001072644302176,
"mostPlausible": 0.08747524001151907,
"alphaCuts": {
"p50": {
"lower": 0.10983099999997875,
"upper": 0.27403866354448264
"lower": 0.08668417849217541,
"upper": 0.08827353325683636
},
"p95": {
"lower": 0.10983099999997875,
"upper": 0.27403866354448264
"lower": 0.085196295997943,
"upper": 0.08981478022606769
},
"p99": {
"lower": 0.10983099999997875,
"upper": 0.2792968083762517
"lower": 0.08449268650925873,
"upper": 0.09056310270952082
}
}
},
"check[include-meta]": {
"mostPlausible": 0.4313421276490078,
"mostPlausible": 0.17368093893555905,
"alphaCuts": {
"p50": {
"lower": 0.2319310000000246,
"upper": 0.9209354557840623
"lower": 0.17235808249234505,
"upper": 0.17501399072522753
},
"p95": {
"lower": 0.2319310000000246,
"upper": 0.9209354557840623
"lower": 0.1698647154489402,
"upper": 0.17758260042495247
},
"p99": {
"lower": 0.2319310000000246,
"upper": 0.9491636618587536
"lower": 0.16868343694884977,
"upper": 0.178826872871734
}
}
},
"check[overlay-on-top]": {
"mostPlausible": 2.4209884301909703,
"mostPlausible": 0.34658090606556824,
"alphaCuts": {
"p50": {
"lower": 2.4209884301909703,
"upper": 2.4209887680862785
"lower": 0.343563635087355,
"upper": 0.3496248120197711
},
"p95": {
"lower": 2.4209884301909703,
"upper": 2.4209887680862785
"lower": 0.33788494447103923,
"upper": 0.3555006559358586
},
"p99": {
"lower": 2.4209884301909703,
"upper": 2.4209887680862785
"lower": 0.33519739588838326,
"upper": 0.35835082770353416
}
}
},
"check[binary-direct]": {
"mostPlausible": 0.24005599110530157,
"mostPlausible": 0.10136432040214981,
"alphaCuts": {
"p50": {
"lower": 0.154731000000304,
"upper": 0.2756826494555559
"lower": 0.10038730449515523,
"upper": 0.10235072528492808
},
"p95": {
"lower": 0.154731000000304,
"upper": 0.41012851350897955
"lower": 0.0985518364498951,
"upper": 0.10425698760463886
},
"p99": {
"lower": 0.154731000000304,
"upper": 0.41896614930102694
"lower": 0.09768455509480933,
"upper": 0.10518292733500886
}
}
},
"snapshot[build-binary]": {
"mostPlausible": 7.446251599999954,
"mostPlausible": 2.848037533444555,
"alphaCuts": {
"p50": {
"lower": 5.3102783288306075,
"upper": 9.582224244110606
"lower": 2.827413825511722,
"upper": 2.868811889350609
},
"p95": {
"lower": 5.3102783288306075,
"upper": 9.582224244110606
"lower": 2.788520370277795,
"upper": 2.9088252291455836
},
"p99": {
"lower": 5.225227783447299,
"upper": 9.667235344457687
"lower": 2.770077133158435,
"upper": 2.9281936091667653
}
}
},
"snapshot[restore-binary]": {
"mostPlausible": 9.853481541731316,
"mostPlausible": 2.9488926124434385,
"alphaCuts": {
"p50": {
"lower": 7.845275634169648,
"upper": 12.375747204692138
"lower": 2.9247809483999716,
"upper": 2.9732031186198484
},
"p95": {
"lower": 4.7317310000000745,
"upper": 21.54997911145655
"lower": 2.8793716893148757,
"upper": 3.020092288118393
},
"p99": {
"lower": 4.7317310000000745,
"upper": 25.40483299999982
"lower": 2.8578657882531426,
"upper": 3.0428172805315117
}
}
}
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@arbiter/core",
"version": "1.0.4",
"version": "1.0.5",
"description": "Arbiter core engine: graph indices, relation/reachability, authorization rule evaluator, DSL/AST, condensed & sharded snapshots, and evidence fusion.",
"license": "ISC",
"author": "",
+100 -16
View File
@@ -28,11 +28,24 @@ const BASELINE_PATH = (() => {
benchmark.config({
measurements: ['timing'],
uncertaintyThreshold: 0.99,
minSamples: 0,
maxSamples: 200,
// Stop collecting once 90% confident (was 0.99 ≈ "confident" immediately,
// so the loop stopped at the first check and the sub-ms checks were
// measured from ~5 noisy samples). A tighter threshold forces the loop to
// keep sampling until variance actually tightens.
uncertaintyThreshold: 0.1,
// Sample-count floor before any early stop (was 0). Sub-ms checks need a
// real sample population for a stable mostPlausible / p95.
minSamples: 200,
// Headroom for noisy actions to run to (was 200, which was the effective
// cap and doubled as the practical floor).
maxSamples: 2000,
overheadCompensation: true,
gcBetweenSamples: true,
// Per-sample GC injected a global.gc() between every iteration, which
// dominated sub-ms timings and produced the wild run-to-run swings
// (check[overlay-on-top] flagged -23%..-83% "faster" across runs of
// unchanged code). GC is left to the runtime; the sample floor + tight
// confidence threshold now stabilize the distribution instead.
gcBetweenSamples: false,
})
let CRASHED = 0
@@ -63,34 +76,64 @@ function buildEngine() {
const engine = buildEngine()
// ── Warmup ────────────────────────────────────────────────────────────
// Sub-ms checks measured without a warmup phase mix first-touch allocation,
// lazy index construction, and JIT compilation into the sample population —
// a bimodal distribution (check[direct-hit] ~4µs vs ~20µs) that flapped the
// regression gate across runs of unchanged code. Run each hot path to a
// steady state before any sampling.
function warmupChecks(a, iterations = 20000) {
const ctx = new PartialGraphContext(a, {
relations: [{ src: 'user:1', relation: 'owner', dst: 'doc:1', possibility: 0.5 }]
})
for (let i = 0; i < iterations; i++) {
a.check('user:1', 'owner', 'doc:1')
a.check('user:1', 'can_read', 'doc:1')
a.check('user:5', 'owner', 'doc:1')
a.check('user:1', 'can_read', 'doc:1', { includeMeta: true })
a.check('user:1', 'owner', 'doc:1', { partialGraphContext: ctx })
}
}
warmupChecks(engine)
// Sub-ms checks are 420µs per call — below reliable single-call timing
// resolution, so one GC tick or context switch inflates a sample (bimodal
// distributions flapped the gate). Each sub-ms action measures a BATCH of
// calls per sample; jitter amortizes across the batch and the relative
// comparison against the baseline (which uses the same BATCH) stays exact.
const BATCH = 100
const directBench = benchmark('check[direct-hit]', () => {
engine.check('user:1', 'owner', 'doc:1')
for (let i = 0; i < BATCH; i++) engine.check('user:1', 'owner', 'doc:1')
})
const unionBench = benchmark('check[union-ttu]', () => {
engine.check('user:1', 'can_read', 'doc:1')
for (let i = 0; i < BATCH; i++) engine.check('user:1', 'can_read', 'doc:1')
})
const deniedBench = benchmark('check[denied-miss]', () => {
engine.check('user:5', 'owner', 'doc:1')
for (let i = 0; i < BATCH; i++) engine.check('user:5', 'owner', 'doc:1')
})
const metaBench = benchmark('check[include-meta]', () => {
engine.check('user:1', 'can_read', 'doc:1', { includeMeta: true })
for (let i = 0; i < BATCH; i++) engine.check('user:1', 'can_read', 'doc:1', { includeMeta: true })
})
const overlayBench = benchmark('check[overlay-on-top]', () => {
const ctx = new PartialGraphContext(engine, {
relations: [{ src: 'user:1', relation: 'owner', dst: 'doc:1', possibility: 0.5 }]
})
engine.check('user:1', 'owner', 'doc:1', { partialGraphContext: ctx })
for (let i = 0; i < BATCH; i++) {
const ctx = new PartialGraphContext(engine, {
relations: [{ src: 'user:1', relation: 'owner', dst: 'doc:1', possibility: 0.5 }]
})
engine.check('user:1', 'owner', 'doc:1', { partialGraphContext: ctx })
}
})
const binaryBench = (() => {
const snap = buildEngine()
snap.enableCondensedSnapshot()
for (let i = 0; i < 5000; i++) snap.check('user:1', 'owner', 'doc:1', { binary: true })
return benchmark('check[binary-direct]', () => {
snap.check('user:1', 'owner', 'doc:1', { binary: true })
for (let i = 0; i < BATCH; i++) snap.check('user:1', 'owner', 'doc:1', { binary: true })
})
})()
@@ -164,9 +207,50 @@ if (SAVE) {
const baseline = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf8'))
const regResult = detectRegressions(current, baseline)
if (!AS_JSON) console.log(formatRegressions(regResult, 'pretty'))
const critical = regResult.regressions.filter(r => r.severity === 'high')
if (critical.length > 0) {
console.error(`${critical.length} critical regression(s): ${critical.map(r => r.name).join(', ')}`)
// ── Ratio-based self-calibration ─────────────────────────────────────
// Absolute timings swing with machine load (a shared runner at load 19
// shifted every action +15..+50%). Comparing each action's RATIO to a cheap
// reference action cancels the load: load scales all actions proportionally,
// while a code regression shifts only the affected action's ratio. The
// reference action itself is still gate-checked absolutely with a loose
// bound (a regression of the reference would otherwise mask every ratio).
const RATIO_REFERENCE = process.env.BENCH_RATIO_REFERENCE || 'check[direct-hit]'
const RATIO_PERCENT = Number(process.env.RATIO_REGRESSION_PERCENT ?? 15)
const REFERENCE_PERCENT = Number(process.env.REFERENCE_REGRESSION_PERCENT ?? 30)
const currentRef = current.actions?.[RATIO_REFERENCE]?.mostPlausible
const baselineRef = baseline.actions?.[RATIO_REFERENCE]?.mostPlausible
const ratioViolations = []
let referenceViolation = false
if (currentRef > 0 && baselineRef > 0) {
if (currentRef > baselineRef * (1 + REFERENCE_PERCENT / 100)) {
referenceViolation = true
ratioViolations.push(
`${RATIO_REFERENCE} (reference) baseline=${baselineRef.toFixed(4)} → current=${currentRef.toFixed(4)} (+${(((currentRef / baselineRef) - 1) * 100).toFixed(1)}%)`
)
}
for (const [name, cur] of Object.entries(current.actions)) {
if (name === RATIO_REFERENCE) continue
const base = baseline.actions?.[name]
if (!base || !(base.mostPlausible > 0)) continue
const curRatio = cur.mostPlausible / currentRef
const baseRatio = base.mostPlausible / baselineRef
if (curRatio > baseRatio * (1 + RATIO_PERCENT / 100)) {
ratioViolations.push(
`${name} ratio ${curRatio.toFixed(3)} → baseline ${baseRatio.toFixed(3)} (+${(((curRatio / baseRatio) - 1) * 100).toFixed(1)}%)`
)
}
}
}
if (ratioViolations.length > 0) {
console.error(`${ratioViolations.length} ratio regression(s):`)
for (const v of ratioViolations) console.error(` - ${v}`)
if (referenceViolation) {
console.error(' (the reference action regressed absolutely — ratios may be unreliable)')
}
process.exit(1)
}
} else if (!SAVE) {
+36
View File
@@ -515,6 +515,9 @@ export class ChainRule extends BaseRule {
if (rule.type === 'chain' && Array.isArray(rule.steps)) {
return this._expandChainRuleFromSrc(srcId, rule, options);
}
if (rule.type === 'tuple_to_userset') {
return this._expandTtuFromSrc(srcId, rule, options);
}
if (rule.type === 'logical') {
if (rule.when || rule.unless || rule.never || rule.requires || rule.always) {
return this._expandDefeasibleFromSrc(srcId, rule, options);
@@ -538,6 +541,39 @@ export class ChainRule extends BaseRule {
return out;
}
/**
* Expand a tuple_to_userset config from a source node. Semantics (forward,
* the common case): src →computed→ intermediate, and intermediate ↔object
* via tupleset (direction decides which end the intermediate sits on). The
* reachable set is the OBJECTS sharing an intermediate with src. Combined
* possibility is the weakest link across the two hops.
*/
_expandTtuFromSrc(srcId, rule, options = null) {
const { tuplesetRelation, computedRelation, tuplesetDirection = 'out', reverse = false } = rule;
if (!tuplesetRelation || !computedRelation) return new Map();
// reverse swaps the roles: src is the object, intermediates come from the
// tupleset side, and computed edges go from intermediate to the user.
const srcAsUser = !reverse;
const computedRel = srcAsUser ? computedRelation : tuplesetRelation;
const tuplesetRel = srcAsUser ? tuplesetRelation : computedRelation;
const computedEdges = this.arbiter.relationManager.getRelationsFromSrc(srcId, computedRel, options);
const out = new Map();
for (const ce of computedEdges || []) {
const intermediateId = ce.dst;
const tsEdges = tuplesetDirection === 'in'
? this.arbiter.relationManager.getRelationsFromSrc(intermediateId, tuplesetRel, options)
: this.arbiter.relationManager.getRelationsToDst(intermediateId, tuplesetRel, options);
for (const te of tsEdges || []) {
const objId = tuplesetDirection === 'in' ? te.dst : te.src;
const poss = Math.min(ce.possibility ?? 1, te.possibility ?? 1);
const rel = (ce.reliability ?? 1) * (te.reliability ?? 1);
const cur = out.get(objId);
if (!cur || poss > cur.possibility) out.set(objId, { possibility: poss, reliability: rel });
}
}
return out;
}
_expandLogicalNodeFromSrc(srcId, node, op, options = null) {
const rules = (node && node.rules) || [];
if (rules.length === 0) return new Map();
+29
View File
@@ -106,6 +106,35 @@ describe('ChainRule condition step (rule-based final hop)', () => {
assert.equal(denied.possibility, 0);
});
it('expands a tuple_to_userset evidence as an intermediate condition step', () => {
arbiter.setRelationConfig('member_of', { type: 'direct' });
arbiter.setRelationConfig('owner', { type: 'direct' });
arbiter.setRelationConfig('doc_route', { type: 'direct' });
arbiter.addNode('group:g', 'group');
arbiter.addNode('doc:1', 'doc');
arbiter.addNode('doc:2', 'doc');
arbiter.addRelation('user:u', 'member_of', 'group:g', { possibility: 1.0 });
arbiter.addRelation('group:g', 'owner', 'doc:1', { possibility: 0.8 });
arbiter.addRelation('doc:1', 'doc_route', 'doc:2', { possibility: 0.9 });
const ttuConfig = {
type: 'tuple_to_userset',
tuplesetRelation: 'owner',
computedRelation: 'member_of',
tuplesetDirection: 'in'
};
const rule = {
type: 'chain',
steps: [{ rule: ttuConfig, conditionStep: true }, 'doc_route']
};
// TTU reachable from user = doc:1 (min(1.0, 0.8) = 0.8), then doc_route → 0.9
const res = evalRule('user:u', 'doc:2', rule);
assert.ok(Math.abs(res.possibility - 0.8) < 1e-9, `expected 0.8, got ${res.possibility} (${res.reason})`);
// Direct expansion unit check
const expanded = chainRule._expandRuleFromSrc(arbiter.resolveNodeId('user:u'), ttuConfig, {});
assert.equal(expanded.size, 1);
assert.equal(arbiter.resolveKey([...expanded.keys()][0]), 'doc:1');
});
it('combines across multiple parallel intermediates (max aggregation)', () => {
arbiter.addNode('group:g2', 'group');
arbiter.addRelation('user:u', 'member_of', 'group:g', { possibility: 0.5 });