bench: complex-query cold-traffic benchmark (normal vs binary)
Prod gating is dominated by rule-based queries, not direct relations. complex-query-bench.js measures cold-traffic latency (distinct subject/object per sample, no cache reuse) across seven complex policy shapes — tuple-to-userset, 2-hop chain, defeasible exclusion, ABAC relational comparator, OWA union, nested comparator + OWA fusion, and a mixed 10-rule union — for both evaluation paths, and enforces binary/ normal decision parity on every query. At 25k and 100k nodes: binary wins every scenario (1.15x-2.0x median speedup), p99 stays sub-0.05ms, and parity mismatches are zero across all scenarios. Binary's early exit wins where a strong rule exists; the earlier direct-relation 'binary slower' observation was a cache-hit artifact (normal serves repeat queries from the rule result cache, binary correctly does not, since thresholds are per-call options). Note: report median, not avg — GC outliers inflate the mean (avg > p95 observed on two rows).
This commit is contained in:
John Dvorak
@@ -0,0 +1,259 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
/**
|
||||||
|
* Complex-query benchmark for @arbiter/core.
|
||||||
|
*
|
||||||
|
* Realistic prod auth gating is dominated by rule-based queries (chains,
|
||||||
|
* tuple-to-userset, defeasible, relational comparators, OWA unions), not
|
||||||
|
* direct relations. This benchmark measures cold-traffic latency (every
|
||||||
|
* sample a distinct subject/object pair — no cache reuse) for both the
|
||||||
|
* normal path and the binary (early-exit) path, and enforces that the
|
||||||
|
* binary decision agrees with the normal decision on every query.
|
||||||
|
*
|
||||||
|
* node --expose-gc benchmarks/complex-query-bench.js
|
||||||
|
* node --expose-gc benchmarks/complex-query-bench.js --size=25000 --samples=20000
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { performance } from 'node:perf_hooks';
|
||||||
|
import { Arbiter } from '../src/core/Arbiter.js';
|
||||||
|
|
||||||
|
function parseArgs(argv) {
|
||||||
|
const args = new Map();
|
||||||
|
for (let i = 2; i < argv.length; i++) {
|
||||||
|
const value = argv[i];
|
||||||
|
if (!value.startsWith('--')) continue;
|
||||||
|
const [key, inline] = value.slice(2).split('=');
|
||||||
|
if (inline !== undefined) args.set(key, inline);
|
||||||
|
else if (argv[i + 1] && !argv[i + 1].startsWith('--')) { args.set(key, argv[i + 1]); i++; }
|
||||||
|
else args.set(key, true);
|
||||||
|
}
|
||||||
|
return args;
|
||||||
|
}
|
||||||
|
|
||||||
|
function createRng(seed) {
|
||||||
|
let state = seed >>> 0;
|
||||||
|
return () => {
|
||||||
|
state = (1664525 * state + 1013904223) >>> 0;
|
||||||
|
return state / 0x100000000;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function randInt(rng, max) { return Math.floor(rng() * max); }
|
||||||
|
function percentile(sorted, p) { return sorted[Math.min(sorted.length - 1, Math.max(0, Math.floor(sorted.length * p) - 1))]; }
|
||||||
|
|
||||||
|
function buildUsers(arbiter, count, prefix) {
|
||||||
|
for (let i = 0; i < count; i++) arbiter.addNode(`${prefix}:${i}`, prefix);
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeQueries(size, rng, pickAllow, pickDeny, count = 4000) {
|
||||||
|
const queries = [];
|
||||||
|
for (let i = 0; i < count; i++) {
|
||||||
|
const allow = i % 2 === 0;
|
||||||
|
const q = allow ? pickAllow() : pickDeny();
|
||||||
|
queries.push({ user: q.user, object: q.object, allow });
|
||||||
|
}
|
||||||
|
return queries;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Scenarios (all complex — no bare direct relations) ────────────────
|
||||||
|
|
||||||
|
const scenarios = [
|
||||||
|
{
|
||||||
|
name: 'tuple_to_userset (group membership)',
|
||||||
|
build: (arbiter, size, rng) => {
|
||||||
|
const groupCount = Math.max(1, Math.floor(size / 10));
|
||||||
|
buildUsers(arbiter, size, 'user');
|
||||||
|
buildUsers(arbiter, groupCount, 'group');
|
||||||
|
buildUsers(arbiter, groupCount, 'resource');
|
||||||
|
for (let i = 0; i < size; i++) arbiter.addRelation(`user:${i}`, 'member', `group:${i % groupCount}`);
|
||||||
|
for (let i = 0; i < groupCount; i++) arbiter.addRelation(`group:${i}`, 'group_access', `resource:${i}`);
|
||||||
|
arbiter.setRelationConfig('member', { type: 'direct' });
|
||||||
|
arbiter.setRelationConfig('group_access', { type: 'direct' });
|
||||||
|
arbiter.setRelationConfig('can_view', { type: 'tuple_to_userset', tuplesetRelation: 'group_access', tuplesetDirection: 'in', computedRelation: 'member' });
|
||||||
|
return makeQueries(size, rng,
|
||||||
|
() => { const id = randInt(rng, size); return { user: `user:${id}`, object: `resource:${id % groupCount}` }; },
|
||||||
|
() => { const id = randInt(rng, size); return { user: `user:${id}`, object: `resource:${(id + 1) % groupCount}` }; });
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'chain (2-hop)',
|
||||||
|
build: (arbiter, size, rng) => {
|
||||||
|
const planCount = Math.max(1, Math.floor(size / 10));
|
||||||
|
buildUsers(arbiter, size, 'user');
|
||||||
|
buildUsers(arbiter, planCount, 'plan');
|
||||||
|
buildUsers(arbiter, planCount, 'resource');
|
||||||
|
for (let i = 0; i < size; i++) arbiter.addRelation(`user:${i}`, 'has_plan', `plan:${i % planCount}`);
|
||||||
|
for (let i = 0; i < planCount; i++) arbiter.addRelation(`plan:${i}`, 'plan_grants', `resource:${i}`);
|
||||||
|
arbiter.setRelationConfig('has_plan', { type: 'direct' });
|
||||||
|
arbiter.setRelationConfig('plan_grants', { type: 'direct' });
|
||||||
|
arbiter.setRelationConfig('can_use', { type: 'chain', steps: [
|
||||||
|
{ relation: 'has_plan', direction: 'out' }, { relation: 'plan_grants', direction: 'out' }], collectValues: false });
|
||||||
|
return makeQueries(size, rng,
|
||||||
|
() => { const id = randInt(rng, size); return { user: `user:${id}`, object: `resource:${id % planCount}` }; },
|
||||||
|
() => { const id = randInt(rng, size); return { user: `user:${id}`, object: `resource:${(id + 1) % planCount}` }; });
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'defeasible exclusion (viewer minus blocked)',
|
||||||
|
build: (arbiter, size, rng) => {
|
||||||
|
buildUsers(arbiter, size, 'user');
|
||||||
|
buildUsers(arbiter, size, 'resource');
|
||||||
|
for (let i = 0; i < size; i++) arbiter.addRelation(`user:${i}`, 'viewer', `resource:${i}`);
|
||||||
|
for (let i = 0; i < Math.floor(size / 4); i++) arbiter.addRelation(`user:${i}`, 'blocked', `resource:${i}`);
|
||||||
|
arbiter.setRelationConfig('viewer', { type: 'direct' });
|
||||||
|
arbiter.setRelationConfig('blocked', { type: 'direct' });
|
||||||
|
arbiter.setRelationConfig('can_view_blocked', { exclusion: [
|
||||||
|
{ type: 'direct', relation: 'viewer' }, { type: 'direct', relation: 'blocked' }] });
|
||||||
|
return makeQueries(size, rng,
|
||||||
|
() => { const id = Math.floor(size / 4) + randInt(rng, size - Math.floor(size / 4)); return { user: `user:${id}`, object: `resource:${id}` }; },
|
||||||
|
() => { const id = randInt(rng, Math.floor(size / 4)); return { user: `user:${id}`, object: `resource:${id}` }; });
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'relational comparator (ABAC age)',
|
||||||
|
build: (arbiter, size, rng) => {
|
||||||
|
buildUsers(arbiter, size, 'user');
|
||||||
|
buildUsers(arbiter, size, 'resource');
|
||||||
|
for (let i = 0; i < size; i++) {
|
||||||
|
arbiter.addRelation(`user:${i}`, 'age', `user:${i}`, 1.0, { value: 18 + (i % 30) });
|
||||||
|
arbiter.addRelation(`resource:${i}`, 'min_age', `resource:${i}`, 1.0, { value: 18 + (i % 15) });
|
||||||
|
}
|
||||||
|
arbiter.setRelationConfig('age', { type: 'direct' });
|
||||||
|
arbiter.setRelationConfig('min_age', { type: 'direct' });
|
||||||
|
arbiter.setRelationConfig('age_ok', { type: 'relational_comparator', comparator: '>=', fallbackBehavior: 'deny',
|
||||||
|
left: { rule: { type: 'direct', relation: 'age' }, extractValue: true, valueRelation: 'age' },
|
||||||
|
right: { rule: { type: 'direct', relation: 'min_age' }, extractValue: true, valueRelation: 'min_age', evaluateFrom: 'object' } });
|
||||||
|
return makeQueries(size, rng,
|
||||||
|
() => { const id = randInt(rng, size); return { user: `user:${id}`, object: `resource:${id % size}` }; },
|
||||||
|
() => { const id = randInt(rng, size); return { user: `user:${id}`, object: `resource:${(id + Math.floor(size / 2)) % size}` }; });
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'OWA union (3 weighted sources)',
|
||||||
|
build: (arbiter, size, rng) => {
|
||||||
|
buildUsers(arbiter, size, 'user');
|
||||||
|
buildUsers(arbiter, size, 'resource');
|
||||||
|
for (let i = 0; i < size; i++) {
|
||||||
|
arbiter.addRelation(`user:${i}`, 'direct_allow', `resource:${i}`, { possibility: i % 2 === 0 ? 0.9 : 0.2 });
|
||||||
|
arbiter.addRelation(`user:${i}`, 'group_allow', `resource:${i}`, { possibility: i % 3 === 0 ? 0.8 : 0.1 });
|
||||||
|
arbiter.addRelation(`user:${i}`, 'manual_allow', `resource:${i}`, { possibility: i % 5 === 0 ? 0.7 : 0.05 });
|
||||||
|
}
|
||||||
|
arbiter.setRelationConfig('direct_allow', { type: 'direct' });
|
||||||
|
arbiter.setRelationConfig('group_allow', { type: 'direct' });
|
||||||
|
arbiter.setRelationConfig('manual_allow', { type: 'direct' });
|
||||||
|
arbiter.setRelationConfig('can_view_owa', { union: {
|
||||||
|
rules: [
|
||||||
|
{ type: 'direct', relation: 'direct_allow' },
|
||||||
|
{ type: 'direct', relation: 'group_allow' },
|
||||||
|
{ type: 'direct', relation: 'manual_allow' }
|
||||||
|
],
|
||||||
|
aggregator: 'owa', owaWeights: [0.7, 0.2, 0.1] } });
|
||||||
|
return makeQueries(size, rng,
|
||||||
|
() => { const id = randInt(rng, size); return { user: `user:${id}`, object: `resource:${id}` }; },
|
||||||
|
() => { const id = randInt(rng, size); return { user: `user:${id}`, object: `resource:${(id + 1) % size}` }; });
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'nested comparator + OWA fusion',
|
||||||
|
build: (arbiter, size, rng) => {
|
||||||
|
buildUsers(arbiter, size, 'user');
|
||||||
|
buildUsers(arbiter, size, 'resource');
|
||||||
|
for (let i = 0; i < size; i++) {
|
||||||
|
const score = i % 2 === 0 ? 20 : 80;
|
||||||
|
arbiter.addRelation(`user:${i}`, 'risk_score', `resource:${i}`, 1.0, { value: score });
|
||||||
|
arbiter.addRelation(`user:${i}`, 'risk_bonus', `resource:${i}`, 1.0, { value: i % 2 === 0 ? 5 : 15 });
|
||||||
|
arbiter.addRelation(`resource:${i}`, 'risk_limit', `resource:${i}`, 1.0, { value: 40 });
|
||||||
|
arbiter.addRelation(`resource:${i}`, 'risk_cap', `resource:${i}`, 1.0, { value: 45 });
|
||||||
|
}
|
||||||
|
arbiter.setRelationConfig('risk_score', { type: 'direct' });
|
||||||
|
arbiter.setRelationConfig('risk_bonus', { type: 'direct' });
|
||||||
|
arbiter.setRelationConfig('risk_limit', { type: 'direct' });
|
||||||
|
arbiter.setRelationConfig('risk_cap', { type: 'direct' });
|
||||||
|
arbiter.setRelationConfig('risk_ok_owa', { type: 'relational_comparator', comparator: '<=', fallbackBehavior: 'deny',
|
||||||
|
left: { rule: { union: { rules: [
|
||||||
|
{ type: 'direct', relation: 'risk_score' }, { type: 'direct', relation: 'risk_bonus' }], aggregator: 'owa', owaWeights: [0.5, 0.5] } },
|
||||||
|
extractValue: true, valueRelation: 'risk_score', aggregator: 'owa', owaWeights: [0.5, 0.5] },
|
||||||
|
right: { rule: { union: { rules: [
|
||||||
|
{ type: 'direct', relation: 'risk_limit' }, { type: 'direct', relation: 'risk_cap' }], aggregator: 'owa', owaWeights: [0.6, 0.4] } },
|
||||||
|
extractValue: true, valueRelation: 'risk_limit', evaluateFrom: 'object', aggregator: 'owa', owaWeights: [0.6, 0.4] } });
|
||||||
|
return makeQueries(size, rng,
|
||||||
|
() => { const id = randInt(rng, Math.ceil(size / 2)) * 2; return { user: `user:${id}`, object: `resource:${id}` }; },
|
||||||
|
() => { const id = randInt(rng, Math.floor(size / 2)) * 2 + 1; return { user: `user:${id}`, object: `resource:${id}` }; });
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'mixed 10-rule union',
|
||||||
|
build: (arbiter, size, rng) => {
|
||||||
|
buildUsers(arbiter, size, 'user');
|
||||||
|
buildUsers(arbiter, size, 'resource');
|
||||||
|
for (let i = 0; i < size; i++) {
|
||||||
|
for (let r = 0; r < 10; r++) {
|
||||||
|
if (i % (r + 1) === 0) arbiter.addRelation(`user:${i}`, `rel${r}`, `resource:${i}`, { possibility: 0.5 + 0.05 * r });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const rules = [];
|
||||||
|
for (let r = 0; r < 10; r++) {
|
||||||
|
arbiter.setRelationConfig(`rel${r}`, { type: 'direct' });
|
||||||
|
rules.push({ type: 'direct', relation: `rel${r}` });
|
||||||
|
}
|
||||||
|
arbiter.setRelationConfig('can_access', { union: rules });
|
||||||
|
return makeQueries(size, rng,
|
||||||
|
() => { const id = randInt(rng, size); return { user: `user:${id}`, object: `resource:${id}` }; },
|
||||||
|
() => { const id = randInt(rng, size); return { user: `user:${id}`, object: `resource:${(id + 1) % size}` }; });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
];
|
||||||
|
|
||||||
|
const args = parseArgs(process.argv);
|
||||||
|
const SIZE = Number(args.get('size') || 25000);
|
||||||
|
const SAMPLES = Number(args.get('samples') || 20000);
|
||||||
|
const SEED = Number(args.get('seed') || 42);
|
||||||
|
|
||||||
|
console.log(`complex_query_bench size=${SIZE} samples=${SAMPLES} (cold traffic)`);
|
||||||
|
console.log('scenario,normal_avg_ms,normal_median_ms,normal_p95_ms,normal_p99_ms,binary_avg_ms,binary_median_ms,binary_p95_ms,binary_p99_ms,binary_speedup,parity_mismatches');
|
||||||
|
|
||||||
|
for (const scenario of scenarios) {
|
||||||
|
const arbiter = new Arbiter();
|
||||||
|
const rng = createRng(SEED);
|
||||||
|
const queries = scenario.build(arbiter, SIZE, rng);
|
||||||
|
|
||||||
|
function run(mode) {
|
||||||
|
const opts = mode === 'binary' ? { binary: true } : {};
|
||||||
|
// Warm the relation caches once, then measure cold per-query latency.
|
||||||
|
const first = queries[0];
|
||||||
|
arbiter.check(first.user, scenario.relation, first.object, opts);
|
||||||
|
const ts = [];
|
||||||
|
for (const q of queries) {
|
||||||
|
const s = performance.now();
|
||||||
|
arbiter.check(q.user, scenario.relation, q.object, opts);
|
||||||
|
ts.push(performance.now() - s);
|
||||||
|
}
|
||||||
|
const sorted = [...ts].sort((a, b) => a - b);
|
||||||
|
return {
|
||||||
|
avg: ts.reduce((a, b) => a + b, 0) / ts.length,
|
||||||
|
median: sorted[Math.floor(sorted.length / 2)],
|
||||||
|
p95: percentile(sorted, 0.95),
|
||||||
|
p99: percentile(sorted, 0.99)
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const normal = run('normal');
|
||||||
|
const binary = run('binary');
|
||||||
|
|
||||||
|
// Parity: binary decision (possibility >= 0.8 threshold) must agree with normal decision.
|
||||||
|
let mismatches = 0;
|
||||||
|
for (const q of queries) {
|
||||||
|
const n = arbiter.check(q.user, scenario.relation, q.object);
|
||||||
|
const b = arbiter.check(q.user, scenario.relation, q.object, { binary: true });
|
||||||
|
const nAllow = n.possibility > 0;
|
||||||
|
const bAllow = b.possibility > 0;
|
||||||
|
if (nAllow !== bAllow) mismatches++;
|
||||||
|
}
|
||||||
|
|
||||||
|
const speedup = binary.median > 0 ? (normal.median / binary.median).toFixed(2) : 'inf';
|
||||||
|
console.log([
|
||||||
|
scenario.name,
|
||||||
|
normal.avg.toFixed(4), normal.median.toFixed(4), normal.p95.toFixed(4), normal.p99.toFixed(4),
|
||||||
|
binary.avg.toFixed(4), binary.median.toFixed(4), binary.p95.toFixed(4), binary.p99.toFixed(4),
|
||||||
|
speedup, mismatches
|
||||||
|
].join(','));
|
||||||
|
}
|
||||||
+2
-1
@@ -48,7 +48,8 @@
|
|||||||
"benchmark:sharded": "node benchmarks/sharded-snapshot-build.js",
|
"benchmark:sharded": "node benchmarks/sharded-snapshot-build.js",
|
||||||
"benchmark:memory": "node benchmarks/memory-breakdown.js",
|
"benchmark:memory": "node benchmarks/memory-breakdown.js",
|
||||||
"benchmark:multi-hop": "node benchmarks/multi-hop-rule-bench.js",
|
"benchmark:multi-hop": "node benchmarks/multi-hop-rule-bench.js",
|
||||||
"benchmark:save": "node --expose-gc scripts/benchmark.js --save"
|
"benchmark:save": "node --expose-gc scripts/benchmark.js --save",
|
||||||
|
"benchmark:complex": "node --expose-gc benchmarks/complex-query-bench.js"
|
||||||
},
|
},
|
||||||
"keywords": [
|
"keywords": [
|
||||||
"zanzibar",
|
"zanzibar",
|
||||||
|
|||||||
Reference in New Issue
Block a user