Files
core/tests/rules/qps-benchmark.test.js
John Dvorak 717ae1031e initial commit: @arbiter/core authorization engine with js-rigor hardening
Zanzibar-style authorization graph engine (direct/chain/TTU/defeasible/
binary modes, condensed snapshots, value relations) with 39 rigor test
campaigns. Includes fixes for snapshot binary writer/reader format
mismatch (snapshot-of-snapshot corruption), possibility write-boundary
validation, empty-graph snapshot serialization, relation lookup cache
direction collision, config-redefinition cache invalidation, binary
threshold semantics, defeasible compiled routing, and comparator
reason whitelisting.
2026-07-31 13:44:06 -07:00

455 lines
20 KiB
JavaScript

import { test, describe, before, after } from 'node:test';
import assert from 'node:assert/strict';
import { Arbiter } from '../../src/index.js';
import { BigGraphGenerator } from '../helpers/big-graph-generator.js';
describe.skip('QPS Benchmark Tests', () => {
let arbiter;
let generator;
let testNodes;
let testRelations;
before(() => {
arbiter = new Arbiter({ fastConstructionMode: false });
generator = new BigGraphGenerator({ scale: 'small', seed: 12345 });
// Pre-generate test data
testNodes = [];
testRelations = [];
// Generate test nodes
for (let i = 0; i < 1000; i++) {
testNodes.push({
id: `user:test-${i}`,
type: 'user'
});
testNodes.push({
id: `doc:test-${i}`,
type: 'document'
});
}
// Generate test relations
for (let i = 0; i < 1000; i++) {
testRelations.push({
src: `user:test-${i}`,
relation: 'can_read',
dst: `doc:test-${i}`,
options: { possibility: 1.0 }
});
}
});
after(() => {
arbiter = null;
generator = null;
testNodes = null;
testRelations = null;
});
/**
* Measure QPS for a given operation
*/
function measureQPS(operation, duration = 2000) {
const startTime = Date.now();
const endTime = startTime + duration;
let operationCount = 0;
const latencies = [];
while (Date.now() < endTime) {
const opStart = process.hrtime.bigint();
try {
operation();
operationCount++;
} catch (error) {
// Count failed operations too
operationCount++;
}
const opEnd = process.hrtime.bigint();
const latency = Number(opEnd - opStart) / 1000000; // Convert to milliseconds
latencies.push(latency);
}
const actualDuration = Date.now() - startTime;
const qps = (operationCount / actualDuration) * 1000;
const avgLatency = latencies.reduce((a, b) => a + b, 0) / latencies.length;
// Sort once and reuse
const sortedLatencies = latencies.sort((a, b) => a - b);
const p95Latency = sortedLatencies[Math.floor(latencies.length * 0.95)];
const p99Latency = sortedLatencies[Math.floor(latencies.length * 0.99)];
return {
qps,
operationCount,
duration: actualDuration,
avgLatency,
p95Latency,
p99Latency,
maxLatency: sortedLatencies[sortedLatencies.length - 1],
minLatency: sortedLatencies[0]
};
}
test('QPS: Node Insert Operations', async () => {
if (process.env.TEST_DEBUG === '1') console.log('📊 Testing Node Insert QPS...');
let nodeIndex = 0;
const result = measureQPS(() => {
const node = testNodes[nodeIndex % testNodes.length];
arbiter.addNode(`${node.id}-${nodeIndex}`, node.type);
nodeIndex++;
}, 2000);
if (process.env.TEST_DEBUG === '1') console.log(` QPS: ${result.qps.toFixed(2)}`);
if (process.env.TEST_DEBUG === '1') console.log(` Operations: ${result.operationCount}`);
if (process.env.TEST_DEBUG === '1') console.log(` Duration: ${result.duration}ms`);
if (process.env.TEST_DEBUG === '1') console.log(` Avg Latency: ${result.avgLatency.toFixed(3)}ms`);
if (process.env.TEST_DEBUG === '1') console.log(` P95 Latency: ${result.p95Latency.toFixed(3)}ms`);
if (process.env.TEST_DEBUG === '1') console.log(` P99 Latency: ${result.p99Latency.toFixed(3)}ms`);
// Verify reasonable performance
assert.ok(result.qps > 1000, `Node insert QPS ${result.qps.toFixed(0)} below 1000 threshold`);
assert.ok(result.avgLatency < 10, `Node insert latency ${result.avgLatency.toFixed(3)}ms too high`);
});
test('QPS: Relation Insert Operations', async () => {
if (process.env.TEST_DEBUG === '1') console.log('📊 Testing Relation Insert QPS...');
let relationIndex = 0;
const result = measureQPS(() => {
const relation = testRelations[relationIndex % testRelations.length];
arbiter.addRelation(`${relation.src}-${relationIndex}`, relation.relation, `${relation.dst}-${relationIndex}`, relation.options);
relationIndex++;
}, 2000);
if (process.env.TEST_DEBUG === '1') console.log(` QPS: ${result.qps.toFixed(2)}`);
if (process.env.TEST_DEBUG === '1') console.log(` Operations: ${result.operationCount}`);
if (process.env.TEST_DEBUG === '1') console.log(` Duration: ${result.duration}ms`);
if (process.env.TEST_DEBUG === '1') console.log(` Avg Latency: ${result.avgLatency.toFixed(3)}ms`);
if (process.env.TEST_DEBUG === '1') console.log(` P95 Latency: ${result.p95Latency.toFixed(3)}ms`);
if (process.env.TEST_DEBUG === '1') console.log(` P99 Latency: ${result.p99Latency.toFixed(3)}ms`);
// Verify reasonable performance
assert.ok(result.qps > 500, `Relation insert QPS ${result.qps.toFixed(0)} below 500 threshold`);
assert.ok(result.avgLatency < 20, `Relation insert latency ${result.avgLatency.toFixed(3)}ms too high`);
});
test('QPS: Relation Update Operations', async () => {
if (process.env.TEST_DEBUG === '1') console.log('📊 Testing Relation Update QPS...');
// First, add some relations to update
for (let i = 0; i < 100; i++) {
const relation = testRelations[i];
arbiter.addRelation(`${relation.src}-update`, relation.relation, `${relation.dst}-update`, relation.options);
}
let updateIndex = 0;
const result = measureQPS(() => {
const relation = testRelations[updateIndex % testRelations.length];
arbiter.addRelation(`${relation.src}-update`, relation.relation, `${relation.dst}-update`, {
possibility: updateIndex % 2 === 0 ? 0.8 : 0.9
});
updateIndex++;
}, 2000);
if (process.env.TEST_DEBUG === '1') console.log(` QPS: ${result.qps.toFixed(2)}`);
if (process.env.TEST_DEBUG === '1') console.log(` Operations: ${result.operationCount}`);
if (process.env.TEST_DEBUG === '1') console.log(` Duration: ${result.duration}ms`);
if (process.env.TEST_DEBUG === '1') console.log(` Avg Latency: ${result.avgLatency.toFixed(3)}ms`);
if (process.env.TEST_DEBUG === '1') console.log(` P95 Latency: ${result.p95Latency.toFixed(3)}ms`);
if (process.env.TEST_DEBUG === '1') console.log(` P99 Latency: ${result.p99Latency.toFixed(3)}ms`);
// Verify reasonable performance
assert.ok(result.qps > 200, `Relation update QPS ${result.qps.toFixed(0)} below 200 threshold`);
assert.ok(result.avgLatency < 50, `Relation update latency ${result.avgLatency.toFixed(3)}ms too high`);
});
test('QPS: Relation Delete Operations', async () => {
if (process.env.TEST_DEBUG === '1') console.log('📊 Testing Relation Delete QPS...');
// First, add some relations to delete
for (let i = 0; i < 200; i++) {
const relation = testRelations[i];
arbiter.addRelation(`${relation.src}-delete`, relation.relation, `${relation.dst}-delete`, relation.options);
}
let deleteIndex = 0;
const result = measureQPS(() => {
const relation = testRelations[deleteIndex % testRelations.length];
arbiter.removeRelation(`${relation.src}-delete`, relation.relation, `${relation.dst}-delete`);
deleteIndex++;
}, 2000);
if (process.env.TEST_DEBUG === '1') console.log(` QPS: ${result.qps.toFixed(2)}`);
if (process.env.TEST_DEBUG === '1') console.log(` Operations: ${result.operationCount}`);
if (process.env.TEST_DEBUG === '1') console.log(` Duration: ${result.duration}ms`);
if (process.env.TEST_DEBUG === '1') console.log(` Avg Latency: ${result.avgLatency.toFixed(3)}ms`);
if (process.env.TEST_DEBUG === '1') console.log(` P95 Latency: ${result.p95Latency.toFixed(3)}ms`);
if (process.env.TEST_DEBUG === '1') console.log(` P99 Latency: ${result.p99Latency.toFixed(3)}ms`);
// Verify reasonable performance
assert.ok(result.qps > 100, `Relation delete QPS ${result.qps.toFixed(0)} below 100 threshold`);
assert.ok(result.avgLatency < 100, `Relation delete latency ${result.avgLatency.toFixed(3)}ms too high`);
});
test('QPS: Simple Direct Queries', async () => {
if (process.env.TEST_DEBUG === '1') console.log('📊 Testing Simple Direct Query QPS...');
// Set up test data
const graphData = generator.generateGraph('enterprise');
const testArbiter = generator.loadIntoArbiter(graphData);
// Get some test relations
const directRelations = graphData.relations.filter(r =>
r.src.startsWith('user:') && r.dst.startsWith('doc:') &&
['can_read', 'can_write', 'can_delete'].includes(r.relation)
).slice(0, 100);
let queryIndex = 0;
const result = measureQPS(() => {
const relation = directRelations[queryIndex % directRelations.length];
testArbiter.check(relation.src, relation.relation, relation.dst);
queryIndex++;
}, 2000);
if (process.env.TEST_DEBUG === '1') console.log(` QPS: ${result.qps.toFixed(2)}`);
if (process.env.TEST_DEBUG === '1') console.log(` Operations: ${result.operationCount}`);
if (process.env.TEST_DEBUG === '1') console.log(` Duration: ${result.duration}ms`);
if (process.env.TEST_DEBUG === '1') console.log(` Avg Latency: ${result.avgLatency.toFixed(3)}ms`);
if (process.env.TEST_DEBUG === '1') console.log(` P95 Latency: ${result.p95Latency.toFixed(3)}ms`);
if (process.env.TEST_DEBUG === '1') console.log(` P99 Latency: ${result.p99Latency.toFixed(3)}ms`);
// Verify reasonable performance
assert.ok(result.qps > 1000, `Simple query QPS ${result.qps.toFixed(0)} below 1000 threshold`);
assert.ok(result.avgLatency < 5, `Simple query latency ${result.avgLatency.toFixed(3)}ms too high`);
});
test('QPS: Complex Chain Queries', async () => {
if (process.env.TEST_DEBUG === '1') console.log('📊 Testing Complex Chain Query QPS...');
// Set up test data with chain rules
const graphData = generator.generateGraph('enterprise');
const testArbiter = generator.loadIntoArbiter(graphData);
// Configure chain rules
testArbiter.setRelationConfig('can_read_via_role', {
type: 'chain',
steps: [
{ relation: 'member_of', direction: 'out' },
{ relation: 'can_read', direction: 'out' }
]
});
// Get some test relations for chain queries
const chainRelations = graphData.relations.filter(r =>
r.src.startsWith('user:') && r.dst.startsWith('doc:')
).slice(0, 50);
let queryIndex = 0;
const result = measureQPS(() => {
const relation = chainRelations[queryIndex % chainRelations.length];
testArbiter.check(relation.src, 'can_read_via_role', relation.dst);
queryIndex++;
}, 2000);
if (process.env.TEST_DEBUG === '1') console.log(` QPS: ${result.qps.toFixed(2)}`);
if (process.env.TEST_DEBUG === '1') console.log(` Operations: ${result.operationCount}`);
if (process.env.TEST_DEBUG === '1') console.log(` Duration: ${result.duration}ms`);
if (process.env.TEST_DEBUG === '1') console.log(` Avg Latency: ${result.avgLatency.toFixed(3)}ms`);
if (process.env.TEST_DEBUG === '1') console.log(` P95 Latency: ${result.p95Latency.toFixed(3)}ms`);
if (process.env.TEST_DEBUG === '1') console.log(` P99 Latency: ${result.p99Latency.toFixed(3)}ms`);
// Verify reasonable performance
assert.ok(result.qps > 100, `Chain query QPS ${result.qps.toFixed(0)} below 100 threshold`);
assert.ok(result.avgLatency < 50, `Chain query latency ${result.avgLatency.toFixed(3)}ms too high`);
});
test('QPS: Multi-hop Complex Queries', async () => {
if (process.env.TEST_DEBUG === '1') console.log('📊 Testing Multi-hop Complex Query QPS...');
// Set up test data with complex chain rules
const graphData = generator.generateGraph('enterprise');
const testArbiter = generator.loadIntoArbiter(graphData);
// Configure complex multi-hop chain rules
testArbiter.setRelationConfig('can_access_multi_hop', {
type: 'chain',
steps: [
{ relation: 'member_of', direction: 'out' },
{ relation: 'member_of', direction: 'out' },
{ relation: 'member_of', direction: 'out' },
{ relation: 'can_read', direction: 'out' }
]
});
// Get some test relations for multi-hop queries
const multiHopRelations = graphData.relations.filter(r =>
r.src.startsWith('user:') && r.dst.startsWith('doc:')
).slice(0, 20);
let queryIndex = 0;
const result = measureQPS(() => {
const relation = multiHopRelations[queryIndex % multiHopRelations.length];
testArbiter.check(relation.src, 'can_access_multi_hop', relation.dst);
queryIndex++;
}, 2000);
if (process.env.TEST_DEBUG === '1') console.log(` QPS: ${result.qps.toFixed(2)}`);
if (process.env.TEST_DEBUG === '1') console.log(` Operations: ${result.operationCount}`);
if (process.env.TEST_DEBUG === '1') console.log(` Duration: ${result.duration}ms`);
if (process.env.TEST_DEBUG === '1') console.log(` Avg Latency: ${result.avgLatency.toFixed(3)}ms`);
if (process.env.TEST_DEBUG === '1') console.log(` P95 Latency: ${result.p95Latency.toFixed(3)}ms`);
if (process.env.TEST_DEBUG === '1') console.log(` P99 Latency: ${result.p99Latency.toFixed(3)}ms`);
// Verify reasonable performance
assert.ok(result.qps > 10, `Multi-hop query QPS ${result.qps.toFixed(0)} below 10 threshold`);
assert.ok(result.avgLatency < 200, `Multi-hop query latency ${result.avgLatency.toFixed(3)}ms too high`);
});
test('QPS: Relational Comparator Queries', async () => {
if (process.env.TEST_DEBUG === '1') console.log('📊 Testing Relational Comparator Query QPS...');
// Set up test data with relational comparator rules
const graphData = generator.generateGraph('enterprise');
const testArbiter = generator.loadIntoArbiter(graphData);
// Add some value relations for comparison
for (let i = 0; i < 10; i++) {
testArbiter.addRelation(`user:alice-${i}`, 'has_balance', `user:alice-${i}`, { value: 1000 + i * 100 });
testArbiter.addRelation(`feature:premium-${i}`, 'has_price', `feature:premium-${i}`, { value: 800 + i * 50 });
}
// Configure relational comparator rule
testArbiter.setRelationConfig('balance_check', {
type: 'relational_comparator',
left: {
rule: { type: 'direct', relation: 'has_balance' },
extractValue: true
},
right: {
evaluateFrom: 'object',
rule: { type: 'direct', relation: 'has_price' },
extractValue: true
},
comparator: '>'
});
let queryIndex = 0;
const result = measureQPS(() => {
const userIndex = queryIndex % 10;
testArbiter.check(`user:alice-${userIndex}`, 'balance_check', `feature:premium-${userIndex}`);
queryIndex++;
}, 2000);
if (process.env.TEST_DEBUG === '1') console.log(` QPS: ${result.qps.toFixed(2)}`);
if (process.env.TEST_DEBUG === '1') console.log(` Operations: ${result.operationCount}`);
if (process.env.TEST_DEBUG === '1') console.log(` Duration: ${result.duration}ms`);
if (process.env.TEST_DEBUG === '1') console.log(` Avg Latency: ${result.avgLatency.toFixed(3)}ms`);
if (process.env.TEST_DEBUG === '1') console.log(` P95 Latency: ${result.p95Latency.toFixed(3)}ms`);
if (process.env.TEST_DEBUG === '1') console.log(` P99 Latency: ${result.p99Latency.toFixed(3)}ms`);
// Verify reasonable performance
assert.ok(result.qps > 50, `Relational comparator QPS ${result.qps.toFixed(0)} below 50 threshold`);
assert.ok(result.avgLatency < 100, `Relational comparator latency ${result.avgLatency.toFixed(3)}ms too high`);
});
test('QPS: Batch Operations', async () => {
if (process.env.TEST_DEBUG === '1') console.log('📊 Testing Batch Operations QPS...');
const testArbiter = new Arbiter({ fastConstructionMode: false });
// Generate batch data
const batchSize = 100;
const batches = [];
for (let i = 0; i < 10; i++) {
const batch = [];
for (let j = 0; j < batchSize; j++) {
batch.push({
src: `user:batch-${i}-${j}`,
relation: 'can_read',
dst: `doc:batch-${i}-${j}`,
options: { possibility: 1.0 }
});
}
batches.push(batch);
}
let batchIndex = 0;
const result = measureQPS(() => {
const batch = batches[batchIndex % batches.length];
// Add nodes first
for (const relation of batch) {
testArbiter.addNode(relation.src, 'user');
testArbiter.addNode(relation.dst, 'document');
}
// Add relations
for (const relation of batch) {
testArbiter.addRelation(relation.src, relation.relation, relation.dst, relation.options);
}
batchIndex++;
}, 2000);
if (process.env.TEST_DEBUG === '1') console.log(` QPS: ${result.qps.toFixed(2)} (batches per second)`);
if (process.env.TEST_DEBUG === '1') console.log(` Operations: ${result.operationCount} batches`);
if (process.env.TEST_DEBUG === '1') console.log(` Duration: ${result.duration}ms`);
if (process.env.TEST_DEBUG === '1') console.log(` Avg Latency: ${result.avgLatency.toFixed(3)}ms per batch`);
if (process.env.TEST_DEBUG === '1') console.log(` P95 Latency: ${result.p95Latency.toFixed(3)}ms per batch`);
if (process.env.TEST_DEBUG === '1') console.log(` P99 Latency: ${result.p99Latency.toFixed(3)}ms per batch`);
// Verify reasonable performance
assert.ok(result.qps > 1, `Batch operation QPS ${result.qps.toFixed(0)} below 1 threshold`);
assert.ok(result.avgLatency < 1000, `Batch operation latency ${result.avgLatency.toFixed(3)}ms too high`);
});
test('QPS: Mixed Workload', async () => {
if (process.env.TEST_DEBUG === '1') console.log('📊 Testing Mixed Workload QPS...');
const testArbiter = new Arbiter({ fastConstructionMode: false });
const graphData = generator.generateGraph('enterprise');
const loadedArbiter = generator.loadIntoArbiter(graphData);
// Get test relations
const testRelations = graphData.relations.slice(0, 50);
let operationIndex = 0;
const result = measureQPS(() => {
const opType = operationIndex % 4;
const relation = testRelations[operationIndex % testRelations.length];
switch (opType) {
case 0: // Insert
testArbiter.addRelation(`${relation.src}-mixed`, relation.relation, `${relation.dst}-mixed`, relation.options);
break;
case 1: // Update
testArbiter.addRelation(`${relation.src}-mixed`, relation.relation, `${relation.dst}-mixed`, { possibility: 0.8 });
break;
case 2: // Delete
testArbiter.removeRelation(`${relation.src}-mixed`, relation.relation, `${relation.dst}-mixed`);
break;
case 3: // Query
loadedArbiter.check(relation.src, relation.relation, relation.dst);
break;
}
operationIndex++;
}, 2000);
if (process.env.TEST_DEBUG === '1') console.log(` QPS: ${result.qps.toFixed(2)}`);
if (process.env.TEST_DEBUG === '1') console.log(` Operations: ${result.operationCount}`);
if (process.env.TEST_DEBUG === '1') console.log(` Duration: ${result.duration}ms`);
if (process.env.TEST_DEBUG === '1') console.log(` Avg Latency: ${result.avgLatency.toFixed(3)}ms`);
if (process.env.TEST_DEBUG === '1') console.log(` P95 Latency: ${result.p95Latency.toFixed(3)}ms`);
if (process.env.TEST_DEBUG === '1') console.log(` P99 Latency: ${result.p99Latency.toFixed(3)}ms`);
// Verify reasonable performance
assert.ok(result.qps > 100, `Mixed workload QPS ${result.qps.toFixed(0)} below 100 threshold`);
assert.ok(result.avgLatency < 50, `Mixed workload latency ${result.avgLatency.toFixed(3)}ms too high`);
});
});