import { test, describe, before, after } from 'node:test'; import assert from 'node:assert/strict'; import { Arbiter } from '../../src/index.js'; import { BigGraphGenerator } from '../helpers/big-graph-generator.js'; describe.skip('Realistic Chain Query Benchmark', () => { let arbiter; let generator; let testArbiter; let graphData; before(() => { arbiter = new Arbiter({ fastConstructionMode: false }); generator = new BigGraphGenerator({ scale: 'medium', seed: 12345 }); // Generate a larger, more realistic graph graphData = generator.generateGraph('enterprise'); testArbiter = generator.loadIntoArbiter(graphData); // Configure chain rules testArbiter.setRelationConfig('can_read_via_role', { type: 'chain', steps: [ { relation: 'member_of', direction: 'out' }, { relation: 'can_read', direction: 'out' } ] }); testArbiter.setRelationConfig('can_access_multi_hop', { type: 'chain', steps: [ { relation: 'member_of', direction: 'out' }, { relation: 'member_of', direction: 'out' }, { relation: 'member_of', direction: 'out' }, { relation: 'can_read', direction: 'out' } ] }); }); after(() => { arbiter = null; generator = null; testArbiter = null; graphData = null; }); /** * Measure QPS for a given operation with cache analysis */ function measureQPSWithCacheAnalysis(operation, duration = 2000, operationName = 'Operation') { const startTime = Date.now(); const endTime = startTime + duration; let operationCount = 0; const latencies = []; let cacheHits = 0; let cacheMisses = 0; // Clear caches before starting if (testArbiter.relationManager && testArbiter.relationManager.chainRule) { testArbiter.relationManager.chainRule.chainResultCache.clear(); testArbiter.relationManager.chainRule.chainPathCache.clear(); } while (Date.now() < endTime) { const opStart = process.hrtime.bigint(); try { const result = operation(); operationCount++; // Check if this was likely a cache hit (very fast execution) const opEnd = process.hrtime.bigint(); const latency = Number(opEnd - opStart) / 1000000; // Convert to milliseconds if (latency < 0.001) { // Less than 1 microsecond suggests cache hit cacheHits++; } else { cacheMisses++; } latencies.push(latency); } catch (error) { operationCount++; cacheMisses++; const opEnd = process.hrtime.bigint(); const latency = Number(opEnd - opStart) / 1000000; latencies.push(latency); } } const actualDuration = Date.now() - startTime; const qps = (operationCount / actualDuration) * 1000; const avgLatency = latencies.reduce((a, b) => a + b, 0) / latencies.length; // Sort once and reuse const sortedLatencies = latencies.sort((a, b) => a - b); const p95Latency = sortedLatencies[Math.floor(latencies.length * 0.95)]; const p99Latency = sortedLatencies[Math.floor(latencies.length * 0.99)]; return { qps, operationCount, duration: actualDuration, avgLatency, p95Latency, p99Latency, maxLatency: sortedLatencies[sortedLatencies.length - 1], minLatency: sortedLatencies[0], cacheHits, cacheMisses, cacheHitRate: cacheHits / operationCount }; } test('Realistic Chain Query - No Cache (Cold Start)', async () => { if (process.env.TEST_DEBUG === '1') console.log('📊 Testing Realistic Chain Query QPS (Cold Start - No Cache)...'); // Generate many unique queries to avoid cache hits const allUsers = graphData.relations .filter(r => r.src.startsWith('user:')) .map(r => r.src) .filter((value, index, self) => self.indexOf(value) === index) .slice(0, 1000); // Use 1000 different users const allDocs = graphData.relations .filter(r => r.dst.startsWith('doc:')) .map(r => r.dst) .filter((value, index, self) => self.indexOf(value) === index) .slice(0, 1000); // Use 1000 different documents let queryIndex = 0; const result = measureQPSWithCacheAnalysis(() => { // Use different user-doc combinations to avoid cache hits const user = allUsers[queryIndex % allUsers.length]; const doc = allDocs[(queryIndex + Math.floor(queryIndex / allUsers.length)) % allDocs.length]; return testArbiter.check(user, 'can_read_via_role', doc); }, 3000, 'Chain Query (Cold)'); if (process.env.TEST_DEBUG === '1') console.log(` QPS: ${result.qps.toFixed(2)}`); if (process.env.TEST_DEBUG === '1') console.log(` Operations: ${result.operationCount}`); if (process.env.TEST_DEBUG === '1') console.log(` Duration: ${result.duration}ms`); if (process.env.TEST_DEBUG === '1') console.log(` Avg Latency: ${result.avgLatency.toFixed(3)}ms`); if (process.env.TEST_DEBUG === '1') console.log(` P95 Latency: ${result.p95Latency.toFixed(3)}ms`); if (process.env.TEST_DEBUG === '1') console.log(` P99 Latency: ${result.p99Latency.toFixed(3)}ms`); if (process.env.TEST_DEBUG === '1') console.log(` Cache Hits: ${result.cacheHits} (${(result.cacheHitRate * 100).toFixed(1)}%)`); if (process.env.TEST_DEBUG === '1') console.log(` Cache Misses: ${result.cacheMisses} (${((1 - result.cacheHitRate) * 100).toFixed(1)}%)`); // More realistic expectations for cold chain queries assert.ok(result.qps > 10, `Cold chain query QPS ${result.qps.toFixed(0)} below 10 threshold`); assert.ok(result.avgLatency < 100, `Cold chain query latency ${result.avgLatency.toFixed(3)}ms too high`); assert.ok(result.cacheHitRate < 0.1, `Cache hit rate ${(result.cacheHitRate * 100).toFixed(1)}% too high for cold start`); }); test('Realistic Multi-hop Chain Query - No Cache', async () => { if (process.env.TEST_DEBUG === '1') console.log('📊 Testing Realistic Multi-hop Chain Query QPS (Cold Start)...'); // Generate many unique queries to avoid cache hits const allUsers = graphData.relations .filter(r => r.src.startsWith('user:')) .map(r => r.src) .filter((value, index, self) => self.indexOf(value) === index) .slice(0, 500); // Use 500 different users const allDocs = graphData.relations .filter(r => r.dst.startsWith('doc:')) .map(r => r.dst) .filter((value, index, self) => self.indexOf(value) === index) .slice(0, 500); // Use 500 different documents let queryIndex = 0; const result = measureQPSWithCacheAnalysis(() => { // Use different user-doc combinations to avoid cache hits const user = allUsers[queryIndex % allUsers.length]; const doc = allDocs[(queryIndex + Math.floor(queryIndex / allUsers.length)) % allDocs.length]; return testArbiter.check(user, 'can_access_multi_hop', doc); }, 3000, 'Multi-hop Chain Query (Cold)'); if (process.env.TEST_DEBUG === '1') console.log(` QPS: ${result.qps.toFixed(2)}`); if (process.env.TEST_DEBUG === '1') console.log(` Operations: ${result.operationCount}`); if (process.env.TEST_DEBUG === '1') console.log(` Duration: ${result.duration}ms`); if (process.env.TEST_DEBUG === '1') console.log(` Avg Latency: ${result.avgLatency.toFixed(3)}ms`); if (process.env.TEST_DEBUG === '1') console.log(` P95 Latency: ${result.p95Latency.toFixed(3)}ms`); if (process.env.TEST_DEBUG === '1') console.log(` P99 Latency: ${result.p99Latency.toFixed(3)}ms`); if (process.env.TEST_DEBUG === '1') console.log(` Cache Hits: ${result.cacheHits} (${(result.cacheHitRate * 100).toFixed(1)}%)`); if (process.env.TEST_DEBUG === '1') console.log(` Cache Misses: ${result.cacheMisses} (${((1 - result.cacheHitRate) * 100).toFixed(1)}%)`); // Multi-hop should be slower than 2-hop assert.ok(result.qps > 5, `Cold multi-hop chain query QPS ${result.qps.toFixed(0)} below 5 threshold`); assert.ok(result.avgLatency < 200, `Cold multi-hop chain query latency ${result.avgLatency.toFixed(3)}ms too high`); assert.ok(result.cacheHitRate < 0.1, `Cache hit rate ${(result.cacheHitRate * 100).toFixed(1)}% too high for cold start`); }); test('Chain Query with Warm Cache (Realistic)', async () => { if (process.env.TEST_DEBUG === '1') console.log('📊 Testing Chain Query QPS with Warm Cache...'); // First, warm up the cache with some queries const warmupQueries = 100; const allUsers = graphData.relations .filter(r => r.src.startsWith('user:')) .map(r => r.src) .filter((value, index, self) => self.indexOf(value) === index) .slice(0, 50); const allDocs = graphData.relations .filter(r => r.dst.startsWith('doc:')) .map(r => r.dst) .filter((value, index, self) => self.indexOf(value) === index) .slice(0, 50); if (process.env.TEST_DEBUG === '1') console.log(` Warming up cache with ${warmupQueries} queries...`); for (let i = 0; i < warmupQueries; i++) { const user = allUsers[i % allUsers.length]; const doc = allDocs[i % allDocs.length]; testArbiter.check(user, 'can_read_via_role', doc); } // Now test with cache hits let queryIndex = 0; const result = measureQPSWithCacheAnalysis(() => { const user = allUsers[queryIndex % allUsers.length]; const doc = allDocs[queryIndex % allDocs.length]; return testArbiter.check(user, 'can_read_via_role', doc); }, 2000, 'Chain Query (Warm Cache)'); if (process.env.TEST_DEBUG === '1') console.log(` QPS: ${result.qps.toFixed(2)}`); if (process.env.TEST_DEBUG === '1') console.log(` Operations: ${result.operationCount}`); if (process.env.TEST_DEBUG === '1') console.log(` Duration: ${result.duration}ms`); if (process.env.TEST_DEBUG === '1') console.log(` Avg Latency: ${result.avgLatency.toFixed(3)}ms`); if (process.env.TEST_DEBUG === '1') console.log(` P95 Latency: ${result.p95Latency.toFixed(3)}ms`); if (process.env.TEST_DEBUG === '1') console.log(` P99 Latency: ${result.p99Latency.toFixed(3)}ms`); if (process.env.TEST_DEBUG === '1') console.log(` Cache Hits: ${result.cacheHits} (${(result.cacheHitRate * 100).toFixed(1)}%)`); if (process.env.TEST_DEBUG === '1') console.log(` Cache Misses: ${result.cacheMisses} (${((1 - result.cacheHitRate) * 100).toFixed(1)}%)`); // With warm cache, should be much faster assert.ok(result.qps > 1000, `Warm cache chain query QPS ${result.qps.toFixed(0)} below 1000 threshold`); assert.ok(result.avgLatency < 5, `Warm cache chain query latency ${result.avgLatency.toFixed(3)}ms too high`); assert.ok(result.cacheHitRate > 0.8, `Cache hit rate ${(result.cacheHitRate * 100).toFixed(1)}% too low for warm cache`); }); test('Graph Statistics', async () => { if (process.env.TEST_DEBUG === '1') console.log('📊 Graph Statistics:'); if (process.env.TEST_DEBUG === '1') console.log(` Total Relations: ${graphData.relations.length}`); if (process.env.TEST_DEBUG === '1') console.log(` Total Nodes: ${graphData.nodes.length}`); const userCount = graphData.nodes.filter(n => n.type === 'user').length; const docCount = graphData.nodes.filter(n => n.type === 'document').length; const groupCount = graphData.nodes.filter(n => n.type === 'group').length; if (process.env.TEST_DEBUG === '1') console.log(` Users: ${userCount}`); if (process.env.TEST_DEBUG === '1') console.log(` Documents: ${docCount}`); if (process.env.TEST_DEBUG === '1') console.log(` Groups: ${groupCount}`); const relationTypes = [...new Set(graphData.relations.map(r => r.relation))]; if (process.env.TEST_DEBUG === '1') console.log(` Relation Types: ${relationTypes.length} (${relationTypes.join(', ')})`); // Verify we have a reasonable graph size assert.ok(graphData.relations.length > 1000, `Graph too small: ${graphData.relations.length} relations`); assert.ok(graphData.nodes.length > 100, `Graph too small: ${graphData.nodes.length} nodes`); }); });