// benchmark-interval-inference.js // Benchmark interval-based inference with relational comparisons import { performance } from 'perf_hooks'; import { Arbiter } from '../src/index.js'; import { PACBayesInference } from '../src/inference/PACBayesInference.js'; import { fileURLToPath } from 'url'; import { dirname, resolve } from 'path'; import fs from 'fs'; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); // Create a realistic financial services graph function createFinancialGraph() { const arbiter = new Arbiter({ enableInference: true, useOptimizedInference: true, useANN: true }); // Account tiers with typical balance ranges const accountTiers = { basic: { min: 1000, max: 25000, count: 100 }, silver: { min: 25000, max: 100000, count: 50 }, gold: { min: 100000, max: 500000, count: 25 }, platinum: { min: 500000, max: 2000000, count: 10 } }; // Create accounts with realistic balance distributions const accounts = []; for (const [tier, config] of Object.entries(accountTiers)) { for (let i = 0; i < config.count; i++) { const balance = config.min + Math.random() * (config.max - config.min); const accountAge = Math.floor(Math.random() * 120); // months const creditScore = 600 + Math.floor(Math.random() * 250); const id = `${tier}_account_${i}`; accounts.push({ id, tier, balance: Math.floor(balance), accountAge, creditScore }); arbiter.addNode(id, 'account', { tier, balance: Math.floor(balance), accountAge, creditScore }); } } // Create financial products with requirements const products = [ { id: 'savings_basic', minBalance: 1000, minCredit: 0 }, { id: 'checking_premium', minBalance: 25000, minCredit: 650 }, { id: 'investment_silver', minBalance: 50000, minCredit: 700 }, { id: 'investment_gold', minBalance: 100000, minCredit: 720 }, { id: 'private_banking', minBalance: 500000, minCredit: 750 }, { id: 'wealth_management', minBalance: 1000000, minCredit: 780 } ]; for (const product of products) { arbiter.addNode(product.id, 'product', { minBalance: product.minBalance, minCredit: product.minCredit }); } // Add access relations based on requirements for (const account of accounts) { for (const product of products) { if (account.balance >= product.minBalance && account.creditScore >= product.minCredit) { arbiter.addRelation(account.id, 'can_access', product.id); } } } // Mock relational rules arbiter.getRelationalRules = (relation) => { if (relation === 'has_min_balance_for') { return [{ type: 'relational-comparison', attribute: 'balance', operator: '>=', threshold: 'object.minBalance' // Would be resolved dynamically }]; } if (relation === 'meets_credit_requirement') { return [{ type: 'relational-comparison', attribute: 'creditScore', operator: '>=', threshold: 'object.minCredit' }]; } return []; }; return { arbiter, accounts, products }; } async function benchmarkIntervalQueries() { console.log('=== Interval Inference Benchmark ===\n'); const { arbiter, accounts, products } = createFinancialGraph(); // Create interval inference engine const inference = new PACBayesInference(arbiter, { intervalConfidence: 0.95, minVotersForInterval: 3, delta: 0.05, k: 15 // neighbors for inference }); arbiter.inferenceEngine = inference; // Record observed decisions let recordedCount = 0; const canAccessRelations = arbiter.relationManager.getRelationsByName('can_access'); for (const rel of canAccessRelations) { if (Math.random() < 0.7) { // Record 70% const subjectKey = arbiter.nodeManager.getNodeKey(rel.src); const objectKey = arbiter.nodeManager.getNodeKey(rel.dst); if (subjectKey && objectKey) { inference.recordDecision(subjectKey, rel.rel, objectKey, 'allow'); recordedCount++; } } } console.log(`Recorded ${recordedCount} access decisions\n`); // Generate embeddings if (arbiter.embeddingManager) { arbiter.embeddingManager.forceRegenerateEmbeddings(); if (arbiter.similarityManager) arbiter.similarityManager.ensureIndexReady(); } // Simulate missing attribute values const testAccounts = accounts.slice(0, 20); // Test with 20 accounts const originalBalances = new Map(); for (const account of testAccounts) { originalBalances.set(account.id, account.balance); // Remove balance attribute to simulate stale/missing data const node = arbiter.nodeManager.getNodeByKey(account.id); if (node && node.data) { delete node.data.balance; } } console.log('=== Testing Interval Estimation ===\n'); // Test interval estimation accuracy let totalError = 0; let validEstimates = 0; for (const account of testAccounts.slice(0, 5)) { // Show first 5 const interval = await inference.estimateAttributeInterval(account.id, 'balance', 10); const actual = originalBalances.get(account.id); console.log(`Account: ${account.id} (${account.tier})`); console.log(` Actual balance: $${actual.toLocaleString()}`); console.log(` Estimated interval: [$${interval.lower?.toLocaleString() || 'N/A'}, $${interval.upper?.toLocaleString() || 'N/A'}]`); console.log(` Confidence: ${(interval.confidence * 100).toFixed(1)}%`); console.log(` Voters: ${interval.voters}`); if (interval.lower && interval.upper) { const inInterval = actual >= interval.lower && actual <= interval.upper; console.log(` Contains actual: ${inInterval ? 'YES' : 'NO'}`); if (inInterval) validEstimates++; const midpoint = (interval.lower + interval.upper) / 2; const error = Math.abs(midpoint - actual) / actual; totalError += error; } console.log(); } console.log(`\nInterval Coverage: ${validEstimates}/${testAccounts.slice(0, 5).length}`); console.log(`Average Error: ${(totalError / 5 * 100).toFixed(1)}%\n`); console.log('=== Benchmarking Authorization Queries ===\n'); // Benchmark authorization checks with missing data const queries = []; for (let i = 0; i < 100; i++) { const account = testAccounts[Math.floor(Math.random() * testAccounts.length)]; const product = products[Math.floor(Math.random() * products.length)]; queries.push({ subject: account.id, object: product.id }); } let allowCount = 0; let denyCount = 0; let undeterminedCount = 0; let totalTime = 0; for (const query of queries) { const start = performance.now(); const result = await inference.check(query.subject, 'can_access', query.object); const elapsed = performance.now() - start; totalTime += elapsed; if (result.outcome === 'allow') allowCount++; else if (result.outcome === 'deny') denyCount++; else undeterminedCount++; } console.log('Query Results:'); console.log(` Allow: ${allowCount} (${(allowCount/queries.length*100).toFixed(1)}%)`); console.log(` Deny: ${denyCount} (${(denyCount/queries.length*100).toFixed(1)}%)`); console.log(` Undetermined: ${undeterminedCount} (${(undeterminedCount/queries.length*100).toFixed(1)}%)`); console.log(`\nPerformance:`); console.log(` Total queries: ${queries.length}`); console.log(` Average latency: ${(totalTime / queries.length).toFixed(2)}ms`); console.log(` QPS: ${(1000 / (totalTime / queries.length)).toFixed(0)}`); // Show cache stats console.log('\n=== Cache Statistics ==='); const cacheStats = inference.getIntervalCacheStats(); console.log(`Cache entries: ${cacheStats.size}`); if (cacheStats.entries.length > 0) { console.log('Sample entries:'); for (const entry of cacheStats.entries.slice(0, 3)) { console.log(` ${entry.key}: [${entry.interval.lower}, ${entry.interval.upper}] (age: ${(entry.age/1000).toFixed(1)}s)`); } } // Show optimization statistics console.log('\n=== Optimization Statistics ==='); const perfStats = inference.getPerformanceStats(); console.log('Inference Performance:'); console.log(` Queries: ${perfStats.inference.queries}`); console.log(` Cache hits: ${perfStats.inference.cacheHits}`); console.log(` Cache hit rate: ${(perfStats.inference.overallCacheHitRate * 100).toFixed(1)}%`); console.log(` Similarity cache hits: ${perfStats.inference.similarityCacheHits}`); console.log('\nSimilarity Manager Performance:'); console.log(` Total searches: ${perfStats.similarity.searches || 0}`); console.log(` Vector cache hits: ${perfStats.similarity.vectorCacheHits || 0}`); console.log(` Vector cache hit rate: ${((perfStats.similarity.vectorCacheHitRate || 0) * 100).toFixed(1)}%`); console.log(` Average search time: ${(perfStats.similarity.avgSearchTime || 0).toFixed(2)}ms`); console.log('\nOptimization Impact:'); console.log(` Redundant searches avoided: ${perfStats.optimization.redundantSearchesAvoided}`); console.log(` Redundant conversions avoided: ${perfStats.optimization.redundantConversionsAvoided}`); console.log(` Vector conversion efficiency: ${((perfStats.optimization.vectorCacheHitRate || 0) * 100).toFixed(1)}%`); } // Run benchmark (async () => { try { await benchmarkIntervalQueries(); } catch (error) { console.error('Benchmark error:', error); } })();