import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { performance } from 'node:perf_hooks'; import { CondensedGraph } from '../src/core/CondensedGraph.js'; import { ShardedSnapshotBuilder } from '../src/core/shards/ShardedSnapshotBuilder.js'; import { ShardedSnapshot } from '../src/core/shards/ShardedSnapshot.js'; import { FileShardStorage } from '../src/core/shards/FileShardStorage.js'; import { DeltaShardBinary } from '../src/core/shards/DeltaShardBinary.js'; import { WaveletShardBinary } from '../src/core/shards/WaveletShardBinary.js'; function parseArgs(argv) { const args = new Map(); for (let i = 2; i < argv.length; i++) { const value = argv[i]; if (!value.startsWith('--')) continue; const [key, inline] = value.slice(2).split('='); if (inline !== undefined) { args.set(key, inline); continue; } const next = argv[i + 1]; if (next && !next.startsWith('--')) { args.set(key, next); i++; } else { args.set(key, true); } } return args; } function makeRng(seed) { let state = seed >>> 0; return () => { state = (1664525 * state + 1013904223) >>> 0; return state / 0x100000000; }; } function randInt(rng, max) { return Math.floor(rng() * max); } function shuffleWithRng(items, rng) { for (let i = items.length - 1; i > 0; i--) { const j = Math.floor(rng() * (i + 1)); [items[i], items[j]] = [items[j], items[i]]; } return items; } function buildSnapshot(graph, bucketSize, dir, shardMode, componentRelations, chainGroupTargetSize, chainGroupTargetCount) { const builder = new ShardedSnapshotBuilder({ bucketSize, includeDirections: ['out', 'in'], shardMode, componentRelations, chainGroupTargetSize, chainGroupTargetCount }); const manifest = builder.build(graph, dir); const storage = new FileShardStorage(dir); const snapshot = new ShardedSnapshot(manifest, storage, { cacheLimit: 64, recentLimit: 256 }); snapshot.initializeSync(); return { snapshot, manifest, storage }; } function buildDeltaLayer(snapshot, relationIds, deltaEdges, rng, dir) { const buckets = new Map(); const relationList = relationIds.filter((relId) => Number.isFinite(relId)); let addCount = 0; let removeCount = 0; for (let i = 0; i < deltaEdges; i++) { const relId = relationList[i % relationList.length]; const srcId = randInt(rng, snapshot.nodeCount); const shardMeta = snapshot._selectShardMeta(relId, 'out', srcId); if (!shardMeta) continue; const localSource = snapshot._localSource(srcId, shardMeta); const key = shardMeta.cacheKey; let entry = buckets.get(key); if (!entry) { entry = { shardMeta, additions: [], removals: [] }; buckets.set(key, entry); } const edges = snapshot.getOutEdgesSync(srcId, relId); if (edges.length && rng() < 0.5) { const edge = edges[randInt(rng, edges.length)]; entry.removals.push({ srcLocal: localSource, otherId: edge.dst }); removeCount++; } else { const dstId = randInt(rng, snapshot.nodeCount); entry.additions.push({ srcLocal: localSource, otherId: dstId, possBits: 65535, relBits: 65535 }); addCount++; } } fs.mkdirSync(dir, { recursive: true }); const shards = []; for (const entry of buckets.values()) { const shardKey = `delta-${entry.shardMeta.key}`; const buffer = DeltaShardBinary.serialize({ relationId: entry.shardMeta.relationId, direction: entry.shardMeta.direction, rangeStart: entry.shardMeta.rangeStart, rangeEnd: entry.shardMeta.rangeEnd, nodeCount: snapshot.nodeCount, additions: entry.additions, removals: entry.removals }); fs.writeFileSync(path.join(dir, shardKey), new Uint8Array(buffer)); shards.push({ key: shardKey, relationId: entry.shardMeta.relationId, direction: entry.shardMeta.direction, rangeStart: entry.shardMeta.rangeStart, rangeEnd: entry.shardMeta.rangeEnd, cacheKey: entry.shardMeta.cacheKey }); } return { shards, storage: new FileShardStorage(dir), addCount, removeCount }; } function timeQueries(runQuery, queries) { const start = performance.now(); let hits = 0; for (const query of queries) { if (runQuery(query)) hits++; } const duration = performance.now() - start; return { duration, hits }; } function sumDeltaBytes(dir) { let total = 0; const entries = fs.readdirSync(dir, { withFileTypes: true }); for (const entry of entries) { if (!entry.isFile()) continue; total += fs.statSync(path.join(dir, entry.name)).size; } return total; } function sumShardBytes(dir, manifest) { let total = 0; if (manifest.nodeTableKey) { const nodeTablePath = path.join(dir, manifest.nodeTableKey); if (fs.existsSync(nodeTablePath)) total += fs.statSync(nodeTablePath).size; } if (manifest.componentKey) { const componentPath = path.join(dir, manifest.componentKey); if (fs.existsSync(componentPath)) total += fs.statSync(componentPath).size; } for (const shard of manifest.shards || []) { const shardPath = path.join(dir, shard.key); if (fs.existsSync(shardPath)) total += fs.statSync(shardPath).size; } return total; } function shardSizeStats(dir, manifest) { const sizes = []; for (const shard of manifest.shards || []) { const shardPath = path.join(dir, shard.key); if (fs.existsSync(shardPath)) sizes.push(fs.statSync(shardPath).size); } sizes.sort((a, b) => a - b); if (!sizes.length) return { p50: 0, p95: 0, p99: 0, max: 0, min: 0 }; const pct = (p) => sizes[Math.min(sizes.length - 1, Math.floor(p * sizes.length))]; return { p50: pct(0.5), p95: pct(0.95), p99: pct(0.99), max: sizes[sizes.length - 1], min: sizes[0] }; } function countShardFiles(manifest) { let total = 0; if (manifest.nodeTableKey) total += 1; if (manifest.componentKey) total += 1; total += (manifest.shards || []).length; return total; } function scenarioOwnerDirect(size, rng) { const graph = new CondensedGraph(); const users = []; const resources = []; for (let i = 0; i < size; i++) { users.push(graph._ensureNode(`user:${i}`)); resources.push(graph._ensureNode(`resource:${i}`)); } for (let i = 0; i < size; i++) { graph.addEdge(users[i], 'owner', resources[i]); } const queries = []; for (let i = 0; i < Math.min(2000, size * 2); i++) { const id = randInt(rng, size); queries.push({ userId: users[id], resourceId: resources[id], expect: true }); } for (let i = 0; i < Math.min(2000, size * 2); i++) { const id = randInt(rng, size); queries.push({ userId: users[id], resourceId: resources[(id + 1) % size], expect: false }); } return { graph, queries: shuffleWithRng(queries, rng), relations: ['owner'] }; } function scenarioTupleToUserset(size, rng) { const graph = new CondensedGraph(); const groupCount = Math.max(1, Math.floor(size / 10)); const users = []; const groups = []; const resources = []; for (let i = 0; i < size; i++) users.push(graph._ensureNode(`user:${i}`)); for (let i = 0; i < groupCount; i++) groups.push(graph._ensureNode(`group:${i}`)); for (let i = 0; i < groupCount; i++) resources.push(graph._ensureNode(`resource:${i}`)); for (let i = 0; i < size; i++) { const groupId = i % groupCount; graph.addEdge(users[i], 'member', groups[groupId]); } for (let i = 0; i < groupCount; i++) { graph.addEdge(groups[i], 'group_access', resources[i]); } const queries = []; for (let i = 0; i < Math.min(2000, size); i++) { const userId = randInt(rng, size); const groupId = userId % groupCount; queries.push({ userId: users[userId], resourceId: resources[groupId], expect: true }); } for (let i = 0; i < Math.min(2000, size); i++) { const userId = randInt(rng, size); const groupId = (userId + 1) % groupCount; queries.push({ userId: users[userId], resourceId: resources[groupId], expect: false }); } return { graph, queries: shuffleWithRng(queries, rng), relations: ['member', 'group_access'] }; } function scenarioBlockedUnless(size, rng) { const graph = new CondensedGraph(); const users = []; const resources = []; for (let i = 0; i < size; i++) { users.push(graph._ensureNode(`user:${i}`)); resources.push(graph._ensureNode(`resource:${i}`)); } for (let i = 0; i < size; i++) { graph.addEdge(users[i], 'viewer', resources[i]); if (i % 5 === 0) graph.addEdge(users[i], 'blocked', resources[i]); } const queries = []; for (let i = 0; i < Math.min(2000, size * 2); i++) { const id = randInt(rng, size); queries.push({ userId: users[id], resourceId: resources[id], expect: id % 5 !== 0 }); } for (let i = 0; i < Math.min(2000, size * 2); i++) { const id = randInt(rng, size); queries.push({ userId: users[id], resourceId: resources[(id + 1) % size], expect: false }); } return { graph, queries: shuffleWithRng(queries, rng), relations: ['viewer', 'blocked'] }; } function scenarioRiskComparator(size, rng) { const graph = new CondensedGraph(); const users = []; const resources = []; for (let i = 0; i < size; i++) { users.push(graph._ensureNode(`user:${i}`)); resources.push(graph._ensureNode(`resource:${i}`)); } for (let i = 0; i < size; i++) { graph.addEdge(users[i], 'risk_score', resources[i], 1.0, { value: (i % 100) / 100 }); graph.addEdge(resources[i], 'risk_limit', resources[i], 1.0, { value: 0.6 }); } const queries = []; for (let i = 0; i < Math.min(2000, size); i++) { const id = randInt(rng, size); queries.push({ userId: users[id], resourceId: resources[id], expect: (id % 100) / 100 <= 0.6 }); } return { graph, queries: shuffleWithRng(queries, rng), relations: ['risk_score', 'risk_limit'] }; } function evaluateOwner(snapshot, relId, query) { return snapshot.findEdgeSync(query.userId, relId, query.resourceId) !== null; } function evaluateTupleToUserset(snapshot, relIds, query) { const groupAccessRel = relIds.group_access; const memberRel = relIds.member; const groupEdges = snapshot.getInEdgesSync(query.resourceId, groupAccessRel); for (const groupEdge of groupEdges) { const groupId = groupEdge.src; const memberEdge = snapshot.findEdgeSync(query.userId, memberRel, groupId); if (memberEdge) return true; } return false; } function evaluateBlocked(snapshot, relIds, query) { const viewerRel = relIds.viewer; const blockedRel = relIds.blocked; const viewer = snapshot.findEdgeSync(query.userId, viewerRel, query.resourceId); if (!viewer) return false; const blocked = snapshot.findEdgeSync(query.userId, blockedRel, query.resourceId); return !blocked; } function evaluateRisk(snapshot, relIds, query) { const scoreRel = relIds.risk_score; const limitRel = relIds.risk_limit; const scoreEdge = snapshot.findEdgeSync(query.userId, scoreRel, query.resourceId); if (!scoreEdge) return false; const limitEdge = snapshot.findEdgeSync(query.resourceId, limitRel, query.resourceId); if (!limitEdge) return false; return scoreEdge.value <= limitEdge.value; } const scenarios = [ { name: 'owner_direct', build: scenarioOwnerDirect, eval: (snapshot, relIds, query) => evaluateOwner(snapshot, relIds.owner, query) }, { name: 'group_tuple_to_userset', build: scenarioTupleToUserset, eval: (snapshot, relIds, query) => evaluateTupleToUserset(snapshot, relIds, query) }, { name: 'blocked_unless', build: scenarioBlockedUnless, eval: (snapshot, relIds, query) => evaluateBlocked(snapshot, relIds, query) }, { name: 'risk_comparator', build: scenarioRiskComparator, eval: (snapshot, relIds, query) => evaluateRisk(snapshot, relIds, query) } ]; const args = parseArgs(process.argv); const size = Number(args.get('size') || 50000); const seed = Number(args.get('seed') || 1337); const bucketSize = Number(args.get('bucket') || 65536); const deltaPowers = String(args.get('delta-powers') || '0,1,2,3').split(',').map((v) => Number(v.trim())); const compactAt = Number(args.get('compact-at') || 0); const shardMode = String(args.get('shard-mode') || 'chain'); const componentRelations = args.get('component-relations') ? String(args.get('component-relations')).split(',').map((v) => v.trim()).filter(Boolean) : ['member', 'group_access', 'viewer', 'blocked', 'risk_score', 'risk_limit', 'owner']; const chainGroupTargetSize = Number(args.get('chain-group-target-size') || 0); const chainGroupTargetCount = Number(args.get('chain-group-target-count') || 0); console.log('B2C delta overlay bench'); console.log(` size: ${size}`); console.log(` bucket: ${bucketSize}`); console.log(` delta powers: ${deltaPowers.join(',')}`); if (compactAt > 0) console.log(` compact at: ${compactAt}`); console.log(` shard mode: ${shardMode}`); console.log(` component relations: ${componentRelations.join(',')}`); console.log(' note: timings are per-query averages; expect noise and cache effects. Compare trends, not single points.'); if (chainGroupTargetSize > 0) console.log(` chain group target size: ${chainGroupTargetSize}`); if (chainGroupTargetCount > 0) console.log(` chain group target count: ${chainGroupTargetCount}`); for (const scenario of scenarios) { const rng = makeRng(seed + scenario.name.length); const { graph, queries, relations } = scenario.build(size, rng); graph.finalizePerfectHash(); graph.finalizeWaveletAdjacency({ dropAdjacencyList: true }); const dir = fs.mkdtempSync(path.join(os.tmpdir(), `b2c-delta-${scenario.name}-`)); const { snapshot, manifest } = buildSnapshot( graph, bucketSize, dir, shardMode, componentRelations, chainGroupTargetSize > 0 ? chainGroupTargetSize : undefined, chainGroupTargetCount > 0 ? chainGroupTargetCount : undefined ); const relIds = {}; for (const rel of relations) { relIds[rel] = graph.getRelationId(rel); } const relationIds = relations.map((rel) => relIds[rel]); const baseBytes = sumShardBytes(dir, manifest); const shardFiles = countShardFiles(manifest); const shardStats = shardSizeStats(dir, manifest); console.log(`Scenario: ${scenario.name}`); console.log('deltaEdges | avg_us_per_query | base_bytes | shard_files | shard_p50 | shard_p95 | shard_p99 | delta_bytes | compact_ms | compact_us_per_edge'); snapshot.setDeltaLayers([]); snapshot.clearDeltaCache(); const baseline = timeQueries((query) => scenario.eval(snapshot, relIds, query), queries); console.log(`0 | ${(baseline.duration / queries.length) * 1000} | ${baseBytes} | ${shardFiles} | ${shardStats.p50} | ${shardStats.p95} | ${shardStats.p99} | 0 | - | -`); for (const power of deltaPowers) { const deltaEdges = Math.max(1, Math.floor(Math.pow(10, power))); const deltaDir = path.join(dir, `delta-${power}`); const layer = buildDeltaLayer(snapshot, relationIds, deltaEdges, rng, deltaDir); snapshot.setDeltaLayers([layer]); snapshot.clearDeltaCache(); const result = timeQueries((query) => scenario.eval(snapshot, relIds, query), queries); const deltaBytes = sumDeltaBytes(deltaDir); let compactMs = '-'; let compactUsPerEdge = '-'; if (compactAt > 0 && deltaEdges >= compactAt) { const compactDir = path.join(dir, `compact-${scenario.name}-${power}`); const start = performance.now(); for (const shardMeta of layer.shards) { const base = snapshot._cacheIndex.get(shardMeta.cacheKey); if (!base) continue; const shard = snapshot._loadShardSync(base.relationId, base.direction, base.rangeStart); if (!shard) continue; const deltaBuffer = layer.storage.getSync(shardMeta.key); if (!deltaBuffer) continue; const deltaShard = DeltaShardBinary.deserialize(deltaBuffer); const rangeSize = shard.rangeEnd - shard.rangeStart; const sources = new Array(rangeSize); for (let localSource = 0; localSource < rangeSize; localSource++) { const range = snapshot._rangeForSource(shard, localSource); const list = []; if (range) { for (let pos = range.start; pos < range.end; pos++) { list.push({ otherId: shard.dstIds[pos], possBits: shard.possBits[pos], relBits: shard.relBits[pos] }); } } sources[localSource] = list; } for (const removal of deltaShard.removals) { const list = sources[removal.srcLocal]; if (!list) continue; const idx = list.findIndex((item) => item.otherId === removal.otherId); if (idx !== -1) list.splice(idx, 1); } for (const addition of deltaShard.additions) { const list = sources[addition.srcLocal] || (sources[addition.srcLocal] = []); list.push({ otherId: addition.otherId, possBits: addition.possBits, relBits: addition.relBits }); } const buffer = WaveletShardBinary.serialize({ relationId: shard.relationId, direction: shard.direction, rangeStart: shard.rangeStart, rangeEnd: shard.rangeEnd, nodeCount: snapshot.nodeCount, sources }); fs.mkdirSync(compactDir, { recursive: true }); fs.writeFileSync(path.join(compactDir, base.key), new Uint8Array(buffer)); } const totalMs = performance.now() - start; compactMs = totalMs.toFixed(2); const denom = Math.max(1, layer.addCount + layer.removeCount); compactUsPerEdge = ((totalMs * 1000) / denom).toFixed(3); } console.log(`${deltaEdges} | ${(result.duration / queries.length) * 1000} | ${baseBytes} | ${shardFiles} | ${shardStats.p50} | ${shardStats.p95} | ${shardStats.p99} | ${deltaBytes} | ${compactMs} | ${compactUsPerEdge}`); } fs.rmSync(dir, { recursive: true, force: true }); }