import { test, describe, before, after } from 'node:test'; import assert from 'node:assert/strict'; import { Arbiter } from '../../src/index.js'; describe.skip('Memory Investigation Tests', () => { let baselineMemory; before(() => { if (global.gc) { global.gc(); } baselineMemory = process.memoryUsage(); }); after(() => { if (global.gc) { global.gc(); } }); /** * Get current memory usage in MB */ function getMemoryUsage() { const memUsage = process.memoryUsage(); return { heapUsed: memUsage.heapUsed / 1024 / 1024, heapTotal: memUsage.heapTotal / 1024 / 1024, external: memUsage.external / 1024 / 1024, rss: memUsage.rss / 1024 / 1024 }; } /** * Measure memory usage for an operation */ function measureMemoryUsage(operation, description) { if (global.gc) global.gc(); const before = getMemoryUsage(); const result = operation(); if (global.gc) global.gc(); const after = getMemoryUsage(); const delta = { heapUsed: after.heapUsed - before.heapUsed, heapTotal: after.heapTotal - before.heapTotal, external: after.external - before.external, rss: after.rss - before.rss }; if (process.env.TEST_DEBUG === '1') console.log(`šŸ“Š ${description}:`); if (process.env.TEST_DEBUG === '1') console.log(` Before: ${before.heapUsed.toFixed(2)}MB heap, ${before.rss.toFixed(2)}MB RSS`); if (process.env.TEST_DEBUG === '1') console.log(` After: ${after.heapUsed.toFixed(2)}MB heap, ${after.rss.toFixed(2)}MB RSS`); if (process.env.TEST_DEBUG === '1') console.log(` Delta: ${delta.heapUsed.toFixed(2)}MB heap, ${delta.rss.toFixed(2)}MB RSS`); return { before, after, delta, result }; } test('Investigate: Node memory scaling in detail', async () => { if (process.env.TEST_DEBUG === '1') console.log('šŸ” Investigating node memory scaling in detail...'); const nodeCounts = [100, 200, 500, 1000, 2000, 5000, 10000]; const memoryResults = []; for (const nodeCount of nodeCounts) { const result = measureMemoryUsage(() => { const arbiter = new Arbiter({ fastConstructionMode: true }); // Disable indices // Add nodes one by one to see incremental growth for (let i = 0; i < nodeCount; i++) { arbiter.addNode(`user:${i}`, 'user'); arbiter.addNode(`doc:${i}`, 'document'); } return arbiter; }, `Adding ${nodeCount * 2} nodes (fast construction mode)`); memoryResults.push({ nodeCount: nodeCount * 2, heapUsed: result.after.heapUsed, heapDelta: result.delta.heapUsed, rss: result.after.rss, rssDelta: result.delta.rss, memoryPerNode: result.delta.heapUsed / (nodeCount * 2) }); } // Analyze detailed scaling if (process.env.TEST_DEBUG === '1') console.log('\nšŸ“ˆ Detailed Node Memory Scaling Analysis:'); if (process.env.TEST_DEBUG === '1') console.log('Node Count | Heap Delta | Memory/Node | Scaling Factor'); if (process.env.TEST_DEBUG === '1') console.log('-----------|------------|-------------|---------------'); for (let i = 0; i < memoryResults.length; i++) { const result = memoryResults[i]; const scalingFactor = i > 0 ? (result.memoryPerNode / memoryResults[i-1].memoryPerNode).toFixed(2) : 'N/A'; if (process.env.TEST_DEBUG === '1') console.log(`${result.nodeCount.toString().padStart(10)} | ${result.heapDelta.toFixed(2).padStart(10)}MB | ${result.memoryPerNode.toFixed(4).padStart(11)}MB | ${scalingFactor.padStart(13)}x`); } // Check for nonlinear patterns const memoryPerNodeValues = memoryResults.map(r => r.memoryPerNode); const minMemoryPerNode = Math.min(...memoryPerNodeValues); const maxMemoryPerNode = Math.max(...memoryPerNodeValues); const memoryVariation = (maxMemoryPerNode - minMemoryPerNode) / minMemoryPerNode; if (process.env.TEST_DEBUG === '1') console.log(`\nMemory per node variation: ${(memoryVariation * 100).toFixed(1)}%`); if (memoryVariation > 0.5) { if (process.env.TEST_DEBUG === '1') console.log('āš ļø WARNING: Significant nonlinear scaling detected!'); } }); test('Investigate: Relation memory scaling in detail', async () => { if (process.env.TEST_DEBUG === '1') console.log('šŸ” Investigating relation memory scaling in detail...'); const relationCounts = [100, 200, 500, 1000, 2000, 5000, 10000]; const memoryResults = []; for (const relationCount of relationCounts) { const result = measureMemoryUsage(() => { const arbiter = new Arbiter({ fastConstructionMode: true }); // Disable indices // Add nodes first for (let i = 0; i < relationCount; i++) { arbiter.addNode(`user:${i}`, 'user'); arbiter.addNode(`doc:${i}`, 'document'); } // Add relations one by one for (let i = 0; i < relationCount; i++) { arbiter.addRelation(`user:${i}`, 'can_read', `doc:${i}`, { possibility: 1.0 }); } return arbiter; }, `Adding ${relationCount} relations (fast construction mode)`); memoryResults.push({ relationCount, heapUsed: result.after.heapUsed, heapDelta: result.delta.heapUsed, rss: result.after.rss, rssDelta: result.delta.rss, memoryPerRelation: result.delta.heapUsed / relationCount }); } // Analyze detailed scaling if (process.env.TEST_DEBUG === '1') console.log('\nšŸ“ˆ Detailed Relation Memory Scaling Analysis:'); if (process.env.TEST_DEBUG === '1') console.log('Rel Count | Heap Delta | Memory/Rel | Scaling Factor'); if (process.env.TEST_DEBUG === '1') console.log('----------|------------|------------|---------------'); for (let i = 0; i < memoryResults.length; i++) { const result = memoryResults[i]; const scalingFactor = i > 0 ? (result.memoryPerRelation / memoryResults[i-1].memoryPerRelation).toFixed(2) : 'N/A'; if (process.env.TEST_DEBUG === '1') console.log(`${result.relationCount.toString().padStart(9)} | ${result.heapDelta.toFixed(2).padStart(10)}MB | ${result.memoryPerRelation.toFixed(4).padStart(10)}MB | ${scalingFactor.padStart(13)}x`); } // Check for nonlinear patterns const memoryPerRelationValues = memoryResults.map(r => r.memoryPerRelation); const minMemoryPerRelation = Math.min(...memoryPerRelationValues); const maxMemoryPerRelation = Math.max(...memoryPerRelationValues); const memoryVariation = (maxMemoryPerRelation - minMemoryPerRelation) / minMemoryPerRelation; if (process.env.TEST_DEBUG === '1') console.log(`\nMemory per relation variation: ${(memoryVariation * 100).toFixed(1)}%`); if (memoryVariation > 0.5) { if (process.env.TEST_DEBUG === '1') console.log('āš ļø WARNING: Significant nonlinear scaling detected!'); } }); test('Investigate: What happens during index building', async () => { if (process.env.TEST_DEBUG === '1') console.log('šŸ” Investigating index building memory impact...'); const relationCounts = [1000, 2000, 5000, 10000]; for (const relationCount of relationCounts) { if (process.env.TEST_DEBUG === '1') console.log(`\n--- Testing with ${relationCount} relations ---`); // Test 1: Fast construction mode (no indices) const resultFast = measureMemoryUsage(() => { const arbiter = new Arbiter({ fastConstructionMode: true }); // Add nodes and relations for (let i = 0; i < relationCount; i++) { arbiter.addNode(`user:${i}`, 'user'); arbiter.addNode(`doc:${i}`, 'document'); arbiter.addRelation(`user:${i}`, 'can_read', `doc:${i}`, { possibility: 1.0 }); } return arbiter; }, `Fast construction mode (${relationCount} relations)`); // Test 2: Normal mode (with indices) const resultNormal = measureMemoryUsage(() => { const arbiter = new Arbiter({ fastConstructionMode: false }); // Add nodes and relations for (let i = 0; i < relationCount; i++) { arbiter.addNode(`user:${i}`, 'user'); arbiter.addNode(`doc:${i}`, 'document'); arbiter.addRelation(`user:${i}`, 'can_read', `doc:${i}`, { possibility: 1.0 }); } return arbiter; }, `Normal mode with indices (${relationCount} relations)`); // Test 3: Build indices manually const resultManual = measureMemoryUsage(() => { const arbiter = new Arbiter({ fastConstructionMode: true }); // Add nodes and relations for (let i = 0; i < relationCount; i++) { arbiter.addNode(`user:${i}`, 'user'); arbiter.addNode(`doc:${i}`, 'document'); arbiter.addRelation(`user:${i}`, 'can_read', `doc:${i}`, { possibility: 1.0 }); } // Build indices manually arbiter.setFastConstructionMode(false); return arbiter; }, `Manual index building (${relationCount} relations)`); const indexOverhead = resultNormal.after.heapUsed - resultFast.after.heapUsed; const manualOverhead = resultManual.after.heapUsed - resultFast.after.heapUsed; if (process.env.TEST_DEBUG === '1') console.log(` Index overhead (normal): ${indexOverhead.toFixed(2)}MB`); if (process.env.TEST_DEBUG === '1') console.log(` Index overhead (manual): ${manualOverhead.toFixed(2)}MB`); if (process.env.TEST_DEBUG === '1') console.log(` Overhead per relation: ${(indexOverhead / relationCount).toFixed(4)}MB`); } }); test('Investigate: Memory usage of internal data structures', async () => { if (process.env.TEST_DEBUG === '1') console.log('šŸ” Investigating internal data structure memory usage...'); const relationCount = 5000; const result = measureMemoryUsage(() => { const arbiter = new Arbiter({ fastConstructionMode: true }); // Add nodes and relations for (let i = 0; i < relationCount; i++) { arbiter.addNode(`user:${i}`, 'user'); arbiter.addNode(`doc:${i}`, 'document'); arbiter.addRelation(`user:${i}`, 'can_read', `doc:${i}`, { possibility: 1.0 }); } return arbiter; }, `Creating arbiter with ${relationCount} relations`); // Inspect internal structures const arbiter = result.result; if (process.env.TEST_DEBUG === '1') console.log('\nšŸ“Š Internal Data Structure Sizes:'); if (process.env.TEST_DEBUG === '1') console.log(` Relations array length: ${arbiter.relations.length}`); if (process.env.TEST_DEBUG === '1') console.log(` Node count: ${arbiter.nodeManager.nodes.size}`); if (process.env.TEST_DEBUG === '1') console.log(` Node ID map size: ${arbiter.nodeIdByKey ? arbiter.nodeIdByKey.size : 'N/A'}`); if (process.env.TEST_DEBUG === '1') console.log(` Key manager string cache: ${arbiter.keyManager ? arbiter.keyManager.stringToId.size : 'N/A'}`); if (process.env.TEST_DEBUG === '1') console.log(` Key manager ID cache: ${arbiter.keyManager ? arbiter.keyManager.idToString.size : 'N/A'}`); if (process.env.TEST_DEBUG === '1') console.log(` Relation keys set size: ${arbiter.relationManager ? arbiter.relationManager._relationKeys.size : 'N/A'}`); // Estimate memory per structure const estimatedMemoryPerRelation = result.delta.heapUsed / relationCount; if (process.env.TEST_DEBUG === '1') console.log(` Estimated memory per relation: ${estimatedMemoryPerRelation.toFixed(4)}MB`); // Check if relations array is the main memory consumer const relationObjectSize = JSON.stringify(arbiter.relations[0]).length; const totalRelationStringSize = relationObjectSize * relationCount; const relationStringMemoryMB = totalRelationStringSize / 1024 / 1024; if (process.env.TEST_DEBUG === '1') console.log(` Relation object string size: ${relationObjectSize} bytes`); if (process.env.TEST_DEBUG === '1') console.log(` Total relation string memory: ${relationStringMemoryMB.toFixed(2)}MB`); if (process.env.TEST_DEBUG === '1') console.log(` Relation string % of total: ${((relationStringMemoryMB / result.delta.heapUsed) * 100).toFixed(1)}%`); }); test('Investigate: JavaScript heap behavior', async () => { if (process.env.TEST_DEBUG === '1') console.log('šŸ” Investigating JavaScript heap behavior...'); const nodeCounts = [100, 500, 1000, 2000, 5000, 10000]; for (const nodeCount of nodeCounts) { const result = measureMemoryUsage(() => { const arbiter = new Arbiter({ fastConstructionMode: true }); for (let i = 0; i < nodeCount; i++) { arbiter.addNode(`user:${i}`, 'user'); } return arbiter; }, `Adding ${nodeCount} nodes`); const heapUtilization = (result.after.heapUsed / result.after.heapTotal) * 100; const heapGrowth = result.after.heapTotal - result.before.heapTotal; if (process.env.TEST_DEBUG === '1') console.log(` ${nodeCount} nodes: ${result.after.heapUsed.toFixed(2)}MB used, ${result.after.heapTotal.toFixed(2)}MB total (${heapUtilization.toFixed(1)}% utilization)`); if (process.env.TEST_DEBUG === '1') console.log(` Heap growth: ${heapGrowth.toFixed(2)}MB`); if (heapUtilization > 80) { if (process.env.TEST_DEBUG === '1') console.log(' āš ļø High heap utilization - may trigger GC'); } } }); test('Investigate: Memory fragmentation', async () => { if (process.env.TEST_DEBUG === '1') console.log('šŸ” Investigating memory fragmentation...'); // Test with different allocation patterns const patterns = [ { name: 'Sequential', count: 5000 }, { name: 'Batched (100)', count: 5000, batchSize: 100 }, { name: 'Batched (500)', count: 5000, batchSize: 500 }, { name: 'Batched (1000)', count: 5000, batchSize: 1000 } ]; for (const pattern of patterns) { const result = measureMemoryUsage(() => { const arbiter = new Arbiter({ fastConstructionMode: true }); if (pattern.batchSize) { // Batched allocation for (let batch = 0; batch < pattern.count; batch += pattern.batchSize) { const batchEnd = Math.min(batch + pattern.batchSize, pattern.count); for (let i = batch; i < batchEnd; i++) { arbiter.addNode(`user:${i}`, 'user'); } // Force GC between batches to see fragmentation if (global.gc) global.gc(); } } else { // Sequential allocation for (let i = 0; i < pattern.count; i++) { arbiter.addNode(`user:${i}`, 'user'); } } return arbiter; }, `${pattern.name} allocation (${pattern.count} nodes)`); const memoryPerNode = result.delta.heapUsed / pattern.count; if (process.env.TEST_DEBUG === '1') console.log(` ${pattern.name}: ${memoryPerNode.toFixed(4)}MB per node`); } }); });