349 lines
15 KiB
JavaScript
349 lines
15 KiB
JavaScript
|
import { test, describe, before, after } from 'node:test';
|
||
|
|
import assert from 'node:assert/strict';
|
||
|
|
import { Arbiter } from '../../src/index.js';
|
||
|
|
|
||
|
|
describe.skip('Memory Investigation Tests', () => {
|
||
|
|
let baselineMemory;
|
||
|
|
|
||
|
|
before(() => {
|
||
|
|
if (global.gc) {
|
||
|
|
global.gc();
|
||
|
|
}
|
||
|
|
baselineMemory = process.memoryUsage();
|
||
|
|
});
|
||
|
|
|
||
|
|
after(() => {
|
||
|
|
if (global.gc) {
|
||
|
|
global.gc();
|
||
|
|
}
|
||
|
|
});
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Get current memory usage in MB
|
||
|
|
*/
|
||
|
|
function getMemoryUsage() {
|
||
|
|
const memUsage = process.memoryUsage();
|
||
|
|
return {
|
||
|
|
heapUsed: memUsage.heapUsed / 1024 / 1024,
|
||
|
|
heapTotal: memUsage.heapTotal / 1024 / 1024,
|
||
|
|
external: memUsage.external / 1024 / 1024,
|
||
|
|
rss: memUsage.rss / 1024 / 1024
|
||
|
|
};
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Measure memory usage for an operation
|
||
|
|
*/
|
||
|
|
function measureMemoryUsage(operation, description) {
|
||
|
|
if (global.gc) global.gc();
|
||
|
|
const before = getMemoryUsage();
|
||
|
|
|
||
|
|
const result = operation();
|
||
|
|
|
||
|
|
if (global.gc) global.gc();
|
||
|
|
const after = getMemoryUsage();
|
||
|
|
|
||
|
|
const delta = {
|
||
|
|
heapUsed: after.heapUsed - before.heapUsed,
|
||
|
|
heapTotal: after.heapTotal - before.heapTotal,
|
||
|
|
external: after.external - before.external,
|
||
|
|
rss: after.rss - before.rss
|
||
|
|
};
|
||
|
|
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(`📊 ${description}:`);
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(` Before: ${before.heapUsed.toFixed(2)}MB heap, ${before.rss.toFixed(2)}MB RSS`);
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(` After: ${after.heapUsed.toFixed(2)}MB heap, ${after.rss.toFixed(2)}MB RSS`);
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(` Delta: ${delta.heapUsed.toFixed(2)}MB heap, ${delta.rss.toFixed(2)}MB RSS`);
|
||
|
|
|
||
|
|
return { before, after, delta, result };
|
||
|
|
}
|
||
|
|
|
||
|
|
test('Investigate: Node memory scaling in detail', async () => {
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log('🔍 Investigating node memory scaling in detail...');
|
||
|
|
|
||
|
|
const nodeCounts = [100, 200, 500, 1000, 2000, 5000, 10000];
|
||
|
|
const memoryResults = [];
|
||
|
|
|
||
|
|
for (const nodeCount of nodeCounts) {
|
||
|
|
const result = measureMemoryUsage(() => {
|
||
|
|
const arbiter = new Arbiter({ fastConstructionMode: true }); // Disable indices
|
||
|
|
|
||
|
|
// Add nodes one by one to see incremental growth
|
||
|
|
for (let i = 0; i < nodeCount; i++) {
|
||
|
|
arbiter.addNode(`user:${i}`, 'user');
|
||
|
|
arbiter.addNode(`doc:${i}`, 'document');
|
||
|
|
}
|
||
|
|
|
||
|
|
return arbiter;
|
||
|
|
}, `Adding ${nodeCount * 2} nodes (fast construction mode)`);
|
||
|
|
|
||
|
|
memoryResults.push({
|
||
|
|
nodeCount: nodeCount * 2,
|
||
|
|
heapUsed: result.after.heapUsed,
|
||
|
|
heapDelta: result.delta.heapUsed,
|
||
|
|
rss: result.after.rss,
|
||
|
|
rssDelta: result.delta.rss,
|
||
|
|
memoryPerNode: result.delta.heapUsed / (nodeCount * 2)
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
// Analyze detailed scaling
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log('\n📈 Detailed Node Memory Scaling Analysis:');
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log('Node Count | Heap Delta | Memory/Node | Scaling Factor');
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log('-----------|------------|-------------|---------------');
|
||
|
|
|
||
|
|
for (let i = 0; i < memoryResults.length; i++) {
|
||
|
|
const result = memoryResults[i];
|
||
|
|
const scalingFactor = i > 0 ?
|
||
|
|
(result.memoryPerNode / memoryResults[i-1].memoryPerNode).toFixed(2) : 'N/A';
|
||
|
|
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(`${result.nodeCount.toString().padStart(10)} | ${result.heapDelta.toFixed(2).padStart(10)}MB | ${result.memoryPerNode.toFixed(4).padStart(11)}MB | ${scalingFactor.padStart(13)}x`);
|
||
|
|
}
|
||
|
|
|
||
|
|
// Check for nonlinear patterns
|
||
|
|
const memoryPerNodeValues = memoryResults.map(r => r.memoryPerNode);
|
||
|
|
const minMemoryPerNode = Math.min(...memoryPerNodeValues);
|
||
|
|
const maxMemoryPerNode = Math.max(...memoryPerNodeValues);
|
||
|
|
const memoryVariation = (maxMemoryPerNode - minMemoryPerNode) / minMemoryPerNode;
|
||
|
|
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(`\nMemory per node variation: ${(memoryVariation * 100).toFixed(1)}%`);
|
||
|
|
|
||
|
|
if (memoryVariation > 0.5) {
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log('⚠️ WARNING: Significant nonlinear scaling detected!');
|
||
|
|
}
|
||
|
|
});
|
||
|
|
|
||
|
|
test('Investigate: Relation memory scaling in detail', async () => {
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log('🔍 Investigating relation memory scaling in detail...');
|
||
|
|
|
||
|
|
const relationCounts = [100, 200, 500, 1000, 2000, 5000, 10000];
|
||
|
|
const memoryResults = [];
|
||
|
|
|
||
|
|
for (const relationCount of relationCounts) {
|
||
|
|
const result = measureMemoryUsage(() => {
|
||
|
|
const arbiter = new Arbiter({ fastConstructionMode: true }); // Disable indices
|
||
|
|
|
||
|
|
// Add nodes first
|
||
|
|
for (let i = 0; i < relationCount; i++) {
|
||
|
|
arbiter.addNode(`user:${i}`, 'user');
|
||
|
|
arbiter.addNode(`doc:${i}`, 'document');
|
||
|
|
}
|
||
|
|
|
||
|
|
// Add relations one by one
|
||
|
|
for (let i = 0; i < relationCount; i++) {
|
||
|
|
arbiter.addRelation(`user:${i}`, 'can_read', `doc:${i}`, { possibility: 1.0 });
|
||
|
|
}
|
||
|
|
|
||
|
|
return arbiter;
|
||
|
|
}, `Adding ${relationCount} relations (fast construction mode)`);
|
||
|
|
|
||
|
|
memoryResults.push({
|
||
|
|
relationCount,
|
||
|
|
heapUsed: result.after.heapUsed,
|
||
|
|
heapDelta: result.delta.heapUsed,
|
||
|
|
rss: result.after.rss,
|
||
|
|
rssDelta: result.delta.rss,
|
||
|
|
memoryPerRelation: result.delta.heapUsed / relationCount
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
// Analyze detailed scaling
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log('\n📈 Detailed Relation Memory Scaling Analysis:');
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log('Rel Count | Heap Delta | Memory/Rel | Scaling Factor');
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log('----------|------------|------------|---------------');
|
||
|
|
|
||
|
|
for (let i = 0; i < memoryResults.length; i++) {
|
||
|
|
const result = memoryResults[i];
|
||
|
|
const scalingFactor = i > 0 ?
|
||
|
|
(result.memoryPerRelation / memoryResults[i-1].memoryPerRelation).toFixed(2) : 'N/A';
|
||
|
|
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(`${result.relationCount.toString().padStart(9)} | ${result.heapDelta.toFixed(2).padStart(10)}MB | ${result.memoryPerRelation.toFixed(4).padStart(10)}MB | ${scalingFactor.padStart(13)}x`);
|
||
|
|
}
|
||
|
|
|
||
|
|
// Check for nonlinear patterns
|
||
|
|
const memoryPerRelationValues = memoryResults.map(r => r.memoryPerRelation);
|
||
|
|
const minMemoryPerRelation = Math.min(...memoryPerRelationValues);
|
||
|
|
const maxMemoryPerRelation = Math.max(...memoryPerRelationValues);
|
||
|
|
const memoryVariation = (maxMemoryPerRelation - minMemoryPerRelation) / minMemoryPerRelation;
|
||
|
|
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(`\nMemory per relation variation: ${(memoryVariation * 100).toFixed(1)}%`);
|
||
|
|
|
||
|
|
if (memoryVariation > 0.5) {
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log('⚠️ WARNING: Significant nonlinear scaling detected!');
|
||
|
|
}
|
||
|
|
});
|
||
|
|
|
||
|
|
test('Investigate: What happens during index building', async () => {
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log('🔍 Investigating index building memory impact...');
|
||
|
|
|
||
|
|
const relationCounts = [1000, 2000, 5000, 10000];
|
||
|
|
|
||
|
|
for (const relationCount of relationCounts) {
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(`\n--- Testing with ${relationCount} relations ---`);
|
||
|
|
|
||
|
|
// Test 1: Fast construction mode (no indices)
|
||
|
|
const resultFast = measureMemoryUsage(() => {
|
||
|
|
const arbiter = new Arbiter({ fastConstructionMode: true });
|
||
|
|
|
||
|
|
// Add nodes and relations
|
||
|
|
for (let i = 0; i < relationCount; i++) {
|
||
|
|
arbiter.addNode(`user:${i}`, 'user');
|
||
|
|
arbiter.addNode(`doc:${i}`, 'document');
|
||
|
|
arbiter.addRelation(`user:${i}`, 'can_read', `doc:${i}`, { possibility: 1.0 });
|
||
|
|
}
|
||
|
|
|
||
|
|
return arbiter;
|
||
|
|
}, `Fast construction mode (${relationCount} relations)`);
|
||
|
|
|
||
|
|
// Test 2: Normal mode (with indices)
|
||
|
|
const resultNormal = measureMemoryUsage(() => {
|
||
|
|
const arbiter = new Arbiter({ fastConstructionMode: false });
|
||
|
|
|
||
|
|
// Add nodes and relations
|
||
|
|
for (let i = 0; i < relationCount; i++) {
|
||
|
|
arbiter.addNode(`user:${i}`, 'user');
|
||
|
|
arbiter.addNode(`doc:${i}`, 'document');
|
||
|
|
arbiter.addRelation(`user:${i}`, 'can_read', `doc:${i}`, { possibility: 1.0 });
|
||
|
|
}
|
||
|
|
|
||
|
|
return arbiter;
|
||
|
|
}, `Normal mode with indices (${relationCount} relations)`);
|
||
|
|
|
||
|
|
// Test 3: Build indices manually
|
||
|
|
const resultManual = measureMemoryUsage(() => {
|
||
|
|
const arbiter = new Arbiter({ fastConstructionMode: true });
|
||
|
|
|
||
|
|
// Add nodes and relations
|
||
|
|
for (let i = 0; i < relationCount; i++) {
|
||
|
|
arbiter.addNode(`user:${i}`, 'user');
|
||
|
|
arbiter.addNode(`doc:${i}`, 'document');
|
||
|
|
arbiter.addRelation(`user:${i}`, 'can_read', `doc:${i}`, { possibility: 1.0 });
|
||
|
|
}
|
||
|
|
|
||
|
|
// Build indices manually
|
||
|
|
arbiter.setFastConstructionMode(false);
|
||
|
|
|
||
|
|
return arbiter;
|
||
|
|
}, `Manual index building (${relationCount} relations)`);
|
||
|
|
|
||
|
|
const indexOverhead = resultNormal.after.heapUsed - resultFast.after.heapUsed;
|
||
|
|
const manualOverhead = resultManual.after.heapUsed - resultFast.after.heapUsed;
|
||
|
|
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(` Index overhead (normal): ${indexOverhead.toFixed(2)}MB`);
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(` Index overhead (manual): ${manualOverhead.toFixed(2)}MB`);
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(` Overhead per relation: ${(indexOverhead / relationCount).toFixed(4)}MB`);
|
||
|
|
}
|
||
|
|
});
|
||
|
|
|
||
|
|
test('Investigate: Memory usage of internal data structures', async () => {
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log('🔍 Investigating internal data structure memory usage...');
|
||
|
|
|
||
|
|
const relationCount = 5000;
|
||
|
|
|
||
|
|
const result = measureMemoryUsage(() => {
|
||
|
|
const arbiter = new Arbiter({ fastConstructionMode: true });
|
||
|
|
|
||
|
|
// Add nodes and relations
|
||
|
|
for (let i = 0; i < relationCount; i++) {
|
||
|
|
arbiter.addNode(`user:${i}`, 'user');
|
||
|
|
arbiter.addNode(`doc:${i}`, 'document');
|
||
|
|
arbiter.addRelation(`user:${i}`, 'can_read', `doc:${i}`, { possibility: 1.0 });
|
||
|
|
}
|
||
|
|
|
||
|
|
return arbiter;
|
||
|
|
}, `Creating arbiter with ${relationCount} relations`);
|
||
|
|
|
||
|
|
// Inspect internal structures
|
||
|
|
const arbiter = result.result;
|
||
|
|
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log('\n📊 Internal Data Structure Sizes:');
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(` Relations array length: ${arbiter.relations.length}`);
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(` Node count: ${arbiter.nodeManager.nodes.size}`);
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(` Node ID map size: ${arbiter.nodeIdByKey ? arbiter.nodeIdByKey.size : 'N/A'}`);
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(` Key manager string cache: ${arbiter.keyManager ? arbiter.keyManager.stringToId.size : 'N/A'}`);
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(` Key manager ID cache: ${arbiter.keyManager ? arbiter.keyManager.idToString.size : 'N/A'}`);
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(` Relation keys set size: ${arbiter.relationManager ? arbiter.relationManager._relationKeys.size : 'N/A'}`);
|
||
|
|
|
||
|
|
// Estimate memory per structure
|
||
|
|
const estimatedMemoryPerRelation = result.delta.heapUsed / relationCount;
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(` Estimated memory per relation: ${estimatedMemoryPerRelation.toFixed(4)}MB`);
|
||
|
|
|
||
|
|
// Check if relations array is the main memory consumer
|
||
|
|
const relationObjectSize = JSON.stringify(arbiter.relations[0]).length;
|
||
|
|
const totalRelationStringSize = relationObjectSize * relationCount;
|
||
|
|
const relationStringMemoryMB = totalRelationStringSize / 1024 / 1024;
|
||
|
|
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(` Relation object string size: ${relationObjectSize} bytes`);
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(` Total relation string memory: ${relationStringMemoryMB.toFixed(2)}MB`);
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(` Relation string % of total: ${((relationStringMemoryMB / result.delta.heapUsed) * 100).toFixed(1)}%`);
|
||
|
|
});
|
||
|
|
|
||
|
|
test('Investigate: JavaScript heap behavior', async () => {
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log('🔍 Investigating JavaScript heap behavior...');
|
||
|
|
|
||
|
|
const nodeCounts = [100, 500, 1000, 2000, 5000, 10000];
|
||
|
|
|
||
|
|
for (const nodeCount of nodeCounts) {
|
||
|
|
const result = measureMemoryUsage(() => {
|
||
|
|
const arbiter = new Arbiter({ fastConstructionMode: true });
|
||
|
|
|
||
|
|
for (let i = 0; i < nodeCount; i++) {
|
||
|
|
arbiter.addNode(`user:${i}`, 'user');
|
||
|
|
}
|
||
|
|
|
||
|
|
return arbiter;
|
||
|
|
}, `Adding ${nodeCount} nodes`);
|
||
|
|
|
||
|
|
const heapUtilization = (result.after.heapUsed / result.after.heapTotal) * 100;
|
||
|
|
const heapGrowth = result.after.heapTotal - result.before.heapTotal;
|
||
|
|
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(` ${nodeCount} nodes: ${result.after.heapUsed.toFixed(2)}MB used, ${result.after.heapTotal.toFixed(2)}MB total (${heapUtilization.toFixed(1)}% utilization)`);
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(` Heap growth: ${heapGrowth.toFixed(2)}MB`);
|
||
|
|
|
||
|
|
if (heapUtilization > 80) {
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(' ⚠️ High heap utilization - may trigger GC');
|
||
|
|
}
|
||
|
|
}
|
||
|
|
});
|
||
|
|
|
||
|
|
test('Investigate: Memory fragmentation', async () => {
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log('🔍 Investigating memory fragmentation...');
|
||
|
|
|
||
|
|
// Test with different allocation patterns
|
||
|
|
const patterns = [
|
||
|
|
{ name: 'Sequential', count: 5000 },
|
||
|
|
{ name: 'Batched (100)', count: 5000, batchSize: 100 },
|
||
|
|
{ name: 'Batched (500)', count: 5000, batchSize: 500 },
|
||
|
|
{ name: 'Batched (1000)', count: 5000, batchSize: 1000 }
|
||
|
|
];
|
||
|
|
|
||
|
|
for (const pattern of patterns) {
|
||
|
|
const result = measureMemoryUsage(() => {
|
||
|
|
const arbiter = new Arbiter({ fastConstructionMode: true });
|
||
|
|
|
||
|
|
if (pattern.batchSize) {
|
||
|
|
// Batched allocation
|
||
|
|
for (let batch = 0; batch < pattern.count; batch += pattern.batchSize) {
|
||
|
|
const batchEnd = Math.min(batch + pattern.batchSize, pattern.count);
|
||
|
|
for (let i = batch; i < batchEnd; i++) {
|
||
|
|
arbiter.addNode(`user:${i}`, 'user');
|
||
|
|
}
|
||
|
|
// Force GC between batches to see fragmentation
|
||
|
|
if (global.gc) global.gc();
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
// Sequential allocation
|
||
|
|
for (let i = 0; i < pattern.count; i++) {
|
||
|
|
arbiter.addNode(`user:${i}`, 'user');
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return arbiter;
|
||
|
|
}, `${pattern.name} allocation (${pattern.count} nodes)`);
|
||
|
|
|
||
|
|
const memoryPerNode = result.delta.heapUsed / pattern.count;
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(` ${pattern.name}: ${memoryPerNode.toFixed(4)}MB per node`);
|
||
|
|
}
|
||
|
|
});
|
||
|
|
});
|