253 lines
11 KiB
JavaScript
253 lines
11 KiB
JavaScript
|
import { test, describe, before, after } from 'node:test';
|
||
|
|
import assert from 'node:assert/strict';
|
||
|
|
import { Arbiter } from '../../src/index.js';
|
||
|
|
|
||
|
|
describe('Clean Memory Scaling Tests', () => {
|
||
|
|
/**
|
||
|
|
* Get current memory usage in MB
|
||
|
|
*/
|
||
|
|
function getMemoryUsage() {
|
||
|
|
const memUsage = process.memoryUsage();
|
||
|
|
return {
|
||
|
|
heapUsed: memUsage.heapUsed / 1024 / 1024,
|
||
|
|
heapTotal: memUsage.heapTotal / 1024 / 1024,
|
||
|
|
external: memUsage.external / 1024 / 1024,
|
||
|
|
rss: memUsage.rss / 1024 / 1024
|
||
|
|
};
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Measure memory usage with proper GC handling
|
||
|
|
*/
|
||
|
|
function measureMemoryUsage(operation, description) {
|
||
|
|
// Force multiple GC cycles to get clean baseline
|
||
|
|
if (global.gc) {
|
||
|
|
global.gc();
|
||
|
|
global.gc();
|
||
|
|
global.gc();
|
||
|
|
}
|
||
|
|
|
||
|
|
const before = getMemoryUsage();
|
||
|
|
|
||
|
|
const result = operation();
|
||
|
|
|
||
|
|
// Force multiple GC cycles to get clean measurement
|
||
|
|
if (global.gc) {
|
||
|
|
global.gc();
|
||
|
|
global.gc();
|
||
|
|
global.gc();
|
||
|
|
}
|
||
|
|
|
||
|
|
const after = getMemoryUsage();
|
||
|
|
|
||
|
|
const delta = {
|
||
|
|
heapUsed: after.heapUsed - before.heapUsed,
|
||
|
|
heapTotal: after.heapTotal - before.heapTotal,
|
||
|
|
external: after.external - before.external,
|
||
|
|
rss: after.rss - before.rss
|
||
|
|
};
|
||
|
|
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(`📊 ${description}:`);
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(` Before: ${before.heapUsed.toFixed(2)}MB heap, ${before.rss.toFixed(2)}MB RSS`);
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(` After: ${after.heapUsed.toFixed(2)}MB heap, ${after.rss.toFixed(2)}MB RSS`);
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(` Delta: ${delta.heapUsed.toFixed(2)}MB heap, ${delta.rss.toFixed(2)}MB RSS`);
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(` Heap utilization: ${((after.heapUsed / after.heapTotal) * 100).toFixed(1)}%`);
|
||
|
|
|
||
|
|
return { before, after, delta, result };
|
||
|
|
}
|
||
|
|
|
||
|
|
test('Clean node memory scaling test', async () => {
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log('🧠 Clean node memory scaling test...');
|
||
|
|
|
||
|
|
const nodeCounts = [100, 200, 500, 1000, 2000, 5000];
|
||
|
|
const memoryResults = [];
|
||
|
|
|
||
|
|
for (const nodeCount of nodeCounts) {
|
||
|
|
const result = measureMemoryUsage(() => {
|
||
|
|
const arbiter = new Arbiter({ fastConstructionMode: true });
|
||
|
|
|
||
|
|
for (let i = 0; i < nodeCount; i++) {
|
||
|
|
arbiter.addNode(`user:${i}`, 'user');
|
||
|
|
}
|
||
|
|
|
||
|
|
return arbiter;
|
||
|
|
}, `Adding ${nodeCount} nodes`);
|
||
|
|
|
||
|
|
memoryResults.push({
|
||
|
|
nodeCount,
|
||
|
|
heapUsed: result.after.heapUsed,
|
||
|
|
heapDelta: result.delta.heapUsed,
|
||
|
|
memoryPerNode: result.delta.heapUsed / nodeCount,
|
||
|
|
heapUtilization: (result.after.heapUsed / result.after.heapTotal) * 100
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
// Analyze scaling
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log('\n📈 Clean Node Memory Scaling Analysis:');
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log('Nodes | Heap Delta | Memory/Node | Heap Util | Scaling Factor');
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log('------|------------|-------------|-----------|---------------');
|
||
|
|
|
||
|
|
for (let i = 0; i < memoryResults.length; i++) {
|
||
|
|
const result = memoryResults[i];
|
||
|
|
const scalingFactor = i > 0 ?
|
||
|
|
(result.memoryPerNode / memoryResults[i-1].memoryPerNode).toFixed(2) : 'N/A';
|
||
|
|
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(`${result.nodeCount.toString().padStart(5)} | ${result.heapDelta.toFixed(2).padStart(10)}MB | ${result.memoryPerNode.toFixed(4).padStart(11)}MB | ${result.heapUtilization.toFixed(1).padStart(8)}% | ${scalingFactor.padStart(13)}x`);
|
||
|
|
}
|
||
|
|
|
||
|
|
// Check for linear scaling
|
||
|
|
const memoryPerNodeValues = memoryResults.map(r => r.memoryPerNode);
|
||
|
|
const avgMemoryPerNode = memoryPerNodeValues.reduce((a, b) => a + b, 0) / memoryPerNodeValues.length;
|
||
|
|
const maxDeviation = Math.max(...memoryPerNodeValues.map(v => Math.abs(v - avgMemoryPerNode)));
|
||
|
|
const deviationPercent = (maxDeviation / avgMemoryPerNode) * 100;
|
||
|
|
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(`\nAverage memory per node: ${avgMemoryPerNode.toFixed(4)}MB`);
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(`Maximum deviation: ${maxDeviation.toFixed(4)}MB (${deviationPercent.toFixed(1)}%)`);
|
||
|
|
|
||
|
|
if (deviationPercent < 20) {
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log('✅ Memory scaling is approximately linear');
|
||
|
|
} else {
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log('⚠️ Memory scaling shows significant nonlinearity');
|
||
|
|
}
|
||
|
|
});
|
||
|
|
|
||
|
|
test('Clean relation memory scaling test', async () => {
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log('🧠 Clean relation memory scaling test...');
|
||
|
|
|
||
|
|
const relationCounts = [100, 200, 500, 1000, 2000, 5000];
|
||
|
|
const memoryResults = [];
|
||
|
|
|
||
|
|
for (const relationCount of relationCounts) {
|
||
|
|
const result = measureMemoryUsage(() => {
|
||
|
|
const arbiter = new Arbiter({ fastConstructionMode: true });
|
||
|
|
|
||
|
|
// Add nodes first
|
||
|
|
for (let i = 0; i < relationCount; i++) {
|
||
|
|
arbiter.addNode(`user:${i}`, 'user');
|
||
|
|
arbiter.addNode(`doc:${i}`, 'document');
|
||
|
|
}
|
||
|
|
|
||
|
|
// Add relations
|
||
|
|
for (let i = 0; i < relationCount; i++) {
|
||
|
|
arbiter.addRelation(`user:${i}`, 'can_read', `doc:${i}`, { possibility: 1.0 });
|
||
|
|
}
|
||
|
|
|
||
|
|
return arbiter;
|
||
|
|
}, `Adding ${relationCount} relations`);
|
||
|
|
|
||
|
|
memoryResults.push({
|
||
|
|
relationCount,
|
||
|
|
heapUsed: result.after.heapUsed,
|
||
|
|
heapDelta: result.delta.heapUsed,
|
||
|
|
memoryPerRelation: result.delta.heapUsed / relationCount,
|
||
|
|
heapUtilization: (result.after.heapUsed / result.after.heapTotal) * 100
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
// Analyze scaling
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log('\n📈 Clean Relation Memory Scaling Analysis:');
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log('Rels | Heap Delta | Memory/Rel | Heap Util | Scaling Factor');
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log('------|------------|------------|-----------|---------------');
|
||
|
|
|
||
|
|
for (let i = 0; i < memoryResults.length; i++) {
|
||
|
|
const result = memoryResults[i];
|
||
|
|
const scalingFactor = i > 0 ?
|
||
|
|
(result.memoryPerRelation / memoryResults[i-1].memoryPerRelation).toFixed(2) : 'N/A';
|
||
|
|
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(`${result.relationCount.toString().padStart(5)} | ${result.heapDelta.toFixed(2).padStart(10)}MB | ${result.memoryPerRelation.toFixed(4).padStart(10)}MB | ${result.heapUtilization.toFixed(1).padStart(8)}% | ${scalingFactor.padStart(13)}x`);
|
||
|
|
}
|
||
|
|
|
||
|
|
// Check for linear scaling
|
||
|
|
const memoryPerRelationValues = memoryResults.map(r => r.memoryPerRelation);
|
||
|
|
const avgMemoryPerRelation = memoryPerRelationValues.reduce((a, b) => a + b, 0) / memoryPerRelationValues.length;
|
||
|
|
const maxDeviation = Math.max(...memoryPerRelationValues.map(v => Math.abs(v - avgMemoryPerRelation)));
|
||
|
|
const deviationPercent = (maxDeviation / avgMemoryPerRelation) * 100;
|
||
|
|
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(`\nAverage memory per relation: ${avgMemoryPerRelation.toFixed(4)}MB`);
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(`Maximum deviation: ${maxDeviation.toFixed(4)}MB (${deviationPercent.toFixed(1)}%)`);
|
||
|
|
|
||
|
|
if (deviationPercent < 20) {
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log('✅ Memory scaling is approximately linear');
|
||
|
|
} else {
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log('⚠️ Memory scaling shows significant nonlinearity');
|
||
|
|
}
|
||
|
|
});
|
||
|
|
|
||
|
|
test('Index memory overhead analysis', async () => {
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log('🧠 Index memory overhead analysis...');
|
||
|
|
|
||
|
|
const relationCounts = [1000, 2000, 5000];
|
||
|
|
|
||
|
|
for (const relationCount of relationCounts) {
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(`\n--- Testing with ${relationCount} relations ---`);
|
||
|
|
|
||
|
|
// Fast construction mode (no indices)
|
||
|
|
const resultFast = measureMemoryUsage(() => {
|
||
|
|
const arbiter = new Arbiter({ fastConstructionMode: true });
|
||
|
|
|
||
|
|
for (let i = 0; i < relationCount; i++) {
|
||
|
|
arbiter.addNode(`user:${i}`, 'user');
|
||
|
|
arbiter.addNode(`doc:${i}`, 'document');
|
||
|
|
arbiter.addRelation(`user:${i}`, 'can_read', `doc:${i}`, { possibility: 1.0 });
|
||
|
|
}
|
||
|
|
|
||
|
|
return arbiter;
|
||
|
|
}, `Fast construction mode (${relationCount} relations)`);
|
||
|
|
|
||
|
|
// Normal mode (with indices)
|
||
|
|
const resultNormal = measureMemoryUsage(() => {
|
||
|
|
const arbiter = new Arbiter({ fastConstructionMode: false });
|
||
|
|
|
||
|
|
for (let i = 0; i < relationCount; i++) {
|
||
|
|
arbiter.addNode(`user:${i}`, 'user');
|
||
|
|
arbiter.addNode(`doc:${i}`, 'document');
|
||
|
|
arbiter.addRelation(`user:${i}`, 'can_read', `doc:${i}`, { possibility: 1.0 });
|
||
|
|
}
|
||
|
|
|
||
|
|
return arbiter;
|
||
|
|
}, `Normal mode with indices (${relationCount} relations)`);
|
||
|
|
|
||
|
|
const indexOverhead = resultNormal.after.heapUsed - resultFast.after.heapUsed;
|
||
|
|
const overheadPerRelation = indexOverhead / relationCount;
|
||
|
|
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(` Index overhead: ${indexOverhead.toFixed(2)}MB`);
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(` Overhead per relation: ${overheadPerRelation.toFixed(4)}MB`);
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(` Overhead percentage: ${((indexOverhead / resultFast.after.heapUsed) * 100).toFixed(1)}%`);
|
||
|
|
}
|
||
|
|
});
|
||
|
|
|
||
|
|
test('Memory efficiency comparison', async () => {
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log('🧠 Memory efficiency comparison...');
|
||
|
|
|
||
|
|
const relationCount = 5000;
|
||
|
|
|
||
|
|
// Test different configurations
|
||
|
|
const configs = [
|
||
|
|
{ name: 'Fast construction', fastMode: true },
|
||
|
|
{ name: 'Normal mode', fastMode: false },
|
||
|
|
{ name: 'Fast + manual indices', fastMode: true, manualIndices: true }
|
||
|
|
];
|
||
|
|
|
||
|
|
for (const config of configs) {
|
||
|
|
const result = measureMemoryUsage(() => {
|
||
|
|
const arbiter = new Arbiter({ fastConstructionMode: config.fastMode });
|
||
|
|
|
||
|
|
for (let i = 0; i < relationCount; i++) {
|
||
|
|
arbiter.addNode(`user:${i}`, 'user');
|
||
|
|
arbiter.addNode(`doc:${i}`, 'document');
|
||
|
|
arbiter.addRelation(`user:${i}`, 'can_read', `doc:${i}`, { possibility: 1.0 });
|
||
|
|
}
|
||
|
|
|
||
|
|
if (config.manualIndices) {
|
||
|
|
arbiter.setFastConstructionMode(false);
|
||
|
|
}
|
||
|
|
|
||
|
|
return arbiter;
|
||
|
|
}, `${config.name} (${relationCount} relations)`);
|
||
|
|
|
||
|
|
const memoryPerRelation = result.delta.heapUsed / relationCount;
|
||
|
|
if (process.env.TEST_DEBUG === '1') console.log(` ${config.name}: ${memoryPerRelation.toFixed(4)}MB per relation`);
|
||
|
|
}
|
||
|
|
});
|
||
|
|
});
|