core/tests/engine/sharded-delta-equivalence.test.js

import { describe, test } from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { CondensedGraph } from '../../src/core/CondensedGraph.js';
import { ShardedSnapshotBuilder } from '../../src/core/shards/ShardedSnapshotBuilder.js';
import { ShardedSnapshot } from '../../src/core/shards/ShardedSnapshot.js';
import { FileShardStorage } from '../../src/core/shards/FileShardStorage.js';
import { DeltaShardBinary } from '../../src/core/shards/DeltaShardBinary.js';
import { WaveletShardBinary } from '../../src/core/shards/WaveletShardBinary.js';

function buildSnapshot(bucketSize = 4) {
  const graph = new CondensedGraph();
  const nodes = [];
  for (let i = 0; i < 6; i++) {
    nodes.push(graph._ensureNode(`node:${i}`));
  }
  graph.addEdge(nodes[0], 'owner', nodes[1]);
  graph.addEdge(nodes[0], 'owner', nodes[2]);
  graph.addEdge(nodes[3], 'owner', nodes[4]);
  graph.addEdge(nodes[5], 'owner', nodes[0]);

  graph.finalizePerfectHash();
  graph.finalizeWaveletAdjacency({ dropAdjacencyList: true });

  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'sharded-delta-eq-'));
  const builder = new ShardedSnapshotBuilder({ bucketSize, includeDirections: ['out', 'in'] });
  const manifest = builder.build(graph, dir);
  const storage = new FileShardStorage(dir);
  const snapshot = new ShardedSnapshot(manifest, storage, { cacheLimit: 8 });
  snapshot.initializeSync();
  return { snapshot, manifest, dir, nodes };
}

function buildBaseEdges(nodes) {
  const edges = new Map();
  const add = (srcIdx, dstIdx) => {
    const srcId = nodes[srcIdx];
    const dstId = nodes[dstIdx];
    const set = edges.get(srcId) || new Set();
    set.add(dstId);
    edges.set(srcId, set);
  };
  add(0, 1);
  add(0, 2);
  add(3, 4);
  add(5, 0);
  return edges;
}

function cloneEdges(edges) {
  const next = new Map();
  for (const [src, set] of edges.entries()) {
    next.set(src, new Set(set));
  }
  return next;
}

function applyOps(edges, ops, mode = 'override') {
  const next = cloneEdges(edges);
  if (mode !== 'union') {
    for (const op of ops) {
      if (op.op !== 'remove') continue;
      const set = next.get(op.srcId) || new Set();
      set.delete(op.dstId);
      if (set.size > 0) next.set(op.srcId, set);
    }
  }
  for (const op of ops) {
    if (op.op !== 'add') continue;
    const set = next.get(op.srcId) || new Set();
    set.add(op.dstId);
    if (set.size > 0) next.set(op.srcId, set);
  }
  return next;
}

function hasEdge(edges, srcId, dstId) {
  const set = edges.get(srcId);
  return set ? set.has(dstId) : false;
}

function buildDeltaLayer(snapshot, dir, entries, mode = 'override') {
  fs.mkdirSync(dir, { recursive: true });
  const merged = new Map();

  for (const entry of entries) {
    const shardMeta = snapshot._selectShardMeta(entry.relationId, entry.direction, entry.srcId);
    assert.ok(shardMeta, 'Missing shard meta for delta entry');
    const localSource = snapshot._localSource(entry.srcId, shardMeta);
    const key = shardMeta.cacheKey;
    let bucket = merged.get(key);
    if (!bucket) {
      bucket = { shardMeta, additions: [], removals: [] };
      merged.set(key, bucket);
    }
    for (const add of entry.additions) {
      bucket.additions.push({
        srcLocal: localSource,
        otherId: add.dstId,
        possBits: add.possBits,
        relBits: add.relBits
      });
    }
    for (const rem of entry.removals) {
      bucket.removals.push({
        srcLocal: localSource,
        otherId: rem.dstId
      });
    }
  }

  const shards = [];
  for (const bucket of merged.values()) {
    const shardMeta = bucket.shardMeta;
    const buffer = DeltaShardBinary.serialize({
      relationId: shardMeta.relationId,
      direction: shardMeta.direction,
      rangeStart: shardMeta.rangeStart,
      rangeEnd: shardMeta.rangeEnd,
      nodeCount: snapshot.nodeCount,
      additions: bucket.additions,
      removals: bucket.removals
    });
    const shardKey = `delta-${shardMeta.key}`;
    fs.writeFileSync(path.join(dir, shardKey), new Uint8Array(buffer));
    shards.push({
      key: shardKey,
      relationId: shardMeta.relationId,
      direction: shardMeta.direction,
      rangeStart: shardMeta.rangeStart,
      rangeEnd: shardMeta.rangeEnd,
      cacheKey: shardMeta.cacheKey
    });
  }

  return { shards, storage: new FileShardStorage(dir), mode };
}

function compactLayer(snapshot, layer, manifest, outputDir, baseDir) {
  fs.rmSync(outputDir, { recursive: true, force: true });
  fs.mkdirSync(outputDir, { recursive: true });

  if (manifest.nodeTableKey) {
    fs.copyFileSync(path.join(baseDir, manifest.nodeTableKey), path.join(outputDir, manifest.nodeTableKey));
  }
  if (manifest.componentKey) {
    fs.copyFileSync(path.join(baseDir, manifest.componentKey), path.join(outputDir, manifest.componentKey));
  }

  for (const shard of manifest.shards || []) {
    const basePath = path.join(baseDir, shard.key);
    fs.copyFileSync(basePath, path.join(outputDir, shard.key));
  }

  for (const shardMeta of layer.shards) {
    const base = snapshot._cacheIndex.get(shardMeta.cacheKey);
    if (!base) continue;
    const shard = snapshot._loadShardSync(base.relationId, base.direction, base.rangeStart);
    if (!shard) continue;

    const deltaBuffer = layer.storage.getSync(shardMeta.key);
    if (!deltaBuffer) continue;
    const deltaShard = DeltaShardBinary.deserialize(deltaBuffer);
    const rangeSize = shard.rangeEnd - shard.rangeStart;
    const sources = new Array(rangeSize);
    for (let localSource = 0; localSource < rangeSize; localSource++) {
      const range = snapshot._rangeForSource(shard, localSource);
      const list = [];
      if (range) {
        for (let pos = range.start; pos < range.end; pos++) {
          list.push({ otherId: shard.dstIds[pos], possBits: shard.possBits[pos], relBits: shard.relBits[pos] });
        }
      }
      sources[localSource] = list;
    }

    for (const removal of deltaShard.removals) {
      const list = sources[removal.srcLocal];
      if (!list) continue;
      let idx = list.findIndex((item) => item.otherId === removal.otherId);
      while (idx !== -1) {
        list.splice(idx, 1);
        idx = list.findIndex((item) => item.otherId === removal.otherId);
      }
    }

    for (const addition of deltaShard.additions) {
      const list = sources[addition.srcLocal] || (sources[addition.srcLocal] = []);
      const idx = list.findIndex((item) => item.otherId === addition.otherId);
      if (idx === -1) {
        list.push({ otherId: addition.otherId, possBits: addition.possBits, relBits: addition.relBits });
      } else {
        list[idx] = { otherId: addition.otherId, possBits: addition.possBits, relBits: addition.relBits };
      }
    }

    const buffer = WaveletShardBinary.serialize({
      relationId: shard.relationId,
      direction: shard.direction,
      rangeStart: shard.rangeStart,
      rangeEnd: shard.rangeEnd,
      nodeCount: snapshot.nodeCount,
      sources
    });
    fs.writeFileSync(path.join(outputDir, base.key), new Uint8Array(buffer));
  }
}

// ADR-003: sharded snapshots are stubs in src/core/shards/ — tests are the spec for when the subsystem is implemented.
describe.skip('Sharded snapshot delta equivalence', () => {
  test('overlay matches compacted base for direct access', () => {
    const { snapshot, manifest, dir, nodes } = buildSnapshot(4);
    const relId = snapshot.relationIdToName.indexOf('owner');

    const base = buildBaseEdges(nodes);
    const ops = [
      { srcId: nodes[0], dstId: nodes[3], op: 'add' },
      { srcId: nodes[3], dstId: nodes[4], op: 'remove' }
    ];
    const expected = applyOps(base, ops, 'override');

    const deltaDir = path.join(dir, 'delta');
    const layer = buildDeltaLayer(snapshot, deltaDir, [
      { relationId: relId, direction: 'out', srcId: nodes[0], additions: [{ dstId: nodes[3], possBits: 65535, relBits: 65535 }], removals: [] },
      { relationId: relId, direction: 'out', srcId: nodes[3], additions: [], removals: [{ dstId: nodes[4] }] }
    ], 'override');

    snapshot.setDeltaLayers([layer]);

    const compactDir = path.join(dir, 'compact');
    compactLayer(snapshot, layer, manifest, compactDir, dir);
    const compactSnapshot = new ShardedSnapshot(manifest, new FileShardStorage(compactDir), { cacheLimit: 8 });
    compactSnapshot.initializeSync();

    for (const src of nodes) {
      for (const dst of nodes) {
        const overlayEdge = snapshot.findEdgeSync(src, relId, dst);
        const compactEdge = compactSnapshot.findEdgeSync(src, relId, dst);
        const expectedEdge = hasEdge(expected, src, dst);
        assert.strictEqual(!!overlayEdge, !!compactEdge);
        assert.strictEqual(!!overlayEdge, expectedEdge);
      }
    }

    fs.rmSync(dir, { recursive: true, force: true });
  });

  test('union overlays never remove base access', () => {
    const { snapshot, dir, nodes } = buildSnapshot(4);
    const relId = snapshot.relationIdToName.indexOf('owner');

    const base = buildBaseEdges(nodes);
    const ops = [
      { srcId: nodes[0], dstId: nodes[1], op: 'remove' },
      { srcId: nodes[2], dstId: nodes[4], op: 'add' }
    ];
    const expected = applyOps(base, ops, 'union');

    const deltaDir = path.join(dir, 'union');
    const layer = buildDeltaLayer(snapshot, deltaDir, [
      { relationId: relId, direction: 'out', srcId: nodes[0], additions: [], removals: [{ dstId: nodes[1] }] },
      { relationId: relId, direction: 'out', srcId: nodes[2], additions: [{ dstId: nodes[4], possBits: 65535, relBits: 65535 }], removals: [] }
    ], 'union');

    snapshot.setDeltaLayers([layer]);
    for (const src of nodes) {
      for (const dst of nodes) {
        const overlayEdge = snapshot.findEdgeSync(src, relId, dst);
        const expectedEdge = hasEdge(expected, src, dst);
        assert.strictEqual(!!overlayEdge, expectedEdge);
      }
    }

    fs.rmSync(dir, { recursive: true, force: true });
  });
});