// Incremental TF-IDF based relation filter for lifelong learning
class IncrementalTFIDFFilter {
  constructor() {
    this.documentFrequency = new Map(); // relation pattern -> number of nodes that have it
    this.nodePatterns = new Map(); // nodeId -> Set of patterns (for tracking what each node has)
    this.totalDocuments = 0; // total number of nodes
    this.minIDF = 0.1; // Minimum IDF threshold (filter out if IDF < this)
    this.maxTF = 50; // Maximum term frequency per node (hard limit)
    this.version = 0; // Version for tracking distribution changes
    this.lastDistributionSnapshot = null; // Snapshot for change detection
    this.minFactsThreshold = 50; // Only use TF-IDF when we have enough facts
    this.updatesSinceLastCheck = 0; // Track updates for periodic distribution checks
    this.distributionCheckInterval = 100; // Check distribution changes every N updates
  }

  // Add a relation incrementally
  addRelation(srcNodeId, rel, dstNode) {
    if (!dstNode) return;
    
    // Create relation patterns
    const relationTypePattern = `${rel}:${dstNode.type}`;
    const specificPattern = `${rel}:${dstNode.key}`;
    
    // Track patterns for source node
    if (!this.nodePatterns.has(srcNodeId)) {
      this.nodePatterns.set(srcNodeId, new Set());
    }
    const patterns = this.nodePatterns.get(srcNodeId);
    
    // Only increment document frequency if this node doesn't already have this pattern
    if (!patterns.has(relationTypePattern)) {
      patterns.add(relationTypePattern);
      this.documentFrequency.set(relationTypePattern, (this.documentFrequency.get(relationTypePattern) || 0) + 1);
    }
    
    if (!patterns.has(specificPattern)) {
      patterns.add(specificPattern);
      this.documentFrequency.set(specificPattern, (this.documentFrequency.get(specificPattern) || 0) + 1);
    }
    
    this.updatesSinceLastCheck++;
    this._checkDistributionChange();
  }

  // Remove a relation incrementally
  removeRelation(srcNodeId, rel, dstNode) {
    if (!dstNode || !this.nodePatterns.has(srcNodeId)) return;
    
    const relationTypePattern = `${rel}:${dstNode.type}`;
    const specificPattern = `${rel}:${dstNode.key}`;
    const patterns = this.nodePatterns.get(srcNodeId);
    
    // Only decrement if this node had this pattern
    if (patterns.has(relationTypePattern)) {
      patterns.delete(relationTypePattern);
      const count = this.documentFrequency.get(relationTypePattern) || 0;
      if (count > 1) {
        this.documentFrequency.set(relationTypePattern, count - 1);
      } else {
        this.documentFrequency.delete(relationTypePattern);
      }
    }
    
    if (patterns.has(specificPattern)) {
      patterns.delete(specificPattern);
      const count = this.documentFrequency.get(specificPattern) || 0;
      if (count > 1) {
        this.documentFrequency.set(specificPattern, count - 1);
      } else {
        this.documentFrequency.delete(specificPattern);
      }
    }
    
    // Clean up empty pattern sets
    if (patterns.size === 0) {
      this.nodePatterns.delete(srcNodeId);
    }
    
    this.updatesSinceLastCheck++;
    this._checkDistributionChange();
  }

  // Update total document count
  setTotalDocuments(count) {
    if (count !== this.totalDocuments) {
      this.totalDocuments = count;
      this.updatesSinceLastCheck++;
      this._checkDistributionChange();
    }
  }

  // Periodically check if distribution has changed significantly
  _checkDistributionChange() {
    if (this.updatesSinceLastCheck < this.distributionCheckInterval) return;
    
    // Create a snapshot of current distribution
    const frequencies = Array.from(this.documentFrequency.values()).sort((a, b) => a - b);
    const currentSnapshot = {
      size: this.documentFrequency.size,
      max: frequencies.length > 0 ? frequencies[frequencies.length - 1] : 0,
      median: frequencies.length > 0 ? frequencies[Math.floor(frequencies.length / 2)] : 0,
      p95: frequencies.length > 0 ? frequencies[Math.floor(frequencies.length * 0.95)] : 0
    };
    
    // Check if distribution changed significantly
    if (this.lastDistributionSnapshot) {
      const sizeChange = Math.abs(currentSnapshot.size - this.lastDistributionSnapshot.size) / Math.max(this.lastDistributionSnapshot.size, 1);
      const maxChange = Math.abs(currentSnapshot.max - this.lastDistributionSnapshot.max) / Math.max(this.lastDistributionSnapshot.max, 1);
      
      // If significant change (>20% in size or >50% in max frequency)
      if (sizeChange > 0.2 || maxChange > 0.5) {
        this.version++;
        // console.log(`📊 TF-IDF distribution shift detected, incrementing version to ${this.version}`);
        // console.log(`   Size: ${this.lastDistributionSnapshot.size} → ${currentSnapshot.size} (${(sizeChange*100).toFixed(1)}%)`);
        // console.log(`   Max freq: ${this.lastDistributionSnapshot.max} → ${currentSnapshot.max} (${(maxChange*100).toFixed(1)}%)`);
      }
    }
    
    this.lastDistributionSnapshot = currentSnapshot;
    this.updatesSinceLastCheck = 0;
  }

  // Check if we should use TF-IDF filtering (enough data)
  shouldUseTFIDF() {
    return this.documentFrequency.size >= this.minFactsThreshold;
  }

  // Calculate IDF for a relation pattern
  getIDF(relationPattern) {
    const df = this.documentFrequency.get(relationPattern) || 0;
    if (df === 0 || this.totalDocuments === 0) return 0;
    return Math.log(this.totalDocuments / df);
  }

  // Check if a relation pattern should be included based on IDF
  shouldInclude(relationPattern) {
    if (!this.shouldUseTFIDF()) return true; // Include everything if not enough data
    const idf = this.getIDF(relationPattern);
    return idf >= this.minIDF;
  }

  // Get TF-IDF score for a relation pattern with given term frequency
  getTFIDF(relationPattern, termFreq) {
    if (!this.shouldUseTFIDF()) return 1.0; // Default score if not using TF-IDF
    const idf = this.getIDF(relationPattern);
    // Use log(1 + tf) to dampen very high frequencies
    const tf = Math.log(1 + Math.min(termFreq, this.maxTF));
    return tf * idf;
  }

  // Get statistics for debugging
  getStats() {
    const frequencies = Array.from(this.documentFrequency.values());
    return {
      totalPatterns: this.documentFrequency.size,
      totalDocuments: this.totalDocuments,
      avgFrequency: frequencies.length > 0 ? frequencies.reduce((sum, f) => sum + f, 0) / frequencies.length : 0,
      maxFrequency: frequencies.length > 0 ? Math.max(...frequencies) : 0,
      minFrequency: frequencies.length > 0 ? Math.min(...frequencies) : 0,
      version: this.version,
      usingTFIDF: this.shouldUseTFIDF(),
      updatesSinceLastCheck: this.updatesSinceLastCheck,
      trackedNodes: this.nodePatterns.size
    };
  }

  // Force a distribution check (for testing)
  forceDistributionCheck() {
    this.updatesSinceLastCheck = this.distributionCheckInterval;
    this._checkDistributionChange();
  }
}

// Quantization helpers
function quantizeValue(val) {
  if (typeof val !== 'number' || isNaN(val)) return String(val);
  if (val <= 0) return 'zero';
  const log10 = Math.floor(Math.log10(val));
  const lower = Math.pow(10, log10);
  const upper = Math.pow(10, log10 + 1) - 1;
  return `${lower}-${upper}`;
}

function quantizeRecency(msAgo) {
  if (msAgo < 60 * 1000) return '<1min';
  if (msAgo < 10 * 60 * 1000) return '<10min';
  if (msAgo < 60 * 60 * 1000) return '<1h';
  if (msAgo < 6 * 60 * 60 * 1000) return '<6h';
  if (msAgo < 24 * 60 * 60 * 1000) return '<1d';
  if (msAgo < 7 * 24 * 60 * 60 * 1000) return '<1w';
  return 'old';
}

export class EmbeddingManager {
  constructor(arbiter) {
    this.arbiter = arbiter;
    this.pendingStaleNodes = new Set();
    
    // Optimization: Cache relation lookups
    this.relationsBySource = new Map(); // nodeId -> relations[]
    this.relationsByTarget = new Map(); // nodeId -> relations[]
    this.relationCacheDirty = true;
    
    // Incremental TF-IDF filter for lifelong learning
    this.tfidfFilter = new IncrementalTFIDFFilter();
    this.maxFactsPerNode = 100; // Hard limit on facts per node
    this.minTFIDFScore = 0.1; // Minimum TF-IDF score to include a fact
    
    // Dual-clock staleness tracking system
    this.globalDistributionClock = 0; // Incremented when TF-IDF distribution shifts
    this.nodeRelationClocks = new Map(); // nodeId -> relation clock value
    this.nodeDistributionClocks = new Map(); // nodeId -> last seen distribution clock
    
    // Batch processing settings
    this.batchSize = 1000;
    this.parallelWorkers = 4;
  }

  // Increment global distribution clock (when TF-IDF distribution changes)
  incrementDistributionClock() {
    this.globalDistributionClock++;
    // console.log(`📊 Global distribution clock incremented to ${this.globalDistributionClock}`);
  }

  // Increment the relation clock for a specific node
  incrementNodeRelationClock(nodeId) {
    const currentClock = this.nodeRelationClocks.get(nodeId) || 0;
    const newClock = currentClock + 1;
    this.nodeRelationClocks.set(nodeId, newClock);
    
    // OPTIMIZATION: No longer update neighbor max clocks proactively
    // Instead, we compute max neighbor clocks on-demand in _getMaxNeighborClock()
    // This eliminates O(neighbors) work for every relation addition
    
    return newClock;
  }

  // Check if a node's embedding is stale based on dual-clock system
  isNodeStale(nodeId) {
    const node = this.arbiter.nodes.get(nodeId);
    if (!node || !node.embedding) return true;
    
    // Check 1: Distribution clock staleness
    const nodeDistClock = this.nodeDistributionClocks.get(nodeId) || 0;
    if (nodeDistClock < this.globalDistributionClock) {
      return true; // Node hasn't seen latest TF-IDF distribution
    }
    
    // Check 2: Neighbor relation clock staleness - computed on-demand
    const nodeRelClock = this.nodeRelationClocks.get(nodeId) || 0;
    const maxNeighborClock = this._getMaxNeighborClock(nodeId);
    if (maxNeighborClock > nodeRelClock) {
      return true; // Node has neighbors with newer relations
    }
    
    return false; // Node is fresh
  }

  // Compute max neighbor clock on-demand (much more efficient than maintaining proactively)
  _getMaxNeighborClock(nodeId) {
    let maxClock = 0;
    
    // Check outgoing neighbors (nodes this node points to)
    const outgoing = this.arbiter.indices.outgoingEdges.get(nodeId);
    if (outgoing) {
      for (const neighborId of outgoing) {
        const neighborClock = this.nodeRelationClocks.get(neighborId) || 0;
        if (neighborClock > maxClock) {
          maxClock = neighborClock;
        }
      }
    }
    
    // Check incoming neighbors (nodes that point to this node)
    const incoming = this.arbiter.indices.incomingEdges.get(nodeId);
    if (incoming) {
      for (const neighborId of incoming) {
        const neighborClock = this.nodeRelationClocks.get(neighborId) || 0;
        if (neighborClock > maxClock) {
          maxClock = neighborClock;
        }
      }
    }
    
    return maxClock;
  }

  // Mark node as fresh with current clocks
  markNodeFresh(nodeId) {
    const node = this.arbiter.nodes.get(nodeId);
    if (node) {
      // Update distribution clock to current global
      this.nodeDistributionClocks.set(nodeId, this.globalDistributionClock);
      
      // Node's relation clock stays the same (only changes when its relations change)
      // But we ensure it exists
      if (!this.nodeRelationClocks.has(nodeId)) {
        this.nodeRelationClocks.set(nodeId, 0);
      }
      
      // Backward compatibility
      node.stale = false;
    }
  }

  // Add a relation to the TF-IDF filter incrementally and update clocks
  addRelationToFilter(srcId, rel, dstId) {
    const dstNode = this.arbiter.nodes.get(dstId);
    this.tfidfFilter.addRelation(srcId, rel, dstNode);
    this.tfidfFilter.setTotalDocuments(this.arbiter.nodes.size);
    
    // Increment relation clocks for both source and destination nodes
    this.incrementNodeRelationClock(srcId);
    this.incrementNodeRelationClock(dstId);
    
    // Check if TF-IDF distribution changed
    if (this.tfidfFilter.version > this.tfidfVersion) {
      this.incrementDistributionClock();
      this.tfidfVersion = this.tfidfFilter.version;
    }
  }

  // Remove a relation from the TF-IDF filter incrementally and update clocks
  removeRelationFromFilter(srcId, rel, dstId) {
    const dstNode = this.arbiter.nodes.get(dstId);
    this.tfidfFilter.removeRelation(srcId, rel, dstNode);
    this.tfidfFilter.setTotalDocuments(this.arbiter.nodes.size);
    
    // Increment relation clocks for both source and destination nodes
    this.incrementNodeRelationClock(srcId);
    this.incrementNodeRelationClock(dstId);
    
    // Check if TF-IDF distribution changed
    if (this.tfidfFilter.version > this.tfidfVersion) {
      this.incrementDistributionClock();
      this.tfidfVersion = this.tfidfFilter.version;
    }
  }

  // Initialize TF-IDF filter from existing relations (one-time setup)
  _initializeTFIDFFilter() {
    // console.log('🔄 Initializing incremental TF-IDF filter...');
    const start = process.hrtime.bigint();
    
    // Check if filter is already properly initialized
    const currentStats = this.tfidfFilter.getStats();
    const expectedPatterns = this.arbiter.relations.length > 0 ? this.arbiter.relations.length * 2 : 0; // Rough estimate
    
    // console.log(`   Current filter state: ${currentStats.totalPatterns} patterns, ${currentStats.trackedNodes} nodes`);
    // console.log(`   Expected patterns (rough): ~${expectedPatterns} (from ${this.arbiter.relations.length} relations)`);
    
    // If filter seems empty but we have relations, reinitialize
    if (currentStats.totalPatterns === 0 && this.arbiter.relations.length > 0) {
      // console.log('   🔧 Filter appears empty, reinitializing...');
      
      // Reset the filter
      this.tfidfFilter = new IncrementalTFIDFFilter();
    }
    
    this.tfidfFilter.setTotalDocuments(this.arbiter.nodes.size);
    
    // Initialize all node clocks
    for (const nodeId of this.arbiter.nodes.keys()) {
      this.nodeRelationClocks.set(nodeId, 0);
      this.nodeDistributionClocks.set(nodeId, 0);
    }
    
    // Add all existing relations if filter is empty
    if (this.tfidfFilter.getStats().totalPatterns === 0) {
      // console.log(`   📝 Adding ${this.arbiter.relations.length} relations to filter...`);
      for (const rel of this.arbiter.relations) {
        const dstNode = this.arbiter.nodes.get(rel.dst);
        this.tfidfFilter.addRelation(rel.src, rel.rel, dstNode);
      }
    }
    
    // Force a distribution check to set initial version
    this.tfidfFilter.forceDistributionCheck();
    this.tfidfVersion = this.tfidfFilter.version;
    this.globalDistributionClock = 1; // Start at 1 after initialization
    
    const initTime = Number(process.hrtime.bigint() - start) / 1_000_000;
    // console.log(`✅ TF-IDF filter initialized in ${initTime.toFixed(2)}ms (v${this.tfidfFilter.version})`);
    
    const stats = this.tfidfFilter.getStats();
    // console.log(`   Patterns: ${stats.totalPatterns}, Nodes: ${stats.trackedNodes}, Using TF-IDF: ${stats.usingTFIDF}`);
    // console.log(`   Clocks: distribution=${this.globalDistributionClock}, nodes=${this.nodeRelationClocks.size}`);
    
    // Verify filter is working
    if (stats.totalPatterns === 0 && this.arbiter.relations.length > 0) {
      // console.log(`⚠️  Warning: TF-IDF filter still empty after initialization!`);
    }
  }

  // Get staleness statistics for debugging
  getStalenessStats() {
    let distributionStale = 0;
    let neighborStale = 0;
    let fresh = 0;
    let noEmbedding = 0;
    
    for (const nodeId of this.arbiter.nodes.keys()) {
      const node = this.arbiter.nodes.get(nodeId);
      if (!node || !node.embedding) {
        noEmbedding++;
        continue;
      }
      
      const nodeDistClock = this.nodeDistributionClocks.get(nodeId) || 0;
      const nodeRelClock = this.nodeRelationClocks.get(nodeId) || 0;
      const maxNeighborClock = this._getMaxNeighborClock(nodeId);
      
      if (nodeDistClock < this.globalDistributionClock) {
        distributionStale++;
      } else if (maxNeighborClock > nodeRelClock) {
        neighborStale++;
      } else {
        fresh++;
      }
    }
    
    return {
      fresh,
      distributionStale,
      neighborStale,
      noEmbedding,
      total: this.arbiter.nodes.size,
      globalDistributionClock: this.globalDistributionClock,
      maxNodeRelationClock: Math.max(...Array.from(this.nodeRelationClocks.values()), 0)
    };
  }

  // Build TF-IDF filter from all relations
  _buildTFIDFFilter() {
    const oldVersion = this.tfidfFilter.version;
    this.tfidfFilter.build(this.arbiter);
    
    // If TF-IDF distribution changed, increment global version
    if (this.tfidfFilter.version > oldVersion) {
      this.incrementDistributionClock();
    }
    
    this.tfidfVersion = this.tfidfFilter.version;
  }

  // Rebuild relation indices for fast lookup
  _ensureRelationCache() {
    if (!this.relationCacheDirty) return;
    
    this.relationsBySource.clear();
    this.relationsByTarget.clear();
    
    // Build indices once instead of scanning every time
    for (const rel of this.arbiter.relations) {
      // Source index
      if (!this.relationsBySource.has(rel.src)) {
        this.relationsBySource.set(rel.src, []);
      }
      this.relationsBySource.get(rel.src).push(rel);
      
      // Target index
      if (!this.relationsByTarget.has(rel.dst)) {
        this.relationsByTarget.set(rel.dst, []);
      }
      this.relationsByTarget.get(rel.dst).push(rel);
    }
    
    this.relationCacheDirty = false;
  }

  markNodeStale(nodeId) {
    this._markTwoStepNeighborsStale(nodeId);
  }

  addPendingStaleNode(nodeId) {
    this.pendingStaleNodes.add(nodeId);
  }

  clearPendingStaleNodes() {
    this.pendingStaleNodes.clear();
  }

  processPendingStaleNodes() {
    if (this.pendingStaleNodes.size === 0) return;
    
    const allAffectedNodes = new Set();
    
    const pendingArray = Array.from(this.pendingStaleNodes);
    var pendingLen = pendingArray.length;
    for (var i = 0; i < pendingLen; i++) {
      const nodeId = pendingArray[i];
      const toStale = new Set([nodeId]);
      
      const outgoing = this.arbiter.indices.outgoingEdges.get(nodeId);
      const incoming = this.arbiter.indices.incomingEdges.get(nodeId);
      
      if (outgoing) {
        const outgoingArray = Array.from(outgoing);
        var outLen = outgoingArray.length;
        for (var j = 0; j < outLen; j++) {
          toStale.add(outgoingArray[j]);
        }
      }
      
      if (incoming) {
        const incomingArray = Array.from(incoming);
        var inLen = incomingArray.length;
        for (var k = 0; k < inLen; k++) {
          toStale.add(incomingArray[k]);
        }
      }
      
      const firstNeighbors = Array.from(toStale);
      var firstLen = firstNeighbors.length;
      for (var l = 0; l < firstLen; l++) {
        const nid = firstNeighbors[l];
        if (nid === nodeId) continue;
        
        const secondOutgoing = this.arbiter.indices.outgoingEdges.get(nid);
        const secondIncoming = this.arbiter.indices.incomingEdges.get(nid);
        
        if (secondOutgoing) {
          const secondOutArray = Array.from(secondOutgoing);
          var secOutLen = secondOutArray.length;
          for (var m = 0; m < secOutLen; m++) {
            toStale.add(secondOutArray[m]);
          }
        }
        
        if (secondIncoming) {
          const secondInArray = Array.from(secondIncoming);
          var secInLen = secondInArray.length;
          for (var n = 0; n < secInLen; n++) {
            toStale.add(secondInArray[n]);
          }
        }
      }
      
      const toStaleArray = Array.from(toStale);
      var staleLen = toStaleArray.length;
      for (var o = 0; o < staleLen; o++) {
        allAffectedNodes.add(toStaleArray[o]);
      }
    }
    
    for (const nid of allAffectedNodes) {
      const node = this.arbiter.nodes.get(nid);
      if (node) node.stale = true;
    }
  }

  processPendingStaleNodesLazy() {
    if (this.pendingStaleNodes.size === 0) return;
    
    const toMark = new Set(this.pendingStaleNodes);
    
    for (const nodeId of this.pendingStaleNodes) {
      const outgoing = this.arbiter.indices.outgoingEdges.get(nodeId) || new Set();
      const incoming = this.arbiter.indices.incomingEdges.get(nodeId) || new Set();
      
      for (const neighbor of outgoing) toMark.add(neighbor);
      for (const neighbor of incoming) toMark.add(neighbor);
    }
    
    for (const nid of toMark) {
      const node = this.arbiter.nodes.get(nid);
      if (node) node.stale = true;
    }
    
    this.pendingStaleNodes.clear();
  }

  markTwoStepNeighborsStale(nodeId) {
    this._markTwoStepNeighborsStale(nodeId);
  }

  _markTwoStepNeighborsStale(nodeId) {
    const toStale = new Set([nodeId]);
    
    const outgoing = this.arbiter.indices.outgoingEdges.get(nodeId) || new Set();
    const incoming = this.arbiter.indices.incomingEdges.get(nodeId) || new Set();
    
    for (const neighbor of outgoing) toStale.add(neighbor);
    for (const neighbor of incoming) toStale.add(neighbor);
    
    const firstNeighbors = Array.from(toStale);
    for (const nid of firstNeighbors) {
      if (nid === nodeId) continue;
      
      const secondOutgoing = this.arbiter.indices.outgoingEdges.get(nid) || new Set();
      const secondIncoming = this.arbiter.indices.incomingEdges.get(nid) || new Set();
      
      for (const neighbor of secondOutgoing) toStale.add(neighbor);
      for (const neighbor of secondIncoming) toStale.add(neighbor);
    }
    
    for (const nid of toStale) {
      const node = this.arbiter.nodes.get(nid);
      if (node) node.stale = true;
    }
  }

  ensureFreshEmbedding(nodeId) {
    const node = this.arbiter.nodes.get(nodeId);
    if (!node) return;
    
    if (this.arbiter.lazyStaleMarking && this.pendingStaleNodes.size > 0) {
      this.processPendingStaleNodesLazy();
    }
    
    if (!node.stale) return;
    
    const situation = this.getSituationFactsFast(nodeId);
    if (!situation) return;
    const embedding = this.arbiter.jsonhash.generateSketch(situation);
    node.embedding = embedding;
    node.stale = false;
  }

  // Batch embedding generation with progress tracking
  regenerateVectorsBatch(options = {}) {
    console.log('🚀 Starting optimized batch embedding generation...');
    const startTime = process.hrtime.bigint();
    
    // Options for controlling what gets rebuilt
    const { 
      forceRebuild = false,  // Force rebuild even if embeddings are fresh
      rebuildSimilarity = true  // Rebuild similarity index after generation
    } = options;
    
    // Ensure relation cache is built once
    this._ensureRelationCache();
    
    // Initialize TF-IDF filter if not already done (one-time setup)
    if (this.tfidfFilter.totalDocuments === 0) {
      this._initializeTFIDFFilter();
    }
    
    // CRITICAL: Disable stale marking during embedding generation
    // Embeddings are auxiliary data and shouldn't trigger neighbor invalidation
    const originalLazyStaleMarking = this.arbiter.lazyStaleMarking;
    this.arbiter.lazyStaleMarking = false;
    
    // Clear any pending stale nodes before we start
    this.clearPendingStaleNodes();
    
    try {
      // Get all nodes that actually need embedding updates (version-based)
      const nodesToUpdate = [];
      let totalNodes = 0;
      let alreadyFresh = 0;
      
      for (const [nodeId, node] of this.arbiter.nodes.entries()) {
        totalNodes++;
        if (forceRebuild || this.isNodeStale(nodeId)) {
          nodesToUpdate.push(nodeId);
        } else {
          alreadyFresh++;
        }
      }
      
      console.log(`Processing ${nodesToUpdate.length} nodes in batches of ${this.batchSize}...`);
      
      // Process in batches with optimized progress reporting
      let processed = 0;
      let totalFactsGenerated = 0;
      const progressInterval = Math.max(1000, Math.floor(nodesToUpdate.length / 20)); // 20 progress updates max
      
      for (let i = 0; i < nodesToUpdate.length; i += this.batchSize) {
        const batch = nodesToUpdate.slice(i, i + this.batchSize);
        
        // Process batch with minimal overhead - no stale marking
        for (const nodeId of batch) {
          this._ensureFreshEmbeddingNoStaleMarking(nodeId);
          processed++;
          
          // Track facts generation for statistics
          const node = this.arbiter.nodes.get(nodeId);
          if (node && node.embedding) {
            // Rough estimate of facts generated (this is approximate)
            const situation = this.getSituationFactsFast(nodeId);
            if (situation) {
              totalFactsGenerated += situation.facts.length;
            }
          }
        }
        
        // Progress update (less frequent for large graphs)
        if (processed % progressInterval === 0 || processed === nodesToUpdate.length) {
          const progress = (processed / nodesToUpdate.length * 100).toFixed(1);
          const rate = Math.round(processed / ((Number(process.hrtime.bigint() - startTime) / 1_000_000)));
          const avgFacts = totalFactsGenerated > 0 ? (totalFactsGenerated / processed).toFixed(1) : '0';
        }
      }
      
      // After processing all embeddings, rebuild similarity index if requested
      if (rebuildSimilarity && nodesToUpdate.length > 0 && this.arbiter.similarityManager) {
        console.log('🔍 Updating similarity index...');
        const similarityStart = process.hrtime.bigint();
        
        this.arbiter.similarityManager.rebuildIndex();
        
        const similarityTime = Number(process.hrtime.bigint() - similarityStart) / 1_000_000;
        console.log(`   Similarity index updated in ${similarityTime.toFixed(2)}ms`);
      }
      
      const embeddingTime = Number(process.hrtime.bigint() - startTime) / 1_000_000;
      console.log(`✅ Embedding generation completed in ${embeddingTime.toFixed(2)}ms`);
      console.log(`   Rate: ${Math.round(nodesToUpdate.length / (embeddingTime / 1000))} embeddings/sec`);
      console.log(`   Facts: ${totalFactsGenerated} total, ${(totalFactsGenerated / nodesToUpdate.length).toFixed(1)} avg per node`);
      
    } finally {
      // Restore original stale marking setting
      this.arbiter.lazyStaleMarking = originalLazyStaleMarking;
    }
  }

  // Legacy method - now uses optimized batch version
  regenerateVectors() {
    this.regenerateVectorsBatch();
  }

  regenerateVectorsForAuthorization() {
    return this.regenerateVectorsBatch();
  }

  regenerateVectorsForSimilarity() {
    return this.regenerateVectorsBatch();
  }

  forceRegenerateEmbeddings() {
    // Mark all nodes as stale first
    for (const node of this.arbiter.nodes.values()) {
      node.stale = true;
    }
    this.regenerateVectorsBatch({ forceRebuild: true });
  }

  // Force regenerate for authorization only (fastest)
  forceRegenerateEmbeddingsForAuthorization() {
    // Mark all nodes as stale first
    for (const node of this.arbiter.nodes.values()) {
      node.stale = true;
    }
    return this.regenerateVectorsBatch({ forceRebuild: true });
  }

  removeNodeEmbedding(nodeId) {
    const node = this.arbiter.nodes.get(nodeId);
    if (node) {
      node.embedding = null;
    }
  }

  flushPendingStaleNodes() {
    if (this.pendingStaleNodes.size > 0) {
      if (this.arbiter.lazyStaleMarking) {
        this.processPendingStaleNodesLazy();
      } else {
        this.processPendingStaleNodes();
      }
      this.pendingStaleNodes.clear();
    }
    return this;
  }

  // Invalidate relation cache when relations change
  invalidateRelationCache() {
    this.relationCacheDirty = true;
  }

  // Get embedding generation statistics
  getEmbeddingStats() {
    let totalNodes = 0;
    let staleNodes = 0;
    let embeddedNodes = 0;
    
    for (const node of this.arbiter.nodes.values()) {
      totalNodes++;
      if (node.stale) staleNodes++;
      if (node.embedding) embeddedNodes++;
    }
    
    return {
      totalNodes,
      staleNodes,
      embeddedNodes,
      freshNodes: embeddedNodes - staleNodes,
      stalePercentage: (staleNodes / totalNodes * 100).toFixed(1),
      embeddedPercentage: (embeddedNodes / totalNodes * 100).toFixed(1)
    };
  }

  // Update getSituationFactsFast to use quantized value and recency for value-carrying relations
  getSituationFactsFast(nodeId, maxDepth = 2, visited = new Set(), prefix = '') {
    const node = this.arbiter.nodes.get(nodeId);
    if (!node) return null;
    if (visited.has(nodeId) || maxDepth < 1) return null;
    visited.add(nodeId);
    
    this._ensureRelationCache();
    
    // Initialize TF-IDF filter if needed (but don't rebuild)
    if (this.tfidfFilter.totalDocuments === 0) {
      this._initializeTFIDFFilter();
    }
    
    const facts = [];
    const outgoingRels = this.relationsBySource.get(nodeId) || [];
    
    // Count term frequencies for this node
    const termFrequencies = new Map(); // pattern -> count
    for (const rel of outgoingRels) {
      const targetNode = this.arbiter.nodes.get(rel.dst);
      if (!targetNode) continue;
      
      const relationTypePattern = `${rel.rel}:${targetNode.type}`;
      const specificPattern = `${rel.rel}:${targetNode.key}`;
      
      termFrequencies.set(relationTypePattern, (termFrequencies.get(relationTypePattern) || 0) + 1);
      termFrequencies.set(specificPattern, (termFrequencies.get(specificPattern) || 0) + 1);
    }
    
    // Create fact candidates with TF-IDF scores
    const factCandidates = [];
    
    for (const rel of outgoingRels) {
      const targetNode = this.arbiter.nodes.get(rel.dst);
      if (!targetNode) continue;
      
      const relationTypePattern = `${rel.rel}:${targetNode.type}`;
      const specificPattern = `${rel.rel}:${targetNode.key}`;
      
      // Quantize value-carrying relations
      if (rel.value !== undefined) {
        // Quantize value
        const valueBucket = quantizeValue(rel.value);
        // Quantize recency (prefer changed_last_at, fallback to updated_last_at)
        const now = Date.now();
        const lastAt = rel.changed_last_at || rel.updated_last_at;
        const msAgo = lastAt ? (now - lastAt) : null;
        const recencyBucket = msAgo !== null ? quantizeRecency(msAgo) : 'unknown';
        // Add fact for value bucket
        facts.push(`${rel.rel}:${valueBucket}`);
        // Add fact for recency bucket
        facts.push(`${rel.rel}:recency:${recencyBucket}`);
        continue;
      }
      
      // Check if patterns should be included based on IDF
      if (!this.tfidfFilter.shouldInclude(relationTypePattern) && 
          !this.tfidfFilter.shouldInclude(specificPattern)) {
        continue;
      }
      
      // Calculate TF-IDF scores for ranking
      const typeTFIDF = this.tfidfFilter.getTFIDF(relationTypePattern, termFrequencies.get(relationTypePattern) || 1);
      const specificTFIDF = this.tfidfFilter.getTFIDF(specificPattern, termFrequencies.get(specificPattern) || 1);
      const maxScore = Math.max(typeTFIDF, specificTFIDF);
      
      // Only include if score meets minimum threshold
      if (maxScore < this.minTFIDFScore) {
        continue;
      }
      
      const fact = prefix ? `${prefix}|${rel.rel}:${targetNode.key}` : `${rel.rel}:${targetNode.key}`;
      
      factCandidates.push({
        fact,
        score: maxScore,
        targetId: rel.dst,
        depth: prefix ? prefix.split('|').length : 0
      });
    }
    
    // Sort by TF-IDF score (descending) and take top facts
    factCandidates.sort((a, b) => b.score - a.score);
    
    // Add top-scoring facts
    for (const candidate of factCandidates.slice(0, this.maxFactsPerNode)) {
      facts.push(candidate.fact);
      
      // Recursively add sub-facts for high-scoring relations
      if (maxDepth > 1 && candidate.depth < 2 && candidate.score > this.minTFIDFScore * 2) {
        const subFacts = this.getSituationFactsFast(candidate.targetId, maxDepth - 1, visited, candidate.fact);
        if (subFacts && subFacts.facts) {
          // Add sub-facts but limit total
          const remainingSlots = this.maxFactsPerNode - facts.length;
          facts.push(...subFacts.facts.slice(0, remainingSlots));
          
          if (facts.length >= this.maxFactsPerNode) {
            break;
          }
        }
      }
    }
    
    return {
      key: node.key,
      type: node.type,
      facts
    };
  }

  // Legacy method for compatibility
  getSituationFacts(nodeId, maxDepth = 2, visited = new Set(), prefix = '') {
    return this.getSituationFactsFast(nodeId, maxDepth, visited, prefix);
  }

  // Embedding generation without stale marking (for batch operations)
  _ensureFreshEmbeddingNoStaleMarking(nodeId) {
    const node = this.arbiter.nodes.get(nodeId);
    if (!node) return;
    
    // Check version-based staleness
    if (!this.isNodeStale(nodeId)) return; // Already fresh
    
    const situation = this.getSituationFactsFast(nodeId);
    if (!situation) return;
    const embedding = this.arbiter.jsonhash.generateSketch(situation);
    node.embedding = embedding;
    this.markNodeFresh(nodeId);
  }
} 