core/src/core/ValueManager.js

/**
 * ValueManager - Manages value intervals, blurring, and decay for relation values
 * 
 * This module handles the sophisticated logic of treating all values as intervals
 * that blur over time based on decaying possibility. It provides lazy recalculation
 * and caching to ensure efficient access to current value states.
 * 
 * NEW: Uses statistical distributions from all instances of a relation type to
 * inform intelligent, data-driven blurring.
 */

import { PriorityQueueFast } from './PriorityQueueFast.js';
import { UnifiedKeyManager } from './UnifiedKeyManager.js';
import { defaultCacheFactory } from './cache.js';

// Default decay configuration
const DEFAULT_DECAY_CONFIG = {
  valueDecayRate: 0.1,
  possibilityDecayRate: 0.1,
  decayPeriod: 'HOUR',
  valueBlurDirection: 'neutral',
  possibilityDecayDirection: 'down',
  baseBlurAmount: 1.0,
  minPossibility: 0.01,
  epsilon: 0.0001,
  // New distribution-based settings
  useDistributionBlur: true,      // Use statistical distribution for blur
  distributionBlurMode: 'adaptive', // 'fixed', 'adaptive', or 'confidence'
  confidenceLevel: 0.95,           // For confidence interval blur
  minSampleSize: 5                 // Minimum samples before using distribution
};

// Period conversions to milliseconds
const PERIOD_TO_MS = {
  SECOND: 1000,
  MINUTE: 1000 * 60,
  HOUR: 1000 * 60 * 60,
  DAY: 1000 * 60 * 60 * 24,
  WEEK: 1000 * 60 * 60 * 24 * 7,
  MONTH: 1000 * 60 * 60 * 24 * 30,
  YEAR: 1000 * 60 * 60 * 24 * 365
};

export class ValueManager {
  constructor(arbiter) {
    this.arbiter = arbiter;
    
    // Initialize unified key manager
    this.keyManager = new UnifiedKeyManager();
    
    // Global decay configurations by relation type
    this.decayConfigs = new Map();
    
    // Cache factory with fallback for minimal/mock arbiters that do not
    // expose cacheFactory (e.g. unit-test MockArbiter instances).
    const cacheFactory = (typeof arbiter.cacheFactory === 'function')
      ? arbiter.cacheFactory
      : defaultCacheFactory;
    
    // Cache for computed blurred intervals to avoid redundant calculations
    this.blurredValueCache = cacheFactory(5000, {
      onEvict: (key, value) => {
        // Track evictions for debugging
        this.blurredValueCacheStats = this.blurredValueCacheStats || { evictions: 0 };
        this.blurredValueCacheStats.evictions++;
      },
      sampleSize: 6,
      sketchEpsilon: 0.01,
      sketchDelta: 0.01
    });
    
    // Cache for relation type statistics
    this.distributionCache = cacheFactory(1000, {
      onEvict: (key, value) => {
        // Track evictions for debugging
        this.distributionCacheStats = this.distributionCacheStats || { evictions: 0 };
        this.distributionCacheStats.evictions++;
      },
      sampleSize: 4,
      sketchEpsilon: 0.02,
      sketchDelta: 0.01
    });
    this.distributionCacheMaxAge = 5 * 60 * 1000; // 5 minutes
    
    // Default configuration
    this.defaultConfig = { ...DEFAULT_DECAY_CONFIG };
    
    // TTL configuration (separate from decay)
    this.ttlConfigs = new Map(); // Key: relationType, Value: ttlMs
    this.defaultTTL = 24 * 60 * 60 * 1000; // 24 hours default TTL

    // Staleness Management
    this.staleValuesQueue = new PriorityQueueFast();
    // Map to quickly find stale items in the queue for updates
    this.staleValueItemsIndex = new Map(); // Key: stale cache key (id of StaleValueItem), Value: StaleValueItem
  }

  /**
   * Convert string key to integer for PriorityQueueFast
   * @private
   */
  _stringToInt(key) {
    return this.keyManager.getStringId(key);
  }

  /**
   * Set decay configuration for a specific relation type
   * @param {string} relationType - The relation type (e.g., 'balance', 'price')
   * @param {Object} config - Decay configuration
   */
  setDecayConfig(relationType, config) {
    this.decayConfigs.set(relationType, {
      ...this.defaultConfig,
      ...config
    });
    // Invalidate distribution cache for this type
    this.distributionCache.delete(relationType);
  }

  /**
   * Get decay configuration for a relation type
   * @param {string} relationType - The relation type
   * @returns {Object} Decay configuration
   */
  getDecayConfig(relationType) {
    return this.decayConfigs.get(relationType) || this.defaultConfig;
  }

  /**
   * Set global default decay configuration
   * @param {Object} config - Default decay configuration
   */
  setDefaultDecayConfig(config) {
    this.defaultConfig = {
      ...this.defaultConfig,
      ...config
    };
  }

  /**
   * Set TTL for a specific relation type
   * @param {string} relationType - The relation type (e.g., 'balance', 'price')
   * @param {number} ttlMs - TTL in milliseconds
   */
  setTTL(relationType, ttlMs) {
    this.ttlConfigs.set(relationType, ttlMs);
  }

  /**
   * Get TTL for a relation type
   * @param {string} relationType - The relation type
   * @returns {number} TTL in milliseconds
   */
  getTTL(relationType) {
    return this.ttlConfigs.get(relationType) || this.defaultTTL;
  }

  /**
   * Check if a relation value has expired based on TTL
   * @private
   */
  _isValueExpired(relation, now = null) {
    const ttl = this.getTTL(relation.rel);
    const ts = now !== null && now !== undefined ? now : Date.now();
    const timestamp = relation.changed_last_at || relation.updated_last_at || ts;
    const age = ts - timestamp;
    return age > ttl;
  }

  /**
   * Get or calculate the blurred value interval for a relation
   * @param {Object} relation - The relation object
   * @param {number|null} [now] - Caller-pinned clock; the wall clock is
   *        only the fallback for unpinned callers.
   * @returns {Object} { interval: {min, max}, possibility: number, reliability: number }
   */
  getBlurredValue(relation, now = null) {
    // If relation has no value, return null interval
    if (relation.value === undefined || relation.value === null) {
      return {
        interval: null,
        possibility: 0,
        reliability: relation.reliability || 1.0
      };
    }

    // Check TTL first - if expired, return null interval
    if (this._isValueExpired(relation, now)) {
      return {
        interval: null,
        possibility: 0,
        reliability: relation.reliability || 1.0
      };
    }
    const possibility = relation.possibility !== undefined ? relation.possibility : 1.0;
    return {
      interval: { min: relation.value, max: relation.value },
      possibility,
      reliability: relation.reliability || 1.0
    };
  }

  /**
   * Get decayed relation with separated value blurring and possibility decay
   * This is the main interface that should be used by other components
   * @param {Object} relation - The relation object
   * @returns {Object} { pointValue, blurredInterval, currentPossibility, originalPossibility, reliability }
   */
  getDecayedRelation(relation, now = null) {
    if (relation.value === undefined || relation.value === null) {
      return {
        pointValue: null,
        blurredInterval: null,
        currentPossibility: 0,
        originalPossibility: relation.possibility !== undefined ? relation.possibility : 1.0,
        reliability: relation.reliability || 1.0,
        decayApplied: false,
        stateId: relation.stateId
      };
    }
    const possibility = relation.possibility !== undefined ? relation.possibility : 1.0;
    return {
      pointValue: relation.value,
      blurredInterval: { min: relation.value, max: relation.value },
      currentPossibility: possibility,
      originalPossibility: possibility,
      reliability: relation.reliability || 1.0,
      decayApplied: false,
      stateId: relation.stateId
    };
  }

  /**
   * Calculate separated decay for value blurring and possibility
   * @private
   */
  _calculateSeparatedDecay(relation, now = null) {
    const pointValue = relation.value;
    const initialPossibility = relation.possibility !== undefined ? relation.possibility : 1.0;
    const reliability = relation.reliability !== undefined ? relation.reliability : 1.0;
    const timestamp = relation.changed_last_at || relation.updated_last_at ||
      (now !== null && now !== undefined ? now : Date.now());
    
    // Get decay configuration
    const config = this._getRelationDecayConfig(relation);
    
    // Calculate age
    const evalNow = now !== null && now !== undefined ? now : Date.now();
    const ageMs = Math.max(0, evalNow - timestamp);
    const periodMs = PERIOD_TO_MS[config.decayPeriod.toUpperCase()] || PERIOD_TO_MS.HOUR;
    const ageInPeriod = ageMs / periodMs;
    
    // No decay if age is 0 or decay rates are 0
    if (ageInPeriod === 0 || (config.valueDecayRate === 0 && config.possibilityDecayRate === 0)) {
      return {
        pointValue,
        blurredInterval: { min: pointValue, max: pointValue },
        currentPossibility: initialPossibility,
        originalPossibility: initialPossibility,
        reliability,
        decayApplied: false
      };
    }
    
    // Calculate possibility decay with better neutral handling
    const possibilityDecayFactor = this._calculateDecayFactor(ageInPeriod, config.possibilityDecayRate, config.decayType);
    const decayedPossibility = this._applyImprovedPossibilityDecay(
      initialPossibility,
      possibilityDecayFactor,
      config.possibilityDecayDirection,
      ageInPeriod
    );
    
    // Calculate value blur (independent of possibility)
    let blurredInterval;
    if (config.useDistributionBlur) {
      blurredInterval = this._calculateDistributionBasedBlur(
        relation,
        pointValue,
        initialPossibility,
        decayedPossibility,
        config
      );
    } else {
      // Use age-based blur instead of possibility-based
      const blurMagnitude = ageInPeriod * config.baseBlurAmount * config.valueDecayRate;
      blurredInterval = this._calculateBlurredInterval(
        pointValue,
        blurMagnitude,
        config.valueBlurDirection,
        config.epsilon
      );
    }
    
    // Check minimum possibility threshold
    const finalPossibility = decayedPossibility < config.minPossibility ? 0 : decayedPossibility;
    
    return {
      pointValue,
      blurredInterval: finalPossibility > 0 ? blurredInterval : null,
      currentPossibility: finalPossibility,
      originalPossibility: initialPossibility,
      reliability,
      decayApplied: ageInPeriod > 0
    };
  }

  /**
   * Improved possibility decay with better neutral handling
   * @private
   */
  _applyImprovedPossibilityDecay(initialPossibility, decayFactor, direction, ageInPeriod) {
    let decayed;
    
    switch (direction) {
      case 'up':
        // Decay towards 1 (increasing confidence over time)
        decayed = 1 - (1 - initialPossibility) * decayFactor;
        break;
      case 'down':
        // Decay towards 0 (decreasing confidence over time)
        decayed = initialPossibility * decayFactor;
        break;
      case 'stable':
        // No decay
        decayed = initialPossibility;
        break;
      case 'neutral':
      default:
        // IMPROVED: Better neutral decay that doesn't drop too quickly
        // Use a gentler decay curve that approaches but doesn't rapidly fall to 0.5
        const targetPossibility = 0.5;
        const decayStrength = Math.min(0.8, ageInPeriod * 0.05); // Cap max decay effect
        decayed = initialPossibility + (targetPossibility - initialPossibility) * decayStrength;
        break;
    }
    
    return Math.max(0, Math.min(1, decayed));
  }

  /**
   * Get blurred values for multiple relations (batch operation)
   * @param {Array} relations - Array of relation objects
   * @returns {Array} Array of blurred value results
   */
  getBlurredValuesBatch(relations) {
    return relations.map(rel => this.getBlurredValue(rel));
  }

  /**
   * Invalidate cache for specific relations
   * @param {Array} relationUniqueKeysWithOldStateIds - Array of { relationUniqueKey, oldStateId, accessTime (optional) }
   */
  invalidateCache(relationUniqueKeysWithOldStateIds) {
    if (!relationUniqueKeysWithOldStateIds) {
      // Clear entire cache and stale queue
      this.blurredValueCache.clear();
      this.distributionCache.clear(); // Keep this? Or make it separate. For now, clear.
      this.staleValuesQueue.clear(); // Assuming PQ has clear() or re-init
      this.staleValueItemsIndex.clear();
    } else {
      const now = Date.now();
      for (const itemToInvalidate of relationUniqueKeysWithOldStateIds) {
        // itemToInvalidate is { relationUniqueKey, oldStateId, accessTime (optional) }
        const oldCacheKey = this._generateCacheKeyForStaleness(itemToInvalidate.relationUniqueKey, itemToInvalidate.oldStateId);

        if (this.blurredValueCache.has(oldCacheKey)) {
          const cachedDataForOldState = this.blurredValueCache.get(oldCacheKey);
          this._deleteFromCache(this.blurredValueCache, oldCacheKey); // Remove from active cache

          // Add to stale queue
          if (!this.staleValueItemsIndex.has(oldCacheKey)) {
            const staleItem = {
              id: oldCacheKey, // ID for the stale queue is the old cache key
              relationUniqueKey: itemToInvalidate.relationUniqueKey,
              itemType: 'relationValue',
              lastAccessTime: itemToInvalidate.accessTime || cachedDataForOldState?.recalculatedAt || now,
              staleSinceTime: now,
              originalRelationStateId: itemToInvalidate.oldStateId
            };
            const intKey = this._stringToInt(oldCacheKey);
            this.staleValuesQueue.push(intKey, now);
            this.staleValueItemsIndex.set(oldCacheKey, staleItem);
          }
        }
      }
    }
  }

  _deleteFromCache(cache, key) {
    if (key === null || key === undefined || !cache) return;
    if (typeof cache.delete === 'function') {
      cache.delete(key);
      return;
    }
    if (typeof cache._deleteKey === 'function') {
      cache._deleteKey(key);
      return;
    }
    if (cache.cache && typeof cache.cache.delete === 'function') {
      cache.cache.delete(key);
    }
  }

  /**
   * Calculate statistics for a relation type based on its local context
   * Uses a weighted combination of global baseline and local similarity-based samples
   * @private
   * @param {Object} relation - The relation object for context
   * @returns {Object | null} Calculated statistics or null
   */
  _calculateRelationTypeStatistics(relation) {
    const relationType = relation.rel;
    const srcNodeId = relation.src.id || relation.src; 
    const dstNodeId = relation.dst.id || relation.dst; 

    // Generate a cache key specific to this local context using numeric IDs
    const srcId = this.keyManager.getStringId(srcNodeId);
    const dstId = this.keyManager.getStringId(dstNodeId);
    const relationTypeId = this.keyManager.getStringId(relationType);
    const cacheKey = `local_stats_type:${relationTypeId}_src:${srcId}_dst:${dstId}`;
    
    const cached = this.distributionCache.get(cacheKey);
    if (cached && (Date.now() - cached.calculatedAt) < this.distributionCacheMaxAge) {
      return cached.stats;
    }

    // Weighted samples collection
    const weightedSamples = new Map(); // key -> {value, weight}
    
    // Configuration for weighting
    const baseWeight = 1.0;         // Weight for global samples
    const localBoostFactor = 2.0;   // Multiplier for local samples
    const similarityPower = 2.0;    // Exponent to emphasize high similarity scores
    
    // 1. Get global baseline sample
    const globalRelations = this.arbiter.relationManager.getRawValueRelationsByName(
      relationType,
      relation // Exclude the current relation
    ) || [];
    
    // Add global samples with base weight
    globalRelations.forEach(rel => {
      const srcId = this.keyManager.getStringId(rel.src);
      const dstId = this.keyManager.getStringId(rel.dst);
      const key = this.keyManager.createCompositeKey(srcId, rel.rel, dstId);
      weightedSamples.set(key, { 
        value: rel.value, 
        weight: baseWeight 
      });
    });
    
    // 2. Get local samples from source and destination contexts
    const defaultSimilarityOptions = {
      count: 5, // Number of similar nodes to consider
      efSearch: 50 // Search effort for ANN
    };
    
    // Process local context samples with similarity-based weights
    const processLocalSamples = (localResults) => {
      localResults.forEach(result => {
        const rel = result.relation;
        const similarityScore = result.similarityScore;
        const srcId = this.keyManager.getStringId(rel.src);
        const dstId = this.keyManager.getStringId(rel.dst);
        const key = this.keyManager.createCompositeKey(srcId, rel.rel, dstId);
        
        // Calculate weight based on similarity
        // Higher similarity = higher weight, with exponential emphasis
        const similarityWeight = Math.pow(similarityScore, similarityPower);
        const weight = baseWeight + (localBoostFactor * similarityWeight);
        
        // If already in samples, use the higher weight
        const existing = weightedSamples.get(key);
        if (!existing || existing.weight < weight) {
          weightedSamples.set(key, { 
            value: rel.value, 
            weight: weight 
          });
        }
      });
    };
    
    // Fetch and process source context
    const srcLocalResults = this.arbiter.relationManager.getRawValueRelationsForLocalContext(
      srcNodeId, 
      'src',
      relationType, 
      relation,
      defaultSimilarityOptions
    ) || [];
    processLocalSamples(srcLocalResults);
    
    // Fetch and process destination context
    const dstLocalResults = this.arbiter.relationManager.getRawValueRelationsForLocalContext(
      dstNodeId, 
      'dst',
      relationType, 
      relation,
      defaultSimilarityOptions
    ) || [];
    processLocalSamples(dstLocalResults);
    
    // Convert to arrays for statistical calculation
    const samples = Array.from(weightedSamples.values());
    
    if (samples.length === 0) {
      this.distributionCache.set(cacheKey, { stats: null, calculatedAt: Date.now() });
      return null;
    }
    
    // Calculate weighted statistics
    const stats = this._calculateWeightedStatistics(samples);
    
    // Cache the statistics
    this.distributionCache.set(cacheKey, {
      stats,
      calculatedAt: Date.now()
    });
    
    return stats;
  }

  /**
   * Calculate statistics from weighted samples
   * @private
   * @param {Array} samples - Array of {value, weight} objects
   * @returns {Object} Statistical summary
   */
  _calculateWeightedStatistics(samples) {
    const n = samples.length;
    if (n === 0) return null;
    
    // Calculate total weight
    const totalWeight = samples.reduce((sum, s) => sum + s.weight, 0);
    
    // Weighted mean
    const weightedSum = samples.reduce((sum, s) => sum + s.value * s.weight, 0);
    const mean = weightedSum / totalWeight;
    
    // Weighted variance and standard deviation
    const weightedSquaredDiff = samples.reduce((sum, s) => 
      sum + s.weight * Math.pow(s.value - mean, 2), 0
    );
    const variance = weightedSquaredDiff / totalWeight;
    const stdDev = Math.sqrt(variance);
    
    // For percentiles, we need to sort by value and accumulate weights
    const sortedSamples = [...samples].sort((a, b) => a.value - b.value);
    
    // Calculate weighted percentiles
    const getWeightedPercentile = (p) => {
      const targetWeight = p * totalWeight;
      let accumulatedWeight = 0;
      
      for (let i = 0; i < sortedSamples.length; i++) {
        accumulatedWeight += sortedSamples[i].weight;
        if (accumulatedWeight >= targetWeight) {
          // Linear interpolation for smoother percentiles
          if (i === 0) return sortedSamples[0].value;
          if (accumulatedWeight === targetWeight) {
            return (sortedSamples[i].value + sortedSamples[i + 1]?.value || sortedSamples[i].value) / 2;
          }
          
          // Interpolate between current and previous value
          const prevWeight = accumulatedWeight - sortedSamples[i].weight;
          const fraction = (targetWeight - prevWeight) / sortedSamples[i].weight;
          
          if (i > 0) {
            return sortedSamples[i - 1].value + 
                   fraction * (sortedSamples[i].value - sortedSamples[i - 1].value);
          }
          return sortedSamples[i].value;
        }
      }
      return sortedSamples[n - 1].value;
    };
    
    // Calculate key percentiles
    const median = getWeightedPercentile(0.5);
    const q1 = getWeightedPercentile(0.25);
    const q3 = getWeightedPercentile(0.75);
    
    // Weighted MAD (Median Absolute Deviation)
    const deviations = samples.map(s => ({
      value: Math.abs(s.value - median),
      weight: s.weight
    }));
    const sortedDeviations = deviations.sort((a, b) => a.value - b.value);
    const mad = getWeightedPercentile(0.5); // This won't work correctly, we need to use sortedDeviations
    
    // Calculate MAD properly
    const getMadFromDeviations = () => {
      const targetWeight = 0.5 * totalWeight;
      let accumulatedWeight = 0;
      
      for (const dev of sortedDeviations) {
        accumulatedWeight += dev.weight;
        if (accumulatedWeight >= targetWeight) {
          return dev.value;
        }
      }
      return sortedDeviations[sortedDeviations.length - 1].value;
    };
    
    const actualMad = getMadFromDeviations();
    
    // Calculate effective sample size (for confidence intervals)
    const effectiveSampleSize = Math.pow(totalWeight, 2) / 
      samples.reduce((sum, s) => sum + Math.pow(s.weight, 2), 0);
    
    return {
      count: n,
      effectiveSampleSize: Math.round(effectiveSampleSize),
      totalWeight,
      mean,
      median,
      stdDev,
      variance,
      mad: actualMad,
      min: sortedSamples[0].value,
      max: sortedSamples[n - 1].value,
      q1,
      q3,
      iqr: q3 - q1,
      p5: getWeightedPercentile(0.05),
      p95: getWeightedPercentile(0.95),
      p10: getWeightedPercentile(0.10),
      p90: getWeightedPercentile(0.90),
      skewness: (mean - median) / (stdDev || 1),
      range: sortedSamples[n - 1].value - sortedSamples[0].value,
      // Additional info about sample composition
      globalSampleWeight: samples.filter(s => s.weight === 1.0).length,
      localSampleWeight: samples.filter(s => s.weight > 1.0).length
    };
  }

  /**
   * Calculate the blurred value interval for a relation
   * @private
   */
  _calculateBlurredValue(relation, now = null) {
    const pointValue = relation.value;
    const initialPossibility = relation.possibility !== undefined ? relation.possibility : 1.0;
    const reliability = relation.reliability !== undefined ? relation.reliability : 1.0;
    const timestamp = relation.changed_last_at || relation.updated_last_at ||
      (now !== null && now !== undefined ? now : Date.now());
    
    // Get decay configuration
    const config = this._getRelationDecayConfig(relation);
    
    // Calculate age
    const evalNow = now !== null && now !== undefined ? now : Date.now();
    const ageMs = Math.max(0, evalNow - timestamp); // Ensure non-negative
    const periodMs = PERIOD_TO_MS[config.decayPeriod.toUpperCase()] || PERIOD_TO_MS.HOUR;
    const ageInPeriod = ageMs / periodMs;
    
    // No decay if age is 0 or decay rates are 0
    if (ageInPeriod === 0 || (config.valueDecayRate === 0 && config.possibilityDecayRate === 0)) {
      return {
        interval: { min: pointValue, max: pointValue },
        possibility: initialPossibility,
        reliability: reliability
      };
    }
    
    // Calculate possibility decay
    const possibilityDecayFactor = this._calculateDecayFactor(ageInPeriod, config.possibilityDecayRate, config.decayType);
    const decayedPossibility = this._applyPossibilityDecay(
      initialPossibility,
      possibilityDecayFactor,
      config.possibilityDecayDirection
    );
    
    // Calculate value blur
    let blurredInterval;
    if (config.useDistributionBlur) {
      // Use distribution-based blur
      blurredInterval = this._calculateDistributionBasedBlur(
        relation,
        pointValue,
        initialPossibility,
        decayedPossibility,
        config
      );
    } else {
      // Use traditional fixed blur
      const blurMagnitude = this._calculateBlurMagnitudeFromPossibility(
        initialPossibility,
        decayedPossibility,
        config.baseBlurAmount
      );
      blurredInterval = this._calculateBlurredInterval(
        pointValue,
        blurMagnitude,
        config.valueBlurDirection,
        config.epsilon
      );
    }
    
    // Check minimum possibility threshold
    const finalPossibility = decayedPossibility < config.minPossibility ? 0 : decayedPossibility;
    
    return {
      interval: finalPossibility > 0 ? blurredInterval : null,
      possibility: finalPossibility,
      reliability: reliability // Reliability doesn't decay in this model
    };
  }

  /**
   * Calculate blur based on statistical distribution
   * @private
   */
  _calculateDistributionBasedBlur(relation, pointValue, initialPossibility, decayedPossibility, config) {
    const stats = this._calculateRelationTypeStatistics(relation);
    
    // Fallback to fixed blur if insufficient data
    if (!stats || stats.count < config.minSampleSize) {
      const blurMagnitude = this._calculateBlurMagnitudeFromPossibility(
        initialPossibility,
        decayedPossibility,
        config.baseBlurAmount
      );
      return this._calculateBlurredInterval(
        pointValue,
        blurMagnitude,
        config.valueBlurDirection,
        config.epsilon
      );
    }
    
    // Calculate lost possibility proportion
    const lostPossibilityProportion = initialPossibility > 0 
      ? Math.max(0, initialPossibility - decayedPossibility) / initialPossibility 
      : 0;
    
    let blurMagnitude;
    
    switch (config.distributionBlurMode) {
      case 'fixed':
        // Use a fixed proportion of the standard deviation
        blurMagnitude = stats.stdDev * lostPossibilityProportion;
        break;
        
      case 'adaptive':
        // Adaptive blur based on value's position in distribution
        const valuePercentile = this._getValuePercentile(pointValue, stats);
        
        // Use different blur strategies based on position
        if (valuePercentile < 0.1 || valuePercentile > 0.9) {
          // Extreme values: use larger blur (IQR-based)
          blurMagnitude = stats.iqr * lostPossibilityProportion;
        } else if (valuePercentile < 0.25 || valuePercentile > 0.75) {
          // Moderate outliers: use MAD-based blur
          blurMagnitude = stats.mad * 2 * lostPossibilityProportion;
        } else {
          // Central values: use standard deviation
          blurMagnitude = stats.stdDev * lostPossibilityProportion;
        }
        break;
        
      case 'confidence':
        // Use confidence interval based blur
        const z = this._getZScore(config.confidenceLevel);
        // Use effective sample size for weighted samples
        const effectiveN = stats.effectiveSampleSize || stats.count;
        const standardError = stats.stdDev / Math.sqrt(effectiveN);
        blurMagnitude = z * standardError * lostPossibilityProportion;
        break;
        
      default:
        // Default to standard deviation
        blurMagnitude = stats.stdDev * lostPossibilityProportion;
    }
    
    // Apply direction-aware blur
    let interval;
    const direction = config.valueBlurDirection;
    
    if (direction === 'auto') {
      // Auto-detect direction based on distribution skewness and value position
      const relativePosition = (pointValue - stats.median) / (stats.iqr || 1);
      
      if (stats.skewness > 0.5 && relativePosition > 0) {
        // Right-skewed distribution, value above median: blur upward
        interval = { min: pointValue, max: pointValue + blurMagnitude * 1.5 };
      } else if (stats.skewness < -0.5 && relativePosition < 0) {
        // Left-skewed distribution, value below median: blur downward
        interval = { min: pointValue - blurMagnitude * 1.5, max: pointValue };
      } else {
        // Symmetric blur
        interval = { 
          min: pointValue - blurMagnitude, 
          max: pointValue + blurMagnitude 
        };
      }
    } else {
      // Use configured direction
      switch (direction) {
        case 'up':
          interval = { min: pointValue, max: pointValue + blurMagnitude };
          break;
        case 'down':
          interval = { min: pointValue - blurMagnitude, max: pointValue };
          break;
        case 'stable':
          const epsilonBlur = blurMagnitude > 0 ? config.epsilon / 2 : 0;
          interval = { min: pointValue - epsilonBlur, max: pointValue + epsilonBlur };
          break;
        case 'neutral':
        default:
          interval = { 
            min: pointValue - blurMagnitude, 
            max: pointValue + blurMagnitude 
          };
          break;
      }
    }
    
    // Constrain interval to observed range (with some margin)
    const rangeMargin = stats.range * 0.1; // 10% margin beyond observed range
    interval.min = Math.max(interval.min, stats.min - rangeMargin);
    interval.max = Math.min(interval.max, stats.max + rangeMargin);
    
    return interval;
  }

  /**
   * Get percentile position of a value in distribution
   * @private
   */
  _getValuePercentile(value, stats) {
    if (value <= stats.min) return 0;
    if (value >= stats.max) return 1;
    
    // Simple linear interpolation between known percentiles
    if (value <= stats.p5) return 0.05 * (value - stats.min) / (stats.p5 - stats.min);
    if (value <= stats.q1) return 0.05 + 0.2 * (value - stats.p5) / (stats.q1 - stats.p5);
    if (value <= stats.median) return 0.25 + 0.25 * (value - stats.q1) / (stats.median - stats.q1);
    if (value <= stats.q3) return 0.5 + 0.25 * (value - stats.median) / (stats.q3 - stats.median);
    if (value <= stats.p95) return 0.75 + 0.2 * (value - stats.q3) / (stats.p95 - stats.q3);
    return 0.95 + 0.05 * (value - stats.p95) / (stats.max - stats.p95);
  }

  /**
   * Get Z-score for confidence level
   * @private
   */
  _getZScore(confidenceLevel) {
    // Common confidence levels
    const zScores = {
      0.90: 1.645,
      0.95: 1.96,
      0.99: 2.576
    };
    return zScores[confidenceLevel] || 1.96; // Default to 95%
  }

  /**
   * Calculate decay factor using different decay types
   * @private
   */
  _calculateDecayFactor(ageInPeriod, decayRate, decayType = 'rational') {
    switch (decayType) {
      case 'exponential':
        // Exponential decay: e^(-decayRate * ageInPeriod)
        return Math.exp(-decayRate * ageInPeriod);
      case 'linear':
        // Linear decay: 1 - decayRate * ageInPeriod (clamped to 0)
        return Math.max(0, 1 - decayRate * ageInPeriod);
      case 'quadratic':
        // Quadratic decay: 1 / (1 + decayRate * ageInPeriod^2)
        return 1 / (1 + decayRate * Math.pow(ageInPeriod, 2));
      case 'rational':
      default:
        // Rational decay: 1 / (1 + decayRate * ageInPeriod)
        return 1 / (1 + decayRate * ageInPeriod);
    }
  }

  /**
   * Apply possibility decay based on direction
   * @private
   */
  _applyPossibilityDecay(initialPossibility, decayFactor, direction) {
    let decayed;
    
    switch (direction) {
      case 'up':
        // Decay towards 1 (increasing confidence over time)
        decayed = 1 - (1 - initialPossibility) * decayFactor;
        break;
      case 'down':
        // Decay towards 0 (decreasing confidence over time)
        decayed = initialPossibility * decayFactor;
        break;
      case 'stable':
        // No decay
        decayed = initialPossibility;
        break;
      case 'neutral':
      default:
        // Decay towards 0.5 (uncertainty)
        decayed = 0.5 + (initialPossibility - 0.5) * decayFactor;
        break;
    }
    
    return Math.max(0, Math.min(1, decayed));
  }

  /**
   * Calculate blurred interval based on possibility loss (traditional method)
   * @private
   */
  _calculateBlurredInterval(pointValue, blurMagnitude, blurDirection, epsilon) {
    // Apply blur based on direction
    let interval;
    switch (blurDirection) {
      case 'up':
        // Blur expands upward
        interval = { min: pointValue, max: pointValue + blurMagnitude };
        break;
      case 'down':
        // Blur expands downward
        interval = { min: pointValue - blurMagnitude, max: pointValue };
        break;
      case 'stable':
        // Minimal blur (just epsilon for numerical stability)
        const epsilonBlur = epsilon / 2;
        interval = { min: pointValue - epsilonBlur, max: pointValue + epsilonBlur };
        break;
      case 'neutral':
      default:
        // Blur expands symmetrically
        interval = { 
          min: pointValue - blurMagnitude / 2, 
          max: pointValue + blurMagnitude / 2 
        };
        break;
    }
    
    return interval;
  }

  _calculateBlurMagnitudeFromPossibility(initialPossibility, decayedPossibility, baseBlurAmount) {
    if (!Number.isFinite(initialPossibility) || initialPossibility <= 0) {
      return baseBlurAmount;
    }
    const lostProportion = Math.max(0, initialPossibility - decayedPossibility) / initialPossibility;
    return baseBlurAmount * lostProportion;
  }

  /**
   * Get decay configuration for a specific relation
   * @private
   */
  _getRelationDecayConfig(relation) {
    // First check if relation has custom decay config
    if (relation.decayConfig) {
      return {
        ...this.defaultConfig,
        ...relation.decayConfig
      };
    }
    
    // Then check relation type config
    const typeConfig = this.decayConfigs.get(relation.rel);
    if (typeConfig) {
      return typeConfig;
    }
    
    // Fall back to default
    return this.defaultConfig;
  }

  /**
   * Generate cache key for a relation
   * @private
   */
  _generateCacheKey(relation) {
    // Use composite key with stateId for better performance
    const baseKey = this.arbiter.keyManager.createCompositeKey(
      relation.src,
      relation.rel,
      relation.dst
    );
    return `${baseKey}_${relation.stateId}`;
  }

  _generateCacheKeyForStaleness(relationUniqueKey, stateId) {
    // relationUniqueKey is like "srcId_relName_dstId"
    return `${relationUniqueKey}_${stateId}`;
  }

  /**
   * Check if cached value is still valid for the given relation
   * @private
   */
  _isCacheValid(cached, relation) {
    // PRIMARY CHECK: Relation's StateId must match what the cache entry was based on
    if (cached.originalRelationStateId !== relation.stateId) return false;
    
    // Optional: Further checks like max cache age, though stateId is the main version control
    const cacheAge = Date.now() - cached.recalculatedAt;
    // Example: const maxCacheAgeForcedRefresh = 24 * 60 * 60 * 1000; // 1 day
    // if (cacheAge > maxCacheAgeForcedRefresh) return false;
    
    return true;
  }

  /**
   * Updates the lastAccessTime for an item if it's in the stale queue.
   * @private
   */
  _updateAccessForStaleItem(cacheKeyForCurrentVersion) {
    // This function's purpose is to update lastAccessTime if an item,
    // currently considered stale (i.e., its *old* version is in staleValuesQueue),
    // is accessed via its *current* version.
    // This is complex because stale queue items are keyed by their *old* stateId.
    // For simplicity, we might not need this if access to current version doesn't affect staleness priority of old version.
    // However, if the intent is "most recently accessed (entity) but oldest stale (version)", this could be relevant.

    // For now, let's assume access to current version does not directly re-prioritize an old version in stale queue.
    // The priority queue already handles lastAccessTime of the stale item itself.
  }

  /**
   * Refreshes a specified number of stale values from the priority queue.
   * "Refreshes" means attempting to compute and cache the value for the *current*
   * version of the relation that the stale item pertained to.
   * @param {number} countToRefresh - The maximum number of stale items to refresh.
   * @returns {number} The number of items still remaining in the stale queue.
   */
  refreshStaleValues(countToRefresh) {
    let refreshedOps = 0;
    for (let i = 0; i < countToRefresh; i++) {
      if (this.staleValuesQueue.isEmpty()) {
        break;
      }

      const intKey = this.staleValuesQueue.pop();
      const cacheKey = this.keyManager.getIdString(intKey);
      
      if (!cacheKey) {
        console.warn(`[ValueManager] No cacheKey found for intKey ${intKey}`);
        continue;
      }
      
      const staleItem = this.staleValueItemsIndex.get(cacheKey);
      if (!staleItem) {
        console.warn(`[ValueManager] No staleItem found for cacheKey ${cacheKey}`);
        continue;
      }
      
      this.staleValueItemsIndex.delete(cacheKey); // Remove from index

      // staleItem.relationUniqueKey is "srcId_relName_dstId"
      const keyParts = staleItem.relationUniqueKey.split('_');
      const srcId = parseInt(keyParts[0], 10);
      const relName = keyParts[1];
      const dstId = parseInt(keyParts[2], 10);

      // Fetch the *current* version of the relation
      const currentRelation = this.arbiter.relationManager.getDirectRelation(srcId, relName, dstId);

      if (currentRelation) {
        // Generate the cache key for the current version of this relation
        const currentCacheKey = this._generateCacheKey(currentRelation);

        // Check if we already have an up-to-date calculation for the current version
        // or if the current version is newer than the one that caused staleness.
        // This avoids recomputing if another process already updated it.
        if (!this.blurredValueCache.has(currentCacheKey) || 
            (this.blurredValueCache.get(currentCacheKey).originalRelationStateId !== currentRelation.stateId)) {
          
          const newBlurredValue = this._calculateSeparatedDecay(currentRelation);
          this.blurredValueCache.set(currentCacheKey, {
            interval: newBlurredValue.blurredInterval,
            possibility: newBlurredValue.currentPossibility,
            recalculatedAt: Date.now(),
            valueSnapshot: currentRelation.value,
            timestampSnapshot: currentRelation.changed_last_at || currentRelation.updated_last_at,
            originalRelationStateId: currentRelation.stateId,
            decayApplied: newBlurredValue.decayApplied
          });
        }
        refreshedOps++;
      } else {
        // The relation might have been deleted entirely. Nothing to refresh.
      }
    }
    return this.staleValuesQueue.size();
  }

  /**
   * Aggregate multiple interval values using interval arithmetic
   * @param {Array} values - Array of { interval, possibility, reliability }
   * @param {string} aggregator - Aggregation method: 'max', 'min', 'sum', 'average'
   * @returns {Object} Aggregated { interval, possibility, reliability }
   */
  aggregateCrispValues(values, aggregator = 'max') {
    // Filter out null intervals
    const validValues = values.filter(v => v.interval !== null);
    
    if (validValues.length === 0) {
      return {
        interval: null,
        possibility: 0,
        reliability: 0
      };
    }
    
    if (validValues.length === 1) {
      return validValues[0];
    }
    
    let aggregatedInterval;
    let aggregatedPossibility;
    let aggregatedReliability;
    
    switch (aggregator) {
      case 'max':
        // For max, take the interval with highest max value
        const maxIdx = validValues.reduce((maxI, v, i) => 
          v.interval.max > validValues[maxI].interval.max ? i : maxI, 0);
        aggregatedInterval = validValues[maxIdx].interval;
        aggregatedPossibility = validValues[maxIdx].possibility;
        aggregatedReliability = validValues[maxIdx].reliability;
        break;
        
      case 'min':
        // For min, take the interval with lowest min value
        const minIdx = validValues.reduce((minI, v, i) => 
          v.interval.min < validValues[minI].interval.min ? i : minI, 0);
        aggregatedInterval = validValues[minIdx].interval;
        aggregatedPossibility = validValues[minIdx].possibility;
        aggregatedReliability = validValues[minIdx].reliability;
        break;
        
      case 'sum':
        // Sum intervals: [sum of mins, sum of maxs]
        aggregatedInterval = {
          min: validValues.reduce((sum, v) => sum + v.interval.min, 0),
          max: validValues.reduce((sum, v) => sum + v.interval.max, 0)
        };
        // Average possibility and multiply reliabilities
        aggregatedPossibility = validValues.reduce((sum, v) => sum + v.possibility, 0) / validValues.length;
        aggregatedReliability = validValues.reduce((prod, v) => prod * v.reliability, 1.0);
        break;
        
      case 'average':
      default:
        // Average intervals: [avg of mins, avg of maxs]
        aggregatedInterval = {
          min: validValues.reduce((sum, v) => sum + v.interval.min, 0) / validValues.length,
          max: validValues.reduce((sum, v) => sum + v.interval.max, 0) / validValues.length
        };
        // Average possibility and reliability
        aggregatedPossibility = validValues.reduce((sum, v) => sum + v.possibility, 0) / validValues.length;
        aggregatedReliability = validValues.reduce((sum, v) => sum + v.reliability, 0) / validValues.length;
        break;
    }
    
    return {
      interval: aggregatedInterval,
      possibility: aggregatedPossibility,
      reliability: aggregatedReliability
    };
  }

  /**
   * Compare two intervals using a comparator
   * @param {Object} leftInterval - { min, max }
   * @param {Object} rightInterval - { min, max }
   * @param {string} comparator - '>', '>=', '<', '<=', '==', '!='
   * @param {number} epsilon - Epsilon for equality comparison
   * @returns {number} Possibility (0-1) that comparison holds
   */
  compareIntervals(leftInterval, rightInterval, comparator, epsilon = null) {
    if (!leftInterval || !rightInterval) {
      return 0; // No comparison possible with null intervals
    }
    
    const eps = epsilon || this.defaultConfig.epsilon;
    
    // Calculate difference interval D = L - R
    const D_min = leftInterval.min - rightInterval.max;
    const D_max = leftInterval.max - rightInterval.min;
    const D_length = D_max - D_min;
    
    // Handle degenerate case
    if (D_length <= 0) {
      const midL = (leftInterval.min + leftInterval.max) / 2;
      const midR = (rightInterval.min + rightInterval.max) / 2;
      switch (comparator) {
        case '>': return midL > midR ? 1 : 0;
        case '>=': return midL >= midR ? 1 : 0;
        case '<': return midL < midR ? 1 : 0;
        case '<=': return midL <= midR ? 1 : 0;
        case '==': return Math.abs(midL - midR) < eps ? 1 : 0;
        case '!=': return Math.abs(midL - midR) >= eps ? 1 : 0;
        default: return 0;
      }
    }
    
    // Calculate possibility based on comparator
    switch (comparator) {
      case '>':
        if (D_min > 0) return 1;
        if (D_max <= 0) return 0;
        return D_max / D_length;
        
      case '>=':
        if (D_min >= 0) return 1;
        if (D_max < 0) return 0;
        return D_max / D_length;
        
      case '<':
        if (D_max < 0) return 1;
        if (D_min >= 0) return 0;
        return -D_min / D_length;
        
      case '<=':
        if (D_max <= 0) return 1;
        if (D_min > 0) return 0;
        return -D_min / D_length;
        
      case '==':
        const overlap_min = Math.max(D_min, -eps);
        const overlap_max = Math.min(D_max, eps);
        if (overlap_max <= overlap_min) return 0;
        return (overlap_max - overlap_min) / D_length;
        
      case '!=':
        const eq_overlap_min = Math.max(D_min, -eps);
        const eq_overlap_max = Math.min(D_max, eps);
        if (eq_overlap_max <= eq_overlap_min) return 1;
        return 1 - (eq_overlap_max - eq_overlap_min) / D_length;
        
      default:
        console.warn('Unknown comparator:', comparator);
        return 0;
    }
  }
}