apophis-fastify/src/test/quality.test.ts

/**
 * Quality Engine Tests — Flake Detection, Mutation Testing, Chaos
 */
import { test } from 'node:test'
import assert from 'node:assert'
import Fastify from 'fastify'
import swagger from '@fastify/swagger'
import apophisPlugin from '../index.js'
import { FlakeDetector } from '../quality/flake.js'
import { runMutationTesting, testMutation, type Mutation } from '../quality/mutation.js'
import * as fc from 'fast-check'
import {
  applyChaosToExecution,
  applyChaosToAllResponses,
  applyChaosToDependencyResponse,
  createChaosEventArbitrary,
  extractDelays,
  sleep,
  hasAppliedChaos,
  formatChaosEvents,
  type ChaosEvent,
} from '../quality/chaos-v3.js'
import type { EvalContext, TestResult, RouteContract } from '../types.js'

// Quality engines require test environment
process.env.NODE_ENV = 'test'

// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------

function makeCtx(overrides?: Partial<EvalContext['response']>): EvalContext {
  return {
    request: { body: {}, headers: {}, query: {}, params: {} },
    response: { body: { id: '1' }, headers: {}, statusCode: 200, ...overrides },
  } as EvalContext
}

function makeTestResult(ok: boolean): TestResult {
  return {
    ok,
    name: 'test',
    id: 1,
    diagnostics: ok ? {} : { error: 'failed' },
  }
}

// ---------------------------------------------------------------------------
// Chaos Tests
// ---------------------------------------------------------------------------

test('applyChaosToExecution: no chaos when events are empty', () => {
  const ctx = makeCtx()
  const result = applyChaosToExecution(ctx, [])
  assert.strictEqual(result.applied, false)
  assert.strictEqual(result.ctx.response.statusCode, 200)
})

test('applyChaosToExecution: inbound error changes status code', () => {
  const ctx = makeCtx()
  const result = applyChaosToExecution(ctx, [
    { type: 'inbound-error', target: 'inbound', statusCode: 503 },
  ])
  assert.strictEqual(result.applied, true)
  assert.strictEqual(result.ctx.response.statusCode, 503)
})

test('applyChaosToExecution: inbound dropout simulates gateway timeout', () => {
  const ctx = makeCtx()
  const result = applyChaosToExecution(ctx, [
    { type: 'inbound-dropout', target: 'inbound' },
  ])
  assert.strictEqual(result.ctx.response.statusCode, 504)
})

test('applyChaosToExecution: inbound corruption truncates response body', () => {
  const ctx = makeCtx({ body: { id: '1', name: 'Alice', email: 'a@b.com' } })
  const result = applyChaosToExecution(ctx, [
    { type: 'inbound-corruption', target: 'inbound', corruptionStrategy: 'truncate' },
  ])
  const body = result.ctx.response.body as Record<string, unknown>
  assert.ok(Object.keys(body).length <= 2)
})

test('applyChaosToExecution: inbound corruption with field-corrupt', () => {
  const ctx = makeCtx({ body: { id: '1', name: 'Alice' } })
  const result = applyChaosToExecution(ctx, [
    { type: 'inbound-corruption', target: 'inbound', corruptionStrategy: 'field-corrupt', corruptionField: 'name' },
  ])
  const body = result.ctx.response.body as Record<string, unknown>
  assert.strictEqual(body.name, null)
})

test('applyChaosToExecution: inbound corruption malformed', () => {
  const ctx = makeCtx({ body: { id: '1' } })
  const result = applyChaosToExecution(ctx, [
    { type: 'inbound-corruption', target: 'inbound', corruptionStrategy: 'malformed' },
  ])
  assert.strictEqual(result.ctx.response.body, '{"broken":')
})

test('applyChaosToExecution: ignores none events', () => {
  const ctx = makeCtx()
  const result = applyChaosToExecution(ctx, [
    { type: 'none', target: 'inbound' },
  ])
  assert.strictEqual(result.applied, false)
})

test('applyChaosToExecution: only applies first non-delay event', () => {
  const ctx = makeCtx()
  const result = applyChaosToExecution(ctx, [
    { type: 'inbound-error', target: 'inbound', statusCode: 503 },
    { type: 'inbound-dropout', target: 'inbound' },
  ])
  assert.strictEqual(result.ctx.response.statusCode, 503)
})

test('applyChaosToDependencyResponse: outbound error changes status', () => {
  const response = { contractName: 'api', statusCode: 200, body: { ok: true } }
  const result = applyChaosToDependencyResponse(response, [
    { type: 'outbound-error', target: 'outbound', contractName: 'api', statusCode: 503 },
  ])
  assert.strictEqual(result.statusCode, 503)
})

test('applyChaosToDependencyResponse: ignores events for other contracts', () => {
  const response = { contractName: 'api', statusCode: 200, body: {} }
  const result = applyChaosToDependencyResponse(response, [
    { type: 'outbound-error', target: 'outbound', contractName: 'other', statusCode: 503 },
  ])
  assert.strictEqual(result.statusCode, 200)
})

test('applyChaosToAllResponses: applies chaos to multiple responses', () => {
  const responses = [
    { contractName: 'a', statusCode: 200, body: {} },
    { contractName: 'b', statusCode: 200, body: {} },
  ]
  const events = [
    { type: 'outbound-error', target: 'outbound', contractName: 'a', statusCode: 503 },
    { type: 'outbound-error', target: 'outbound', contractName: 'b', statusCode: 504 },
  ] as ChaosEvent[]
  const result = applyChaosToAllResponses(responses, events)
  assert.strictEqual(result[0]!.statusCode, 503)
  assert.strictEqual(result[1]!.statusCode, 504)
})

test('extractDelays: computes total delay', () => {
  const events = [
    { type: 'inbound-delay', target: 'inbound', delayMs: 100 },
    { type: 'outbound-delay', target: 'outbound', delayMs: 200 },
  ] as ChaosEvent[]
  const result = extractDelays(events)
  assert.strictEqual(result.totalMs, 300)
  assert.strictEqual(result.events.length, 2)
})

test('sleep: resolves after specified ms', async () => {
  const start = Date.now()
  await sleep(10)
  const elapsed = Date.now() - start
  assert.ok(elapsed >= 8 && elapsed < 100)
})

test('hasAppliedChaos: detects applied chaos', () => {
  assert.strictEqual(hasAppliedChaos([{ type: 'none', target: 'inbound' }]), false)
  assert.strictEqual(hasAppliedChaos([{ type: 'inbound-error', target: 'inbound' }]), true)
})

test('formatChaosEvents: formats events for diagnostics', () => {
  const events = [
    { type: 'inbound-error', target: 'inbound', statusCode: 503 },
  ] as ChaosEvent[]
  const formatted = formatChaosEvents(events)
  assert.ok(formatted.includes('inbound-error'))
  assert.ok(formatted.includes('503'))
})

test('formatChaosEvents: returns "No chaos applied" for empty events', () => {
  assert.strictEqual(formatChaosEvents([]), 'No chaos applied')
  assert.strictEqual(formatChaosEvents([{ type: 'none', target: 'inbound' }]), 'No chaos applied')
})

test('createChaosEventArbitrary: generates deterministic events with seed', () => {
  const arb = createChaosEventArbitrary(
    { probability: 1, delay: { probability: 1, minMs: 100, maxMs: 200 } },
    []
  )
  const sample1 = fc.sample(arb, { seed: 42, numRuns: 5 })
  const sample2 = fc.sample(arb, { seed: 42, numRuns: 5 })
  assert.deepStrictEqual(sample1, sample2)
})

test('createChaosEventArbitrary: returns empty array when no config', () => {
  const arb = createChaosEventArbitrary(undefined, [])
  const sample = fc.sample(arb, { numRuns: 10 })
  assert.ok(sample.every((events) => events.length === 0 || events.every((e) => e.type === 'none')))
})

test('createChaosEventArbitrary: generates outbound events for contracts', () => {
  const arb = createChaosEventArbitrary(
    {
      probability: 1,
      error: { probability: 1, statusCode: 503 },
      outbound: [
        {
          target: 'api',
          error: {
            probability: 1,
            responses: [{ statusCode: 500 }],
          },
        },
      ],
    },
    ['api']
  )
  const sample = fc.sample(arb, { numRuns: 20 })
  const hasOutbound = sample.some((events) =>
    events.some((e) => e.target === 'outbound')
  )
  assert.ok(hasOutbound, 'expected some outbound events')
})

// ---------------------------------------------------------------------------
// Flake Detection Tests
// ---------------------------------------------------------------------------

test('FlakeDetector: deterministic failure returns high confidence', async () => {
  const detector = new FlakeDetector({ sameSeedReruns: 1, seedVariations: 2 })
  const report = await detector.detectFlake(
    makeTestResult(false),
    async () => ({ passed: false }),
    42
  )
  assert.strictEqual(report.isFlaky, false)
  assert.strictEqual(report.confidence, 'high')
  assert.strictEqual(report.reruns.length, 3)
})

test('FlakeDetector: passing same-seed rerun flags flaky', async () => {
  const detector = new FlakeDetector({ sameSeedReruns: 1, seedVariations: 0 })
  let callCount = 0
  const report = await detector.detectFlake(
    makeTestResult(false),
    async () => {
      callCount++
      return { passed: callCount === 1 }
    },
    42
  )
  assert.strictEqual(report.isFlaky, true)
  assert.strictEqual(report.confidence, 'low')
})

test('FlakeDetector: medium confidence when some pass', async () => {
  const detector = new FlakeDetector({ sameSeedReruns: 1, seedVariations: 2 })
  let callCount = 0
  const report = await detector.detectFlake(
    makeTestResult(false),
    async () => {
      callCount++
      return { passed: callCount % 2 === 0 }
    },
    42
  )
  assert.strictEqual(report.isFlaky, true)
  assert.strictEqual(report.confidence, 'medium')
})

test('FlakeDetector: uses Date.now() when no seed provided', async () => {
  const detector = new FlakeDetector({ sameSeedReruns: 0, seedVariations: 1 })
  const report = await detector.detectFlake(
    makeTestResult(false),
    async () => ({ passed: false })
  )
  assert.strictEqual(report.reruns.length, 1)
  assert.ok(report.reruns[0]!.seed > 0)
})

// ---------------------------------------------------------------------------
// Mutation Testing Tests
// ---------------------------------------------------------------------------

function makeContract(ensures: string[], requires: string[] = []): RouteContract {
  return {
    path: '/items',
    method: 'POST',
    category: 'constructor',
    requires,
    ensures,
    invariants: [],
    regexPatterns: {},
    validateRuntime: false,
    schema: {
      body: {
        type: 'object',
        properties: { name: { type: 'string' } },
      },
      response: {
        201: {
          type: 'object',
          properties: { id: { type: 'string' } },
        },
      },
    },
  }
}

test('testMutation: detects flipped operator mutation', async () => {
  const fastify = Fastify()
  await fastify.register(swagger)
  await fastify.register(apophisPlugin)
  fastify.post('/items', {
    schema: {
      body: { type: 'object', properties: { name: { type: 'string' } } },
      response: { 201: { type: 'object', properties: { id: { type: 'string' } } } },
      'x-ensures': ['response_code(this) == 201'],
    },
  }, async (request, reply) => {
    reply.status(201)
    return { id: '1' }
  })
  await fastify.ready()

  const mutation: Mutation = {
    id: 'm1',
    route: 'POST /items',
    original: 'response_code(this) == 201',
    mutated: 'response_code(this) != 201',
    type: 'flip-operator',
  }

  const killed = await testMutation(fastify, makeContract(['response_code(this) == 201']), mutation)
  assert.strictEqual(killed, true)
})

test('runMutationTesting: produces report with score', async () => {
  const fastify = Fastify()
  await fastify.register(swagger)
  await fastify.register(apophisPlugin)
  fastify.post('/items', {
    schema: {
      body: { type: 'object', properties: { name: { type: 'string' } } },
      response: { 201: { type: 'object', properties: { id: { type: 'string' } } } },
      'x-ensures': ['response_code(this) == 201'],
    },
  }, async (request, reply) => {
    reply.status(201)
    return { id: '1' }
  })
  await fastify.ready()

  const report = await runMutationTesting(fastify, { runs: 5, seed: 42, maxMutationsPerContract: 3 })
  assert.ok(typeof report.score === 'number')
  assert.ok(report.score >= 0 && report.score <= 100)
  assert.ok(report.mutations.length > 0)
  assert.ok(report.durationMs > 0)
})

test('runMutationTesting: skips routes without contracts', async () => {
  const fastify = Fastify()
  await fastify.register(swagger)
  await fastify.register(apophisPlugin)
  fastify.get('/health', {}, async () => 'ok')
  await fastify.ready()

  const report = await runMutationTesting(fastify, { runs: 5, seed: 42 })
  assert.strictEqual(report.mutations.length, 0)
  assert.strictEqual(report.score, 0)
})

test('runMutationTesting: filters by route', async () => {
  const fastify = Fastify()
  await fastify.register(swagger)
  await fastify.register(apophisPlugin)
  fastify.post('/items', {
    schema: {
      body: { type: 'object', properties: { name: { type: 'string' } } },
      response: { 201: { type: 'object', properties: { id: { type: 'string' } } } },
      'x-ensures': ['response_code(this) == 201'],
    },
  }, async (request, reply) => {
    reply.status(201)
    return { id: '1' }
  })
  fastify.post('/other', {
    schema: {
      body: { type: 'object', properties: { name: { type: 'string' } } },
      response: { 201: { type: 'object', properties: { id: { type: 'string' } } } },
      'x-ensures': ['response_code(this) == 201'],
    },
  }, async (request, reply) => {
    reply.status(201)
    return { id: '2' }
  })
  await fastify.ready()

  const report = await runMutationTesting(fastify, { runs: 5, seed: 42, routes: ['/items'] })
  const itemsMutations = report.mutations.filter((m) => m.mutation.route.includes('/items'))
  const otherMutations = report.mutations.filter((m) => m.mutation.route.includes('/other'))
  assert.ok(itemsMutations.length > 0)
  assert.strictEqual(otherMutations.length, 0)
})

test('runMutationTesting: identifies weak contracts', async () => {
  const fastify = Fastify()
  await fastify.register(swagger)
  await fastify.register(apophisPlugin)
  fastify.post('/items', {
    schema: {
      body: { type: 'object', properties: { name: { type: 'string' } } },
      response: { 201: { type: 'object', properties: { id: { type: 'string' } } } },
      'x-ensures': ['response_code(this) == 201', 'response_body(this).id != null'],
    },
  }, async (request, reply) => {
    reply.status(201)
    return { id: '1' }
  })
  await fastify.ready()

  const report = await runMutationTesting(fastify, { runs: 5, seed: 42 })
  // All mutations should be killed since the handler always returns 201 with id
  assert.strictEqual(report.weakContracts.length, 0)
})