refactor: remove depth/generationProfile tiering, use explicit runs

- Replace TestDepth ('quick'|'standard'|'thorough') with explicit runs:number
- Remove GenerationProfile type and all generationProfile parameters
- schema-to-arbitrary.ts now uses fixed defaults (string≤128, array≤10, props≤6)
- Delete src/cli/core/generation-profile.ts
- Remove --generation-profile CLI flag from verify and qualify
- Remove depth field from PresetDefinition and all preset scaffolds
- Remove generationProfiles from Config interface
- Update all test files: depth:'quick' → runs:10
- Remove depth from all fixture configs
- Update builders.ts, mutation.ts, outbound-mock-runtime.ts
- Build: clean | Tests: 849 pass, 0 fail
This commit is contained in:
John Dvorak
2026-04-30 13:47:24 -07:00
parent 3e5758dd54
commit 6295c476dc
38 changed files with 130 additions and 526 deletions
@@ -1,7 +1,6 @@
/**
* APOPHIS configuration for broken-behavior fixture.
*/
export default {
mode: "verify",
profiles: {
@@ -15,7 +14,6 @@ export default {
presets: {
"safe-ci": {
name: "safe-ci",
depth: "quick",
timeout: 5000,
parallel: false,
chaos: false,
@@ -2,11 +2,9 @@
* LEGACY APOPHIS configuration (old-style, for migration tests).
* This uses deprecated field names that should be detected by `apophis migrate`.
*/
export default {
// Deprecated: 'mode' used to be 'testMode'
testMode: "verify",
// Deprecated: 'profiles' used to be 'testProfiles'
testProfiles: {
quick: {
@@ -17,7 +15,6 @@ export default {
routeFilter: ["GET /legacy"],
},
},
// Deprecated: 'presets' used to be 'testPresets'
testPresets: {
"safe-ci": {
@@ -28,7 +25,6 @@ export default {
maxDuration: 5000,
},
},
// Deprecated: 'environments' used to be 'envPolicies'
envPolicies: {
local: {
@@ -2,7 +2,6 @@
* Root-level APOPHIS config for monorepo.
* Packages can override with their own configs.
*/
export default {
mode: "verify",
profiles: {
@@ -20,7 +19,6 @@ export default {
presets: {
"safe-ci": {
name: "safe-ci",
depth: "quick",
timeout: 5000,
parallel: false,
chaos: false,
@@ -1,7 +1,6 @@
/**
* APOPHIS configuration for observe-config fixture.
*/
export default {
mode: "observe",
profiles: {
@@ -15,7 +14,6 @@ export default {
presets: {
"observe-safe": {
name: "observe-safe",
depth: "quick",
timeout: 5000,
parallel: false,
chaos: false,
@@ -1,7 +1,6 @@
/**
* APOPHIS configuration for protocol-lab fixture.
*/
export default {
mode: "qualify",
profiles: {
@@ -15,7 +14,6 @@ export default {
presets: {
deep: {
name: "deep",
depth: "deep",
timeout: 30000,
parallel: false,
chaos: true,
@@ -1,7 +1,6 @@
/**
* APOPHIS configuration for tiny-fastify fixture.
*/
export default {
mode: "verify",
profiles: {
@@ -15,7 +14,6 @@ export default {
presets: {
"safe-ci": {
name: "safe-ci",
depth: "quick",
timeout: 5000,
parallel: false,
chaos: false,
@@ -11,7 +11,6 @@ export default {
presets: {
"safe-ci": {
name: "safe-ci",
depth: "quick",
timeout: 5000,
parallel: false,
chaos: false,
@@ -11,7 +11,6 @@ export default {
presets: {
"safe-ci": {
name: "safe-ci",
depth: "quick",
timeout: 5000,
parallel: false,
chaos: false,
@@ -11,7 +11,6 @@ export default {
presets: {
"safe-ci": {
name: "safe-ci",
depth: "quick",
timeout: 5000,
parallel: false,
chaos: false,
-4
View File
@@ -17,7 +17,6 @@ export interface ScaffoldResult {
export function safeCiScaffold(): ScaffoldResult {
const preset: PresetDefinition = {
name: 'safe-ci',
depth: 'quick',
timeout: 5000,
parallel: false,
chaos: false,
@@ -95,7 +94,6 @@ If \`apophis verify\` says "No behavioral contracts found", it means your routes
export function platformObserveScaffold(): ScaffoldResult {
const preset: PresetDefinition = {
name: 'platform-observe',
depth: 'standard',
timeout: 10000,
parallel: true,
chaos: false,
@@ -180,7 +178,6 @@ This project was scaffolded with \`apophis init --preset platform-observe\`.
export function llmSafeScaffold(): ScaffoldResult {
const preset: PresetDefinition = {
name: 'llm-safe',
depth: 'quick',
timeout: 3000,
parallel: false,
chaos: false,
@@ -258,7 +255,6 @@ If \`apophis verify\` says "No behavioral contracts found", it means your routes
export function protocolLabScaffold(): ScaffoldResult {
const preset: PresetDefinition = {
name: 'protocol-lab',
depth: 'deep',
timeout: 15000,
parallel: false,
chaos: true,
+1 -26
View File
@@ -19,7 +19,7 @@
import type { CliContext } from '../../core/context.js'
import { loadConfig } from '../../core/config-loader.js'
import { PolicyEngine, detectEnvironment } from '../../core/policy-engine.js'
import { resolveGenerationProfileOverride, GenerationProfileResolutionError } from '../../core/generation-profile.js'
import { SUCCESS, BEHAVIORAL_FAILURE, USAGE_ERROR, INTERNAL_ERROR } from '../../core/exit-codes.js'
import type { CommandResult, Artifact, FailureRecord } from '../../core/types.js'
import { classifyError, ErrorTaxonomy } from '../../core/error-taxonomy.js'
@@ -54,13 +54,6 @@ function isReplayCompatibleRoute(route: string): boolean {
return ROUTE_IDENTITY_PATTERN.test(route)
}
function coerceDepth(value: unknown): TestConfig['depth'] {
if (value === 'quick' || value === 'standard' || value === 'thorough') {
return value
}
return 'standard'
}
function coerceTimeout(value: unknown): number | undefined {
return typeof value === 'number' ? value : undefined
}
@@ -71,7 +64,6 @@ function coerceTimeout(value: unknown): number | undefined {
export interface QualifyOptions {
profile?: string
generationProfile?: string
seed?: number
config?: string
cwd?: string
@@ -529,7 +521,6 @@ export async function qualifyCommand(
): Promise<CommandResult> {
const {
profile,
generationProfile,
seed: explicitSeed,
config: configPath,
cwd,
@@ -558,7 +549,6 @@ export async function qualifyCommand(
}
const config = loadResult.config
const resolvedGenerationProfile = resolveGenerationProfileOverride(generationProfile, config)
// 2. Run policy engine checks
const policyEngine = new PolicyEngine({
@@ -600,12 +590,9 @@ export async function qualifyCommand(
// 6. Build stateful config
const presetName = profileDef?.preset
const preset = presetName ? config.presets?.[presetName] : undefined
const presetDepth = coerceDepth((preset as { depth?: unknown } | undefined)?.depth)
const presetTimeout = coerceTimeout((preset as { timeout?: unknown } | undefined)?.timeout)
const statefulConfig: TestConfig | undefined = gates.stateful
? {
depth: presetDepth,
generationProfile: resolvedGenerationProfile,
seed,
timeout: presetTimeout,
routes: profileDef?.routes,
@@ -752,12 +739,6 @@ export async function qualifyCommand(
}
}
} catch (error) {
if (error instanceof GenerationProfileResolutionError) {
return {
exitCode: USAGE_ERROR,
message: error.message,
}
}
const message = error instanceof Error ? error.message : String(error)
return {
exitCode: INTERNAL_ERROR,
@@ -780,7 +761,6 @@ export async function handleQualify(
): Promise<number> {
const options: QualifyOptions = {
profile: ctx.options.profile || undefined,
generationProfile: ctx.options.generationProfile,
seed: undefined,
config: ctx.options.config || undefined,
cwd: ctx.cwd,
@@ -798,11 +778,6 @@ export async function handleQualify(
}
}
const generationProfileIdx = args.indexOf('--generation-profile')
if (generationProfileIdx !== -1 && args[generationProfileIdx + 1]) {
options.generationProfile = args[generationProfileIdx + 1]
}
const result = await qualifyCommand(options, ctx)
const format = options.format || ctx.options.format || 'human'
const machineMode = format === 'json' || format === 'ndjson' || format === 'json-summary' || format === 'ndjson-summary'
+2 -16
View File
@@ -22,7 +22,7 @@
import type { CliContext } from '../../core/context.js'
import { loadConfig, findWorkspacePackages } from '../../core/config-loader.js'
import { PolicyEngine, detectEnvironment } from '../../core/policy-engine.js'
import { resolveGenerationProfileOverride, GenerationProfileResolutionError } from '../../core/generation-profile.js'
import { SUCCESS, BEHAVIORAL_FAILURE, USAGE_ERROR, INTERNAL_ERROR } from '../../core/exit-codes.js'
import type { CommandResult, Artifact, FailureRecord, RouteResult, WorkspaceRun, WorkspaceResult } from '../../core/types.js'
import { classifyError, ErrorTaxonomy } from '../../core/error-taxonomy.js'
@@ -54,7 +54,6 @@ function isReplayCompatibleRoute(route: string): boolean {
export interface VerifyOptions {
profile?: string
generationProfile?: string
routes?: string
seed?: number
changed?: boolean
@@ -381,7 +380,6 @@ export async function verifyCommand(
): Promise<CommandResult> {
const {
profile,
generationProfile,
routes: routesFlag,
seed: explicitSeed,
changed,
@@ -412,7 +410,6 @@ export async function verifyCommand(
}
const config = loadResult.config
const resolvedGenerationProfile = resolveGenerationProfileOverride(generationProfile, config)
// 2a. Resolve profile — if explicitly requested but missing, list available ones
if (profile && !config.profiles?.[profile]) {
@@ -551,12 +548,7 @@ export async function verifyCommand(
message: `Config validation failed: ${message}`,
}
}
if (error instanceof GenerationProfileResolutionError) {
return {
exitCode: USAGE_ERROR,
message,
}
}
return {
exitCode: INTERNAL_ERROR,
message: `Internal error in verify command: ${message}`,
@@ -578,7 +570,6 @@ export async function handleVerify(
): Promise<number> {
const options: VerifyOptions = {
profile: ctx.options.profile || undefined,
generationProfile: ctx.options.generationProfile,
routes: undefined,
seed: undefined,
changed: false,
@@ -610,11 +601,6 @@ export async function handleVerify(
options.changed = true
}
const generationProfileIdx = args.indexOf('--generation-profile')
if (generationProfileIdx !== -1 && args[generationProfileIdx + 1]) {
options.generationProfile = args[generationProfileIdx + 1]
}
const workspaceMode = args.includes('--workspace')
if (workspaceMode) {
+4 -35
View File
@@ -30,7 +30,6 @@ export interface Config {
environments?: Record<string, EnvironmentPolicy>;
profiles?: Record<string, ProfileDefinition>;
presets?: Record<string, PresetDefinition>;
generationProfiles?: Record<string, 'quick' | 'standard' | 'thorough' | { base: 'quick' | 'standard' | 'thorough' }>;
[key: string]: unknown;
}
@@ -107,11 +106,6 @@ const CONFIG_SCHEMA: Record<string, SchemaField> = {
optional: true,
properties: {},
},
generationProfiles: {
type: 'object',
optional: true,
properties: {},
},
packs: {
type: 'array',
optional: true,
@@ -151,7 +145,6 @@ const PROFILE_SCHEMA: Record<string, SchemaField> = {
// Schema for PresetDefinition values (inside presets.<name>)
const PRESET_SCHEMA: Record<string, SchemaField> = {
name: { type: 'string', optional: false },
depth: { type: 'string', optional: true, enumValues: ['quick', 'standard', 'deep'] },
timeout: { type: 'number', optional: true, min: 0 },
parallel: { type: 'boolean', optional: true },
chaos: { type: 'boolean', optional: true },
@@ -160,12 +153,9 @@ const PRESET_SCHEMA: Record<string, SchemaField> = {
sampling: { type: 'number', optional: true },
blocking: { type: 'boolean', optional: true },
sinks: { type: 'object', optional: true },
runs: { type: 'number', optional: true, min: 1 },
};
const GENERATION_PROFILE_ALIAS_SCHEMA: Record<string, SchemaField> = {
base: { type: 'string', optional: false, enumValues: ['quick', 'standard', 'thorough'] },
}
// ---------------------------------------------------------------------------
// Config discovery
// ---------------------------------------------------------------------------
@@ -259,7 +249,6 @@ function getDynamicContainerSchema(path: string): Record<string, SchemaField> |
if (path === 'profiles') return PROFILE_SCHEMA;
if (path === 'presets') return PRESET_SCHEMA;
if (path === 'environments') return ENVIRONMENT_POLICY_SCHEMA;
if (path === 'generationProfiles') return GENERATION_PROFILE_ALIAS_SCHEMA;
return null;
}
@@ -267,7 +256,7 @@ function getDynamicContainerSchema(path: string): Record<string, SchemaField> |
* Check if a path is inside a dynamic container (e.g., profiles.foo, presets.bar).
*/
function isInsideDynamicContainer(path: string): boolean {
return path.startsWith('profiles.') || path.startsWith('presets.') || path.startsWith('environments.') || path.startsWith('generationProfiles.');
return path.startsWith('profiles.') || path.startsWith('presets.') || path.startsWith('environments.');
}
/**
@@ -379,18 +368,11 @@ export function validateConfigAgainstSchema(
// Handle dynamic containers: profiles, presets, environments
// The keys are user-defined names; their values have specific schemas
const isDynamicContainer = path === 'profiles' || path === 'presets' || path === 'environments' || path === 'generationProfiles';
const isDynamicContainer = path === 'profiles' || path === 'presets' || path === 'environments';
if (!fieldSchema && isDynamicContainer) {
const childSchema = getDynamicContainerSchema(path);
const fieldValue = obj[key];
if (path === 'generationProfiles' && typeof fieldValue === 'string') {
validateType(
fieldValue,
{ type: 'string', optional: false, enumValues: ['quick', 'standard', 'thorough'] },
currentPath,
key,
);
} else if (childSchema && fieldValue !== null && typeof fieldValue === 'object') {
if (childSchema && fieldValue !== null && typeof fieldValue === 'object') {
// Validate the dynamic container value against its specific schema
validateConfigAgainstSchema(fieldValue, childSchema, currentPath);
} else if (childSchema) {
@@ -633,19 +615,6 @@ export function validateConfigSemantics(config: Config): void {
);
}
}
if (preset.depth !== undefined) {
const validDepths = ['quick', 'standard', 'deep'];
const depthValue = preset.depth;
if (typeof depthValue === 'string' && !validDepths.includes(depthValue as string)) {
throw new ConfigValidationError(
`Preset "${presetName}" has invalid depth: "${depthValue}"`,
`presets.${presetName}.depth`,
'depth',
depthValue,
`Must be one of: ${validDepths.join(', ')}.`,
);
}
}
}
}
-5
View File
@@ -101,10 +101,6 @@ export function createContext(options: Record<string, unknown> = {}): CliContext
? options.color
: 'auto';
const generationProfile = typeof options.generationProfile === 'string'
? options.generationProfile
: undefined;
return {
cwd,
env: {
@@ -119,7 +115,6 @@ export function createContext(options: Record<string, unknown> = {}): CliContext
options: {
config: typeof options.config === 'string' ? options.config : undefined,
profile: typeof options.profile === 'string' ? options.profile : undefined,
generationProfile,
format,
color,
quiet: options.quiet === true,
-56
View File
@@ -1,56 +0,0 @@
import type { Config } from './config-loader.js'
export type ResolvedGenerationProfile = 'quick' | 'standard' | 'thorough'
export class GenerationProfileResolutionError extends Error {
constructor(message: string) {
super(message)
this.name = 'GenerationProfileResolutionError'
}
}
function isBuiltInProfile(value: string): value is ResolvedGenerationProfile {
return value === 'quick' || value === 'standard' || value === 'thorough' || value === 'deep'
}
function normalizeProfile(value: string): ResolvedGenerationProfile {
if (value === 'deep') return 'thorough'
return value as ResolvedGenerationProfile
}
export function resolveGenerationProfileOverride(
rawProfile: string | undefined,
config: Config,
): ResolvedGenerationProfile | undefined {
if (!rawProfile) {
return undefined
}
if (isBuiltInProfile(rawProfile)) {
return normalizeProfile(rawProfile)
}
const aliases = config.generationProfiles
if (!aliases) {
throw new GenerationProfileResolutionError(
`Unknown generation profile "${rawProfile}". Use one of: quick, standard, deep, or define an alias in config.generationProfiles.`,
)
}
const alias = aliases[rawProfile]
if (!alias) {
const available = Object.keys(aliases).join(', ') || 'none'
throw new GenerationProfileResolutionError(
`Unknown generation profile "${rawProfile}". Built-ins: quick, standard, deep. Config aliases: ${available}.`,
)
}
const target = typeof alias === 'string' ? alias : alias.base
if (!isBuiltInProfile(target)) {
throw new GenerationProfileResolutionError(
`Invalid generation profile alias "${rawProfile}". Alias must resolve to quick, standard, or deep.`,
)
}
return normalizeProfile(target)
}
+2 -9
View File
@@ -22,7 +22,6 @@ const HELP_HEADER = `
${pc.dim('Global Options:')}
--config <path> Config file path
--profile <name> Profile name from config
--generation-profile <name> Generation budget profile (built-in or config alias)
--cwd <path> Working directory override
--format <mode> Output format: human | json | ndjson (default: human)
--color <mode> Color mode: auto | always | never (default: auto)
@@ -71,7 +70,6 @@ function getCommandHelp(command: string): string {
${pc.dim('Options:')}
--profile <name> Profile name from config
--generation-profile <name> Generation budget profile (built-in or config alias)
--routes <filter> Route filter pattern
--seed <number> Deterministic seed
--changed Filter to git-modified routes
@@ -103,7 +101,6 @@ function getCommandHelp(command: string): string {
${pc.dim('Options:')}
--profile <name> Profile name from config
--generation-profile <name> Generation budget profile (built-in or config alias)
--seed <number> Deterministic seed
${pc.dim('Examples:')}
@@ -225,7 +222,6 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
// Global flags
cli.option('--config <path>', 'Config file path');
cli.option('--profile <name>', 'Profile name from config');
cli.option('--generation-profile <name>', 'Generation budget profile (built-in or config alias)');
cli.option('--cwd <path>', 'Working directory override');
cli.option('--format <mode>', 'Output format: human | json | ndjson', { default: 'human' });
cli.option('--color <mode>', 'Color mode: auto | always | never', { default: 'auto' });
@@ -270,7 +266,6 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
break;
case 'verify':
cmd.option('--profile <name>', 'Profile name from config');
cmd.option('--generation-profile <name>', 'Generation budget profile (built-in or config alias)');
cmd.option('--routes <filter>', 'Route filter pattern');
cmd.option('--seed <number>', 'Deterministic seed');
cmd.option('--changed', 'Filter to git-modified routes');
@@ -281,7 +276,6 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
break;
case 'qualify':
cmd.option('--profile <name>', 'Profile name from config');
cmd.option('--generation-profile <name>', 'Generation budget profile (built-in or config alias)');
cmd.option('--seed <number>', 'Deterministic seed');
break;
case 'replay':
@@ -373,16 +367,15 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
// Handle unknown flags
const knownGlobalFlags = new Set([
'--config', '--profile', '--cwd', '--format', '--color',
'--generation-profile',
'--quiet', '--verbose', '--artifact-dir', '--workspace',
'-v', '--version', '-h', '--help',
]);
const commandSpecificFlags: Record<string, Set<string>> = {
init: new Set(['--preset', '--force', '--noninteractive']),
verify: new Set(['--profile', '--generation-profile', '--routes', '--seed', '--changed', '--workspace']),
verify: new Set(['--profile', '--routes', '--seed', '--changed', '--workspace']),
observe: new Set(['--profile', '--check-config', '--workspace']),
qualify: new Set(['--profile', '--generation-profile', '--seed', '--workspace']),
qualify: new Set(['--profile', '--seed', '--workspace']),
replay: new Set(['--artifact']),
doctor: new Set(['--mode', '--strict', '--workspace']),
migrate: new Set(['--check', '--dry-run', '--write']),
-3
View File
@@ -29,7 +29,6 @@ export interface CliContext {
options: {
config: string | undefined;
profile: string | undefined;
generationProfile?: string;
format: OutputFormat;
color: ColorMode;
quiet: boolean;
@@ -132,7 +131,6 @@ export interface ProfileDefinition {
* required: ["name"],
* properties: {
* name: { type: "string" },
* depth: { type: "string", enum: ["quick", "standard", "deep"] },
* timeout: { type: "number" },
* parallel: { type: "boolean" },
* chaos: { type: "boolean" },
@@ -143,7 +141,6 @@ export interface ProfileDefinition {
*/
export interface PresetDefinition {
name: string;
depth?: "quick" | "standard" | "deep";
timeout?: number;
parallel?: boolean;
chaos?: boolean;
+31 -55
View File
@@ -7,13 +7,9 @@ import type { Arbitrary } from 'fast-check'
import * as fc from 'fast-check'
import { CONTENT_TYPE } from '../infrastructure/http-executor.js'
export type GenerationProfile = 'quick' | 'standard' | 'thorough'
export interface SchemaToArbOptions {
/** 'request' skips readOnly, 'response' skips writeOnly */
readonly context: 'request' | 'response'
/** Generation budget profile: quick favors speed, thorough favors breadth */
readonly generationProfile?: GenerationProfile
}
interface ContextCache {
@@ -29,45 +25,32 @@ const patternRegexCache = new Map<string, RegExp>()
const STABLE_SCHEMA_CACHE_LIMIT = 512
const PATTERN_REGEX_CACHE_LIMIT = 256
function normalizeProfile(profile: GenerationProfile | undefined): GenerationProfile {
return profile ?? 'standard'
// Fixed defaults for generated data size (previously controlled by generationProfile tier)
const DEFAULT_STRING_MAX = 128
const DEFAULT_ARRAY_MAX = 10
const DEFAULT_ADDITIONAL_PROPS_MAX = 6
function defaultStringMaxLength(): number | undefined {
return DEFAULT_STRING_MAX
}
function defaultStringMaxLength(profile: GenerationProfile): number | undefined {
if (profile === 'quick') return 48
if (profile === 'standard') return 128
return undefined
function defaultArrayMaxLength(): number | undefined {
return DEFAULT_ARRAY_MAX
}
function defaultArrayMaxLength(profile: GenerationProfile): number | undefined {
if (profile === 'quick') return 4
if (profile === 'standard') return 10
return undefined
function additionalPropsMaxKeys(): number {
return DEFAULT_ADDITIONAL_PROPS_MAX
}
function additionalPropsMaxKeys(profile: GenerationProfile): number {
if (profile === 'quick') return 3
if (profile === 'standard') return 6
return 10
}
function buildFallbackAnyArb(profile: GenerationProfile): Arbitrary<unknown> {
if (profile === 'thorough') {
return fc.anything()
}
const stringMax = profile === 'quick' ? 24 : 64
const arrayMax = profile === 'quick' ? 3 : 6
const dictMax = profile === 'quick' ? 2 : 4
function buildFallbackAnyArb(): Arbitrary<unknown> {
return fc.oneof(
fc.constant(null),
fc.boolean(),
fc.integer(),
fc.double({ noNaN: true }),
fc.string({ maxLength: stringMax }),
fc.array(fc.string({ maxLength: 16 }), { maxLength: arrayMax }),
fc.dictionary(fc.string({ maxLength: 16 }), fc.string({ maxLength: 24 }), { maxKeys: dictMax }),
fc.string({ maxLength: 64 }),
fc.array(fc.string({ maxLength: 16 }), { maxLength: 6 }),
fc.dictionary(fc.string({ maxLength: 16 }), fc.string({ maxLength: 24 }), { maxKeys: 4 }),
)
}
@@ -106,7 +89,6 @@ const getObject = (schema: unknown, key: string): Record<string, unknown> | unde
const buildStringArb = (
schema: Record<string, unknown>,
profile: GenerationProfile,
): Arbitrary<string> => {
const minLength = getNumber(schema, 'minLength')
const maxLength = getNumber(schema, 'maxLength')
@@ -149,7 +131,7 @@ const buildStringArb = (
if (minLength !== undefined) constraints.minLength = minLength
if (maxLength !== undefined) constraints.maxLength = maxLength
else {
const capped = defaultStringMaxLength(profile)
const capped = defaultStringMaxLength()
if (capped !== undefined) constraints.maxLength = capped
}
@@ -298,14 +280,13 @@ function getSchemaFingerprint(schema: Record<string, unknown>): string | undefin
function getStableCachedArbitrary(
schema: Record<string, unknown>,
context: SchemaToArbOptions['context'],
profile: GenerationProfile,
): Arbitrary<unknown> | undefined {
const fingerprint = getSchemaFingerprint(schema)
if (!fingerprint) {
return undefined
}
const key = `${context}:${profile}:${fingerprint}`
const key = `${context}:${fingerprint}`
const cached = stableSchemaArbitraryCache.get(key)
if (!cached) {
return undefined
@@ -319,7 +300,6 @@ function getStableCachedArbitrary(
function setStableCachedArbitrary(
schema: Record<string, unknown>,
context: SchemaToArbOptions['context'],
profile: GenerationProfile,
arbitrary: Arbitrary<unknown>,
): void {
const fingerprint = getSchemaFingerprint(schema)
@@ -327,7 +307,7 @@ function setStableCachedArbitrary(
return
}
const key = `${context}:${profile}:${fingerprint}`
const key = `${context}:${fingerprint}`
if (stableSchemaArbitraryCache.has(key)) {
stableSchemaArbitraryCache.delete(key)
}
@@ -353,19 +333,18 @@ const buildIntegerArb = (schema: Record<string, unknown>): Arbitrary<number> =>
const buildArrayArb = (
schema: Record<string, unknown>,
options: SchemaToArbOptions,
profile: GenerationProfile,
): Arbitrary<unknown[]> => {
const itemsSchema = getObject(schema, 'items')
const itemArb = itemsSchema !== undefined
? convertSchemaInternal(itemsSchema, options, false)
: buildFallbackAnyArb(profile)
: buildFallbackAnyArb()
const minItems = getNumber(schema, 'minItems')
const maxItems = getNumber(schema, 'maxItems')
const constraints: { minLength?: number; maxLength?: number } = {}
if (minItems !== undefined) constraints.minLength = minItems
if (maxItems !== undefined) constraints.maxLength = maxItems
else {
const capped = defaultArrayMaxLength(profile)
const capped = defaultArrayMaxLength()
if (capped !== undefined) constraints.maxLength = capped
}
return fc.array(itemArb, constraints)
@@ -374,7 +353,6 @@ const buildArrayArb = (
const buildObjectArb = (
schema: Record<string, unknown>,
options: SchemaToArbOptions,
profile: GenerationProfile,
): Arbitrary<Record<string, unknown>> => {
const properties = getObject(schema, 'properties') ?? {}
const required = new Set(getArray(schema, 'required') as string[] ?? [])
@@ -398,14 +376,14 @@ const buildObjectArb = (
const baseArb = fc.record(arbs)
if (additionalProperties === true) {
const extraValueArb = buildFallbackAnyArb(profile)
const keyMaxLength = profile === 'quick' ? 16 : 32
const extraValueArb = buildFallbackAnyArb()
const keyMaxLength = 32
return fc.tuple(
baseArb,
fc.dictionary(
fc.string({ maxLength: keyMaxLength }),
extraValueArb,
{ maxKeys: additionalPropsMaxKeys(profile) },
{ maxKeys: additionalPropsMaxKeys() },
),
).map(([base, extra]) => ({
...base,
@@ -418,7 +396,6 @@ const buildObjectArb = (
const buildMultipartArb = (
schema: Record<string, unknown>,
profile: GenerationProfile,
): Arbitrary<{ fields: Record<string, unknown>; files: Record<string, { originalname: string; mimetype: string; size: number; buffer: Buffer } | { originalname: string; mimetype: string; size: number; buffer: Buffer }[]> }> => {
const fieldsSchema = getObject(schema, 'x-multipart-fields') ?? {}
const filesSchema = getObject(schema, 'x-multipart-files') ?? {}
@@ -426,7 +403,7 @@ const buildMultipartArb = (
const fieldArbs: Record<string, Arbitrary<unknown>> = {}
for (const [key, propSchema] of Object.entries(fieldsSchema)) {
if (isObject(propSchema)) {
fieldArbs[key] = convertSchemaInternal(propSchema, { context: 'request', generationProfile: profile }, false)
fieldArbs[key] = convertSchemaInternal(propSchema, { context: 'request' }, false)
}
}
@@ -463,7 +440,6 @@ const convertSchemaInternal = (
options: SchemaToArbOptions,
useStableCache: boolean,
): Arbitrary<unknown> => {
const profile = normalizeProfile(options.generationProfile)
const cacheKey = options.context
const cachedBySchema = schemaArbitraryCache.get(schema)
const cached = cachedBySchema?.[cacheKey]
@@ -472,7 +448,7 @@ const convertSchemaInternal = (
}
if (useStableCache) {
const stableCached = getStableCachedArbitrary(schema, cacheKey, profile)
const stableCached = getStableCachedArbitrary(schema, cacheKey)
if (stableCached) {
const contextCache = cachedBySchema ?? {}
contextCache[cacheKey] = stableCached
@@ -489,11 +465,11 @@ const convertSchemaInternal = (
let arb: Arbitrary<unknown>
if (contentType === CONTENT_TYPE.MULTIPART) {
arb = buildMultipartArb(schema, profile)
arb = buildMultipartArb(schema)
} else if (enumValues !== undefined && enumValues.length > 0) {
arb = fc.constantFrom(...enumValues)
} else if (type === 'string') {
arb = buildStringArb(schema, profile)
arb = buildStringArb(schema)
} else if (type === 'integer') {
arb = buildIntegerArb(schema)
} else if (type === 'number') {
@@ -501,11 +477,11 @@ const convertSchemaInternal = (
} else if (type === 'boolean') {
arb = fc.boolean()
} else if (type === 'array') {
arb = buildArrayArb(schema, options, profile)
arb = buildArrayArb(schema, options)
} else if (type === 'object') {
arb = buildObjectArb(schema, options, profile)
arb = buildObjectArb(schema, options)
} else {
arb = buildFallbackAnyArb(profile)
arb = buildFallbackAnyArb()
}
if (nullable === true) {
@@ -516,7 +492,7 @@ const convertSchemaInternal = (
contextCache[cacheKey] = arb
schemaArbitraryCache.set(schema, contextCache)
if (useStableCache) {
setStableCachedArbitrary(schema, cacheKey, profile, arb)
setStableCachedArbitrary(schema, cacheKey, arb)
}
return arb
+5 -9
View File
@@ -220,7 +220,6 @@ function setNestedValue(obj: Record<string, unknown>, path: string, value: unkno
function createConditionalDependencyArbitrary(
contract: ResolvedOutboundContract,
request: Record<string, unknown>,
generationProfile: 'quick' | 'standard' | 'thorough',
): Arbitrary<DependencyResponseSample> {
const statuses = Object.keys(contract.response).map(Number)
if (statuses.length === 0) {
@@ -229,7 +228,7 @@ function createConditionalDependencyArbitrary(
return fc.integer({ min: 0, max: statuses.length - 1 }).chain((statusIndex) => {
const statusCode = statuses[statusIndex]!
const schema = contract.response[statusCode]
const bodyArb = convertSchema(schema ?? {}, { context: 'response', generationProfile })
const bodyArb = convertSchema(schema ?? {}, { context: 'response' })
return bodyArb.map((rawBody) => ({
contractName: contract.name,
statusCode,
@@ -239,11 +238,10 @@ function createConditionalDependencyArbitrary(
}
function createRequestArbitrary(
route: RouteContract,
generationProfile: 'quick' | 'standard' | 'thorough',
): Arbitrary<Record<string, unknown>> {
const bodySchema = route.schema?.body as Record<string, unknown> | undefined
const bodyArb = bodySchema !== undefined
? convertSchema(bodySchema, { context: 'request', generationProfile })
? convertSchema(bodySchema, { context: 'request' })
: fc.constant({})
const pathParams = route.path.match(/:([a-zA-Z_][a-zA-Z0-9_]*)/g) ?? []
const pathParamArbs: Record<string, fc.Arbitrary<string>> = {}
@@ -264,11 +262,10 @@ function createRequestArbitrary(
function createConditionalDependenciesArbitrary(
contracts: ResolvedOutboundContract[],
request: Record<string, unknown>,
generationProfile: 'quick' | 'standard' | 'thorough',
): Arbitrary<ReadonlyArray<DependencyResponseSample>> {
if (contracts.length === 0) return fc.constant([])
const arbs = contracts.map((contract) =>
createConditionalDependencyArbitrary(contract, request, generationProfile)
createConditionalDependencyArbitrary(contract, request)
)
return fc.tuple(...arbs)
}
@@ -292,11 +289,10 @@ export function createTripleBoundaryArbitrary(
route: RouteContract,
contracts: ResolvedOutboundContract[],
chaosConfig: ChaosConfig,
generationProfile: 'quick' | 'standard' | 'thorough' = 'standard',
): Arbitrary<TripleBoundaryCommand> {
const requestArb = createRequestArbitrary(route, generationProfile)
const requestArb = createRequestArbitrary(route)
return requestArb.chain((request) => {
const depArb = createConditionalDependenciesArbitrary(contracts, request, generationProfile)
const depArb = createConditionalDependenciesArbitrary(contracts, request)
const chaosArb = createChaosEventArbitrary(route, contracts, chaosConfig)
return fc.tuple(depArb, chaosArb).map(([dependencyResponses, chaosEvents]) => ({
route,
+1 -2
View File
@@ -29,7 +29,6 @@ export interface OutboundMockRuntime {
interface OutboundMockOptions {
readonly contracts: ResolvedOutboundContract[]
readonly mode: 'example' | 'property'
readonly generationProfile?: 'quick' | 'standard' | 'thorough'
readonly overrides?: Record<string, {
readonly forceStatus?: number
readonly headers?: Record<string, string>
@@ -77,7 +76,7 @@ export function createOutboundMockRuntime(opts: OutboundMockOptions): OutboundMo
const schema = contract.response[statusCode]
if (!schema) return null
// Generate base response from schema
const arb = convertSchema(schema, { context: 'response', generationProfile: opts.generationProfile })
const arb = convertSchema(schema, { context: 'response' })
const samples = fc.sample(arb, { numRuns: 1, seed: opts.seed + calls.length })
let body = samples[0] ?? null
if (typeof body !== 'object' || body === null) return body
+1 -1
View File
@@ -27,7 +27,7 @@ import { parse } from '../formula/parser.js'
// ---------------------------------------------------------------------------
const normalizeTestConfig = (opts: TestConfig = {}): TestConfig => ({
depth: opts.depth ?? 'standard',
runs: opts.runs,
scope: opts.scope,
seed: opts.seed,
timeout: opts.timeout,
-4
View File
@@ -80,7 +80,6 @@ export function oauth21ProfilePack(opts: PackOptions = {}): Partial<Config> {
presets: {
'protocol-lab': {
name: 'protocol-lab',
depth: 'deep',
timeout: opts.timeout ?? 15000,
parallel: false,
chaos: true,
@@ -88,7 +87,6 @@ export function oauth21ProfilePack(opts: PackOptions = {}): Partial<Config> {
},
'safe-ci': {
name: 'safe-ci',
depth: 'quick',
timeout: opts.timeout ?? 5000,
parallel: false,
chaos: false,
@@ -118,7 +116,6 @@ export function rfc8628DeviceAuthorizationPack(opts: PackOptions = {}): Partial<
presets: {
'protocol-lab': {
name: 'protocol-lab',
depth: 'deep',
timeout: opts.timeout ?? 20000,
parallel: false,
chaos: true,
@@ -148,7 +145,6 @@ export function rfc8693TokenExchangePack(opts: PackOptions = {}): Partial<Config
presets: {
'protocol-lab': {
name: 'protocol-lab',
depth: 'deep',
timeout: opts.timeout ?? 15000,
parallel: false,
chaos: true,
+7 -7
View File
@@ -7,7 +7,7 @@
* If the test suite passes, the mutation "survives" — indicating a gap in coverage.
*
* Usage:
* const report = await runMutationTesting(fastify, { depth: 'quick' })
* const report = await runMutationTesting(fastify, { runs: 10 })
* console.log(`Mutation score: ${report.score}%`)
*/
import type { FastifyInstance } from 'fastify'
@@ -44,7 +44,7 @@ export interface MutationReport {
readonly weakContracts: string[] // contracts that survived all mutations
}
export interface MutationConfig {
readonly depth?: TestConfig['depth']
readonly runs?: number
readonly seed?: number
/** Max mutations per contract (default: 5) */
readonly maxMutationsPerContract?: number
@@ -214,7 +214,7 @@ export async function runMutationTesting(
const suite = await runPetitTestsWithMutation(
fastify as unknown as FastifyInjectInstance,
{
depth: config.depth ?? 'quick',
runs: config.runs ?? 10,
seed: config.seed,
},
mutatedContract
@@ -260,13 +260,13 @@ export async function runMutationTesting(
*/
async function runPetitTestsWithMutation(
fastify: FastifyInjectInstance,
config: { depth?: TestConfig['depth']; seed?: number },
config: { runs?: number; seed?: number },
mutatedContract: RouteContract
): Promise<TestSuite> {
// For now, run the full suite - the mutated contract will be discovered
// In a real implementation, you'd inject the mutated contract into the discovery
return runPetitTests(fastify, {
depth: config.depth ?? 'quick',
runs: config.runs ?? 10,
seed: config.seed,
routes: [`${mutatedContract.method} ${mutatedContract.path}`],
})
@@ -279,14 +279,14 @@ export async function testMutation(
fastify: FastifyInstance,
contract: RouteContract,
mutation: Mutation,
config: Pick<MutationConfig, 'depth' | 'seed'> = {}
config: Pick<MutationConfig, 'runs' | 'seed'> = {}
): Promise<boolean> {
const mutatedContract = applyMutation(contract, mutation)
try {
const suite = await runPetitTestsWithMutation(
fastify as unknown as FastifyInjectInstance,
{
depth: config.depth ?? 'quick',
runs: config.runs ?? 10,
seed: config.seed,
},
mutatedContract
+7 -11
View File
@@ -57,7 +57,7 @@ async function expectLoadConfigError(
test('schema: accepts minimal valid configs', () => {
validateConfigAgainstSchema({}, CONFIG_SCHEMA);
validateConfigAgainstSchema({ mode: 'verify', routes: ['GET /users'], seed: 42 }, CONFIG_SCHEMA);
validateConfigAgainstSchema({ presets: { quick: { depth: 'quick', timeout: 5000 } } }, CONFIG_SCHEMA);
validateConfigAgainstSchema({ presets: { quick: { runs: 10, timeout: 5000 } } }, CONFIG_SCHEMA);
});
test('schema: rejects unknown keys with guidance', () => {
@@ -131,11 +131,7 @@ test('schema: rejects enum and numeric range violations with clear guidance', ()
path: 'profiles.default.mode',
expectedGuidance: ['verify', 'observe', 'qualify'],
},
{
value: { presets: { quick: { depth: 'super-deep' } } },
path: 'presets.quick.depth',
expectedGuidance: ['quick', 'standard', 'deep'],
},
] as const;
for (const c of enumCases) {
@@ -215,9 +211,9 @@ test('semantic: validates cross-reference and value rules', () => {
{ value: { seed: 3.14 }, path: 'seed', guidance: 'integer' },
{ value: { presets: { quick: { timeout: -100 } } }, path: 'presets.quick.timeout', guidance: 'non-negative' },
{
value: { presets: { quick: { depth: 'super-deep' } } },
path: 'presets.quick.depth',
guidance: 'quick, standard, deep',
value: { presets: { quick: { timeout: -100 } } },
path: 'presets.quick.timeout',
guidance: 'non-negative',
},
] as const;
@@ -232,7 +228,7 @@ test('semantic: validates cross-reference and value rules', () => {
{ routes: ['GET /users', 'POST /items'] },
{ seed: -42 },
{ seed: 0 },
{ presets: { quick: { timeout: 0, depth: 'standard' } } },
{ presets: { quick: { timeout: 0 } } },
];
for (const value of acceptCases) {
@@ -322,7 +318,7 @@ test('loadConfig: resolves profile and preset and applies profile overrides', as
},
presets: {
quick: {
depth: 'quick',
runs: 10,
timeout: 5000,
parallel: false,
},
+1 -164
View File
@@ -23,12 +23,10 @@
* - Property and state model-based testing focused on confidence
* - Iterative small steps with rapid feedback loops
*/
import { test } from 'node:test';
import assert from 'node:assert';
import { writeFileSync, readFileSync } from 'node:fs';
import { resolve } from 'node:path';
import {
migrateCommand,
detectAllLegacyPatterns,
@@ -36,31 +34,25 @@ import {
type MigrateOptions,
type MigrationItem,
} from '../../cli/commands/migrate/index.js';
import {
rewriteConfigFile,
detectLegacyConfigFields,
detectLegacyFieldsNoEquivalent,
detectMixedLegacyModernFields,
} from '../../cli/commands/migrate/rewriters/config-rewriter.js';
import {
rewriteRouteAnnotations,
detectLegacyRouteAnnotations,
detectAmbiguousRoutePatterns,
} from '../../cli/commands/migrate/rewriters/route-rewriter.js';
import {
rewriteCodePatterns,
detectLegacyCodePatterns,
detectAmbiguousCodePatterns,
} from '../../cli/commands/migrate/rewriters/code-rewriter.js';
import { createTempDir, cleanup, makeCtx } from './helpers.js';
test('migrate --check detects broad legacy config field set', async () => {
const dir = createTempDir();
try {
const legacyConfig = `export default {
testMode: "verify",
@@ -82,12 +74,9 @@ test('migrate --check detects broad legacy config field set', async () => {
},
},
};`;
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
const ctx = makeCtx({ cwd: dir });
const result = await migrateCommand({ check: true }, ctx);
assert.strictEqual(result.exitCode, 1, 'Should exit 1 when legacy patterns are found');
const legacyNames = result.items.map((item) => item.legacy);
assert.ok(legacyNames.includes('testMode'), 'Should detect testMode');
@@ -103,29 +92,23 @@ test('migrate --check detects broad legacy config field set', async () => {
cleanup(dir);
}
});
// ---------------------------------------------------------------------------
// Test 1: Mixed legacy and modern config detection
// ---------------------------------------------------------------------------
test('migrate detects mixed legacy and modern config fields', async () => {
const dir = createTempDir();
try {
// Config with both legacy and modern fields present
const mixedConfig = `export default {
// Legacy field
testMode: "verify",
// Modern field (conflicts with legacy)
mode: "observe",
profiles: {
quick: {
preset: "safe-ci",
},
},
// Legacy container
testProfiles: {
old: {
@@ -133,22 +116,17 @@ test('migrate detects mixed legacy and modern config fields', async () => {
},
},
};`;
writeFileSync(resolve(dir, 'apophis.config.js'), mixedConfig);
const ctx = makeCtx({ cwd: dir });
const result = await migrateCommand({ check: true }, ctx);
// Should detect legacy patterns
assert.strictEqual(result.exitCode, 1, 'Should exit 1 when legacy patterns found');
assert.ok(result.items.length > 0, 'Should detect legacy items');
// Check that mixed fields are reported
const legacyNames = result.items.map((item) => item.legacy);
assert.ok(legacyNames.includes('testMode'), 'Should detect testMode');
assert.ok(legacyNames.includes('testProfiles'), 'Should detect testProfiles');
assert.ok(legacyNames.includes('usesPreset'), 'Should detect usesPreset');
// Verify guidance mentions the conflict
const testModeItem = result.items.find((item) => item.legacy === 'testMode');
assert.ok(testModeItem, 'Should have testMode item');
@@ -157,19 +135,15 @@ test('migrate detects mixed legacy and modern config fields', async () => {
cleanup(dir);
}
});
// ---------------------------------------------------------------------------
// Test 2: Dry-run shows exact rewrites
// ---------------------------------------------------------------------------
test('migrate dry-run shows exact file path, line number, legacy text, replacement text', async () => {
const dir = createTempDir();
try {
const legacyConfig = `export default {
// Line 2
testMode: "verify",
profiles: {
quick: {
// Line 7
@@ -177,36 +151,27 @@ test('migrate dry-run shows exact file path, line number, legacy text, replaceme
},
},
};`;
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
const ctx = makeCtx({ cwd: dir });
const result = await migrateCommand({ dryRun: true }, ctx);
assert.strictEqual(result.exitCode, 1, 'Should exit 1 when legacy patterns found');
assert.ok(result.message, 'Should have output message');
// Verify dry-run output contains exact details
assert.ok(result.message.includes('Dry run'), 'Should indicate dry run');
assert.ok(result.message.includes('testMode'), 'Should show legacy text');
assert.ok(result.message.includes('mode'), 'Should show replacement text');
assert.ok(result.message.includes('usesPreset'), 'Should show usesPreset');
assert.ok(result.message.includes('preset'), 'Should show preset replacement');
// Verify file path is shown
assert.ok(result.message.includes('apophis.config.js'), 'Should show file path');
// Verify line numbers are shown
assert.ok(result.message.includes(':2') || result.message.includes(': 2'), 'Should show line number');
// Verify total count
assert.ok(result.message.includes('Total:'), 'Should show total count');
assert.ok(result.message.includes('3'), 'Should show correct total (3 items)');
// Verify files would be modified
assert.ok(result.filesWouldBeModified, 'Should list files that would be modified');
assert.strictEqual(result.filesWouldBeModified.length, 1, 'Should show 1 file would be modified');
// Verify file was NOT modified
const content = readFileSync(resolve(dir, 'apophis.config.js'), 'utf-8');
assert.ok(content.includes('testMode'), 'File should still have testMode');
@@ -215,14 +180,11 @@ test('migrate dry-run shows exact file path, line number, legacy text, replaceme
cleanup(dir);
}
});
// ---------------------------------------------------------------------------
// Test 3: Write performs rewrites correctly
// ---------------------------------------------------------------------------
test('migrate write performs rewrites correctly', async () => {
const dir = createTempDir();
try {
const legacyConfig = `export default {
testMode: "verify",
@@ -232,16 +194,12 @@ test('migrate write performs rewrites correctly', async () => {
},
},
};`;
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
const ctx = makeCtx({ cwd: dir });
const result = await migrateCommand({ write: true }, ctx);
assert.strictEqual(result.exitCode, 1, 'Should exit 1 when rewrites performed');
assert.ok(result.completed.length > 0, 'Should have completed items');
assert.ok(result.filesModified && result.filesModified.length > 0, 'Should list modified files');
// Verify file WAS modified
const content = readFileSync(resolve(dir, 'apophis.config.js'), 'utf-8');
assert.ok(!content.includes('testMode'), 'File should not have testMode');
@@ -254,35 +212,26 @@ test('migrate write performs rewrites correctly', async () => {
cleanup(dir);
}
});
// ---------------------------------------------------------------------------
// Test 4: Ambiguous rewrite stops and shows context
// ---------------------------------------------------------------------------
test('migrate ambiguous rewrite stops and shows surrounding context', async () => {
const dir = createTempDir();
try {
// Create a file with an ambiguous code pattern
const ambiguousCode = `import Fastify from 'fastify';
const app = Fastify();
// This is ambiguous: what does oldApi() mean here?
app.register(oldApi());
export default app;`;
writeFileSync(resolve(dir, 'app.js'), ambiguousCode);
// Also create a config file so migration has something to work with
const config = `export default {
mode: "verify",
};`;
writeFileSync(resolve(dir, 'apophis.config.js'), config);
const ctx = makeCtx({ cwd: dir });
const result = await migrateCommand({ write: true }, ctx);
// Should stop with exit code 2 (USAGE_ERROR) because ambiguous patterns found
assert.strictEqual(result.exitCode, 2, 'Should exit 2 when ambiguous patterns found in write mode');
assert.ok(result.remaining.length > 0, 'Should have remaining items');
@@ -290,21 +239,17 @@ export default app;`;
assert.ok(result.message.includes('Ambiguous'), 'Should mention ambiguous patterns');
assert.ok(result.message.includes('oldApi()'), 'Should show the ambiguous pattern');
assert.ok(result.message.includes('manual choice'), 'Should mention manual choice');
// Verify context is shown (surrounding lines)
assert.ok(result.message.includes('app.register'), 'Should show surrounding context');
} finally {
cleanup(dir);
}
});
// ---------------------------------------------------------------------------
// Test 5: Legacy field with no equivalent emits guidance
// ---------------------------------------------------------------------------
test('migrate legacy field with no direct equivalent emits human guidance', async () => {
const dir = createTempDir();
try {
// Config with a legacy field that has no direct equivalent
const legacyConfig = `export default {
@@ -317,16 +262,12 @@ test('migrate legacy field with no direct equivalent emits human guidance', asyn
// This field is deprecated with no direct equivalent
legacyField: true,
};`;
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
const ctx = makeCtx({ cwd: dir });
const result = await migrateCommand({ check: true }, ctx);
// Should detect the legacy field with no equivalent
assert.strictEqual(result.exitCode, 1, 'Should exit 1 when legacy patterns found');
assert.ok(result.items.length > 0, 'Should detect legacy items');
const legacyFieldItem = result.items.find((item) => item.legacy === 'legacyField');
assert.ok(legacyFieldItem, 'Should detect legacyField');
assert.ok(legacyFieldItem.guidance, 'Should have guidance for legacyField');
@@ -343,14 +284,11 @@ test('migrate legacy field with no direct equivalent emits human guidance', asyn
cleanup(dir);
}
});
// ---------------------------------------------------------------------------
// Test 6: Partial migration reports completed and remaining
// ---------------------------------------------------------------------------
test('migrate partial migration reports completed and remaining items', async () => {
const dir = createTempDir();
try {
const legacyConfig = `export default {
testMode: "verify",
@@ -360,12 +298,9 @@ test('migrate partial migration reports completed and remaining items', async ()
},
},
};`;
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
const ctx = makeCtx({ cwd: dir });
const result = await migrateCommand({ write: true }, ctx);
assert.ok(result.completed.length > 0, 'Should have completed items');
assert.ok(result.message, 'Should have output message');
assert.ok(result.message.includes('Completed'), 'Should mention completed');
@@ -374,20 +309,16 @@ test('migrate partial migration reports completed and remaining items', async ()
cleanup(dir);
}
});
// ---------------------------------------------------------------------------
// Test 7: Preserves comments/formatting where feasible
// ---------------------------------------------------------------------------
test('migrate preserves comments and formatting where feasible', async () => {
const dir = createTempDir();
try {
// Config with specific formatting (comments, indentation)
const legacyConfig = `export default {
// This is a comment about testMode
testMode: "verify",
/*
* Block comment about testProfiles
*/
@@ -398,19 +329,14 @@ test('migrate preserves comments and formatting where feasible', async () => {
},
},
};`;
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
const ctx = makeCtx({ cwd: dir });
const result = await migrateCommand({ write: true }, ctx);
const content = readFileSync(resolve(dir, 'apophis.config.js'), 'utf-8');
// Verify comments are preserved
assert.ok(content.includes('// This is a comment about testMode'), 'Should preserve line comment');
assert.ok(content.includes('Block comment about testProfiles'), 'Should preserve block comment');
assert.ok(content.includes('// Inline comment'), 'Should preserve inline comment');
// Verify replacements were made
assert.ok(content.includes('mode:'), 'Should have mode');
assert.ok(content.includes('profiles:'), 'Should have profiles');
@@ -419,14 +345,11 @@ test('migrate preserves comments and formatting where feasible', async () => {
cleanup(dir);
}
});
// ---------------------------------------------------------------------------
// Test 8: Migrate exits 0 when config is already modern
// ---------------------------------------------------------------------------
test('migrate exits 0 when config is already modern', async () => {
const dir = createTempDir();
try {
const modernConfig = `export default {
mode: "verify",
@@ -438,7 +361,7 @@ test('migrate exits 0 when config is already modern', async () => {
},
presets: {
"safe-ci": {
depth: "quick",
,
timeout: 5000,
},
},
@@ -448,12 +371,9 @@ test('migrate exits 0 when config is already modern', async () => {
},
},
};`;
writeFileSync(resolve(dir, 'apophis.config.js'), modernConfig);
const ctx = makeCtx({ cwd: dir });
const result = await migrateCommand({ check: true }, ctx);
assert.strictEqual(result.exitCode, 0, 'Should exit 0 for modern config');
assert.strictEqual(result.items.length, 0, 'Should have no items');
assert.ok(result.message, 'Should have message');
@@ -462,35 +382,25 @@ test('migrate exits 0 when config is already modern', async () => {
cleanup(dir);
}
});
// ---------------------------------------------------------------------------
// Test 9: Migrate exits 2 when ambiguous in write mode
// ---------------------------------------------------------------------------
test('migrate exits 2 when ambiguous patterns found in write mode', async () => {
const dir = createTempDir();
try {
const config = `export default {
mode: "verify",
};`;
writeFileSync(resolve(dir, 'apophis.config.js'), config);
// Create app with an ambiguous pattern
const code = `import Fastify from 'fastify';
const app = Fastify();
// Ambiguous pattern
app.register(oldApi());
export default app;`;
writeFileSync(resolve(dir, 'app.js'), code);
const ctx = makeCtx({ cwd: dir });
const result = await migrateCommand({ write: true }, ctx);
// Should exit 2 because ambiguous patterns found
assert.strictEqual(result.exitCode, 2, 'Should exit 2 when ambiguous patterns found in write mode');
assert.ok(result.remaining.length > 0, 'Should have remaining ambiguous items');
@@ -499,14 +409,11 @@ export default app;`;
cleanup(dir);
}
});
// ---------------------------------------------------------------------------
// Test 10: Migrate emits guidance for each legacy field
// ---------------------------------------------------------------------------
test('migrate emits guidance for each legacy field', async () => {
const dir = createTempDir();
try {
const legacyConfig = `export default {
testMode: "verify",
@@ -516,14 +423,10 @@ test('migrate emits guidance for each legacy field', async () => {
},
},
};`;
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
const ctx = makeCtx({ cwd: dir });
const result = await migrateCommand({ check: true }, ctx);
assert.ok(result.items.length > 0, 'Should have items');
for (const item of result.items) {
assert.ok(item.guidance, `Item ${item.legacy} should have guidance`);
assert.ok(
@@ -535,14 +438,11 @@ test('migrate emits guidance for each legacy field', async () => {
cleanup(dir);
}
});
// ---------------------------------------------------------------------------
// Test 11: Config rewriter replaces legacy fields
// ---------------------------------------------------------------------------
test('config rewriter replaces legacy fields', () => {
const dir = createTempDir();
try {
const content = `export default {
testMode: "verify",
@@ -552,17 +452,13 @@ test('config rewriter replaces legacy fields', () => {
},
},
};`;
writeFileSync(resolve(dir, 'test.config.js'), content);
const items = detectLegacyConfigFields(content, 'test.config.js');
assert.strictEqual(items.length, 3, 'Should detect 3 legacy fields');
const result = rewriteConfigFile(
resolve(dir, 'test.config.js'),
items,
);
assert.strictEqual(result.modified, true, 'Should modify content');
assert.ok(result.content.includes('mode:'), 'Should have mode');
assert.ok(result.content.includes('profiles:'), 'Should have profiles');
@@ -572,35 +468,28 @@ test('config rewriter replaces legacy fields', () => {
cleanup(dir);
}
});
// ---------------------------------------------------------------------------
// Test 12: Route rewriter detects x-validate-runtime annotation
// ---------------------------------------------------------------------------
test('route rewriter detects x-validate-runtime annotation', () => {
const dir = createTempDir();
try {
const content = `export default {
schema: {
'x-validate-runtime': true,
},
};`;
writeFileSync(resolve(dir, 'test.routes.js'), content);
const items = detectLegacyRouteAnnotations(content, 'test.routes.js');
assert.strictEqual(items.length, 1, 'Should detect 1 legacy annotation');
const firstItem = items[0];
assert.ok(firstItem, 'Expected one migration item');
assert.strictEqual(firstItem.legacy, 'x-validate-runtime');
assert.strictEqual(firstItem.replacement, 'runtime');
const result = rewriteRouteAnnotations(
resolve(dir, 'test.routes.js'),
items,
);
assert.strictEqual(result.modified, true, 'Should modify content');
assert.ok(result.content.includes("'runtime'"), 'Should have runtime');
assert.ok(!result.content.includes('x-validate-runtime'), 'Should not have legacy annotation');
@@ -608,34 +497,25 @@ test('route rewriter detects x-validate-runtime annotation', () => {
cleanup(dir);
}
});
// ---------------------------------------------------------------------------
// Test 13: Code rewriter detects legacy patterns
// ---------------------------------------------------------------------------
test('code rewriter detects legacy patterns', () => {
const dir = createTempDir();
try {
const content = `import Fastify from 'fastify';
const app = Fastify();
app.register(contract());
app.register(stateful());
app.register(scenario());
export default app;`;
writeFileSync(resolve(dir, 'test.app.js'), content);
const items = detectLegacyCodePatterns(content, 'test.app.js');
assert.strictEqual(items.length, 3, 'Should detect 3 legacy patterns');
const result = rewriteCodePatterns(
resolve(dir, 'test.app.js'),
items,
);
assert.strictEqual(result.modified, true, 'Should modify content');
assert.ok(result.content.includes("verify({ kind: 'contract' })"), 'Should have verify');
assert.ok(result.content.includes("qualify({ kind: 'stateful' })"), 'Should have qualify stateful');
@@ -644,28 +524,21 @@ export default app;`;
cleanup(dir);
}
});
// ---------------------------------------------------------------------------
// Test 14: Dry-run default mode (safe by default)
// ---------------------------------------------------------------------------
test('migrate defaults to dry-run mode (safe by default)', async () => {
const dir = createTempDir();
try {
const legacyConfig = `export default {
testMode: "verify",
};`;
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
const ctx = makeCtx({ cwd: dir });
// No mode specified — should default to dry-run
const result = await migrateCommand({}, ctx);
assert.strictEqual(result.exitCode, 1, 'Should exit 1 in dry-run mode');
assert.ok(result.message?.includes('Dry run'), 'Should indicate dry run');
// Verify file was NOT modified
const content = readFileSync(resolve(dir, 'apophis.config.js'), 'utf-8');
assert.ok(content.includes('testMode'), 'File should still have testMode');
@@ -673,38 +546,30 @@ test('migrate defaults to dry-run mode (safe by default)', async () => {
cleanup(dir);
}
});
// ---------------------------------------------------------------------------
// Test 15: Mixed legacy/modern field detection at rewriter level
// ---------------------------------------------------------------------------
test('config rewriter detects mixed legacy and modern fields', () => {
const dir = createTempDir();
try {
const content = `export default {
// Both legacy and modern present
testMode: "verify",
mode: "observe",
testProfiles: {
quick: {
usesPreset: "safe-ci",
},
},
profiles: {
modern: {
preset: "safe-ci",
},
},
};`;
writeFileSync(resolve(dir, 'test.config.js'), content);
const mixedReports = detectMixedLegacyModernFields(content, 'test.config.js');
assert.ok(mixedReports.length > 0, 'Should detect mixed fields');
const testModeReport = mixedReports.find((r) => r.legacy === 'testMode');
assert.ok(testModeReport, 'Should report testMode as mixed');
assert.ok(testModeReport.guidance.includes('testMode'), 'Guidance should mention testMode');
@@ -713,14 +578,11 @@ test('config rewriter detects mixed legacy and modern fields', () => {
cleanup(dir);
}
});
// ---------------------------------------------------------------------------
// Test 16: Ambiguous route pattern detection
// ---------------------------------------------------------------------------
test('route rewriter detects ambiguous route patterns with context', () => {
const dir = createTempDir();
try {
const content = `export default {
schema: {
@@ -728,12 +590,9 @@ test('route rewriter detects ambiguous route patterns with context', () => {
'x-validate': true,
},
};`;
writeFileSync(resolve(dir, 'test.routes.js'), content);
const items = detectAmbiguousRoutePatterns(content, 'test.routes.js');
assert.strictEqual(items.length, 1, 'Should detect 1 ambiguous pattern');
const firstItem = items[0];
assert.ok(firstItem, 'Expected one migration item');
assert.strictEqual(firstItem.legacy, 'x-validate');
@@ -744,28 +603,20 @@ test('route rewriter detects ambiguous route patterns with context', () => {
cleanup(dir);
}
});
// ---------------------------------------------------------------------------
// Test 17: Ambiguous code pattern detection with context
// ---------------------------------------------------------------------------
test('code rewriter detects ambiguous code patterns with surrounding context', () => {
const dir = createTempDir();
try {
const content = `import Fastify from 'fastify';
const app = Fastify();
// Ambiguous pattern
app.register(oldApi());
export default app;`;
writeFileSync(resolve(dir, 'test.app.js'), content);
const items = detectAmbiguousCodePatterns(content, 'test.app.js');
assert.strictEqual(items.length, 1, 'Should detect 1 ambiguous pattern');
const firstItem = items[0];
assert.ok(firstItem, 'Expected one migration item');
assert.strictEqual(firstItem.legacy, 'oldApi()');
@@ -777,42 +628,32 @@ export default app;`;
cleanup(dir);
}
});
// ---------------------------------------------------------------------------
// Test 18: Legacy fixture detection
// ---------------------------------------------------------------------------
test('migrate detects legacy patterns in fixture config', async () => {
const ctx = makeCtx({ cwd: 'src/cli/__fixtures__/legacy-config' });
const result = await migrateCommand({ check: true }, ctx);
assert.strictEqual(result.exitCode, 1, 'Should detect legacy patterns in fixture');
assert.ok(result.items.length > 0, 'Should find legacy items');
const legacyNames = result.items.map((item) => item.legacy);
assert.ok(legacyNames.includes('testMode'), 'Should detect testMode in fixture');
assert.ok(legacyNames.includes('testProfiles'), 'Should detect testProfiles in fixture');
assert.ok(legacyNames.includes('testPresets'), 'Should detect testPresets in fixture');
assert.ok(legacyNames.includes('envPolicies'), 'Should detect envPolicies in fixture');
});
// ---------------------------------------------------------------------------
// Test 19: JSON output format
// ---------------------------------------------------------------------------
test('migrate outputs JSON format with all fields', async () => {
const dir = createTempDir();
try {
const legacyConfig = `export default {
testMode: "verify",
};`;
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
const ctx = makeCtx({ cwd: dir, options: { ...makeCtx().options, format: 'json' } });
const result = await migrateCommand({ check: true }, ctx);
assert.strictEqual(result.exitCode, 1, 'Should exit 1');
assert.ok(result.items.length > 0, 'Should have items');
assert.ok(result.totalRewrites, 'Should have totalRewrites');
@@ -821,18 +662,14 @@ test('migrate outputs JSON format with all fields', async () => {
cleanup(dir);
}
});
// ---------------------------------------------------------------------------
// Test 20: No files found returns usage error
// ---------------------------------------------------------------------------
test('migrate returns usage error when no files found', async () => {
const dir = createTempDir();
try {
const ctx = makeCtx({ cwd: dir });
const result = await migrateCommand({ check: true }, ctx);
assert.strictEqual(result.exitCode, 2, 'Should exit 2 when no files found');
assert.ok(result.message?.includes('No config or app files found'), 'Should mention no files found');
} finally {
+7 -7
View File
@@ -152,7 +152,7 @@ test('contract runner supports cross-operation APOSTL ensures', async () => {
await fastify.register(apophisPlugin, {})
registerItemApi(fastify)
await fastify.ready()
const result = await fastify.apophis.contract({ depth: 'quick', seed: 7 })
const result = await fastify.apophis.contract({ runs: 10, seed: 7 })
const failures = result.tests.filter((entry: TestResult) => !entry.ok)
assert.strictEqual(failures.length, 0)
} finally {
@@ -169,7 +169,7 @@ test('stateful runner supports cross-operation APOSTL ensures', async () => {
await fastify.register(apophisPlugin, {})
registerItemApi(fastify)
await fastify.ready()
const result = await fastify.apophis.stateful({ depth: 'quick', seed: 11 })
const result = await fastify.apophis.stateful({ runs: 10, seed: 11 })
const failures = result.tests.filter((entry: TestResult) => !entry.ok)
assert.strictEqual(failures.length, 0)
} finally {
@@ -217,7 +217,7 @@ test('contract runner applies chaos injection when configured', async () => {
}, async () => ({ ok: true }))
await fastify.ready()
const result = await fastify.apophis.contract({
depth: 'quick',
runs: 10,
seed: 3,
chaos: {
probability: 1,
@@ -241,7 +241,7 @@ test('contract runner supports previous(...) with response_body(this) path place
await fastify.register(apophisPlugin, {})
registerPlanApi(fastify)
await fastify.ready()
const result = await fastify.apophis.contract({ depth: 'quick', seed: 17 })
const result = await fastify.apophis.contract({ runs: 10, seed: 17 })
const failures = result.tests.filter((entry: TestResult) => !entry.ok)
assert.strictEqual(failures.length, 0)
} finally {
@@ -258,7 +258,7 @@ test('stateful runner supports previous(...) with response_body(this) path place
await fastify.register(apophisPlugin, {})
registerPlanApi(fastify)
await fastify.ready()
const result = await fastify.apophis.stateful({ depth: 'quick', seed: 19 })
const result = await fastify.apophis.stateful({ runs: 10, seed: 19 })
const failures = result.tests.filter((entry: TestResult) => !entry.ok)
assert.strictEqual(failures.length, 0)
} finally {
@@ -406,7 +406,7 @@ test('contract runner validates hypermedia links with route_exists', async () =>
})
registerHypermediaApi(fastify)
await fastify.ready()
const result = await fastify.apophis.contract({ depth: 'quick', seed: 23 })
const result = await fastify.apophis.contract({ runs: 10, seed: 23 })
const failures = result.tests.filter((entry: TestResult) => !entry.ok)
if (failures.length > 0) {
console.log('Contract failures:', failures.map((f: TestResult) => ({
@@ -441,7 +441,7 @@ test('stateful runner validates cross-route relationships', async () => {
})
registerHypermediaApi(fastify)
await fastify.ready()
const result = await fastify.apophis.stateful({ depth: 'quick', seed: 29 })
const result = await fastify.apophis.stateful({ runs: 10, seed: 29 })
const failures = result.tests.filter((entry: TestResult) => !entry.ok)
if (failures.length > 0) {
console.log('Stateful failures:', failures.map((f: TestResult) => ({
+2 -2
View File
@@ -36,7 +36,7 @@ test('APOPHIS_DEBUG=1 logs requests and responses', async () => {
originalDebug(msg, _obj)
}
const result = await fastify.apophis.contract({ depth: 'quick' })
const result = await fastify.apophis.contract({ runs: 10 })
assert.ok(result.tests.length > 0, 'should have tests')
// Should have logged at least one request and one response
@@ -80,7 +80,7 @@ test('APOPHIS_DEBUG=0 does not log requests', async () => {
originalDebug(msg, _obj)
}
const result = await fastify.apophis.contract({ depth: 'quick' })
const result = await fastify.apophis.contract({ runs: 10 })
assert.ok(result.tests.length > 0, 'should have tests')
// Should not have any request/response logs
+3 -3
View File
@@ -30,7 +30,7 @@ test('example: minimal API compiles and runs', async () => {
await fastify.ready()
const result = await fastify.apophis.contract({ depth: 'quick' })
const result = await fastify.apophis.contract({ runs: 10 })
assert.ok(result.tests.length > 0, 'should have test results')
console.log('Minimal example:', result.summary)
} finally {
@@ -111,7 +111,7 @@ test('example: CRUD API with contracts compiles and runs', async () => {
await fastify.ready()
const result = await fastify.apophis.contract({ depth: 'quick' })
const result = await fastify.apophis.contract({ runs: 10 })
assert.ok(result.tests.length > 0, 'should have test results')
console.log('CRUD example:', result.summary)
} finally {
@@ -149,7 +149,7 @@ test('example: prefix registration works', async () => {
const itemContract = contracts.find(c => c.path === '/api/v1/items')
assert.ok(itemContract, 'should discover prefixed route')
const result = await fastify.apophis.contract({ depth: 'quick' })
const result = await fastify.apophis.contract({ runs: 10 })
assert.ok(result.tests.length > 0, 'should run tests on prefixed routes')
} finally {
await fastify.close()
+9 -9
View File
@@ -243,7 +243,7 @@ test('petit-runner executes tests against real API', async () => {
]
const fastifyWithRoutes = Object.assign(fastify, { routes: mockRoutes })
const result = await runPetitTests(fastifyWithRoutes as any, {
depth: 'quick',
runs: 10,
scope: undefined,
seed: undefined
})
@@ -367,7 +367,7 @@ test('full integration: plugin + routes + test execution', async () => {
const createUserContract = contracts.find(c => c.path === '/users' && c.method === 'POST')
assert.ok(createUserContract, 'create user contract should exist')
assert.strictEqual(createUserContract.category, 'constructor')
const testResult = await fastify.apophis.contract({ depth: 'quick' })
const testResult = await fastify.apophis.contract({ runs: 10 })
assert.ok(Array.isArray(testResult.tests), 'tests should be an array')
assert.ok(testResult.tests.length > 0, 'tests should not be empty')
await fastify.apophis.cleanup()
@@ -439,7 +439,7 @@ test('mode filtering: stateful mode only runs constructor/mutator routes', async
}, async () => ({ status: 'ok' }))
await fastify.ready()
// Run in stateful mode
const result = await fastify.apophis.contract({ depth: 'quick' })
const result = await fastify.apophis.contract({ runs: 10 })
// In stateful mode, utility routes should be excluded
// The test should only run constructor and mutator routes
assert.ok(Array.isArray(result.tests), 'tests should be an array')
@@ -474,7 +474,7 @@ test('failing contract produces ContractViolation with suggestion', async () =>
return { status: 'created' } // Returns 200, not 201
})
await fastify.ready()
const result = await fastify.apophis.contract({ depth: 'quick' })
const result = await fastify.apophis.contract({ runs: 10 })
// Find the failing test
const failingTests = result.tests.filter(t => !t.ok)
assert.ok(failingTests.length > 0, 'should have at least one failing test')
@@ -647,7 +647,7 @@ test('integration: contract routes option limits tested routes', async () => {
}, async () => ({ ok: true }))
await fastify.ready()
const result = await fastify.apophis.contract({
depth: 'quick',
runs: 10,
routes: ['GET /included'],
})
const includedTests = result.tests.filter(t => t.name.includes('GET /included'))
@@ -673,7 +673,7 @@ test('integration: contract variants are tagged and run in declared order', asyn
}, async () => ({ ok: true }))
await fastify.ready()
const result = await fastify.apophis.contract({
depth: 'quick',
runs: 10,
variants: [
{ name: 'json', headers: { accept: 'application/json' } },
{ name: 'xml', headers: { accept: 'application/xml' } },
@@ -715,7 +715,7 @@ test('integration: variant headers override scope headers', async () => {
}, async () => ({ ok: true }))
await fastify.ready()
const result = await fastify.apophis.contract({
depth: 'quick',
runs: 10,
variants: [
{ name: 'xml', headers: { accept: 'application/xml' } },
],
@@ -744,7 +744,7 @@ test('integration: route-level x-variants are extracted and executed', async ()
}, async () => ({ ok: true }))
await fastify.ready()
// No call-site variants; route-level variants should drive execution
const result = await fastify.apophis.contract({ depth: 'quick' })
const result = await fastify.apophis.contract({ runs: 10 })
const jsonTests = result.tests.filter((t) => t.name.includes('[variant:json]'))
const xmlTests = result.tests.filter((t) => t.name.includes('[variant:xml]'))
assert.ok(jsonTests.length > 0, 'route json variant should produce tests')
@@ -781,7 +781,7 @@ test('integration: inferred contracts are guarded by status code', async () => {
return { error: 'not found' }
})
await fastify.ready()
const result = await fastify.apophis.contract({ depth: 'quick' })
const result = await fastify.apophis.contract({ runs: 10 })
// Should pass because the inferred const contract is guarded:
// response_code(this) == 200 => response_body(this).status == "success"
// The 404 response doesn't trigger the antecedent, so the implication holds.
+7 -10
View File
@@ -7,7 +7,7 @@
import { convertSchema } from '../domain/schema-to-arbitrary.js'
import { lookupCache, storeCache } from '../incremental/cache.js'
import type { ApiCommand } from '../domain/stateful.js'
import type { DepthConfig, RouteContract } from '../types.js'
import type { RouteContract, RunConfig } from '../types.js'
import * as fc from 'fast-check'
const buildCommand = (route: RouteContract, data: unknown): ApiCommand => ({
@@ -19,11 +19,10 @@ const buildCommand = (route: RouteContract, data: unknown): ApiCommand => ({
export const generateCommands = (
routes: RouteContract[],
depth: DepthConfig,
runConfig: RunConfig,
seed?: number,
generationProfile: 'quick' | 'standard' | 'thorough' = 'standard',
): { commands: ApiCommand[][], cacheHits: number, cacheMisses: number } => {
const commandsPerRoute = Math.max(1, Math.floor(depth.contractRuns / Math.max(routes.length, 1)))
const commandsPerRoute = Math.max(1, Math.floor(runConfig.contractRuns / Math.max(routes.length, 1)))
let cacheHits = 0
let cacheMisses = 0
const allCommands = routes.map((route) => {
@@ -40,7 +39,7 @@ export const generateCommands = (
cacheMisses++
const bodySchema = route.schema?.body as Record<string, unknown> | undefined
const bodyArb = bodySchema !== undefined
? convertSchema(bodySchema, { context: 'request', generationProfile })
? convertSchema(bodySchema, { context: 'request' })
: fc.constant({})
const pathParams = route.path.match(/:([a-zA-Z_][a-zA-Z0-9_]*)/g) ?? []
const pathParamArbs: Record<string, fc.Arbitrary<string>> = {}
@@ -82,7 +81,7 @@ import { buildPetitSuite, filterPetitRoutes } from './route-filter.js'
import { executePetitCommandStep } from './petit-command-step.js'
import { runTripleBoundaryPropertyTest } from './triple-boundary-runner.js'
import { makeTrackedResource } from '../domain/state-operations.js'
import { resolveDepth, resolveGenerationProfile } from '../types.js'
import { resolveRuns } from '../types.js'
import type {
EvalContext,
FastifyInjectInstance,
@@ -133,9 +132,8 @@ export const runPetitTests = async (
}
}
const depth = resolveDepth(config.depth ?? 'standard')
const generationProfile = config.generationProfile ?? resolveGenerationProfile(config.depth)
const { commands: commandGroups, cacheHits, cacheMisses } = generateCommands(routes, depth, config.seed, generationProfile)
const runConfig = resolveRuns(config.runs)
const { commands: commandGroups, cacheHits, cacheMisses } = generateCommands(routes, runConfig, config.seed)
const allCommands = commandGroups.flat()
const rng = config.seed !== undefined ? new SeededRng(config.seed) : undefined
@@ -181,7 +179,6 @@ export const runPetitTests = async (
? createOutboundMockRuntime({
contracts: outboundContractRegistry.resolve(Array.from(outboundNames)),
mode: config.outboundMocks?.mode ?? 'example',
generationProfile,
overrides: config.outboundMocks?.overrides,
unmatched: config.outboundMocks?.unmatched ?? 'error',
seed: config.seed !== undefined ? hashCombine(config.seed, 0x6d6f636b) : Math.floor(Math.random() * 0xffffffff),
+6 -6
View File
@@ -7,7 +7,7 @@ import apophisPlugin from '../index.js'
import type { TestResult } from '../types.js'
type TestFastifyInstance = FastifyInstance & {
apophis: {
contract: (opts?: { depth?: string; scope?: string; seed?: number }) => Promise<any>
contract: (opts?: { runs?: number; scope?: string; seed?: number }) => Promise<any>
spec: () => Record<string, unknown>
}
}
@@ -44,19 +44,19 @@ test('scope isolation: routes with x-scope are filtered by scope parameter', asy
}, async () => ({ user: true }))
await fastify.ready()
// Test with no scope - should discover all 3 routes
const allResult = await fastify.apophis.contract({ depth: 'quick', scope: undefined })
const allResult = await fastify.apophis.contract({ runs: 10, scope: undefined })
const allPaths = new Set(allResult.tests.map((t: TestResult) => t.name.split(' ')[1]))
assert.ok(allPaths.has('/public'), 'public route should be in all scope')
assert.ok(allPaths.has('/admin'), 'admin route should be in all scope')
assert.ok(allPaths.has('/user'), 'user route should be in all scope')
// Test with admin scope - should only get public + admin
const adminResult = await fastify.apophis.contract({ depth: 'quick', scope: 'admin' })
const adminResult = await fastify.apophis.contract({ runs: 10, scope: 'admin' })
const adminPaths = new Set(adminResult.tests.map((t: TestResult) => t.name.split(' ')[1]))
assert.ok(adminPaths.has('/public'), 'public route should be in admin scope')
assert.ok(adminPaths.has('/admin'), 'admin route should be in admin scope')
assert.ok(!adminPaths.has('/user'), 'user route should NOT be in admin scope')
// Test with user scope - should only get public + user
const userResult = await fastify.apophis.contract({ depth: 'quick', scope: 'user' })
const userResult = await fastify.apophis.contract({ runs: 10, scope: 'user' })
const userPaths = new Set(userResult.tests.map((t: TestResult) => t.name.split(' ')[1]))
assert.ok(userPaths.has('/public'), 'public route should be in user scope')
assert.ok(!userPaths.has('/admin'), 'admin route should NOT be in user scope')
@@ -88,7 +88,7 @@ test('scope isolation: scope headers are passed to requests', async () => {
headers: { 'x-custom-header': 'test-value' },
metadata: {}
})
await fastify.apophis.contract({ depth: 'quick', scope: 'test' })
await fastify.apophis.contract({ runs: 10, scope: 'test' })
assert.strictEqual(receivedHeaders['x-custom-header'], 'test-value', 'scope header should be passed to request')
} finally {
await fastify.close()
@@ -130,7 +130,7 @@ test('scope isolation: non-matching scope returns empty test suite', async () =>
}, async () => ({ ok: true }))
await fastify.ready()
// Test with non-matching scope
const result = await fastify.apophis.contract({ depth: 'quick', scope: 'other' })
const result = await fastify.apophis.contract({ runs: 10, scope: 'other' })
assert.strictEqual(result.tests.length, 0, 'no tests should run for non-matching scope')
assert.strictEqual(result.summary.passed, 0, 'no tests should pass')
assert.strictEqual(result.summary.failed, 0, 'no tests should fail')
+2 -2
View File
@@ -32,7 +32,7 @@ test('serverless: fastify.ready() without listen works', async () => {
await fastify.ready()
// Should be able to run tests
const result = await fastify.apophis.contract({ depth: 'quick' })
const result = await fastify.apophis.contract({ runs: 10 })
assert.ok(result.tests.length > 0, 'should have tests')
// Should be able to get spec
@@ -115,7 +115,7 @@ test('serverless: multiple ready() calls are safe', async () => {
// Second ready() should be safe (idempotent)
await fastify.ready()
const result = await fastify.apophis.contract({ depth: 'quick' })
const result = await fastify.apophis.contract({ runs: 10 })
assert.ok(result.tests.length > 0, 'should still work after multiple ready() calls')
} finally {
await fastify.close()
+8 -8
View File
@@ -22,7 +22,7 @@ test('stateful runner handles empty routes', async () => {
const result = await runStatefulTests(fastify as any, {
depth: 'quick',
runs: 10,
scope: undefined,
seed: 42,
})
@@ -62,7 +62,7 @@ test('stateful runner executes commands', async () => {
const result = await runStatefulTests(fastify as any, {
depth: 'quick',
runs: 10,
scope: undefined,
seed: 42,
})
@@ -103,7 +103,7 @@ test('stateful runner detects status code violations', async () => {
const result = await runStatefulTests(fastify as any, {
depth: 'quick',
runs: 10,
scope: undefined,
seed: 42,
})
@@ -139,7 +139,7 @@ test('stateful runner evaluates APOSTL formulas', async () => {
const result = await runStatefulTests(fastify as any, {
depth: 'quick',
runs: 10,
scope: undefined,
seed: 42,
})
@@ -189,7 +189,7 @@ test('stateful runner tracks resource state', async () => {
const result = await runStatefulTests(fastify as any, {
depth: 'quick',
runs: 10,
scope: undefined,
seed: 42,
})
@@ -264,7 +264,7 @@ test('stateful runner substitutes path params from resource state', async () =>
await fastify.ready()
const result = await runStatefulTests(fastify as any, {
depth: 'quick',
runs: 10,
scope: undefined,
seed: 42,
})
@@ -308,7 +308,7 @@ test('stateful runner supports config-level variants', async () => {
await fastify.ready()
const result = await runStatefulTests(fastify as any, {
depth: 'quick',
runs: 10,
scope: undefined,
seed: 42,
variants: [
@@ -359,7 +359,7 @@ test('stateful runner supports route-level x-variants', async () => {
await fastify.ready()
const result = await runStatefulTests(fastify as any, {
depth: 'quick',
runs: 10,
scope: undefined,
seed: 42,
})
+7 -10
View File
@@ -6,7 +6,7 @@
* generate → execute → validate → update → check-invariants
*/
import type { ExtensionRegistry } from '../extension/types.js'
import { resolveDepth, resolveGenerationProfile } from '../types.js'
import { resolveRuns } from '../types.js'
import { discoverRoutes } from '../domain/discovery.js'
import { convertSchema } from '../domain/schema-to-arbitrary.js'
import { SeededRng } from '../infrastructure/seeded-rng.js'
@@ -22,7 +22,7 @@ import type { OutboundContractRegistry } from '../domain/outbound-contracts.js'
import * as fc from 'fast-check'
import type { ModelState } from '../domain/stateful.js'
import type { CleanupManager } from '../infrastructure/cleanup-manager.js'
import type { DepthConfig, EvalContext, FastifyInjectInstance, RouteContract, ScopeRegistry, TestConfig, TestResult, TestSuite } from '../types.js'
import type { EvalContext, FastifyInjectInstance, RouteContract, ScopeRegistry, TestConfig, TestResult, TestSuite } from '../types.js'
// Pure: hash helpers for deterministic sub-seeds
// ---------------------------------------------------------------------------
@@ -61,7 +61,6 @@ class ApiOperation implements StatefulApiOperation {
// ============================================================================
const createCommandArbitrary = (
routes: RouteContract[],
generationProfile: 'quick' | 'standard' | 'thorough',
): { arb: fc.Arbitrary<ApiOperation>, cacheHits: number, cacheMisses: number } => {
let cacheHits = 0
let cacheMisses = 0
@@ -74,7 +73,7 @@ const createCommandArbitrary = (
cacheMisses++
const bodySchema = route.schema?.body as Record<string, unknown> | undefined
const arb = bodySchema !== undefined
? convertSchema(bodySchema, { context: 'request', generationProfile })
? convertSchema(bodySchema, { context: 'request' })
: fc.constant({})
return arb.map((params) => new ApiOperation(route, params as Record<string, unknown>))
})
@@ -93,8 +92,7 @@ export const runStatefulTests = async (
outboundContractRegistry?: OutboundContractRegistry
): Promise<TestSuite> => {
const startTime = Date.now()
const depth = resolveDepth(config.depth ?? 'standard')
const generationProfile = config.generationProfile ?? resolveGenerationProfile(config.depth)
const runConfig = resolveRuns(config.runs)
if (extensionRegistry) {
await extensionRegistry.runSuiteStartHooks(config)
}
@@ -151,7 +149,7 @@ export const runStatefulTests = async (
variantName ? `[variant:${variantName}] ${name}` : name
// Get scope headers for test requests
const baseScopeHeaders = scopeRegistry?.getHeaders(config.scope ?? null) ?? {}
const { arb: commandArb, cacheHits, cacheMisses } = createCommandArbitrary(routes, generationProfile)
const { arb: commandArb, cacheHits, cacheMisses } = createCommandArbitrary(routes)
let allResults: TestResult[] = []
let globalTestId = 0
// Create seeded RNG for reproducible path param selection
@@ -178,7 +176,6 @@ export const runStatefulTests = async (
suiteMockRuntime = createOutboundMockRuntime({
contracts: allResolved,
mode: config.outboundMocks?.mode ?? 'example',
generationProfile,
overrides: config.outboundMocks?.overrides,
unmatched: config.outboundMocks?.unmatched ?? 'error',
seed: outboundSeed,
@@ -186,7 +183,7 @@ export const runStatefulTests = async (
suiteMockRuntime.install()
}
// Run property-based stateful tests per variant
const numRuns = depth.statefulRuns
const numRuns = runConfig.statefulRuns
const seed = config.seed
let counterexampleOutput: string | undefined
const hashString = (s: string): number => {
@@ -276,7 +273,7 @@ export const runStatefulTests = async (
}
try {
const prop = fc.asyncProperty(
fc.array(commandArb, { minLength: 1, maxLength: depth.maxCommands }),
fc.array(commandArb, { minLength: 1, maxLength: runConfig.maxCommands }),
async (cmds) => {
await runSequence(cmds)
return true
+4 -5
View File
@@ -22,7 +22,7 @@ import type {
TestConfig,
TestResult,
} from '../types.js'
import { resolveDepth, resolveGenerationProfile } from '../types.js'
import { resolveRuns } from '../types.js'
export const runTripleBoundaryPropertyTest = async (
route: RouteContract,
@@ -39,10 +39,9 @@ export const runTripleBoundaryPropertyTest = async (
if (!config.chaos) return []
const results: TestResult[] = []
const generationProfile = config.generationProfile ?? resolveGenerationProfile(config.depth)
const arbitrary = createTripleBoundaryArbitrary(route, contracts, config.chaos, generationProfile)
const depth = resolveDepth(config.depth ?? 'standard')
const numRuns = Math.max(10, Math.floor(depth.contractRuns / 2))
const arbitrary = createTripleBoundaryArbitrary(route, contracts, config.chaos)
const runConfig = resolveRuns(config.runs)
const numRuns = Math.max(10, Math.floor(runConfig.contractRuns / 2))
const property = fc.asyncProperty(arbitrary, async (cmd) => {
const testId = testIdBase + results.length + 1
+2 -5
View File
@@ -39,18 +39,15 @@ export type {
// Formula and test configuration types
export type {
TestDepth,
TestConfig,
ResolvedOutboundContract,
OutboundChaosConfig,
ChaosConfig,
DepthConfig,
RunConfig,
} from './types/formula.js'
export {
DEPTH_CONFIGS,
resolveDepth,
resolveGenerationProfile,
resolveRuns,
} from './types/formula.js'
// Extension types
+1 -1
View File
@@ -201,7 +201,7 @@ export interface ApophisTestDecorations {
// Forward declarations to avoid circular deps — these are defined in sibling modules
export interface TestConfig {
readonly depth?: import('./formula.js').TestDepth
readonly runs?: number
readonly scope?: string
readonly seed?: number
readonly timeout?: number
+10 -34
View File
@@ -7,11 +7,8 @@
// Test: Configuration
// ============================================================================
export type TestDepth = 'quick' | 'standard' | 'thorough' | { runs: number }
export interface TestConfig {
readonly depth?: TestDepth
readonly generationProfile?: GenerationProfile
readonly runs?: number
readonly scope?: string
readonly seed?: number
readonly timeout?: number
@@ -204,49 +201,28 @@ export interface ChaosConfig {
}
// ============================================================================
// Depth Configuration
// Test run configuration
// ============================================================================
export interface DepthConfig {
export interface RunConfig {
readonly contractRuns: number
readonly propertyRuns: number
readonly statefulRuns: number
readonly maxCommands: number
}
export type GenerationProfile = 'quick' | 'standard' | 'thorough'
const DEFAULT_RUNS = 50
export const DEPTH_CONFIGS: Record<'quick' | 'standard' | 'thorough', DepthConfig> = {
quick: { contractRuns: 10, propertyRuns: 50, statefulRuns: 5, maxCommands: 10 },
standard: { contractRuns: 50, propertyRuns: 100, statefulRuns: 20, maxCommands: 30 },
thorough: { contractRuns: 200, propertyRuns: 1000, statefulRuns: 100, maxCommands: 50 }
}
export function resolveDepth(depth: TestDepth): DepthConfig {
if (typeof depth === 'string') {
return DEPTH_CONFIGS[depth]
}
export function resolveRuns(runs: number | undefined): RunConfig {
const r = runs ?? DEFAULT_RUNS
return {
contractRuns: depth.runs,
propertyRuns: depth.runs,
statefulRuns: Math.max(1, Math.floor(depth.runs / 10)),
maxCommands: Math.max(5, Math.floor(depth.runs / 5)),
contractRuns: r,
propertyRuns: r * 2,
statefulRuns: Math.max(1, Math.floor(r / 10)),
maxCommands: Math.max(5, Math.floor(r / 2)),
}
}
export function resolveGenerationProfile(depth: TestDepth | undefined): GenerationProfile {
if (depth === undefined) {
return 'standard'
}
if (typeof depth === 'string') {
return depth
}
if (depth.runs <= 25) return 'quick'
if (depth.runs >= 250) return 'thorough'
return 'standard'
}
// ============================================================================
// Test: Results
// ============================================================================