From 099576f12a03846cb669596c9aa10f52f079b115 Mon Sep 17 00:00:00 2001 From: John Dvorak Date: Thu, 30 Apr 2026 13:56:39 -0700 Subject: [PATCH] Remove generation profile tiering; use explicit runs count --- README.md | 2 +- SKILL.md | 2 +- docs/chaos.md | 2 +- docs/cli.md | 15 --------------- docs/examples/crud-api.ts | 4 ++-- docs/examples/minimal.ts | 2 +- docs/llm-safe-adoption.md | 1 - docs/observe.md | 1 - docs/performance.md | 10 +++++----- docs/qualify.md | 17 +++++++++++------ docs/verify.md | 22 +++++++++++++++++----- package.json | 2 +- scripts/bench/cli.mjs | 19 +------------------ 13 files changed, 41 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index 2f29b57..49bfef4 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ See [docs/getting-started.md](docs/getting-started.md) for the full walkthrough. ## Trust and Safety - **Deterministic replay**: Every failure includes a seed and a one-command replay. -- **Generation profile aliases**: Control test budget with `--generation-profile quick|standard|deep`. +- **Explicit test budget**: Control how many tests run with `runs: 10` in your preset. - **CI-safe default path**: `verify` is deterministic and safe for CI pipelines. - **Machine-readable output**: `--format json-summary` and `--format ndjson-summary` for CI dashboards. - **Production-safe observe path**: `observe` is non-blocking by default. Blocking behavior requires explicit break-glass policy. diff --git a/SKILL.md b/SKILL.md index 3acf37e..afb54b5 100644 --- a/SKILL.md +++ b/SKILL.md @@ -123,7 +123,7 @@ app.post('/users', { }) await app.ready() -const suite = await app.apophis.contract({ depth: 'standard' }) +const suite = await app.apophis.contract({ runs: 50 }) ``` ## API Surface diff --git a/docs/chaos.md b/docs/chaos.md index cc394cc..39f6080 100644 --- a/docs/chaos.md +++ b/docs/chaos.md @@ -8,7 +8,7 @@ Chaos testing applies the invariant-driven verification approach from [Invariant ```javascript const result = await fastify.apophis.contract({ - depth: 'standard', + runs: 50, chaos: { probability: 0.1, // 10% of requests get chaos delay: { probability: 1, minMs: 100, maxMs: 500 }, diff --git a/docs/cli.md b/docs/cli.md index 92d05dc..4d1d531 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -10,7 +10,6 @@ Every command accepts these flags: |---|---|---| | `--config ` | Config file path | Auto-detect | | `--profile ` | Profile name from config | First profile | -| `--generation-profile ` | Generation budget profile (built-in: quick, standard, deep) | Depth-derived | | `--cwd ` | Working directory override | `process.cwd()` | | `--format ` | Output format: `human`, `json`, `ndjson`, `json-summary`, `ndjson-summary` | `human` | | `--color ` | Color mode: `auto`, `always`, `never` | `auto` | @@ -62,7 +61,6 @@ apophis verify --profile quick --routes "POST /users" | Flag | Description | |---|---| | `--profile ` | Profile name from config | -| `--generation-profile ` | Override generation budget for this run | | `--routes ` | Route filter pattern (comma-separated, supports wildcards) | | `--seed ` | Deterministic seed (generated and printed if omitted) | | `--changed` | Filter to git-modified routes only | @@ -121,7 +119,6 @@ apophis qualify --profile oauth-nightly --seed 42 | Flag | Description | |---|---| | `--profile ` | Profile name from config | -| `--generation-profile ` | Override generation budget for this run | | `--seed ` | Deterministic seed (generated and printed if omitted) | **Examples:** @@ -129,18 +126,6 @@ apophis qualify --profile oauth-nightly --seed 42 ```bash apophis qualify --profile oauth-nightly --seed 42 apophis qualify --profile lifecycle-deep -apophis qualify --profile oauth-nightly --generation-profile quick -``` - -You can define aliases in config: - -```js -export default { - generationProfiles: { - pr: 'quick', - nightly: { base: 'thorough' }, - }, -} ``` ### `apophis replay` diff --git a/docs/examples/crud-api.ts b/docs/examples/crud-api.ts index 9b1411a..089c15a 100644 --- a/docs/examples/crud-api.ts +++ b/docs/examples/crud-api.ts @@ -153,11 +153,11 @@ fastify.delete('/users/:id', { await fastify.ready() // Run contract tests (all non-utility routes, property-based) -const result = await fastify.apophis.contract({ depth: 'standard' }) +const result = await fastify.apophis.contract({ runs: 50 }) console.log('Contract tests:', result.summary) // Run stateful tests (constructor→mutator→destructor sequences) -const stateful = await fastify.apophis.stateful({ depth: 'standard', seed: 42 }) +const stateful = await fastify.apophis.stateful({ runs: 50, seed: 42 }) console.log('Stateful tests:', stateful.summary) // Validate a single route diff --git a/docs/examples/minimal.ts b/docs/examples/minimal.ts index 483e2ff..f08be0e 100644 --- a/docs/examples/minimal.ts +++ b/docs/examples/minimal.ts @@ -22,5 +22,5 @@ fastify.get('/health', { await fastify.ready() // Run contract tests -const result = await fastify.apophis.contract({ depth: 'quick' }) +const result = await fastify.apophis.contract({ runs: 10 }) console.log(result.summary) diff --git a/docs/llm-safe-adoption.md b/docs/llm-safe-adoption.md index bcd7628..954284f 100644 --- a/docs/llm-safe-adoption.md +++ b/docs/llm-safe-adoption.md @@ -86,7 +86,6 @@ export default { presets: { 'llm-safe': { name: 'llm-safe', - depth: 'quick', timeout: 3000, parallel: false, chaos: false, diff --git a/docs/observe.md b/docs/observe.md index 720aedb..8dac7c7 100644 --- a/docs/observe.md +++ b/docs/observe.md @@ -135,7 +135,6 @@ export default { presets: { 'platform-observe': { name: 'platform-observe', - depth: 'standard', timeout: 10000, parallel: true, chaos: false, diff --git a/docs/performance.md b/docs/performance.md index 4694b99..7e4ecaf 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -23,8 +23,8 @@ BENCH_RUNS=12 BENCH_WARMUP=3 npm run benchmark:cli # Increase inner-loop work for micro-benchmarks BENCH_INNER_ITERS=5000 npm run benchmark:hot -# Benchmark generation profile matrix -BENCH_GENERATION_PROFILES=quick,standard,thorough npm run benchmark:all +# Benchmark with varying test counts +BENCH_RUNS=10,50,200 npm run benchmark:all ``` ## Capture CPU Profile for Qualify @@ -41,10 +41,10 @@ This writes Chrome-compatible CPU profiles to `.profiles/qualify.cpuprofile` and - CLI benchmark uses spawned `node dist/cli/index.js` commands so startup costs are included. - Hot path benchmark runs in-process for lower-noise function-level comparisons. - Use fixed `--seed` for qualify benchmarks to keep runs deterministic. -- Generation now adapts to depth: `quick` favors bounded payload generation speed, `thorough` keeps broader generation. +- Schema generation uses fixed defaults (string≤128, array≤10) regardless of run count. -You can override generation per run: +You can override runs per preset: ```bash -apophis qualify --profile oauth-nightly --generation-profile quick --seed 42 +apophis qualify --profile oauth-nightly --seed 42 ``` diff --git a/docs/qualify.md b/docs/qualify.md index 7fa9c7e..5f1fe86 100644 --- a/docs/qualify.md +++ b/docs/qualify.md @@ -97,7 +97,7 @@ APOPHIS tracks created resources and runs cleanup after test completion. Run stateful tests via the API: ```javascript -const stateful = await fastify.apophis.stateful({ depth: 'standard', seed: 42 }) +const stateful = await fastify.apophis.stateful({ runs: 50, seed: 42 }) console.log('Stateful tests:', stateful.summary) ``` @@ -203,7 +203,7 @@ export default { presets: { 'protocol-lab': { name: 'protocol-lab', - depth: 'deep', + runs: 200, timeout: 15000, parallel: false, chaos: true, @@ -258,10 +258,15 @@ Run qualify across all packages in a monorepo workspace: apophis qualify --workspace --profile oauth-nightly ``` -## `--generation-profile` Flag +## Test Budget -Control test data generation depth independently from the qualification profile: +The `runs` field in your preset controls how many property-based tests execute per route. Default is 50. Lower for faster CI feedback, higher for deeper exploration: -```bash -apophis qualify --profile oauth-nightly --generation-profile quick +```javascript +presets: { + 'protocol-lab': { + runs: 200, + timeout: 15000 + } +} ``` diff --git a/docs/verify.md b/docs/verify.md index cceefa1..7d966b5 100644 --- a/docs/verify.md +++ b/docs/verify.md @@ -165,7 +165,7 @@ export default { }, presets: { 'safe-ci': { - depth: 'quick', + runs: 10, timeout: 5000 } } @@ -184,10 +184,22 @@ apophis verify --workspace --profile quick --format json Output includes per-package pass/fail summaries. Fails if any package fails. -## `--generation-profile` Flag +## Test Budget -Control test data generation depth independently from the verification profile: +The `runs` field in your preset controls how many property-based tests execute per route. Default is 50. Lower for faster CI feedback, higher for deeper exploration: -```bash -apophis verify --profile quick --generation-profile quick +```javascript +profiles: { + quick: { + mode: 'verify', + preset: 'safe-ci', + routes: ['POST /users'] + } +}, +presets: { + 'safe-ci': { + runs: 10, + timeout: 5000 + } +} ``` diff --git a/package.json b/package.json index b7d8679..8cd3fe9 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,7 @@ "benchmark:cli": "npm run build && node scripts/bench/cli.mjs", "benchmark:hot": "npm run build && node scripts/bench/hot-paths.mjs", "profile:qualify": "npm run build && mkdir -p .profiles && node --cpu-prof --cpu-prof-dir=.profiles --cpu-prof-name=qualify.cpuprofile dist/cli/index.js qualify --cwd src/cli/__fixtures__/protocol-lab --profile oauth-nightly --seed 42 --quiet", - "profile:qualify:quick": "npm run build && mkdir -p .profiles && node --cpu-prof --cpu-prof-dir=.profiles --cpu-prof-name=qualify-quick.cpuprofile dist/cli/index.js qualify --cwd src/cli/__fixtures__/protocol-lab --profile oauth-nightly --generation-profile quick --seed 42 --quiet", + "profile:qualify:quick": "npm run build && mkdir -p .profiles && node --cpu-prof --cpu-prof-dir=.profiles --cpu-prof-name=qualify-quick.cpuprofile dist/cli/index.js qualify --cwd src/cli/__fixtures__/protocol-lab --profile oauth-nightly --seed 42 --quiet", "clean": "rm -rf dist", "apophis:verify": "apophis verify --profile quick", "apophis:doctor": "apophis doctor" diff --git a/scripts/bench/cli.mjs b/scripts/bench/cli.mjs index 8e38ad6..21f74ef 100644 --- a/scripts/bench/cli.mjs +++ b/scripts/bench/cli.mjs @@ -8,30 +8,13 @@ const __dirname = fileURLToPath(new URL('.', import.meta.url)) const repoRoot = resolve(__dirname, '..', '..') const options = getBenchOptions() -const generationProfiles = (process.env.BENCH_GENERATION_PROFILES ?? 'default,quick,standard,thorough') - .split(',') - .map((value) => value.trim()) - .filter(Boolean) - -function withGenerationProfile(baseArgs, profile) { - if (profile === 'default') { - return baseArgs - } - return [...baseArgs, '--generation-profile', profile] -} const scenarios = [ { name: 'cli.help', args: ['--help'] }, { name: 'cli.version', args: ['--version'] }, { name: 'cli.doctor', args: ['doctor', '--cwd', 'src/cli/__fixtures__/tiny-fastify', '--quiet'] }, { name: 'cli.observe.check', args: ['observe', '--cwd', 'src/cli/__fixtures__/observe-config', '--profile', 'staging-observe', '--check-config', '--quiet'] }, - ...generationProfiles.map((profile) => ({ - name: `cli.qualify.profile[${profile}]`, - args: withGenerationProfile( - ['qualify', '--cwd', 'src/cli/__fixtures__/protocol-lab', '--profile', 'oauth-nightly', '--seed', '42', '--quiet'], - profile, - ), - })), + { name: 'cli.qualify', args: ['qualify', '--cwd', 'src/cli/__fixtures__/protocol-lab', '--profile', 'oauth-nightly', '--seed', '42', '--quiet'] }, ] async function run() {