Remove generation profile tiering; use explicit runs count
This commit is contained in:
+1
-1
@@ -8,7 +8,7 @@ Chaos testing applies the invariant-driven verification approach from [Invariant
|
||||
|
||||
```javascript
|
||||
const result = await fastify.apophis.contract({
|
||||
depth: 'standard',
|
||||
runs: 50,
|
||||
chaos: {
|
||||
probability: 0.1, // 10% of requests get chaos
|
||||
delay: { probability: 1, minMs: 100, maxMs: 500 },
|
||||
|
||||
-15
@@ -10,7 +10,6 @@ Every command accepts these flags:
|
||||
|---|---|---|
|
||||
| `--config <path>` | Config file path | Auto-detect |
|
||||
| `--profile <name>` | Profile name from config | First profile |
|
||||
| `--generation-profile <name>` | Generation budget profile (built-in: quick, standard, deep) | Depth-derived |
|
||||
| `--cwd <path>` | Working directory override | `process.cwd()` |
|
||||
| `--format <mode>` | Output format: `human`, `json`, `ndjson`, `json-summary`, `ndjson-summary` | `human` |
|
||||
| `--color <mode>` | Color mode: `auto`, `always`, `never` | `auto` |
|
||||
@@ -62,7 +61,6 @@ apophis verify --profile quick --routes "POST /users"
|
||||
| Flag | Description |
|
||||
|---|---|
|
||||
| `--profile <name>` | Profile name from config |
|
||||
| `--generation-profile <name>` | Override generation budget for this run |
|
||||
| `--routes <filter>` | Route filter pattern (comma-separated, supports wildcards) |
|
||||
| `--seed <number>` | Deterministic seed (generated and printed if omitted) |
|
||||
| `--changed` | Filter to git-modified routes only |
|
||||
@@ -121,7 +119,6 @@ apophis qualify --profile oauth-nightly --seed 42
|
||||
| Flag | Description |
|
||||
|---|---|
|
||||
| `--profile <name>` | Profile name from config |
|
||||
| `--generation-profile <name>` | Override generation budget for this run |
|
||||
| `--seed <number>` | Deterministic seed (generated and printed if omitted) |
|
||||
|
||||
**Examples:**
|
||||
@@ -129,18 +126,6 @@ apophis qualify --profile oauth-nightly --seed 42
|
||||
```bash
|
||||
apophis qualify --profile oauth-nightly --seed 42
|
||||
apophis qualify --profile lifecycle-deep
|
||||
apophis qualify --profile oauth-nightly --generation-profile quick
|
||||
```
|
||||
|
||||
You can define aliases in config:
|
||||
|
||||
```js
|
||||
export default {
|
||||
generationProfiles: {
|
||||
pr: 'quick',
|
||||
nightly: { base: 'thorough' },
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
### `apophis replay`
|
||||
|
||||
@@ -153,11 +153,11 @@ fastify.delete('/users/:id', {
|
||||
await fastify.ready()
|
||||
|
||||
// Run contract tests (all non-utility routes, property-based)
|
||||
const result = await fastify.apophis.contract({ depth: 'standard' })
|
||||
const result = await fastify.apophis.contract({ runs: 50 })
|
||||
console.log('Contract tests:', result.summary)
|
||||
|
||||
// Run stateful tests (constructor→mutator→destructor sequences)
|
||||
const stateful = await fastify.apophis.stateful({ depth: 'standard', seed: 42 })
|
||||
const stateful = await fastify.apophis.stateful({ runs: 50, seed: 42 })
|
||||
console.log('Stateful tests:', stateful.summary)
|
||||
|
||||
// Validate a single route
|
||||
|
||||
@@ -22,5 +22,5 @@ fastify.get('/health', {
|
||||
await fastify.ready()
|
||||
|
||||
// Run contract tests
|
||||
const result = await fastify.apophis.contract({ depth: 'quick' })
|
||||
const result = await fastify.apophis.contract({ runs: 10 })
|
||||
console.log(result.summary)
|
||||
|
||||
@@ -86,7 +86,6 @@ export default {
|
||||
presets: {
|
||||
'llm-safe': {
|
||||
name: 'llm-safe',
|
||||
depth: 'quick',
|
||||
timeout: 3000,
|
||||
parallel: false,
|
||||
chaos: false,
|
||||
|
||||
@@ -135,7 +135,6 @@ export default {
|
||||
presets: {
|
||||
'platform-observe': {
|
||||
name: 'platform-observe',
|
||||
depth: 'standard',
|
||||
timeout: 10000,
|
||||
parallel: true,
|
||||
chaos: false,
|
||||
|
||||
+5
-5
@@ -23,8 +23,8 @@ BENCH_RUNS=12 BENCH_WARMUP=3 npm run benchmark:cli
|
||||
# Increase inner-loop work for micro-benchmarks
|
||||
BENCH_INNER_ITERS=5000 npm run benchmark:hot
|
||||
|
||||
# Benchmark generation profile matrix
|
||||
BENCH_GENERATION_PROFILES=quick,standard,thorough npm run benchmark:all
|
||||
# Benchmark with varying test counts
|
||||
BENCH_RUNS=10,50,200 npm run benchmark:all
|
||||
```
|
||||
|
||||
## Capture CPU Profile for Qualify
|
||||
@@ -41,10 +41,10 @@ This writes Chrome-compatible CPU profiles to `.profiles/qualify.cpuprofile` and
|
||||
- CLI benchmark uses spawned `node dist/cli/index.js` commands so startup costs are included.
|
||||
- Hot path benchmark runs in-process for lower-noise function-level comparisons.
|
||||
- Use fixed `--seed` for qualify benchmarks to keep runs deterministic.
|
||||
- Generation now adapts to depth: `quick` favors bounded payload generation speed, `thorough` keeps broader generation.
|
||||
- Schema generation uses fixed defaults (string≤128, array≤10) regardless of run count.
|
||||
|
||||
You can override generation per run:
|
||||
You can override runs per preset:
|
||||
|
||||
```bash
|
||||
apophis qualify --profile oauth-nightly --generation-profile quick --seed 42
|
||||
apophis qualify --profile oauth-nightly --seed 42
|
||||
```
|
||||
|
||||
+11
-6
@@ -97,7 +97,7 @@ APOPHIS tracks created resources and runs cleanup after test completion.
|
||||
Run stateful tests via the API:
|
||||
|
||||
```javascript
|
||||
const stateful = await fastify.apophis.stateful({ depth: 'standard', seed: 42 })
|
||||
const stateful = await fastify.apophis.stateful({ runs: 50, seed: 42 })
|
||||
console.log('Stateful tests:', stateful.summary)
|
||||
```
|
||||
|
||||
@@ -203,7 +203,7 @@ export default {
|
||||
presets: {
|
||||
'protocol-lab': {
|
||||
name: 'protocol-lab',
|
||||
depth: 'deep',
|
||||
runs: 200,
|
||||
timeout: 15000,
|
||||
parallel: false,
|
||||
chaos: true,
|
||||
@@ -258,10 +258,15 @@ Run qualify across all packages in a monorepo workspace:
|
||||
apophis qualify --workspace --profile oauth-nightly
|
||||
```
|
||||
|
||||
## `--generation-profile` Flag
|
||||
## Test Budget
|
||||
|
||||
Control test data generation depth independently from the qualification profile:
|
||||
The `runs` field in your preset controls how many property-based tests execute per route. Default is 50. Lower for faster CI feedback, higher for deeper exploration:
|
||||
|
||||
```bash
|
||||
apophis qualify --profile oauth-nightly --generation-profile quick
|
||||
```javascript
|
||||
presets: {
|
||||
'protocol-lab': {
|
||||
runs: 200,
|
||||
timeout: 15000
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
+17
-5
@@ -165,7 +165,7 @@ export default {
|
||||
},
|
||||
presets: {
|
||||
'safe-ci': {
|
||||
depth: 'quick',
|
||||
runs: 10,
|
||||
timeout: 5000
|
||||
}
|
||||
}
|
||||
@@ -184,10 +184,22 @@ apophis verify --workspace --profile quick --format json
|
||||
|
||||
Output includes per-package pass/fail summaries. Fails if any package fails.
|
||||
|
||||
## `--generation-profile` Flag
|
||||
## Test Budget
|
||||
|
||||
Control test data generation depth independently from the verification profile:
|
||||
The `runs` field in your preset controls how many property-based tests execute per route. Default is 50. Lower for faster CI feedback, higher for deeper exploration:
|
||||
|
||||
```bash
|
||||
apophis verify --profile quick --generation-profile quick
|
||||
```javascript
|
||||
profiles: {
|
||||
quick: {
|
||||
mode: 'verify',
|
||||
preset: 'safe-ci',
|
||||
routes: ['POST /users']
|
||||
}
|
||||
},
|
||||
presets: {
|
||||
'safe-ci': {
|
||||
runs: 10,
|
||||
timeout: 5000
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Reference in New Issue
Block a user