Compare commits
13 Commits
v2.0.0
...
31530fe899
| Author | SHA1 | Date | |
|---|---|---|---|
| 31530fe899 | |||
| 457a3495ab | |||
| 7e9b940974 | |||
| 099576f12a | |||
| 6295c476dc | |||
| 3e5758dd54 | |||
| c71184333a | |||
| 5921b1437f | |||
| 115d3465b1 | |||
| dc7a4205ec | |||
| bf7376b5ad | |||
| 8d7382417d | |||
| 6c39bd0a6c |
@@ -4,6 +4,8 @@ Behavioral confidence for Fastify services.
|
|||||||
|
|
||||||
APOPHIS checks whether route behavior holds across operations, states, and protocol flows.
|
APOPHIS checks whether route behavior holds across operations, states, and protocol flows.
|
||||||
|
|
||||||
|
Inspired by [Invariant-Driven Automated Testing](https://arxiv.org/abs/2602.23922) (Malhado Ribeiro, 2021): instead of only checking payload shape, APOPHIS encodes intended behavior as executable contracts and verifies them with property-based and stateful testing.
|
||||||
|
|
||||||
Supported Node.js versions: 20.x and 22.x.
|
Supported Node.js versions: 20.x and 22.x.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@@ -12,6 +14,8 @@ apophis init --preset safe-ci
|
|||||||
apophis verify --profile quick --routes "POST /users"
|
apophis verify --profile quick --routes "POST /users"
|
||||||
```
|
```
|
||||||
|
|
||||||
|
`x-ensures` is an OpenAPI schema extension for behavioral contracts — statements about what a route must guarantee.
|
||||||
|
|
||||||
## Cross-Route Failure Example
|
## Cross-Route Failure Example
|
||||||
|
|
||||||
Add one behavioral contract next to a route schema. APOPHIS can verify cross-route behavior, such as whether a resource created by one route is retrievable through another.
|
Add one behavioral contract next to a route schema. APOPHIS can verify cross-route behavior, such as whether a resource created by one route is retrievable through another.
|
||||||
@@ -19,6 +23,8 @@ Add one behavioral contract next to a route schema. APOPHIS can verify cross-rou
|
|||||||
**Route:**
|
**Route:**
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
|
import crypto from 'crypto';
|
||||||
|
|
||||||
app.post('/users', {
|
app.post('/users', {
|
||||||
schema: {
|
schema: {
|
||||||
'x-category': 'constructor',
|
'x-category': 'constructor',
|
||||||
@@ -29,7 +35,7 @@ app.post('/users', {
|
|||||||
}
|
}
|
||||||
}, async (request, reply) => {
|
}, async (request, reply) => {
|
||||||
const { name } = request.body;
|
const { name } = request.body;
|
||||||
const id = `usr-${Date.now()}`;
|
const id = `usr-${crypto.createHash('sha256').update(name).digest('hex').slice(0, 8)}`;
|
||||||
reply.status(201);
|
reply.status(201);
|
||||||
return { id, name };
|
return { id, name };
|
||||||
});
|
});
|
||||||
@@ -47,7 +53,7 @@ Expected
|
|||||||
response_code(GET /users/{response_body(this).id}) == 200
|
response_code(GET /users/{response_body(this).id}) == 200
|
||||||
|
|
||||||
Observed
|
Observed
|
||||||
GET /users/usr-123 returned 404
|
GET /users/usr-7d865e returned 404
|
||||||
|
|
||||||
Why this matters
|
Why this matters
|
||||||
The resource created by POST /users is not retrievable.
|
The resource created by POST /users is not retrievable.
|
||||||
@@ -80,6 +86,9 @@ apophis init --preset safe-ci
|
|||||||
|
|
||||||
# 3. Verify
|
# 3. Verify
|
||||||
apophis verify --profile quick --routes "POST /users"
|
apophis verify --profile quick --routes "POST /users"
|
||||||
|
|
||||||
|
# 4. Doctor
|
||||||
|
apophis doctor
|
||||||
```
|
```
|
||||||
|
|
||||||
See [docs/getting-started.md](docs/getting-started.md) for the full walkthrough.
|
See [docs/getting-started.md](docs/getting-started.md) for the full walkthrough.
|
||||||
@@ -87,9 +96,12 @@ See [docs/getting-started.md](docs/getting-started.md) for the full walkthrough.
|
|||||||
## Trust and Safety
|
## Trust and Safety
|
||||||
|
|
||||||
- **Deterministic replay**: Every failure includes a seed and a one-command replay.
|
- **Deterministic replay**: Every failure includes a seed and a one-command replay.
|
||||||
|
- **Explicit test budget**: Control how many tests run with `runs: 10` in your preset.
|
||||||
- **CI-safe default path**: `verify` is deterministic and safe for CI pipelines.
|
- **CI-safe default path**: `verify` is deterministic and safe for CI pipelines.
|
||||||
|
- **Machine-readable output**: `--format json-summary` and `--format ndjson-summary` for CI dashboards.
|
||||||
- **Production-safe observe path**: `observe` is non-blocking by default. Blocking behavior requires explicit break-glass policy.
|
- **Production-safe observe path**: `observe` is non-blocking by default. Blocking behavior requires explicit break-glass policy.
|
||||||
- **Qualify path gated away from prod**: `qualify` is blocked in production by default.
|
- **Qualify path gated away from prod**: `qualify` is blocked in production by default.
|
||||||
|
- **Monorepo workspace support**: `--workspace` fans out `verify` and `doctor` across all packages.
|
||||||
- **Explicit environment boundaries**: Config rejects unknown keys and unsafe environment mixes.
|
- **Explicit environment boundaries**: Config rejects unknown keys and unsafe environment mixes.
|
||||||
|
|
||||||
## LLM-Safe
|
## LLM-Safe
|
||||||
@@ -110,10 +122,11 @@ See [docs/llm-safe-adoption.md](docs/llm-safe-adoption.md) for templates and CI
|
|||||||
- [Verify Mode](docs/verify.md) — Deterministic contract verification
|
- [Verify Mode](docs/verify.md) — Deterministic contract verification
|
||||||
- [Observe Mode](docs/observe.md) — Runtime visibility and drift detection
|
- [Observe Mode](docs/observe.md) — Runtime visibility and drift detection
|
||||||
- [Qualify Mode](docs/qualify.md) — Scenarios, stateful testing, chaos
|
- [Qualify Mode](docs/qualify.md) — Scenarios, stateful testing, chaos
|
||||||
|
- [Quality Engines](docs/quality.md) — Chaos injection, flake detection, mutation testing
|
||||||
- [Performance](docs/performance.md) — Repeatable benchmarks and CPU profiling
|
- [Performance](docs/performance.md) — Repeatable benchmarks and CPU profiling
|
||||||
- [LLM-Safe Adoption](docs/llm-safe-adoption.md) — Scaffolds and CI guards
|
- [LLM-Safe Adoption](docs/llm-safe-adoption.md) — Scaffolds and CI guards
|
||||||
- [Protocol Extensions](docs/protocol-extensions-spec.md) — JWT, X.509, SPIFFE, WIMSE
|
- [Protocol Extensions](docs/attic/protocol-extensions-spec.md) — JWT, X.509, SPIFFE, WIMSE
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|
||||||
ISC
|
MIT
|
||||||
|
|||||||
@@ -1,49 +1,86 @@
|
|||||||
---
|
---
|
||||||
name: apophis-fastify
|
name: apophis-fastify
|
||||||
description: Use this skill when adding or improving APOPHIS contract-driven testing for Fastify APIs: route schemas, APOSTL x-requires/x-ensures formulas, property and stateful checks, replayable failures, runtime observe hooks, variants, scenarios, and operator-facing adoption guidance.
|
description: Use this skill when adding or improving APOPHIS contract-driven testing for Fastify APIs. This tool finds real implementation bugs—resources that appear to create but cannot be retrieved, updates that silently fail to persist, deletions that leave data visible, cross-tenant leakage, and broken state transitions. Use it to encode intended behavior as executable contracts and verify them continuously, not to paper over failures.
|
||||||
---
|
---
|
||||||
|
|
||||||
# apophis-fastify
|
# apophis-fastify
|
||||||
|
|
||||||
APOPHIS verifies API behavior across operations, state changes, protocol flows, and dependencies. Use it when schema validation is not enough to answer whether an endpoint did the right thing.
|
APOPHIS finds real API behavior bugs that schema validation misses. It verifies that operations cause the state changes, isolation guarantees, and dependency interactions the service actually depends on.
|
||||||
|
|
||||||
|
Inspired by [Invariant-Driven Automated Testing](https://arxiv.org/abs/2602.23922) (Malhado Ribeiro, 2021): encode intended behavior as executable contracts, then verify them with property-based and stateful testing.
|
||||||
|
|
||||||
|
## Core Purpose
|
||||||
|
|
||||||
|
APOPHIS is a correctness tool, not a coverage checkbox. Its job is to find where implementation diverges from intended behavior.
|
||||||
|
|
||||||
|
**What APOPHIS finds:**
|
||||||
|
- POST creates a resource that GET cannot retrieve
|
||||||
|
- PUT updates a field but subsequent GET returns stale data
|
||||||
|
- DELETE removes a resource but it still appears in listings
|
||||||
|
- Cross-tenant data leakage through shared collections
|
||||||
|
- Auth boundaries that return success instead of 401/403
|
||||||
|
- Error paths that violate declared retry semantics
|
||||||
|
- State transitions that skip required steps
|
||||||
|
|
||||||
|
**What APOPHIS does NOT do:**
|
||||||
|
- Prove the whole system correct (it finds counterexamples, not proofs)
|
||||||
|
- Replace integration tests, security review, or domain judgment
|
||||||
|
- Validate JSON Schema (Fastify + Swagger already do this)
|
||||||
|
|
||||||
|
**Critical rule for LLMs:** Do not weaken contracts to make tests pass. A failing contract is a signal that the implementation has a bug or the contract captures wrong intent. Investigate first. Fix the bug if the contract is correct. Fix the contract only if the implementation is right and the contract is wrong.
|
||||||
|
|
||||||
## When To Use
|
## When To Use
|
||||||
|
|
||||||
Use this skill when the operator asks to:
|
Use this skill when the operator asks to:
|
||||||
|
|
||||||
1. Add contract testing, API behavior checks, property tests, stateful tests, or Fastify route verification.
|
1. Add contract testing, API behavior checks, property tests, stateful tests, or Fastify route verification.
|
||||||
2. Improve confidence beyond JSON Schema validation.
|
2. Find real bugs in create/read/update/delete behavior, auth boundaries, tenant isolation, redirects, timeouts, streaming, or external dependency behavior.
|
||||||
3. Check create/read/update/delete behavior, auth boundaries, tenant isolation, redirects, timeouts, streaming, or external dependency behavior.
|
3. Make an API safer for AI-assisted refactoring, CI, or release qualification.
|
||||||
4. Make an API safer for AI-assisted refactoring, CI, or release qualification.
|
4. Evaluate whether APOPHIS would help a project, even if the operator does not name APOPHIS directly.
|
||||||
5. Evaluate whether APOPHIS would help a project, even if the operator does not name APOPHIS directly.
|
|
||||||
|
|
||||||
Read `README.md`, `docs/getting-started.md`, or command-specific docs only when the task needs details not present here.
|
Read `README.md`, `docs/getting-started.md`, or command-specific docs only when the task needs details not present here.
|
||||||
|
|
||||||
## Operator Explanation
|
## Operator Explanation
|
||||||
|
|
||||||
Describe APOPHIS as replayable behavioral checks for declared API contracts.
|
Describe APOPHIS as a bug-finding tool for API behavior.
|
||||||
|
|
||||||
Short explanation:
|
Short explanation:
|
||||||
|
|
||||||
> APOPHIS turns intended API behavior into executable contracts. It checks whether operations cause the state changes, isolation guarantees, and dependency interactions the service depends on, instead of only checking payload shape.
|
> APOPHIS encodes intended API behavior as executable contracts, then automatically finds cases where the implementation violates that behavior. It catches bugs that schema validation cannot: create-not-retrievable, update-not-persisted, delete-still-visible, cross-tenant leakage, and broken state transitions.
|
||||||
|
|
||||||
Use these points when relevant:
|
Use these points when relevant:
|
||||||
|
|
||||||
1. It catches failures schema validation misses: create-not-retrievable, update-not-persisted, delete-still-visible, cross-tenant leakage, and inconsistent error behavior.
|
1. It finds real bugs schema validation misses: resources that create but cannot be read, updates that do not persist, deletes that leave data visible.
|
||||||
2. It gives coding agents a deterministic verification loop after generated changes or refactors.
|
2. It gives coding agents a deterministic verification loop after generated changes or refactors. Run contracts after every change; failures indicate regressions.
|
||||||
3. It reduces review burden by converting agreed behavior into repeatable checks.
|
3. It prevents silent breakage. When a refactoring breaks a state transition or isolation guarantee, contracts fail immediately with a reproducible seed.
|
||||||
4. It improves CI triage with fixed seeds, replay artifacts, and machine-readable output.
|
4. It improves CI triage with fixed seeds, replay artifacts, and machine-readable output.
|
||||||
5. It supports incremental adoption: start with the highest-risk routes, add high-signal formulas, run, fix, and tighten.
|
5. It supports incremental adoption: start with the highest-risk routes, add high-signal behavioral formulas, run, fix real bugs, and tighten.
|
||||||
|
|
||||||
Do not overclaim:
|
Do not overclaim:
|
||||||
|
|
||||||
1. Do not say APOPHIS proves the whole system correct.
|
1. Do not say APOPHIS proves the whole system correct.
|
||||||
2. Do not say contracts replace integration tests, security review, or domain judgment.
|
2. Do not say contracts replace integration tests, security review, or domain judgment.
|
||||||
3. Say explicitly that schema quality and formula quality determine test quality.
|
3. Say explicitly that behavioral contracts—not structural checks—determine bug-finding quality.
|
||||||
|
|
||||||
Good operator ask:
|
Good operator ask:
|
||||||
|
|
||||||
> I can add APOPHIS to the five highest-risk routes first, encode the expected behavior as contracts, run the verifier, and show concrete failures or confidence gaps. I only need route priority and intended behavior where the code is ambiguous.
|
> I can add APOPHIS to the five highest-risk routes first, encode the expected behavior as contracts, run the verifier, and show concrete failures where implementation diverges from intent. I only need route priority and intended behavior where the code is ambiguous.
|
||||||
|
|
||||||
|
## Failure Discipline
|
||||||
|
|
||||||
|
**Never paper over a failing contract.**
|
||||||
|
|
||||||
|
When APOPHIS reports a contract violation, follow this order:
|
||||||
|
|
||||||
|
1. **Replay the failure** with the reported seed to confirm it is real.
|
||||||
|
2. **Investigate the implementation.** Does the handler actually do what the contract expects? Is there a race condition? A missing database commit? A wrong query?
|
||||||
|
3. **Fix the implementation** if the contract correctly describes intended behavior.
|
||||||
|
4. **Fix the contract** only if investigation proves the implementation is correct and the contract over-constrains or misdescribes behavior.
|
||||||
|
5. **Never** weaken a contract because "the test is flaky" or "it is too strict" without first proving the implementation is correct.
|
||||||
|
|
||||||
|
**If a contract fails intermittently**, that is a bug. Intermittent failures indicate nondeterminism: race conditions, uncommitted transactions, time-dependent logic, or randomness in handlers. Do not remove the contract. Isolate the nondeterminism and fix it.
|
||||||
|
|
||||||
|
**If a contract fails only under chaos**, that is a resilience bug. The service does not handle the failure mode correctly. Fix the handler or the contract's error-path expectations.
|
||||||
|
|
||||||
## Context Discipline
|
## Context Discipline
|
||||||
|
|
||||||
@@ -53,7 +90,7 @@ Treat context as a finite budget.
|
|||||||
2. Prefer targeted file reads and symbol searches over loading whole directories.
|
2. Prefer targeted file reads and symbol searches over loading whole directories.
|
||||||
3. Track routes touched, contracts added, seeds used, failures found, and unresolved domain questions.
|
3. Track routes touched, contracts added, seeds used, failures found, and unresolved domain questions.
|
||||||
4. Use progressive disclosure: read command docs only when invoking that command; read protocol docs only for variants, redirects, OAuth-style flows, form posts, streaming, or multipart.
|
4. Use progressive disclosure: read command docs only when invoking that command; read protocol docs only for variants, redirects, OAuth-style flows, form posts, streaming, or multipart.
|
||||||
5. Run small loops: annotate one route group, run the narrowest verification, fix, then widen.
|
5. Run small loops: annotate one route group, run the narrowest verification, fix real bugs, then widen.
|
||||||
|
|
||||||
## Default Workflow
|
## Default Workflow
|
||||||
|
|
||||||
@@ -67,8 +104,8 @@ When entering a Fastify codebase:
|
|||||||
6. Add `x-category` where auto-categorization could be ambiguous.
|
6. Add `x-category` where auto-categorization could be ambiguous.
|
||||||
7. Add `x-requires` for preconditions and `x-ensures` for postconditions.
|
7. Add `x-requires` for preconditions and `x-ensures` for postconditions.
|
||||||
8. Run a focused APOPHIS check, then broader contract or stateful verification.
|
8. Run a focused APOPHIS check, then broader contract or stateful verification.
|
||||||
9. Fix real behavior failures or tighten weak contracts.
|
9. **Fix real behavior failures or tighten weak contracts.** Do not weaken passing contracts to avoid work.
|
||||||
10. Report what changed, what ran, what failed, and what needs operator judgment.
|
10. Report what changed, what ran, what failed, what bugs were found, and what needs operator judgment.
|
||||||
|
|
||||||
## Fast Start
|
## Fast Start
|
||||||
|
|
||||||
@@ -76,6 +113,7 @@ When entering a Fastify codebase:
|
|||||||
import Fastify from 'fastify'
|
import Fastify from 'fastify'
|
||||||
import swagger from '@fastify/swagger'
|
import swagger from '@fastify/swagger'
|
||||||
import apophis from 'apophis-fastify'
|
import apophis from 'apophis-fastify'
|
||||||
|
import crypto from 'crypto'
|
||||||
|
|
||||||
const app = Fastify()
|
const app = Fastify()
|
||||||
await app.register(swagger)
|
await app.register(swagger)
|
||||||
@@ -85,12 +123,15 @@ app.post('/users', {
|
|||||||
schema: {
|
schema: {
|
||||||
'x-category': 'constructor',
|
'x-category': 'constructor',
|
||||||
'x-requires': [
|
'x-requires': [
|
||||||
'request_headers(this).x-tenant-id != null'
|
// Precondition: user must not already exist
|
||||||
|
'response_code(GET /users/{request_body(this).email}) == 404'
|
||||||
],
|
],
|
||||||
'x-ensures': [
|
'x-ensures': [
|
||||||
'status:201',
|
// Behavioral: created resource must be retrievable
|
||||||
'response_body(this).id != null',
|
|
||||||
'response_code(GET /users/{response_body(this).id}) == 200',
|
'response_code(GET /users/{response_body(this).id}) == 200',
|
||||||
|
// Behavioral: round-trip equality
|
||||||
|
'response_body(this) == request_body(this)',
|
||||||
|
// Behavioral: cross-route field persistence
|
||||||
'response_body(GET /users/{response_body(this).id}).email == request_body(this).email'
|
'response_body(GET /users/{response_body(this).id}).email == request_body(this).email'
|
||||||
],
|
],
|
||||||
body: {
|
body: {
|
||||||
@@ -114,12 +155,13 @@ app.post('/users', {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}, async (req, reply) => {
|
}, async (req, reply) => {
|
||||||
|
const id = `usr-${crypto.createHash('sha256').update(req.body.email).digest('hex').slice(0, 8)}`
|
||||||
reply.status(201)
|
reply.status(201)
|
||||||
return { id: 'usr-1', ...req.body }
|
return { id, ...req.body }
|
||||||
})
|
})
|
||||||
|
|
||||||
await app.ready()
|
await app.ready()
|
||||||
const suite = await app.apophis.contract({ depth: 'standard' })
|
const suite = await app.apophis.contract({ runs: 50 })
|
||||||
```
|
```
|
||||||
|
|
||||||
## API Surface
|
## API Surface
|
||||||
@@ -141,25 +183,34 @@ Test-only helpers:
|
|||||||
4. `fastify.apophis.test.disableOutboundMocks()`
|
4. `fastify.apophis.test.disableOutboundMocks()`
|
||||||
5. `fastify.apophis.test.getOutboundCalls(...)`
|
5. `fastify.apophis.test.getOutboundCalls(...)`
|
||||||
|
|
||||||
## Contract Quality
|
## Contract Quality: Behavioral, Not Structural
|
||||||
|
|
||||||
Minimum:
|
**Structural checks are useless.** Fastify + `@fastify/swagger` already enforce status codes, required fields, and types. Behavioral contracts find what schemas cannot.
|
||||||
|
|
||||||
1. Each mutating route has a status expectation.
|
**Minimum behavioral baseline:**
|
||||||
2. Each response with identity has key field non-null checks.
|
|
||||||
|
1. Constructor routes verify cross-route retrievability.
|
||||||
|
2. Mutator routes verify state-change visibility.
|
||||||
|
3. Destructor routes verify unavailability after deletion.
|
||||||
|
|
||||||
```apostl
|
```apostl
|
||||||
status:201
|
// Constructor: resource must be retrievable after creation
|
||||||
response_body(this).id != null
|
response_code(GET /users/{response_body(this).id}) == 200
|
||||||
|
|
||||||
|
// Mutator: changed field must persist
|
||||||
|
response_body(GET /users/{request_params(this).id}).status == request_body(this).status
|
||||||
|
|
||||||
|
// Destructor: deleted resource must not be retrievable
|
||||||
|
response_code(GET /users/{request_params(this).id}) == 404
|
||||||
```
|
```
|
||||||
|
|
||||||
Production baseline:
|
**Production baseline:**
|
||||||
|
|
||||||
1. Constructor routes check that created resources are retrievable.
|
1. Constructor routes check that created resources are retrievable.
|
||||||
2. Mutator routes check that persisted state reflects the mutation.
|
2. Mutator routes check that persisted state reflects the mutation.
|
||||||
3. Destructor routes check that deleted resources are unavailable or marked inactive.
|
3. Destructor routes check that deleted resources are unavailable or marked inactive.
|
||||||
|
|
||||||
High-confidence contracts add:
|
**High-confidence contracts add:**
|
||||||
|
|
||||||
1. Tenant isolation.
|
1. Tenant isolation.
|
||||||
2. Auth and permission behavior.
|
2. Auth and permission behavior.
|
||||||
@@ -172,37 +223,32 @@ High-confidence contracts add:
|
|||||||
|
|
||||||
Constructor routes, such as `POST /collection`:
|
Constructor routes, such as `POST /collection`:
|
||||||
|
|
||||||
1. Response has identity.
|
1. Resource is retrievable after creation.
|
||||||
2. Created resource is retrievable.
|
2. Persisted fields reflect request fields.
|
||||||
3. Persisted fields reflect request fields.
|
|
||||||
|
|
||||||
```apostl
|
```apostl
|
||||||
status:201
|
|
||||||
response_body(this).id != null
|
|
||||||
response_code(GET /items/{response_body(this).id}) == 200
|
response_code(GET /items/{response_body(this).id}) == 200
|
||||||
response_body(GET /items/{response_body(this).id}).name == request_body(this).name
|
response_body(GET /items/{response_body(this).id}).name == request_body(this).name
|
||||||
```
|
```
|
||||||
|
|
||||||
Mutator routes, such as `PUT`, `PATCH`, or action `POST`:
|
Mutator routes, such as `PUT`, `PATCH`, or action `POST`:
|
||||||
|
|
||||||
1. Mutation succeeds with expected code.
|
1. Changed field actually changed and persists.
|
||||||
2. Changed field actually changed.
|
2. Unrelated invariants still hold.
|
||||||
3. Unrelated invariants still hold.
|
|
||||||
|
|
||||||
```apostl
|
```apostl
|
||||||
status:200
|
response_body(GET /items/{request_params(this).id}).status == request_body(this).status
|
||||||
response_body(this).status == request_body(this).status
|
previous(response_body(this).version) < response_body(this).version
|
||||||
response_body(this).updatedAt != null
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Destructor routes:
|
Destructor routes:
|
||||||
|
|
||||||
1. Delete returns expected code.
|
1. Follow-up retrieval fails or shows a domain-specific inactive state.
|
||||||
2. Follow-up retrieval fails or shows a domain-specific inactive state.
|
2. Previous state is preserved if the API returns deleted data.
|
||||||
|
|
||||||
```apostl
|
```apostl
|
||||||
status:204 || status:200
|
|
||||||
response_code(GET /items/{request_params(this).id}) == 404
|
response_code(GET /items/{request_params(this).id}) == 404
|
||||||
|
response_body(this) == previous(response_body(GET /items/{request_params(this).id}))
|
||||||
```
|
```
|
||||||
|
|
||||||
Observer routes:
|
Observer routes:
|
||||||
@@ -254,7 +300,7 @@ Use these patterns when they match the API:
|
|||||||
6. Error consistency: expected error status implies expected error payload fields.
|
6. Error consistency: expected error status implies expected error payload fields.
|
||||||
|
|
||||||
```apostl
|
```apostl
|
||||||
if status:401 then response_body(this).error != null else true
|
if status:401 then response_body(this).error.length > 0 else true
|
||||||
if request_headers(this).x-tenant-id != null then response_headers(this).x-tenant-id == request_headers(this).x-tenant-id else true
|
if request_headers(this).x-tenant-id != null then response_headers(this).x-tenant-id == request_headers(this).x-tenant-id else true
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -312,8 +358,9 @@ await app.apophis.scenario({
|
|||||||
steps: [
|
steps: [
|
||||||
{
|
{
|
||||||
name: 'authorize',
|
name: 'authorize',
|
||||||
request: { method: 'GET', url: '/oauth/authorize?client_id=web&response_type=code' },
|
request: { method: 'GET', url: '/oauth/authorize?client_id=web&response_type=code&state=abc123' },
|
||||||
expect: ['status:200', 'response_payload(this).code != null'],
|
// Behavioral: state parameter round-trips for CSRF protection
|
||||||
|
expect: ['response_payload(this).state == request_query(this).state'],
|
||||||
capture: { code: 'response_payload(this).code' }
|
capture: { code: 'response_payload(this).code' }
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -321,9 +368,10 @@ await app.apophis.scenario({
|
|||||||
request: {
|
request: {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
url: '/oauth/token',
|
url: '/oauth/token',
|
||||||
form: { grant_type: 'authorization_code', code: '$authorize.code' }
|
form: { grant_type: 'authorization_code', code: '$authorize.code', scope: 'read' }
|
||||||
},
|
},
|
||||||
expect: ['status:200', 'response_payload(this).access_token != null']
|
// Behavioral: issued token preserves the requested scope
|
||||||
|
expect: ['response_payload(this).scope == request_body(this).scope']
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
})
|
})
|
||||||
@@ -343,24 +391,49 @@ Prefer deterministic verification for CI, regression triage, and AI-generated ch
|
|||||||
1. Capture and reuse seeds from verify and qualify runs.
|
1. Capture and reuse seeds from verify and qualify runs.
|
||||||
2. Use replay artifacts for failure triage before changing production logic.
|
2. Use replay artifacts for failure triage before changing production logic.
|
||||||
3. Preserve route identity as `METHOD /path` in notes and reports.
|
3. Preserve route identity as `METHOD /path` in notes and reports.
|
||||||
4. If a failure is not reproducible, check for source drift, external dependencies, time, randomness, and insufficient cleanup before weakening the contract.
|
4. **If a failure is not reproducible, treat it as a bug, not a flaky test.** Check for source drift, external dependencies, time, randomness, and insufficient cleanup. Do not weaken the contract without proving the implementation is correct.
|
||||||
5. Treat nondeterminism as a quality issue to isolate.
|
5. Treat nondeterminism as a quality issue to isolate and fix.
|
||||||
|
|
||||||
Operator framing:
|
Operator framing:
|
||||||
|
|
||||||
> The failing seed gives us a reproducible behavioral example. I'll replay it first so we can distinguish a real regression from source drift or nondeterministic app state.
|
> The failing seed gives us a reproducible behavioral counterexample. I'll replay it first to confirm the bug, then investigate the implementation before changing anything.
|
||||||
|
|
||||||
|
## Progressive Complexity
|
||||||
|
|
||||||
|
Start with behavioral contracts and add depth only where it pays off:
|
||||||
|
|
||||||
|
**Level 1 — Cross-route behavior**: Every constructor checks retrievability.
|
||||||
|
```apostl
|
||||||
|
response_code(GET /users/{response_body(this).id}) == 200
|
||||||
|
```
|
||||||
|
|
||||||
|
**Level 2 — State persistence**: Mutators check that changes are visible.
|
||||||
|
```apostl
|
||||||
|
response_body(GET /users/{request_params(this).id}).email == request_body(this).email
|
||||||
|
```
|
||||||
|
|
||||||
|
**Level 3 — Isolation and boundaries**: Tenant, auth, and idempotency checks.
|
||||||
|
```apostl
|
||||||
|
if request_headers(this).x-tenant-id != null then response_headers(this).x-tenant-id == request_headers(this).x-tenant-id else true
|
||||||
|
```
|
||||||
|
|
||||||
|
**Level 4 — Protocol and dependency flows**: Variants, scenarios, outbound contracts, and chaos.
|
||||||
|
|
||||||
|
Add level 1 before level 4. Do not skip level 1 for resource APIs.
|
||||||
|
|
||||||
## Anti-Patterns
|
## Anti-Patterns
|
||||||
|
|
||||||
Do not:
|
Do not:
|
||||||
|
|
||||||
1. Assert only `status:200` everywhere.
|
1. Assert only `status:200` everywhere. Schema validation already checks this.
|
||||||
2. Duplicate JSON Schema checks while ignoring behavior.
|
2. Check `response_body(this).id != null` when the schema already requires `id`.
|
||||||
3. Encode route internals instead of API-observable outcomes.
|
3. Duplicate JSON Schema checks while ignoring cross-route behavior.
|
||||||
4. Ignore delete/retrieve or update/retrieve relationships.
|
4. Encode route internals instead of API-observable outcomes.
|
||||||
5. Treat stateful mode as optional for resource APIs.
|
5. Ignore delete/retrieve or update/retrieve relationships.
|
||||||
6. Ask the operator to review every formula before running; run first when intent is clear, then ask about ambiguous domain behavior.
|
6. Treat stateful mode as optional for resource APIs.
|
||||||
7. Load every doc file before making a small change.
|
7. **Weaken a contract to make a test pass without proving the implementation is correct.**
|
||||||
|
8. Ask the operator to review every formula before running; run first when intent is clear, then ask about ambiguous domain behavior.
|
||||||
|
9. Load every doc file before making a small change.
|
||||||
|
|
||||||
## Verification Commands
|
## Verification Commands
|
||||||
|
|
||||||
@@ -378,7 +451,7 @@ Then execute APOPHIS from the project test harness or CLI as appropriate. For mo
|
|||||||
1. `README.md` for canonical usage.
|
1. `README.md` for canonical usage.
|
||||||
2. `docs/getting-started.md` for quick setup.
|
2. `docs/getting-started.md` for quick setup.
|
||||||
3. `docs/cli.md` and command docs for CLI flags and machine output.
|
3. `docs/cli.md` and command docs for CLI flags and machine output.
|
||||||
4. `docs/protocol-extensions-spec.md` for protocol-specific direction.
|
4. `docs/attic/protocol-extensions-spec.md` for protocol-specific direction.
|
||||||
|
|
||||||
## Final Check
|
## Final Check
|
||||||
|
|
||||||
@@ -388,5 +461,6 @@ For each route, ask:
|
|||||||
2. What must be true after this call?
|
2. What must be true after this call?
|
||||||
3. What related call should now behave differently?
|
3. What related call should now behave differently?
|
||||||
4. What isolation, security, dependency, or protocol expectation should not regress?
|
4. What isolation, security, dependency, or protocol expectation should not regress?
|
||||||
|
5. If a contract fails, is the implementation wrong or is the contract wrong?
|
||||||
|
|
||||||
Write those expectations as formulas and run them continuously.
|
Write those expectations as behavioral formulas, run them continuously, and treat every failure as a bug to investigate—not an obstacle to remove.
|
||||||
|
|||||||
+4
-1
@@ -1,6 +1,9 @@
|
|||||||
## Outbound Contract-Driven Mocking Spec
|
## Outbound Contract-Driven Mocking Spec
|
||||||
|
|
||||||
Status: Proposed
|
Status: Implemented (Phase 1)
|
||||||
|
|
||||||
|
Phase 1 (implemented): Schema parsing (`x-outbound`), mock runtime, imperative API (`enableOutboundMocks`, `getOutboundCalls`), fetch patching.
|
||||||
|
Phase 2 (pending): APOSTL extensions `outbound_calls(this)` and `outbound_last(this)` for contract assertions.
|
||||||
Date: 2026-04-27
|
Date: 2026-04-27
|
||||||
|
|
||||||
This document supersedes Arbiter's local draft at `~/Business/workspace/Arbiter/docs/APOPHIS_OUTBOUND_MOCK_PROPOSAL.md` and its interim adapter at `~/Business/workspace/Arbiter/src/server/server/services/StripeFetchAdapter.js`.
|
This document supersedes Arbiter's local draft at `~/Business/workspace/Arbiter/docs/APOPHIS_OUTBOUND_MOCK_PROPOSAL.md` and its interim adapter at `~/Business/workspace/Arbiter/src/server/server/services/StripeFetchAdapter.js`.
|
||||||
@@ -1,6 +1,10 @@
|
|||||||
# APOPHIS Plugin Contract System Specification
|
# APOPHIS Plugin Contract System Specification
|
||||||
|
|
||||||
## Status: Active design; target version to be assigned
|
## Status: Partially implemented
|
||||||
|
|
||||||
|
- Registry, types, and registration API: **implemented**
|
||||||
|
- Runner integration (merging plugin contracts into route execution): **pending**
|
||||||
|
- Built-in contracts for `@fastify/auth`, `@fastify/compress`, `@fastify/cors`, `@fastify/rate-limit`: **registered but not yet applied**
|
||||||
|
|
||||||
**Note**: Plugin contracts are complementary to Protocol Extensions (see `docs/protocol-extensions-spec.md`). Protocol extensions add domain-specific predicates (JWT, X.509, SPIFFE); plugin contracts add hook-phase behavioral contracts for Fastify plugins.
|
**Note**: Plugin contracts are complementary to Protocol Extensions (see `docs/protocol-extensions-spec.md`). Protocol extensions add domain-specific predicates (JWT, X.509, SPIFFE); plugin contracts add hook-phase behavioral contracts for Fastify plugins.
|
||||||
|
|
||||||
@@ -374,8 +374,10 @@ import { mkdirSync, rmSync } from 'fs'
|
|||||||
import { tmpdir } from 'os'
|
import { tmpdir } from 'os'
|
||||||
import { join } from 'path'
|
import { join } from 'path'
|
||||||
|
|
||||||
|
let testCounter = 0
|
||||||
|
|
||||||
export function createTestWorkspace() {
|
export function createTestWorkspace() {
|
||||||
const dir = join(tmpdir(), `apophis-test-${Date.now()}`)
|
const dir = join(tmpdir(), `apophis-test-${++testCounter}`)
|
||||||
mkdirSync(dir, { recursive: true })
|
mkdirSync(dir, { recursive: true })
|
||||||
|
|
||||||
return {
|
return {
|
||||||
@@ -113,7 +113,7 @@ See [docs/llm-safe-adoption.md](docs/llm-safe-adoption.md) for templates and CI
|
|||||||
## Operator Resources
|
## Operator Resources
|
||||||
|
|
||||||
- [Troubleshooting matrix](docs/troubleshooting.md) — Categorized failure classes with resolution steps
|
- [Troubleshooting matrix](docs/troubleshooting.md) — Categorized failure classes with resolution steps
|
||||||
- [Adoption certification scorecard](docs/adoption-certification-scorecard.md) — Review template for team rollout
|
- [Adoption certification scorecard](adoption-certification-scorecard.md) — Review template for team rollout
|
||||||
|
|
||||||
## CTAs
|
## CTAs
|
||||||
|
|
||||||
|
|||||||
@@ -1,16 +1,18 @@
|
|||||||
# APOPHIS Protocol Extensions Specification
|
# APOPHIS Protocol Extensions Specification
|
||||||
|
|
||||||
## Status: Active design; shipped baseline: v2.x; remaining targets listed per feature
|
## Status: Active design; shipped baseline: v2.0.0; remaining targets listed per feature
|
||||||
|
|
||||||
## 1. Overview
|
## 1. Overview
|
||||||
|
|
||||||
This specification defines protocol-specific extensions for APOPHIS, driven by the Arbiter team's requirements for testing OAuth 2.1, WIMSE S2S, Transaction Tokens (RFC 8693), SPIFFE/SPIRE, and related security protocols.
|
This specification defines protocol-specific extensions for APOPHIS, driven by the Arbiter team's requirements for testing OAuth 2.1, WIMSE S2S, Transaction Tokens (RFC 8693), SPIFFE/SPIRE, and related security protocols.
|
||||||
|
|
||||||
|
APOPHIS is grounded in [Invariant-Driven Automated Testing](https://arxiv.org/abs/2602.23922) (Malhado Ribeiro, 2021). Protocol extensions add domain-specific predicates (JWT, X.509, SPIFFE) to the core invariant framework.
|
||||||
|
|
||||||
Arbiter maintains 58 protocol conformance test files covering 138 behaviors across 7 specifications. These extensions bridge the gap between declarative APOSTL contracts and the domain-specific predicates required for security protocol validation.
|
Arbiter maintains 58 protocol conformance test files covering 138 behaviors across 7 specifications. These extensions bridge the gap between declarative APOSTL contracts and the domain-specific predicates required for security protocol validation.
|
||||||
|
|
||||||
### 1.1 Current Shipped vs Not-Shipped Snapshot
|
### 1.1 Current Shipped vs Not-Shipped Snapshot
|
||||||
|
|
||||||
**Shipped in v2.x:**
|
**Shipped in v2.0.0:**
|
||||||
|
|
||||||
- `contract({ variants })` for multi-header/media negotiation execution.
|
- `contract({ variants })` for multi-header/media negotiation execution.
|
||||||
- `fastify.apophis.scenario(...)` for multi-step capture/rebind flows.
|
- `fastify.apophis.scenario(...)` for multi-step capture/rebind flows.
|
||||||
@@ -166,12 +168,15 @@ jwtExtension({
|
|||||||
The JWT extension maintains state across a test run:
|
The JWT extension maintains state across a test run:
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
interface JwtExtensionState {
|
/**
|
||||||
/** Track seen JTIs for replay detection */
|
* JWT extension state across a test run.
|
||||||
seenJtis: Set<string>
|
* @property {Set<string>} seenJtis - Track seen JTIs for replay detection
|
||||||
/** Cached decoded JWTs */
|
* @property {Map<string, DecodedJwt>} decodedCache - Cached decoded JWTs
|
||||||
decodedCache: Map<string, DecodedJwt>
|
*/
|
||||||
}
|
const jwtExtensionState = {
|
||||||
|
seenJtis: new Set(),
|
||||||
|
decodedCache: new Map()
|
||||||
|
};
|
||||||
```
|
```
|
||||||
|
|
||||||
### 3.5 Example Contracts
|
### 3.5 Example Contracts
|
||||||
@@ -234,16 +239,19 @@ await fastify.apophis.time.set('2026-04-25T12:00:00Z');
|
|||||||
### 4.4 Implementation
|
### 4.4 Implementation
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
interface TimeControl {
|
/**
|
||||||
/** Advance simulated time by milliseconds */
|
* Time control for deterministic testing.
|
||||||
advance(ms: number): void
|
* @property {function(number): void} advance - Advance simulated time by milliseconds
|
||||||
/** Set simulated time to specific timestamp */
|
* @property {function(string): void} set - Set simulated time to specific ISO timestamp
|
||||||
set(isoString: string): void
|
* @property {function(): number} now - Get current simulated time
|
||||||
/** Get current simulated time */
|
* @property {function(): void} reset - Reset to real time
|
||||||
now(): number
|
*/
|
||||||
/** Reset to real time */
|
const timeControl = {
|
||||||
reset(): void
|
advance(ms) { /* ... */ },
|
||||||
}
|
set(isoString) { /* ... */ },
|
||||||
|
now() { return Date.now(); },
|
||||||
|
reset() { /* ... */ }
|
||||||
|
};
|
||||||
```
|
```
|
||||||
|
|
||||||
The `now()` predicate returns simulated time when time mocking is enabled, or the host wall clock outside deterministic test mode. Deterministic runs must inject or freeze time.
|
The `now()` predicate returns simulated time when time mocking is enabled, or the host wall clock outside deterministic test mode. Deterministic runs must inject or freeze time.
|
||||||
@@ -288,11 +296,17 @@ previous(observer).jwt_claims(this).jti # last observer's JWT ID
|
|||||||
Extension state tracks tokens across requests:
|
Extension state tracks tokens across requests:
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
interface StatefulExtensionState {
|
/**
|
||||||
seenTokens: Set<string>
|
* Stateful extension state tracking tokens across requests.
|
||||||
consumedTokens: Set<string>
|
* @property {Set<string>} seenTokens - Tokens observed in previous requests
|
||||||
categoryHistory: Map<string, EvalContext> // category -> last context
|
* @property {Set<string>} consumedTokens - Tokens that have been consumed
|
||||||
}
|
* @property {Map<string, EvalContext>} categoryHistory - category -> last context
|
||||||
|
*/
|
||||||
|
const statefulExtensionState = {
|
||||||
|
seenTokens: new Set(),
|
||||||
|
consumedTokens: new Set(),
|
||||||
|
categoryHistory: new Map()
|
||||||
|
};
|
||||||
```
|
```
|
||||||
|
|
||||||
### 5.4 Example Contracts
|
### 5.4 Example Contracts
|
||||||
@@ -522,14 +536,14 @@ We acknowledge these are too complex or inappropriate for Apophis:
|
|||||||
|
|
||||||
## 14. Implementation Plan
|
## 14. Implementation Plan
|
||||||
|
|
||||||
### Phase 1: JWT + Time Control (P0)
|
### Phase 1: JWT + Time Control (P0) — Shipped in v2.0.0
|
||||||
**Target**: v1.3.0
|
**Status**: Complete
|
||||||
**Files**:
|
**Files**:
|
||||||
- `src/extensions/jwt.ts` — JWT extension implementation
|
- `src/extensions/jwt.ts` — JWT extension implementation
|
||||||
- `src/extensions/time.ts` — Time control extension
|
- `src/extensions/time.ts` — Time control extension
|
||||||
- `src/extensions/stateful.ts` — Stateful predicates extension
|
- `src/extensions/stateful.ts` — Stateful predicates extension
|
||||||
- `src/test/jwt-extension.test.ts` — JWT tests
|
- `src/test/protocol-extensions.test.ts` — Protocol extension tests
|
||||||
- `src/test/time-extension.test.ts` — Time control tests
|
- `src/test/cli/protocol-conformance-p2.test.ts` — Protocol conformance tests
|
||||||
|
|
||||||
**Tests**:
|
**Tests**:
|
||||||
- Decode Base64URL claims without verification
|
- Decode Base64URL claims without verification
|
||||||
@@ -539,27 +553,25 @@ We acknowledge these are too complex or inappropriate for Apophis:
|
|||||||
- `now()` predicate with mocked time
|
- `now()` predicate with mocked time
|
||||||
- `apophis.time.advance()` in stateful tests
|
- `apophis.time.advance()` in stateful tests
|
||||||
|
|
||||||
### Phase 2: X.509 + SPIFFE (P1)
|
### Phase 2: X.509 + SPIFFE (P1) — Shipped in v2.0.0
|
||||||
**Target**: v1.3.1
|
**Status**: Complete
|
||||||
**Files**:
|
**Files**:
|
||||||
- `src/extensions/x509.ts` — X.509 extension
|
- `src/extensions/x509.ts` — X.509 extension
|
||||||
- `src/extensions/spiffe.ts` — SPIFFE extension
|
- `src/extensions/spiffe.ts` — SPIFFE extension
|
||||||
- `src/test/x509-extension.test.ts` — X.509 tests
|
- `src/test/protocol-extensions.test.ts` — Protocol extension tests
|
||||||
- `src/test/spiffe-extension.test.ts` — SPIFFE tests
|
|
||||||
|
|
||||||
### Phase 3: Token Hash + HTTP Signature (P2)
|
### Phase 3: Token Hash + HTTP Signature (P2) — Shipped in v2.0.0
|
||||||
**Target**: v1.3.2
|
**Status**: Complete
|
||||||
**Files**:
|
**Files**:
|
||||||
- `src/extensions/token-hash.ts` — Token hash extension
|
- `src/extensions/token-hash.ts` — Token hash extension
|
||||||
- `src/extensions/http-signature.ts` — HTTP signature extension
|
- `src/extensions/http-signature.ts` — HTTP signature extension
|
||||||
- `src/test/token-hash-extension.test.ts` — Token hash tests
|
- `src/test/protocol-extensions.test.ts` — Protocol extension tests
|
||||||
- `src/test/http-signature-extension.test.ts` — HTTP signature tests
|
|
||||||
|
|
||||||
### Phase 4: Request Context (P2)
|
### Phase 4: Request Context (P2) — Shipped in v2.0.0
|
||||||
**Target**: v1.3.3
|
**Status**: Complete
|
||||||
**Files**:
|
**Files**:
|
||||||
- `src/extensions/request-context.ts` — Request context predicates
|
- `src/extensions/request-context.ts` — Request context predicates
|
||||||
- `src/test/request-context-extension.test.ts` — Request context tests
|
- `src/test/protocol-extensions.test.ts` — Protocol extension tests
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -135,7 +135,7 @@ fastify.get('/wimse/wit', {
|
|||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
See `docs/protocol-extensions-spec.md` for full JWT extension configuration.
|
See `docs/attic/protocol-extensions-spec.md` for full JWT extension configuration.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -144,7 +144,7 @@ See `docs/protocol-extensions-spec.md` for full JWT extension configuration.
|
|||||||
`getToken` runs per request. Handle refresh inline:
|
`getToken` runs per request. Handle refresh inline:
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
let cachedToken: string | null = null
|
let cachedToken = null
|
||||||
|
|
||||||
const auth = createAuthExtension({
|
const auth = createAuthExtension({
|
||||||
name: 'jwt-with-refresh',
|
name: 'jwt-with-refresh',
|
||||||
|
|||||||
@@ -28,10 +28,10 @@ Each entry is keyed by a hash of the route's path, method, and schema. If the sc
|
|||||||
|
|
||||||
| Environment | Cache | Reason |
|
| Environment | Cache | Reason |
|
||||||
|-------------|-------|--------|
|
|-------------|-------|--------|
|
||||||
| `production` | Disabled | No file I/O, no cache hits needed |
|
| `production` | Enabled by default | Set `APOPHIS_DISABLE_CACHE=1` to opt-out |
|
||||||
| `test` | Disabled | Tests should be deterministic, no cache pollution |
|
| `test` | Enabled by default | Set `APOPHIS_DISABLE_CACHE=1` to opt-out |
|
||||||
| `development` | Enabled | Speeds up iterative testing |
|
| `development` | Enabled by default | Speeds up iterative testing |
|
||||||
| default | Enabled | Backward compatible |
|
| default | Enabled by default | Backward compatible |
|
||||||
|
|
||||||
## Cache Invalidation
|
## Cache Invalidation
|
||||||
|
|
||||||
|
|||||||
+38
-34
@@ -2,19 +2,20 @@
|
|||||||
|
|
||||||
Inject controlled failures into contract tests to validate resilience guarantees.
|
Inject controlled failures into contract tests to validate resilience guarantees.
|
||||||
|
|
||||||
|
Chaos testing applies the invariant-driven verification approach from [Invariant-Driven Automated Testing](https://arxiv.org/abs/2602.23922) (Malhado Ribeiro, 2021) under adverse conditions: if a contract must hold, it should still hold when dependencies fail, responses are delayed, or payloads are corrupted.
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
```typescript
|
```javascript
|
||||||
const result = await fastify.apophis.contract({
|
const result = await fastify.apophis.contract({
|
||||||
depth: 'standard',
|
runs: 50,
|
||||||
chaos: {
|
chaos: {
|
||||||
probability: 0.1, // 10% of requests get chaos
|
delay: { probability: 0.1, minMs: 100, maxMs: 500 },
|
||||||
delay: { probability: 1, minMs: 100, maxMs: 500 },
|
error: { probability: 0.1, statusCode: 503 },
|
||||||
error: { probability: 1, statusCode: 503 },
|
dropout: { probability: 0.05 },
|
||||||
dropout: { probability: 1 },
|
corruption: { probability: 0.1 },
|
||||||
corruption: { probability: 1 },
|
|
||||||
},
|
},
|
||||||
})
|
});
|
||||||
```
|
```
|
||||||
|
|
||||||
## Event Types
|
## Event Types
|
||||||
@@ -24,16 +25,18 @@ const result = await fastify.apophis.contract({
|
|||||||
Adds artificial latency. Tests timeout contracts:
|
Adds artificial latency. Tests timeout contracts:
|
||||||
|
|
||||||
```apostl
|
```apostl
|
||||||
timeout_occurred(this) == false
|
|
||||||
response_time(this) < 1000
|
response_time(this) < 1000
|
||||||
```
|
```
|
||||||
|
|
||||||
|
**Note**: Delay events are generated by the chaos arbitrary but the inbound delay handler is currently a no-op. Use this for timeout contract documentation; actual delay injection requires the outbound delay strategy or a custom handler.
|
||||||
|
|
||||||
### Error
|
### Error
|
||||||
|
|
||||||
Forces HTTP status codes. Tests error-handling contracts:
|
Forces HTTP status codes. Tests error-handling contracts:
|
||||||
|
|
||||||
```apostl
|
```apostl
|
||||||
if status:503 then response_body(this).retry_after != null
|
// Behavioral: when the service is unavailable, the client receives a valid retry signal
|
||||||
|
if status:503 then response_headers(this).retry-after > 0
|
||||||
```
|
```
|
||||||
|
|
||||||
### Dropout
|
### Dropout
|
||||||
@@ -41,7 +44,8 @@ if status:503 then response_body(this).retry_after != null
|
|||||||
Simulates network failure (status 0). Tests fallback contracts:
|
Simulates network failure (status 0). Tests fallback contracts:
|
||||||
|
|
||||||
```apostl
|
```apostl
|
||||||
status:200 || status:0
|
// Behavioral: partial failure must still return previously cached data
|
||||||
|
if status:0 then response_body(this).cached_data == previous(response_body(GET /cache/{request_params(this).key}))
|
||||||
```
|
```
|
||||||
|
|
||||||
### Corruption
|
### Corruption
|
||||||
@@ -49,38 +53,39 @@ status:200 || status:0
|
|||||||
Mutates response bodies. Tests parsing robustness:
|
Mutates response bodies. Tests parsing robustness:
|
||||||
|
|
||||||
```apostl
|
```apostl
|
||||||
response_body(this).id != null
|
// Behavioral: corrupted requests maintain traceability for debugging
|
||||||
|
if status:400 then response_body(this).request_id == request_headers(this).x-request-id
|
||||||
```
|
```
|
||||||
|
|
||||||
## Content-Type Aware Corruption
|
## Corruption Strategies
|
||||||
|
|
||||||
Built-in strategies for common formats:
|
Built-in strategies are content-type agnostic:
|
||||||
|
|
||||||
| Content-Type | Strategy | Effect |
|
| Strategy | Effect |
|
||||||
|-------------|----------|--------|
|
|----------|--------|
|
||||||
| `application/json` | Truncate or null field | Removes fields or sets random field to null |
|
| `truncate` | Cuts response body short |
|
||||||
| `application/x-ndjson` | Chunk corrupt | Corrupts one NDJSON chunk |
|
| `malformed` | Invalidates structural boundaries (e.g., unclosed JSON, bad headers) |
|
||||||
| `text/event-stream` | Event corrupt | Adds malformed SSE line |
|
| `field-corrupt` | Replaces a random field value with corrupted data |
|
||||||
| `multipart/form-data` | Field corrupt | Replaces field with corrupted data |
|
|
||||||
| `text/plain` | Truncate | Cuts string in half |
|
Extension strategies can add content-type-specific behavior if needed.
|
||||||
|
|
||||||
## Custom Corruption via Extensions
|
## Custom Corruption via Extensions
|
||||||
|
|
||||||
```typescript
|
```javascript
|
||||||
const myExtension = {
|
const myExtension = {
|
||||||
name: 'custom-corrupt',
|
name: 'custom-corrupt',
|
||||||
corruptionStrategies: {
|
corruptionStrategies: {
|
||||||
'application/vnd.api+json': (data) => ({
|
'application/vnd.api+json': (data) => ({
|
||||||
...data as object,
|
...data,
|
||||||
corrupted: true,
|
corrupted: true,
|
||||||
}),
|
}),
|
||||||
'text/*': (data) => `CORRUPTED:${String(data)}`,
|
'text/*': (data) => `CORRUPTED:${String(data)}`,
|
||||||
},
|
},
|
||||||
}
|
};
|
||||||
|
|
||||||
await fastify.register(apophis, {
|
await fastify.register(apophis, {
|
||||||
extensions: [myExtension],
|
extensions: [myExtension],
|
||||||
})
|
});
|
||||||
```
|
```
|
||||||
|
|
||||||
Extension strategies take precedence over built-ins. Wildcard patterns (`text/*`) match any subtype.
|
Extension strategies take precedence over built-ins. Wildcard patterns (`text/*`) match any subtype.
|
||||||
@@ -90,7 +95,7 @@ Extension strategies take precedence over built-ins. Wildcard patterns (`text/*`
|
|||||||
Low-level contract chaos APIs require `NODE_ENV=test`. For CLI qualification, environment policy controls whether chaos gates may run.
|
Low-level contract chaos APIs require `NODE_ENV=test`. For CLI qualification, environment policy controls whether chaos gates may run.
|
||||||
|
|
||||||
```
|
```
|
||||||
Error: Chaos mode is only available in test environment.
|
Error: chaos is only available in test environment. Set NODE_ENV=test to enable quality features.
|
||||||
```
|
```
|
||||||
|
|
||||||
## Interpreting Results
|
## Interpreting Results
|
||||||
@@ -100,7 +105,7 @@ Failed tests include chaos events in diagnostics:
|
|||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
"statusCode": 503,
|
"statusCode": 503,
|
||||||
"error": "Contract violation: status:200",
|
"error": "Contract violation: if status:503 then response_headers(this).retry-after > 0",
|
||||||
"chaosEvents": [
|
"chaosEvents": [
|
||||||
{
|
{
|
||||||
"type": "error",
|
"type": "error",
|
||||||
@@ -118,26 +123,25 @@ Failed tests include chaos events in diagnostics:
|
|||||||
|
|
||||||
1. **Start small**: `probability: 0.05` (5% of requests)
|
1. **Start small**: `probability: 0.05` (5% of requests)
|
||||||
2. **Test one failure mode at a time**: Comment out other chaos types
|
2. **Test one failure mode at a time**: Comment out other chaos types
|
||||||
3. **Verify contracts handle chaos**: `if status:503 then response_body(this).error != null`
|
3. **Verify contracts handle chaos**: `if status:503 then response_code(GET /health) == 200`
|
||||||
4. **Use seeds for reproducibility**: `seed: 42` makes chaos deterministic
|
4. **Use seeds for reproducibility**: `seed: 42` makes chaos deterministic
|
||||||
|
|
||||||
## Example: Testing Retry Logic
|
## Example: Testing Retry Logic
|
||||||
|
|
||||||
```typescript
|
```javascript
|
||||||
fastify.get('/data', {
|
fastify.get('/data', {
|
||||||
schema: {
|
schema: {
|
||||||
'x-ensures': [
|
'x-ensures': [
|
||||||
'if status:503 then response_headers(this).retry-after != null',
|
'if status:503 then response_headers(this).retry-after > 0',
|
||||||
'redirect_count(this) <= 3',
|
'redirect_count(this) <= 3',
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
}, handler)
|
}, handler);
|
||||||
|
|
||||||
// Test
|
// Test
|
||||||
const result = await fastify.apophis.contract({
|
const result = await fastify.apophis.contract({
|
||||||
chaos: {
|
chaos: {
|
||||||
probability: 0.2,
|
error: { probability: 0.2, statusCode: 503 },
|
||||||
error: { probability: 1, statusCode: 503 },
|
|
||||||
},
|
},
|
||||||
})
|
});
|
||||||
```
|
```
|
||||||
|
|||||||
+39
-23
@@ -10,15 +10,16 @@ Every command accepts these flags:
|
|||||||
|---|---|---|
|
|---|---|---|
|
||||||
| `--config <path>` | Config file path | Auto-detect |
|
| `--config <path>` | Config file path | Auto-detect |
|
||||||
| `--profile <name>` | Profile name from config | First profile |
|
| `--profile <name>` | Profile name from config | First profile |
|
||||||
| `--generation-profile <name>` | Generation budget profile (built-in or config alias) | Depth-derived |
|
|
||||||
| `--cwd <path>` | Working directory override | `process.cwd()` |
|
| `--cwd <path>` | Working directory override | `process.cwd()` |
|
||||||
| `--format <mode>` | Output format: `human`, `json`, `ndjson`, `json-summary`, `ndjson-summary` | `human` |
|
| `--format <mode>` | Output format: `human`, `json`, `ndjson`, `json-summary`, `ndjson-summary` | `human` |
|
||||||
| `--color <mode>` | Color mode: `auto`, `always`, `never` | `auto` |
|
| `--color <mode>` | Color mode: `auto`, `always`, `never` | `auto` |
|
||||||
| `--quiet` | Suppress non-error output | false |
|
| `--quiet` | Suppress non-error output | false |
|
||||||
| `--verbose` | Enable verbose logging | false |
|
| `--verbose` | Enable verbose logging | false |
|
||||||
| `--artifact-dir <path>` | Directory for artifact output | `reports/apophis/` |
|
| `--artifact-dir <path>` | Directory for artifact output. Artifacts written on failure or when explicitly configured. | `reports/apophis/` |
|
||||||
| `--workspace` | Run supported commands across workspace packages | false |
|
| `--workspace` | Run supported commands across workspace packages | false |
|
||||||
|
|
||||||
|
Note: `json-summary` and `ndjson-summary` are only supported by `verify` and `qualify` commands.
|
||||||
|
|
||||||
## Commands
|
## Commands
|
||||||
|
|
||||||
### `apophis init`
|
### `apophis init`
|
||||||
@@ -37,8 +38,8 @@ apophis init --preset safe-ci
|
|||||||
|
|
||||||
| Flag | Description |
|
| Flag | Description |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `--preset <name>` | Preset name: `safe-ci`, `platform-observe`, `llm-safe`, `protocol-lab` |
|
| `-p, --preset <name>` | Preset name: `safe-ci`, `platform-observe`, `llm-safe`, `protocol-lab` |
|
||||||
| `--force` | Overwrite existing files |
|
| `-f, --force` | Overwrite existing files |
|
||||||
| `--noninteractive` | Skip all prompts, require explicit flags |
|
| `--noninteractive` | Skip all prompts, require explicit flags |
|
||||||
|
|
||||||
**Examples:**
|
**Examples:**
|
||||||
@@ -60,10 +61,10 @@ apophis verify --profile quick --routes "POST /users"
|
|||||||
| Flag | Description |
|
| Flag | Description |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `--profile <name>` | Profile name from config |
|
| `--profile <name>` | Profile name from config |
|
||||||
| `--generation-profile <name>` | Override generation budget for this run |
|
|
||||||
| `--routes <filter>` | Route filter pattern (comma-separated, supports wildcards) |
|
| `--routes <filter>` | Route filter pattern (comma-separated, supports wildcards) |
|
||||||
| `--seed <number>` | Deterministic seed (generated and printed if omitted) |
|
| `--seed <number>` | Deterministic seed (generated and printed if omitted) |
|
||||||
| `--changed` | Filter to git-modified routes only |
|
| `--changed` | Filter to git-modified routes only |
|
||||||
|
| `--workspace` | Run across all workspace packages |
|
||||||
| `--format <mode>` | Output format: `human`, `json`, `ndjson`, `json-summary`, `ndjson-summary` |
|
| `--format <mode>` | Output format: `human`, `json`, `ndjson`, `json-summary`, `ndjson-summary` |
|
||||||
|
|
||||||
**Examples:**
|
**Examples:**
|
||||||
@@ -118,7 +119,6 @@ apophis qualify --profile oauth-nightly --seed 42
|
|||||||
| Flag | Description |
|
| Flag | Description |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `--profile <name>` | Profile name from config |
|
| `--profile <name>` | Profile name from config |
|
||||||
| `--generation-profile <name>` | Override generation budget for this run |
|
|
||||||
| `--seed <number>` | Deterministic seed (generated and printed if omitted) |
|
| `--seed <number>` | Deterministic seed (generated and printed if omitted) |
|
||||||
|
|
||||||
**Examples:**
|
**Examples:**
|
||||||
@@ -126,18 +126,6 @@ apophis qualify --profile oauth-nightly --seed 42
|
|||||||
```bash
|
```bash
|
||||||
apophis qualify --profile oauth-nightly --seed 42
|
apophis qualify --profile oauth-nightly --seed 42
|
||||||
apophis qualify --profile lifecycle-deep
|
apophis qualify --profile lifecycle-deep
|
||||||
apophis qualify --profile oauth-nightly --generation-profile quick
|
|
||||||
```
|
|
||||||
|
|
||||||
You can define aliases in config:
|
|
||||||
|
|
||||||
```js
|
|
||||||
export default {
|
|
||||||
generationProfiles: {
|
|
||||||
pr: 'quick',
|
|
||||||
nightly: { base: 'thorough' },
|
|
||||||
},
|
|
||||||
}
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### `apophis replay`
|
### `apophis replay`
|
||||||
@@ -171,6 +159,7 @@ apophis doctor [--mode verify|observe|qualify] [--strict]
|
|||||||
|---|---|
|
|---|---|
|
||||||
| `--mode <mode>` | Filter checks to a specific mode |
|
| `--mode <mode>` | Filter checks to a specific mode |
|
||||||
| `--strict` | Treat warnings as failures |
|
| `--strict` | Treat warnings as failures |
|
||||||
|
| `--workspace` | Run across all workspace packages |
|
||||||
|
|
||||||
**Checks:**
|
**Checks:**
|
||||||
|
|
||||||
@@ -210,6 +199,31 @@ apophis migrate --dry-run
|
|||||||
apophis migrate --write
|
apophis migrate --write
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Common Tasks
|
||||||
|
|
||||||
|
### CI workflow with machine output
|
||||||
|
```bash
|
||||||
|
apophis verify --profile ci --format json-summary --artifact-dir reports/apophis
|
||||||
|
```
|
||||||
|
|
||||||
|
### Monorepo workspace verification
|
||||||
|
```bash
|
||||||
|
apophis verify --workspace --profile quick
|
||||||
|
apophis doctor --workspace
|
||||||
|
```
|
||||||
|
|
||||||
|
### Replay a failure
|
||||||
|
```bash
|
||||||
|
apophis replay --artifact reports/apophis/failure-*.json
|
||||||
|
```
|
||||||
|
|
||||||
|
## Gotchas
|
||||||
|
|
||||||
|
- `--changed` requires a git repository
|
||||||
|
- `migrate` defaults to `--dry-run` (safe by default)
|
||||||
|
- `--workspace` is fully implemented by `verify` and `doctor`. `observe` and `qualify` accept the flag but run in the current package only.
|
||||||
|
- Seeds ensure deterministic generation; handler nondeterminism (e.g., `Date.now()`) can still cause replay divergence
|
||||||
|
|
||||||
## Exit Codes
|
## Exit Codes
|
||||||
|
|
||||||
| Code | Meaning |
|
| Code | Meaning |
|
||||||
@@ -226,10 +240,12 @@ apophis migrate --write
|
|||||||
|---|---|---|---|---|
|
|---|---|---|---|---|
|
||||||
| `verify` | enabled | enabled | optional | optional, usually off |
|
| `verify` | enabled | enabled | optional | optional, usually off |
|
||||||
| `observe` | optional | optional | enabled | enabled |
|
| `observe` | optional | optional | enabled | enabled |
|
||||||
| `qualify: scenario` | enabled | enabled | enabled with allowlist | disabled by default |
|
| `qualify` | enabled | enabled | optional | disabled by default |
|
||||||
| `qualify: stateful` | enabled | enabled | synthetic-only | disabled by default |
|
| `chaos` | enabled | enabled | optional | disabled by default |
|
||||||
| `qualify: chaos` | enabled | enabled | canary-only | disabled by default |
|
|
||||||
| outbound mocks | enabled | enabled | allowlisted only | disabled by default |
|
|
||||||
| runtime throw-on-violation | optional | optional | exceptional | disabled by default |
|
| runtime throw-on-violation | optional | optional | exceptional | disabled by default |
|
||||||
|
|
||||||
Operational rule: Production must never inherit qualify capabilities accidentally from a generic config file.
|
Notes:
|
||||||
|
- `qualify` is gated as a whole. The code does not distinguish scenario, stateful, and chaos sub-modes in environment policy.
|
||||||
|
- `chaos` on protected routes requires `allowChaosOnProtected: true`.
|
||||||
|
- `observe` blocking requires `allowBlocking: true`.
|
||||||
|
- Production must never inherit qualify capabilities accidentally from a generic config file.
|
||||||
|
|||||||
+44
-29
@@ -1,24 +1,29 @@
|
|||||||
import Fastify from 'fastify'
|
import Fastify from 'fastify'
|
||||||
import apophisPlugin from 'apophis-fastify'
|
import apophisPlugin from 'apophis-fastify'
|
||||||
|
import crypto from 'crypto'
|
||||||
|
|
||||||
const fastify = Fastify()
|
const fastify = Fastify()
|
||||||
|
|
||||||
await fastify.register(apophisPlugin, {
|
await fastify.register(apophisPlugin, {
|
||||||
runtime: 'error', // Validate contracts on every request
|
runtime: 'error',
|
||||||
cleanup: true, // Auto-cleanup resources on exit
|
cleanup: true,
|
||||||
})
|
})
|
||||||
|
|
||||||
// In-memory store for demo
|
|
||||||
const users = new Map<string, { id: string; email: string; name: string }>()
|
const users = new Map<string, { id: string; email: string; name: string }>()
|
||||||
|
|
||||||
// CREATE — constructor
|
// CREATE — constructor
|
||||||
|
// Behavioral: the created user must be retrievable.
|
||||||
|
// Note: we do not write 'status:201' or 'response_body(this).id != null'.
|
||||||
|
// The schema already validates status codes and required fields.
|
||||||
|
// Contracts should test behavior across operations, not structure.
|
||||||
fastify.post('/users', {
|
fastify.post('/users', {
|
||||||
schema: {
|
schema: {
|
||||||
'x-category': 'constructor',
|
'x-category': 'constructor',
|
||||||
'x-ensures': [
|
'x-ensures': [
|
||||||
'status:201',
|
// Round-trip: the server returns exactly what we sent (no mutation, no drops)
|
||||||
'response_body(this).id != null',
|
'response_body(this) == request_body(this)',
|
||||||
'response_body(this).email == request_body(this).email',
|
// Cross-route: the created user must be retrievable
|
||||||
|
'response_code(GET /users/{response_body(this).id}) == 200',
|
||||||
],
|
],
|
||||||
body: {
|
body: {
|
||||||
type: 'object',
|
type: 'object',
|
||||||
@@ -40,7 +45,7 @@ fastify.post('/users', {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}, async (req, reply) => {
|
}, async (req, reply) => {
|
||||||
const id = `usr-${Date.now()}`
|
const id = `usr-${crypto.createHash('sha256').update(req.body.email).digest('hex').slice(0, 8)}`
|
||||||
const user = { id, email: req.body.email, name: req.body.name }
|
const user = { id, email: req.body.email, name: req.body.name }
|
||||||
users.set(id, user)
|
users.set(id, user)
|
||||||
reply.status(201)
|
reply.status(201)
|
||||||
@@ -48,19 +53,21 @@ fastify.post('/users', {
|
|||||||
})
|
})
|
||||||
|
|
||||||
// READ — observer
|
// READ — observer
|
||||||
|
// Behavioral: the returned user must match the requested id.
|
||||||
fastify.get('/users/:id', {
|
fastify.get('/users/:id', {
|
||||||
schema: {
|
schema: {
|
||||||
'x-category': 'observer',
|
'x-category': 'observer',
|
||||||
'x-requires': ['users:id'],
|
'x-requires': [
|
||||||
|
// Precondition: the user must exist for this read to be valid
|
||||||
|
'response_code(GET /users/{request_params(this).id}) == 200'
|
||||||
|
],
|
||||||
'x-ensures': [
|
'x-ensures': [
|
||||||
'status:200',
|
// The returned id must match the requested id (no mix-up)
|
||||||
'response_body(this).id == request_params(this).id',
|
'response_body(this).id == request_params(this).id',
|
||||||
],
|
],
|
||||||
params: {
|
params: {
|
||||||
type: 'object',
|
type: 'object',
|
||||||
properties: {
|
properties: { id: { type: 'string' } },
|
||||||
id: { type: 'string' }
|
|
||||||
},
|
|
||||||
required: ['id']
|
required: ['id']
|
||||||
},
|
},
|
||||||
response: {
|
response: {
|
||||||
@@ -83,19 +90,21 @@ fastify.get('/users/:id', {
|
|||||||
})
|
})
|
||||||
|
|
||||||
// UPDATE — mutator
|
// UPDATE — mutator
|
||||||
|
// Behavioral: after update, the change must be visible on read.
|
||||||
fastify.put('/users/:id', {
|
fastify.put('/users/:id', {
|
||||||
schema: {
|
schema: {
|
||||||
'x-category': 'mutator',
|
'x-category': 'mutator',
|
||||||
'x-requires': ['users:id'],
|
'x-requires': [
|
||||||
|
// The user must exist before updating
|
||||||
|
'response_code(GET /users/{request_params(this).id}) == 200'
|
||||||
|
],
|
||||||
'x-ensures': [
|
'x-ensures': [
|
||||||
'status:200',
|
// Cross-route: after update, reading the user shows the new data
|
||||||
'response_body(this).id == request_params(this).id',
|
'response_body(GET /users/{request_params(this).id}).email == request_body(this).email',
|
||||||
],
|
],
|
||||||
params: {
|
params: {
|
||||||
type: 'object',
|
type: 'object',
|
||||||
properties: {
|
properties: { id: { type: 'string' } },
|
||||||
id: { type: 'string' }
|
|
||||||
},
|
|
||||||
required: ['id']
|
required: ['id']
|
||||||
},
|
},
|
||||||
body: {
|
body: {
|
||||||
@@ -131,34 +140,40 @@ fastify.put('/users/:id', {
|
|||||||
})
|
})
|
||||||
|
|
||||||
// DELETE — destructor
|
// DELETE — destructor
|
||||||
|
// Behavioral: after deletion, the user must no longer exist.
|
||||||
fastify.delete('/users/:id', {
|
fastify.delete('/users/:id', {
|
||||||
schema: {
|
schema: {
|
||||||
'x-category': 'destructor',
|
'x-category': 'destructor',
|
||||||
'x-requires': ['users:id'],
|
'x-requires': [
|
||||||
'x-ensures': ['status:204'],
|
// The user must exist before deleting
|
||||||
|
'response_code(GET /users/{request_params(this).id}) == 200'
|
||||||
|
],
|
||||||
|
'x-ensures': [
|
||||||
|
// After deletion, the user is gone
|
||||||
|
'response_code(GET /users/{request_params(this).id}) == 404',
|
||||||
|
// The deleted user data is returned (matches pre-deletion read)
|
||||||
|
'response_body(this) == previous(response_body(GET /users/{request_params(this).id}))',
|
||||||
|
],
|
||||||
params: {
|
params: {
|
||||||
type: 'object',
|
type: 'object',
|
||||||
properties: {
|
properties: { id: { type: 'string' } },
|
||||||
id: { type: 'string' }
|
|
||||||
},
|
|
||||||
required: ['id']
|
required: ['id']
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}, async (req, reply) => {
|
}, async (req, reply) => {
|
||||||
|
const user = users.get(req.params.id)
|
||||||
users.delete(req.params.id)
|
users.delete(req.params.id)
|
||||||
reply.status(204)
|
reply.status(200)
|
||||||
|
return user
|
||||||
})
|
})
|
||||||
|
|
||||||
await fastify.ready()
|
await fastify.ready()
|
||||||
|
|
||||||
// Run contract tests (all non-utility routes, property-based)
|
const result = await fastify.apophis.contract({ runs: 50 })
|
||||||
const result = await fastify.apophis.contract({ depth: 'standard' })
|
|
||||||
console.log('Contract tests:', result.summary)
|
console.log('Contract tests:', result.summary)
|
||||||
|
|
||||||
// Run stateful tests (constructor→mutator→destructor sequences)
|
const stateful = await fastify.apophis.stateful({ runs: 50, seed: 42 })
|
||||||
const stateful = await fastify.apophis.stateful({ depth: 'standard', seed: 42 })
|
|
||||||
console.log('Stateful tests:', stateful.summary)
|
console.log('Stateful tests:', stateful.summary)
|
||||||
|
|
||||||
// Validate a single route
|
|
||||||
const check = await fastify.apophis.check('POST', '/users')
|
const check = await fastify.apophis.check('POST', '/users')
|
||||||
console.log('POST /users check:', check.ok ? 'PASS' : 'FAIL')
|
console.log('POST /users check:', check.ok ? 'PASS' : 'FAIL')
|
||||||
|
|||||||
@@ -6,21 +6,30 @@ const fastify = Fastify()
|
|||||||
// APOPHIS auto-registers @fastify/swagger
|
// APOPHIS auto-registers @fastify/swagger
|
||||||
await fastify.register(apophisPlugin, {})
|
await fastify.register(apophisPlugin, {})
|
||||||
|
|
||||||
fastify.get('/health', {
|
// Behavioral contract: what you send is what you get back.
|
||||||
|
// This is not a structural test — the schema already validates shape.
|
||||||
|
// This checks that the server does not mutate or drop fields.
|
||||||
|
fastify.post('/echo', {
|
||||||
schema: {
|
schema: {
|
||||||
'x-category': 'observer',
|
'x-category': 'observer',
|
||||||
'x-ensures': ['status:200'],
|
'x-ensures': [
|
||||||
|
'response_body(this) == request_body(this)'
|
||||||
|
],
|
||||||
|
body: {
|
||||||
|
type: 'object',
|
||||||
|
properties: { message: { type: 'string' } }
|
||||||
|
},
|
||||||
response: {
|
response: {
|
||||||
200: {
|
200: {
|
||||||
type: 'object',
|
type: 'object',
|
||||||
properties: { status: { type: 'string' } }
|
properties: { message: { type: 'string' } }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}, async () => ({ status: 'ok' }))
|
}, async (req) => req.body)
|
||||||
|
|
||||||
await fastify.ready()
|
await fastify.ready()
|
||||||
|
|
||||||
// Run contract tests
|
// Run contract tests
|
||||||
const result = await fastify.apophis.contract({ depth: 'quick' })
|
const result = await fastify.apophis.contract({ runs: 10 })
|
||||||
console.log(result.summary)
|
console.log(result.summary)
|
||||||
|
|||||||
@@ -280,8 +280,8 @@ app.get('/users/:id', {
|
|||||||
type: 'object',
|
type: 'object',
|
||||||
properties: { id: { type: 'string' } },
|
properties: { id: { type: 'string' } },
|
||||||
'x-ensures': [
|
'x-ensures': [
|
||||||
// Standard APOSTL + extension predicates
|
// Behavioral: returned user must match the requested id
|
||||||
'status:200',
|
'response_body(this).id == request_params(this).id',
|
||||||
'graph_check(this).user.can_read_user == true',
|
'graph_check(this).user.can_read_user == true',
|
||||||
'partial_graph(this).tenant.accessible == true',
|
'partial_graph(this).tenant.accessible == true',
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -284,8 +284,8 @@ fastify.get('/api/resource', {
|
|||||||
'x-ensures': [
|
'x-ensures': [
|
||||||
'timeout_occurred(this) == false',
|
'timeout_occurred(this) == false',
|
||||||
'redirect_count(this) == 0',
|
'redirect_count(this) == 0',
|
||||||
'response_code(this) == 200',
|
// Behavioral: created resource must be retrievable
|
||||||
'response_body(this).id != null',
|
'response_code(GET /api/resource/{response_body(this).id}) == 200',
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
}, handler)
|
}, handler)
|
||||||
|
|||||||
+179
-106
@@ -2,6 +2,8 @@
|
|||||||
|
|
||||||
Get from install to your first behavioral bug in 10 minutes.
|
Get from install to your first behavioral bug in 10 minutes.
|
||||||
|
|
||||||
|
APOPHIS is inspired by [Invariant-Driven Automated Testing](https://arxiv.org/abs/2602.23922) (Malhado Ribeiro, 2021): instead of only validating request and response shape, encode intended behavior as executable contracts and let the tool find violations automatically.
|
||||||
|
|
||||||
## Prerequisites
|
## Prerequisites
|
||||||
|
|
||||||
- Node.js 20.x or 22.x
|
- Node.js 20.x or 22.x
|
||||||
@@ -30,6 +32,8 @@ This creates:
|
|||||||
Pick one important route. Add an `x-ensures` clause that checks behavior across operations:
|
Pick one important route. Add an `x-ensures` clause that checks behavior across operations:
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
|
import crypto from 'crypto';
|
||||||
|
|
||||||
app.post('/users', {
|
app.post('/users', {
|
||||||
schema: {
|
schema: {
|
||||||
'x-category': 'constructor',
|
'x-category': 'constructor',
|
||||||
@@ -40,27 +44,20 @@ app.post('/users', {
|
|||||||
}
|
}
|
||||||
}, async (request, reply) => {
|
}, async (request, reply) => {
|
||||||
const { name } = request.body;
|
const { name } = request.body;
|
||||||
const id = `usr-${Date.now()}`;
|
const id = `usr-${crypto.createHash('sha256').update(name).digest('hex').slice(0, 8)}`;
|
||||||
reply.status(201);
|
reply.status(201);
|
||||||
return { id, name };
|
return { id, name };
|
||||||
});
|
});
|
||||||
```
|
```
|
||||||
|
|
||||||
|
> **Warning:** Using `Date.now()` or `Math.random()` in handlers breaks determinism and replay. Use a stable function of the input instead. APOPHIS does not proactively detect nondeterministic handlers; it warns only when a replay diverges from the original run.
|
||||||
|
|
||||||
## Step 4: Run Verify
|
## Step 4: Run Verify
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
apophis verify --profile quick --routes "POST /users"
|
apophis verify --profile quick --routes "POST /users"
|
||||||
```
|
```
|
||||||
|
|
||||||
APOPHIS will:
|
|
||||||
|
|
||||||
1. Discover routes from your Fastify app
|
|
||||||
2. Filter to `POST /users`
|
|
||||||
3. Generate test data from the schema
|
|
||||||
4. Execute the route
|
|
||||||
5. Check the behavioral contract
|
|
||||||
6. Print pass/fail, seed, and replay command
|
|
||||||
|
|
||||||
## Example Failure
|
## Example Failure
|
||||||
|
|
||||||
If your `GET /users/:id` handler has a bug (always returns 404), APOPHIS catches it:
|
If your `GET /users/:id` handler has a bug (always returns 404), APOPHIS catches it:
|
||||||
@@ -75,7 +72,7 @@ Expected
|
|||||||
response_code(GET /users/{response_body(this).id}) == 200
|
response_code(GET /users/{response_body(this).id}) == 200
|
||||||
|
|
||||||
Observed
|
Observed
|
||||||
GET /users/usr-123 returned 404
|
GET /users/usr-7d865e returned 404
|
||||||
|
|
||||||
Why this matters
|
Why this matters
|
||||||
The resource created by POST /users is not retrievable.
|
The resource created by POST /users is not retrievable.
|
||||||
@@ -97,114 +94,190 @@ apophis replay --artifact reports/apophis/failure-2026-04-28T12-30-22Z.json
|
|||||||
|
|
||||||
Fix the bug in your handler. Re-run verify. The failure should now pass.
|
Fix the bug in your handler. Re-run verify. The failure should now pass.
|
||||||
|
|
||||||
|
## Behavioral vs Structural Contracts
|
||||||
|
|
||||||
|
APOPHIS contracts should verify **behavior**, not structure. Fastify and `@fastify/swagger` already enforce status codes, required fields, and types. Behavioral contracts catch what schemas cannot:
|
||||||
|
|
||||||
|
| Structural (avoid) | Behavioral (prefer) |
|
||||||
|
|---|---|
|
||||||
|
| `status:200` | `response_body(this) == request_body(this)` |
|
||||||
|
| `response_body(this).id != null` | `response_code(GET /users/{response_body(this).id}) == 200` |
|
||||||
|
| `response_body(this).name != null` | `response_body(GET /users/{id}).name == previous(response_body(this).name)` |
|
||||||
|
|
||||||
|
**Good behavioral patterns (from the paper):**
|
||||||
|
- **Constructor precondition**: Resource must not exist before creation
|
||||||
|
```apostl
|
||||||
|
response_code(GET /users/{request_body(this).email}) == 404
|
||||||
|
```
|
||||||
|
- **Round-trip equality**: POST response matches the request body
|
||||||
|
```apostl
|
||||||
|
response_body(this) == request_body(this)
|
||||||
|
```
|
||||||
|
- **Cross-route retrievability**: Creating a resource makes it readable via GET
|
||||||
|
```apostl
|
||||||
|
response_code(GET /users/{response_body(this).id}) == 200
|
||||||
|
```
|
||||||
|
- **State-change verification**: DELETE causes subsequent GET to return 404
|
||||||
|
```apostl
|
||||||
|
response_code(GET /users/{request_params(this).id}) == 404
|
||||||
|
```
|
||||||
|
- **Previous state preservation**: DELETE returns the last known state
|
||||||
|
```apostl
|
||||||
|
response_body(this) == previous(response_body(GET /users/{request_params(this).id}))
|
||||||
|
```
|
||||||
|
- **Invariant over collections**: All resources satisfy a cross-resource constraint
|
||||||
|
```apostl
|
||||||
|
for t in response_body(GET /tournaments) :-
|
||||||
|
response_body(GET /tournaments/{t.id}/players).length <= t.capacity
|
||||||
|
```
|
||||||
|
|
||||||
|
**Anti-patterns to avoid:**
|
||||||
|
- Checking status codes (handled by schema validation)
|
||||||
|
- Checking field existence (handled by schema validation)
|
||||||
|
- Checking field types (handled by schema validation)
|
||||||
|
|
||||||
## Next Steps
|
## Next Steps
|
||||||
|
|
||||||
- Add more routes to your profile: `apophis verify --profile quick --routes "POST /users,PUT /users/:id"`
|
- Add more routes to your profile: `apophis verify --profile quick --routes "POST /users,PUT /users/:id"`
|
||||||
|
- Use wildcards to match route patterns: `apophis verify --routes 'POST /api/*'`
|
||||||
- Run all routes: `apophis verify --profile quick`
|
- Run all routes: `apophis verify --profile quick`
|
||||||
- Run only changed routes in CI: `apophis verify --profile ci --changed`
|
- Run only changed routes in CI: `apophis verify --profile ci --changed`
|
||||||
- Add observe mode for runtime drift detection: see [docs/observe.md](docs/observe.md)
|
- Requires a git repository.
|
||||||
- Add qualify mode for scenario, stateful, and chaos checks: see [docs/qualify.md](docs/qualify.md)
|
- Use machine-readable output in CI: `apophis verify --profile ci --format json-summary`
|
||||||
|
- Add observe mode for runtime drift detection: see [observe.md](observe.md)
|
||||||
|
- Add qualify mode for scenario, stateful, and chaos checks: see [qualify.md](qualify.md)
|
||||||
|
|
||||||
|
## Variants
|
||||||
|
|
||||||
|
Test the same route with different headers or content types:
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
await fastify.apophis.contract({
|
||||||
|
variants: [
|
||||||
|
{ name: 'json', headers: { accept: 'application/json' } },
|
||||||
|
{ name: 'xml', headers: { accept: 'application/xml' } }
|
||||||
|
]
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
Or declare variants in the route schema:
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
app.get('/users', {
|
||||||
|
schema: {
|
||||||
|
'x-variants': [
|
||||||
|
{ name: 'json', headers: { accept: 'application/json' } }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
## Plugin Options
|
||||||
|
|
||||||
|
When registering the APOPHIS plugin, you can pass these options:
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
await fastify.register(apophis, {
|
||||||
|
// Swagger config passthrough (if @fastify/swagger is not already registered)
|
||||||
|
swagger: { openapi: { info: { title: 'API', version: '1.0.0' } } },
|
||||||
|
|
||||||
|
// Runtime contract validation hooks: 'off', 'warn', or 'error'
|
||||||
|
// Only active in non-production environments
|
||||||
|
runtime: 'warn',
|
||||||
|
|
||||||
|
// Automatically clean up tracked resources after tests
|
||||||
|
cleanup: true,
|
||||||
|
|
||||||
|
// Global timeout in milliseconds for all requests
|
||||||
|
timeout: 5000,
|
||||||
|
|
||||||
|
// Tenant isolation scopes
|
||||||
|
scopes: {
|
||||||
|
tenant1: { headers: { 'x-tenant-id': '1' } },
|
||||||
|
tenant2: { headers: { 'x-tenant-id': '2' } },
|
||||||
|
},
|
||||||
|
|
||||||
|
// Auth and protocol extensions
|
||||||
|
extensions: [jwtAuth, apiKeyAuth],
|
||||||
|
|
||||||
|
// Plugin hook-phase contracts
|
||||||
|
pluginContracts: {
|
||||||
|
'rate-limit': { appliesTo: 'POST /users', ensures: ['status != 429'] },
|
||||||
|
},
|
||||||
|
|
||||||
|
// Outbound dependency contracts
|
||||||
|
outboundContracts: {
|
||||||
|
'payment-api': {
|
||||||
|
target: 'https://payments.example.com',
|
||||||
|
method: 'POST',
|
||||||
|
response: { 200: { type: 'object', properties: { id: { type: 'string' } } } }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
## Schema Annotations
|
||||||
|
|
||||||
|
APOPHIS reads these OpenAPI schema extensions:
|
||||||
|
|
||||||
|
| Annotation | Location | Description |
|
||||||
|
|---|---|---|
|
||||||
|
| `x-category` | Top-level | Route classification: `constructor`, `mutator`, `observer`, `destructor`, `utility` |
|
||||||
|
| `x-ensures` | Top-level or `response[statusCode]` | Post-condition contracts (APOSTL formulas) |
|
||||||
|
| `x-requires` | Top-level or `response[statusCode]` | Pre-condition contracts (APOSTL formulas) |
|
||||||
|
| `x-variants` | Top-level | Request variants for content-type negotiation or feature flags |
|
||||||
|
| `x-timeout` | Top-level or `response[statusCode]` | Per-route timeout in milliseconds |
|
||||||
|
| `x-outbound` | Top-level | Outbound dependency contracts for this route |
|
||||||
|
| `x-streaming` | Top-level | Mark route as streaming (populates `chunks` and `streamDurationMs` in eval context) |
|
||||||
|
| `x-validate-runtime` | Top-level or `response[statusCode]` | Toggle runtime validation for this route (default: true) |
|
||||||
|
| `x-extension-config` | Top-level | Per-route config for extensions (e.g., `{ jwt: { verify: false } }`) |
|
||||||
|
|
||||||
|
Annotations can be placed on the top-level schema or nested inside `response[statusCode]`. Nested annotations take precedence for that status code.
|
||||||
|
|
||||||
|
## Programmatic API
|
||||||
|
|
||||||
|
After registration, `fastify.apophis` provides:
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
// Run contract tests for all routes
|
||||||
|
const suite = await fastify.apophis.contract({ runs: 50, seed: 42 })
|
||||||
|
|
||||||
|
// Run stateful tests
|
||||||
|
const stateful = await fastify.apophis.stateful({ runs: 50, seed: 42 })
|
||||||
|
|
||||||
|
// Run a single scenario
|
||||||
|
const scenario = await fastify.apophis.scenario({
|
||||||
|
name: 'oauth-basic',
|
||||||
|
steps: [...]
|
||||||
|
})
|
||||||
|
|
||||||
|
// Check a single route
|
||||||
|
const result = await fastify.apophis.check('GET', '/users/:id')
|
||||||
|
|
||||||
|
// Get enriched OpenAPI spec with contract metadata
|
||||||
|
const spec = fastify.apophis.spec()
|
||||||
|
|
||||||
|
// Clean up tracked resources
|
||||||
|
await fastify.apophis.cleanup()
|
||||||
|
|
||||||
|
// Test-only utilities (NODE_ENV=test only)
|
||||||
|
fastify.apophis.test.registerPluginContracts('name', spec)
|
||||||
|
fastify.apophis.test.registerOutboundContracts({ ... })
|
||||||
|
fastify.apophis.test.enableOutboundMocks({ mode: 'example' })
|
||||||
|
fastify.apophis.test.disableOutboundMocks()
|
||||||
|
const calls = fastify.apophis.test.getOutboundCalls('payment-api')
|
||||||
|
```
|
||||||
|
|
||||||
## Config Reference
|
## Config Reference
|
||||||
|
|
||||||
```javascript
|
For the full configuration reference, see [CLI Reference](cli.md).
|
||||||
// apophis.config.js
|
|
||||||
export default {
|
|
||||||
mode: 'verify',
|
|
||||||
profile: 'quick',
|
|
||||||
profiles: {
|
|
||||||
quick: {
|
|
||||||
name: 'quick',
|
|
||||||
mode: 'verify',
|
|
||||||
preset: 'safe-ci',
|
|
||||||
routes: ['POST /users']
|
|
||||||
},
|
|
||||||
ci: {
|
|
||||||
name: 'ci',
|
|
||||||
mode: 'verify',
|
|
||||||
preset: 'safe-ci',
|
|
||||||
routes: []
|
|
||||||
}
|
|
||||||
},
|
|
||||||
presets: {
|
|
||||||
'safe-ci': {
|
|
||||||
name: 'safe-ci',
|
|
||||||
depth: 'quick',
|
|
||||||
timeout: 5000,
|
|
||||||
parallel: false,
|
|
||||||
chaos: false,
|
|
||||||
observe: false
|
|
||||||
}
|
|
||||||
},
|
|
||||||
environments: {
|
|
||||||
local: {
|
|
||||||
name: 'local',
|
|
||||||
allowVerify: true,
|
|
||||||
allowObserve: true,
|
|
||||||
allowQualify: false,
|
|
||||||
allowChaos: false,
|
|
||||||
allowBlocking: true,
|
|
||||||
requireSink: false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
```
|
|
||||||
|
|
||||||
## Monorepo Workspaces
|
## Monorepo Workspaces
|
||||||
|
|
||||||
APOPHIS supports workspace-wide operations with the `--workspace` flag.
|
Use `--workspace` to run verify or doctor across all packages:
|
||||||
|
|
||||||
### Root package.json scripts
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"scripts": {
|
|
||||||
"apophis:verify": "apophis verify --workspace --profile quick",
|
|
||||||
"apophis:doctor": "apophis doctor --workspace",
|
|
||||||
"apophis:qualify": "apophis qualify --workspace --profile ci"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Workspace fan-out
|
|
||||||
|
|
||||||
Run verify across all packages:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
apophis verify --workspace --profile quick --format json
|
apophis verify --workspace --profile quick --format json
|
||||||
```
|
```
|
||||||
|
|
||||||
Output is package-attributed:
|
See [CLI Reference](cli.md) for workspace output format and exit codes.
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"exitCode": 0,
|
|
||||||
"runs": [
|
|
||||||
{
|
|
||||||
"package": "api",
|
|
||||||
"cwd": "/repo/packages/api",
|
|
||||||
"artifact": { ... }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"package": "web",
|
|
||||||
"cwd": "/repo/packages/web",
|
|
||||||
"artifact": { ... }
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Supported commands
|
|
||||||
|
|
||||||
- `apophis verify --workspace`
|
|
||||||
- `apophis doctor --workspace`
|
|
||||||
|
|
||||||
## Exit Codes
|
|
||||||
|
|
||||||
| Code | Meaning |
|
|
||||||
|---|---|
|
|
||||||
| 0 | Success |
|
|
||||||
| 1 | Behavioral / qualification failure |
|
|
||||||
| 2 | Usage, config, or environment safety violation |
|
|
||||||
| 3 | Internal APOPHIS error |
|
|
||||||
| 130 | Interrupted (SIGINT) |
|
|
||||||
|
|||||||
@@ -2,6 +2,8 @@
|
|||||||
|
|
||||||
APOPHIS is designed to be safe and predictable for LLM-generated Fastify services.
|
APOPHIS is designed to be safe and predictable for LLM-generated Fastify services.
|
||||||
|
|
||||||
|
It applies the invariant-driven approach from [Invariant-Driven Automated Testing](https://arxiv.org/abs/2602.23922) (Malhado Ribeiro, 2021) to LLM-assisted development: constrained vocabulary, deterministic replay, and executable contracts give coding agents a verifiable loop between generated changes and behavioral correctness.
|
||||||
|
|
||||||
## Why APOPHIS Is Good for LLM-Generated Services
|
## Why APOPHIS Is Good for LLM-Generated Services
|
||||||
|
|
||||||
Coding agents benefit from:
|
Coding agents benefit from:
|
||||||
@@ -18,10 +20,10 @@ Use `apophis init` with a preset:
|
|||||||
|
|
||||||
| Preset | Use Case |
|
| Preset | Use Case |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `safe-ci` | General CI-safe setup |
|
| `safe-ci` | Minimal CI-safe preset (default) |
|
||||||
| `llm-safe` | Ultra-minimal for LLM-generated code |
|
| `llm-safe` | Minimal preset for LLM-generated codebases |
|
||||||
| `platform-observe` | Observe-mode policy and runtime drift reporting |
|
| `platform-observe` | Production-ready with observe mode |
|
||||||
| `protocol-lab` | Multi-step flows and stateful testing |
|
| `protocol-lab` | Multi-step flow and stateful testing |
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
apophis init --preset llm-safe
|
apophis init --preset llm-safe
|
||||||
@@ -84,7 +86,6 @@ export default {
|
|||||||
presets: {
|
presets: {
|
||||||
'llm-safe': {
|
'llm-safe': {
|
||||||
name: 'llm-safe',
|
name: 'llm-safe',
|
||||||
depth: 'quick',
|
|
||||||
timeout: 3000,
|
timeout: 3000,
|
||||||
parallel: false,
|
parallel: false,
|
||||||
chaos: false,
|
chaos: false,
|
||||||
@@ -108,6 +109,8 @@ export default {
|
|||||||
### Route Template with Behavioral Contract
|
### Route Template with Behavioral Contract
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
|
import crypto from 'crypto';
|
||||||
|
|
||||||
app.post('/users', {
|
app.post('/users', {
|
||||||
schema: {
|
schema: {
|
||||||
'x-category': 'constructor',
|
'x-category': 'constructor',
|
||||||
@@ -134,7 +137,7 @@ app.post('/users', {
|
|||||||
}
|
}
|
||||||
}, async (request, reply) => {
|
}, async (request, reply) => {
|
||||||
const { name } = request.body;
|
const { name } = request.body;
|
||||||
const id = `usr-${Date.now()}`;
|
const id = `usr-${crypto.createHash('sha256').update(name).digest('hex').slice(0, 8)}`;
|
||||||
reply.status(201);
|
reply.status(201);
|
||||||
return { id, name };
|
return { id, name };
|
||||||
});
|
});
|
||||||
|
|||||||
+50
-4
@@ -2,6 +2,8 @@
|
|||||||
|
|
||||||
Runtime visibility and drift detection without blocking by default.
|
Runtime visibility and drift detection without blocking by default.
|
||||||
|
|
||||||
|
Observe extends the invariant framework from [Invariant-Driven Automated Testing](https://arxiv.org/abs/2602.23922) (Malhado Ribeiro, 2021) to production environments: contracts run continuously against live traffic to detect behavioral drift without affecting requests.
|
||||||
|
|
||||||
## What Observe Does
|
## What Observe Does
|
||||||
|
|
||||||
`apophis observe` validates your runtime observe configuration:
|
`apophis observe` validates your runtime observe configuration:
|
||||||
@@ -65,14 +67,38 @@ profiles: {
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
The `platform-observe` preset enables sampling at the preset level. Fine-tune per route with `x-observe-sampling` in your route schema.
|
The `platform-observe` preset enables sampling. Configure the rate explicitly:
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
profiles: {
|
||||||
|
'staging-observe': {
|
||||||
|
mode: 'observe',
|
||||||
|
preset: 'platform-observe',
|
||||||
|
routes: [],
|
||||||
|
sampling: 1.0 // 100% of requests observed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
## Staging vs Production
|
## Staging vs Production
|
||||||
|
|
||||||
| Environment | Blocking | Sampling | Sink Required |
|
| Environment | Blocking | Sampling | Sink Required |
|
||||||
|---|---|---|---|
|
|---|---|---|---|
|
||||||
| Staging | No (default) | 10% | Yes |
|
| Staging | No (default) | 100% | Yes |
|
||||||
| Production | No (default) | 1% | Yes |
|
| Production | No (default) | 100% | Yes |
|
||||||
|
|
||||||
|
Default is `1.0` (100%). Configure lower rates for production explicitly:
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
profiles: {
|
||||||
|
'prod-observe': {
|
||||||
|
mode: 'observe',
|
||||||
|
preset: 'platform-observe',
|
||||||
|
routes: [],
|
||||||
|
sampling: 0.1 // 10% of requests observed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
## `--check-config` Flag
|
## `--check-config` Flag
|
||||||
|
|
||||||
@@ -109,7 +135,6 @@ export default {
|
|||||||
presets: {
|
presets: {
|
||||||
'platform-observe': {
|
'platform-observe': {
|
||||||
name: 'platform-observe',
|
name: 'platform-observe',
|
||||||
depth: 'standard',
|
|
||||||
timeout: 10000,
|
timeout: 10000,
|
||||||
parallel: true,
|
parallel: true,
|
||||||
chaos: false,
|
chaos: false,
|
||||||
@@ -138,3 +163,24 @@ export default {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Sink Endpoint Configuration
|
||||||
|
|
||||||
|
Configure the reporting sink endpoint in your observe config:
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
observe: {
|
||||||
|
sink: {
|
||||||
|
endpoint: 'http://collector.internal:4318'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Monorepo Validation
|
||||||
|
|
||||||
|
For monorepos, use `apophis doctor --workspace` to validate observe configuration across all workspace packages. `observe` itself does not support `--workspace`; use `doctor` to check config in each package.
|
||||||
|
|
||||||
|
## Mode Mismatch
|
||||||
|
|
||||||
|
Profiles configured for `verify` mode will be rejected by `apophis observe`. Only profiles with `mode: 'observe'` are valid.
|
||||||
|
```
|
||||||
|
|||||||
+5
-5
@@ -23,8 +23,8 @@ BENCH_RUNS=12 BENCH_WARMUP=3 npm run benchmark:cli
|
|||||||
# Increase inner-loop work for micro-benchmarks
|
# Increase inner-loop work for micro-benchmarks
|
||||||
BENCH_INNER_ITERS=5000 npm run benchmark:hot
|
BENCH_INNER_ITERS=5000 npm run benchmark:hot
|
||||||
|
|
||||||
# Benchmark generation profile matrix
|
# Benchmark with varying test counts
|
||||||
BENCH_GENERATION_PROFILES=quick,standard,thorough npm run benchmark:all
|
BENCH_RUNS=10,50,200 npm run benchmark:all
|
||||||
```
|
```
|
||||||
|
|
||||||
## Capture CPU Profile for Qualify
|
## Capture CPU Profile for Qualify
|
||||||
@@ -41,10 +41,10 @@ This writes Chrome-compatible CPU profiles to `.profiles/qualify.cpuprofile` and
|
|||||||
- CLI benchmark uses spawned `node dist/cli/index.js` commands so startup costs are included.
|
- CLI benchmark uses spawned `node dist/cli/index.js` commands so startup costs are included.
|
||||||
- Hot path benchmark runs in-process for lower-noise function-level comparisons.
|
- Hot path benchmark runs in-process for lower-noise function-level comparisons.
|
||||||
- Use fixed `--seed` for qualify benchmarks to keep runs deterministic.
|
- Use fixed `--seed` for qualify benchmarks to keep runs deterministic.
|
||||||
- Generation now adapts to depth: `quick` favors bounded payload generation speed, `thorough` keeps broader generation.
|
- Schema generation uses fixed defaults (string≤128, array≤10) regardless of run count.
|
||||||
|
|
||||||
You can override generation per run:
|
You can override runs per preset:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
apophis qualify --profile oauth-nightly --generation-profile quick --seed 42
|
apophis qualify --profile oauth-nightly --seed 42
|
||||||
```
|
```
|
||||||
|
|||||||
+74
-34
@@ -2,6 +2,8 @@
|
|||||||
|
|
||||||
Run scenario, stateful, and chaos checks against non-production Fastify services.
|
Run scenario, stateful, and chaos checks against non-production Fastify services.
|
||||||
|
|
||||||
|
Qualify extends the invariant-driven approach from [Invariant-Driven Automated Testing](https://arxiv.org/abs/2602.23922) (Malhado Ribeiro, 2021) with multi-step protocol flows, stateful sequences, and controlled fault injection.
|
||||||
|
|
||||||
## What Qualify Does
|
## What Qualify Does
|
||||||
|
|
||||||
`apophis qualify` runs deeper testing than verify:
|
`apophis qualify` runs deeper testing than verify:
|
||||||
@@ -49,6 +51,40 @@ profiles: {
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Scenario Definitions
|
||||||
|
|
||||||
|
Scenarios are multi-step flows with capture and rebind:
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
await fastify.apophis.scenario({
|
||||||
|
name: 'oauth-basic',
|
||||||
|
steps: [
|
||||||
|
{
|
||||||
|
name: 'authorize',
|
||||||
|
request: { method: 'GET', url: '/oauth/authorize?client_id=web&response_type=code&state=abc123' },
|
||||||
|
// Behavioral: state parameter round-trips for CSRF protection
|
||||||
|
expect: ['response_payload(this).state == request_query(this).state'],
|
||||||
|
capture: { code: 'response_payload(this).code' }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'token',
|
||||||
|
request: {
|
||||||
|
method: 'POST',
|
||||||
|
url: '/oauth/token',
|
||||||
|
form: { grant_type: 'authorization_code', code: '$authorize.code', scope: 'read' }
|
||||||
|
},
|
||||||
|
// Behavioral: issued token preserves the requested scope
|
||||||
|
expect: ['response_payload(this).scope == request_body(this).scope']
|
||||||
|
}
|
||||||
|
]
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
Scenario behavior:
|
||||||
|
1. Cookie jar persists `Set-Cookie` values across steps.
|
||||||
|
2. Step-level `headers.cookie` overrides jar values for that step.
|
||||||
|
3. `form` sends `application/x-www-form-urlencoded` payloads.
|
||||||
|
|
||||||
## Stateful Testing
|
## Stateful Testing
|
||||||
|
|
||||||
Stateful tests generate sequences of operations and track resources:
|
Stateful tests generate sequences of operations and track resources:
|
||||||
@@ -58,7 +94,18 @@ Stateful tests generate sequences of operations and track resources:
|
|||||||
3. **Observer**: Read resources (GET)
|
3. **Observer**: Read resources (GET)
|
||||||
4. **Destructor**: Remove resources (DELETE)
|
4. **Destructor**: Remove resources (DELETE)
|
||||||
|
|
||||||
APOPHIS automatically tracks created resources and cleans them up after testing.
|
APOPHIS tracks created resources and runs cleanup after test completion.
|
||||||
|
|
||||||
|
Run stateful tests via the API:
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
const stateful = await fastify.apophis.stateful({ runs: 50, seed: 42 })
|
||||||
|
console.log('Stateful tests:', stateful.summary)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Route Transparency
|
||||||
|
|
||||||
|
Artifacts include `executedRoutes` and `skippedRoutes` arrays. `skippedRoutes` contains reasons such as mode mismatch, environment policy, or route filter exclusion.
|
||||||
|
|
||||||
## Chaos and Adversity
|
## Chaos and Adversity
|
||||||
|
|
||||||
@@ -67,7 +114,9 @@ Chaos testing injects controlled failures:
|
|||||||
- **Delay**: Slow responses
|
- **Delay**: Slow responses
|
||||||
- **Error**: Return error status codes
|
- **Error**: Return error status codes
|
||||||
- **Dropout**: Connection failures
|
- **Dropout**: Connection failures
|
||||||
- **Corruption**: Malformed response bodies
|
- **Truncate**: Truncated response bodies
|
||||||
|
- **Malformed**: Invalid JSON or content-type
|
||||||
|
- **Field-corrupt**: Random field mutation in response objects
|
||||||
|
|
||||||
Configure chaos in your preset:
|
Configure chaos in your preset:
|
||||||
|
|
||||||
@@ -84,36 +133,6 @@ presets: {
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
## Profile Examples
|
|
||||||
|
|
||||||
### oauth-nightly
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
profiles: {
|
|
||||||
'oauth-nightly': {
|
|
||||||
name: 'oauth-nightly',
|
|
||||||
mode: 'qualify',
|
|
||||||
preset: 'protocol-lab',
|
|
||||||
routes: [],
|
|
||||||
seed: 42
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### lifecycle-deep
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
profiles: {
|
|
||||||
'lifecycle-deep': {
|
|
||||||
name: 'lifecycle-deep',
|
|
||||||
mode: 'qualify',
|
|
||||||
preset: 'protocol-lab',
|
|
||||||
routes: [],
|
|
||||||
seed: 42
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Non-Prod Boundaries
|
## Non-Prod Boundaries
|
||||||
|
|
||||||
Qualify mode is gated away from production by default:
|
Qualify mode is gated away from production by default:
|
||||||
@@ -122,7 +141,7 @@ Qualify mode is gated away from production by default:
|
|||||||
|---|---|---|---|
|
|---|---|---|---|
|
||||||
| local | enabled | enabled | enabled |
|
| local | enabled | enabled | enabled |
|
||||||
| test/CI | enabled | enabled | enabled |
|
| test/CI | enabled | enabled | enabled |
|
||||||
| staging | enabled with allowlist | synthetic-only | canary-only |
|
| staging | enabled with allowlist | enabled | blocked on protected routes |
|
||||||
| production | disabled by default | disabled by default | disabled by default |
|
| production | disabled by default | disabled by default | disabled by default |
|
||||||
|
|
||||||
## Machine Output for CI
|
## Machine Output for CI
|
||||||
@@ -186,7 +205,7 @@ export default {
|
|||||||
presets: {
|
presets: {
|
||||||
'protocol-lab': {
|
'protocol-lab': {
|
||||||
name: 'protocol-lab',
|
name: 'protocol-lab',
|
||||||
depth: 'deep',
|
runs: 200,
|
||||||
timeout: 15000,
|
timeout: 15000,
|
||||||
parallel: false,
|
parallel: false,
|
||||||
chaos: true,
|
chaos: true,
|
||||||
@@ -224,3 +243,24 @@ export default {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Gate Execution Counts
|
||||||
|
|
||||||
|
Human output shows per-gate execution counts (scenario, stateful, chaos, adversity) so you can verify which gates actually ran.
|
||||||
|
|
||||||
|
## Zero-Execution Guardrail
|
||||||
|
|
||||||
|
Qualify exits with code 1 if zero checks executed. This prevents silent passes when all routes are filtered out or gates are disabled.
|
||||||
|
|
||||||
|
## Test Budget
|
||||||
|
|
||||||
|
The `runs` field in your preset controls how many property-based tests execute per route. Default is 50. Lower for faster CI feedback, higher for deeper exploration:
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
presets: {
|
||||||
|
'protocol-lab': {
|
||||||
|
runs: 200,
|
||||||
|
timeout: 15000
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|||||||
+226
@@ -0,0 +1,226 @@
|
|||||||
|
# Quality Engines
|
||||||
|
|
||||||
|
APOPHIS includes three quality engines for advanced testing: chaos injection, flake detection, and mutation testing. All require `NODE_ENV=test`.
|
||||||
|
|
||||||
|
## Chaos Injection
|
||||||
|
|
||||||
|
Inject controlled failures into contract tests to validate resilience guarantees. Chaos events are generated by fast-check alongside test data, making them shrinkable — when a test fails, fast-check finds the minimal chaos event that causes the failure.
|
||||||
|
|
||||||
|
### Usage
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
const result = await fastify.apophis.contract({
|
||||||
|
runs: 50,
|
||||||
|
chaos: {
|
||||||
|
delay: { probability: 0.1, minMs: 100, maxMs: 500 },
|
||||||
|
error: { probability: 0.1, statusCode: 503 },
|
||||||
|
dropout: { probability: 0.05 },
|
||||||
|
corruption: { probability: 0.1 },
|
||||||
|
},
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
### Event Types
|
||||||
|
|
||||||
|
| Type | Effect | Tests |
|
||||||
|
|------|--------|-------|
|
||||||
|
| `delay` | Artificial latency | `response_time(this) < 1000` |
|
||||||
|
| `error` | Forces HTTP status code | Error-handling contracts |
|
||||||
|
| `dropout` | Network failure (status 0 or 504) | Fallback contracts |
|
||||||
|
| `corruption` | Mutates response bodies | Parsing robustness |
|
||||||
|
|
||||||
|
### Corruption Strategies
|
||||||
|
|
||||||
|
| Strategy | Effect |
|
||||||
|
|----------|--------|
|
||||||
|
| `truncate` | Cuts response body in half |
|
||||||
|
| `malformed` | Returns invalid JSON (`{"broken":`) |
|
||||||
|
| `field-corrupt` | Sets a random field to `null` |
|
||||||
|
|
||||||
|
### Programmatic API
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
import {
|
||||||
|
applyChaosToExecution,
|
||||||
|
createChaosEventArbitrary,
|
||||||
|
formatChaosEvents,
|
||||||
|
} from 'apophis-fastify'
|
||||||
|
|
||||||
|
// Apply pre-generated chaos events to a context
|
||||||
|
const result = applyChaosToExecution(ctx, events)
|
||||||
|
|
||||||
|
// Generate deterministic chaos events
|
||||||
|
const arb = createChaosEventArbitrary(config, contractNames)
|
||||||
|
const events = fc.sample(arb, { numRuns: 1, seed: 42 })[0]
|
||||||
|
|
||||||
|
// Format for diagnostics
|
||||||
|
console.log(formatChaosEvents(events))
|
||||||
|
```
|
||||||
|
|
||||||
|
### Best Practices
|
||||||
|
|
||||||
|
1. Start small: `probability: 0.05` (5% of requests)
|
||||||
|
2. Test one failure mode at a time
|
||||||
|
3. Verify contracts handle chaos: `if status:503 then response_code(GET /health) == 200`
|
||||||
|
4. Use seeds for reproducibility: `seed: 42`
|
||||||
|
|
||||||
|
## Flake Detection
|
||||||
|
|
||||||
|
Automatically rerun failing tests with varied seeds to detect non-deterministic contracts. A "flake" is a test that fails on one run but passes on another with the same or different seed.
|
||||||
|
|
||||||
|
### Usage
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
import { FlakeDetector } from 'apophis-fastify'
|
||||||
|
|
||||||
|
const detector = new FlakeDetector({
|
||||||
|
sameSeedReruns: 1, // Rerun with same seed
|
||||||
|
seedVariations: 3, // Try 3 additional seeds
|
||||||
|
})
|
||||||
|
|
||||||
|
const report = await detector.detectFlake(
|
||||||
|
originalFailingResult,
|
||||||
|
async (seed) => {
|
||||||
|
const suite = await fastify.apophis.contract({ seed })
|
||||||
|
return { passed: suite.summary.failed === 0 }
|
||||||
|
},
|
||||||
|
originalSeed
|
||||||
|
)
|
||||||
|
|
||||||
|
if (report.isFlaky) {
|
||||||
|
console.log(`Flaky with ${report.confidence} confidence`)
|
||||||
|
console.log('Reruns:', report.reruns)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Report Structure
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
{
|
||||||
|
isFlaky: true,
|
||||||
|
confidence: 'high', // 'high' | 'medium' | 'low'
|
||||||
|
reruns: [
|
||||||
|
{ seed: 42, passed: false },
|
||||||
|
{ seed: 43, passed: true },
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Confidence Scoring
|
||||||
|
|
||||||
|
| Pass Rate | Confidence |
|
||||||
|
|-----------|------------|
|
||||||
|
| 0% pass | `high` (deterministic failure) |
|
||||||
|
| < 50% pass | `medium` |
|
||||||
|
| >= 50% pass | `low` (likely flaky) |
|
||||||
|
|
||||||
|
## Mutation Testing
|
||||||
|
|
||||||
|
Measure contract strength by injecting synthetic bugs. A "mutation" is a small change to a contract (e.g., flip `==` to `!=`). If the test suite catches the mutation (fails), the mutation is "killed". If it passes, the mutation "survives" — indicating weak coverage.
|
||||||
|
|
||||||
|
### Usage
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
import { runMutationTesting } from 'apophis-fastify/quality/mutation'
|
||||||
|
|
||||||
|
const report = await runMutationTesting(fastify, {
|
||||||
|
runs: 10,
|
||||||
|
seed: 42,
|
||||||
|
maxMutationsPerContract: 5,
|
||||||
|
routes: ['/items'], // Optional: only test these routes
|
||||||
|
})
|
||||||
|
|
||||||
|
console.log(`Mutation score: ${report.score}%`)
|
||||||
|
console.log(`Killed: ${report.killed}, Survived: ${report.survived}`)
|
||||||
|
console.log('Weak contracts:', report.weakContracts)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Mutation Operators
|
||||||
|
|
||||||
|
| Type | Example |
|
||||||
|
|------|---------|
|
||||||
|
| `flip-operator` | `== 201` → `!= 201` |
|
||||||
|
| `change-number` | `== 200` → `== 201` |
|
||||||
|
| `remove-clause` | `A && B` → `A` |
|
||||||
|
| `negate-boolean` | `== true` → `== false` |
|
||||||
|
| `swap-variable` | `response_body` → `request_body` |
|
||||||
|
| `remove-ensures` | Remove one ensures clause entirely |
|
||||||
|
|
||||||
|
### Report Structure
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
{
|
||||||
|
score: 85, // 0-100
|
||||||
|
killed: 17,
|
||||||
|
survived: 3,
|
||||||
|
durationMs: 4500,
|
||||||
|
weakContracts: ['POST /items'], // Routes where no mutations were killed
|
||||||
|
mutations: [
|
||||||
|
{
|
||||||
|
mutation: {
|
||||||
|
id: 'm0',
|
||||||
|
route: 'POST /items',
|
||||||
|
original: 'response_code(this) == 201',
|
||||||
|
mutated: 'response_code(this) != 201',
|
||||||
|
type: 'flip-operator',
|
||||||
|
},
|
||||||
|
killed: true,
|
||||||
|
durationMs: 120,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Single Mutation Test
|
||||||
|
|
||||||
|
Test a specific mutation without running the full suite:
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
import { testMutation } from 'apophis-fastify/quality/mutation'
|
||||||
|
|
||||||
|
const killed = await testMutation(fastify, contract, mutation, {
|
||||||
|
runs: 10,
|
||||||
|
seed: 42,
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
## Environment Guard
|
||||||
|
|
||||||
|
All quality engines require `NODE_ENV=test`:
|
||||||
|
|
||||||
|
```
|
||||||
|
Error: chaos is only available in test environment.
|
||||||
|
Set NODE_ENV=test to enable quality features.
|
||||||
|
```
|
||||||
|
|
||||||
|
This prevents accidental execution in production or development.
|
||||||
|
|
||||||
|
## Integration Example
|
||||||
|
|
||||||
|
Run all three engines in a CI pipeline:
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
// 1. Standard contract tests
|
||||||
|
const suite = await fastify.apophis.contract({ runs: 50, seed: 42 })
|
||||||
|
|
||||||
|
// 2. Chaos tests
|
||||||
|
const chaosSuite = await fastify.apophis.contract({
|
||||||
|
runs: 50,
|
||||||
|
seed: 42,
|
||||||
|
chaos: { error: { probability: 0.1, statusCode: 503 } },
|
||||||
|
})
|
||||||
|
|
||||||
|
// 3. Flake detection on failures
|
||||||
|
for (const test of suite.tests.filter(t => !t.ok)) {
|
||||||
|
const report = await detector.detectFlake(test, rerunFn, 42)
|
||||||
|
if (report.isFlaky) {
|
||||||
|
console.warn(`Flaky test detected: ${test.name}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. Mutation testing
|
||||||
|
const mutationReport = await runMutationTesting(fastify, { runs: 10 })
|
||||||
|
if (mutationReport.score < 80) {
|
||||||
|
console.warn(`Low mutation score: ${mutationReport.score}%`)
|
||||||
|
}
|
||||||
|
```
|
||||||
@@ -31,7 +31,7 @@ APOPHIS classifies failures into six categories. Lower categories take precedenc
|
|||||||
|
|
||||||
**Symptoms**
|
**Symptoms**
|
||||||
- `Unexpected token` in formula output
|
- `Unexpected token` in formula output
|
||||||
- `Unterminated string` in x-ensures clause
|
- `Unterminated string literal` in x-ensures clause
|
||||||
- `Missing this` in operation call
|
- `Missing this` in operation call
|
||||||
|
|
||||||
**Resolution**
|
**Resolution**
|
||||||
@@ -88,12 +88,12 @@ APOPHIS classifies failures into six categories. Lower categories take precedenc
|
|||||||
**Symptoms**
|
**Symptoms**
|
||||||
- `Plugin decorator already added`
|
- `Plugin decorator already added`
|
||||||
- `Duplicate route registration`
|
- `Duplicate route registration`
|
||||||
- `No behavioral contracts found`
|
- `No behavioral contracts found. Schema-only routes are not enough for verify. Add x-ensures or x-requires to route schemas. See docs/getting-started.md for examples.`
|
||||||
|
|
||||||
**Resolution**
|
**Resolution**
|
||||||
1. Ensure the APOPHIS plugin is registered exactly once in the Fastify app.
|
1. Ensure the APOPHIS plugin is registered exactly once in the Fastify app.
|
||||||
2. Check for multiple imports or plugin registrations in test vs production entry points.
|
2. Check for multiple imports or plugin registrations in test vs production entry points.
|
||||||
3. If `No behavioral contracts found`, add `x-ensures` or `x-requires` to route schemas.
|
3. If `No behavioral contracts found. Schema-only routes are not enough for verify. Add x-ensures or x-requires to route schemas. See docs/getting-started.md for examples.`, add `x-ensures` or `x-requires` to route schemas.
|
||||||
4. Run `apophis doctor` to verify route discovery matches expectations.
|
4. Run `apophis doctor` to verify route discovery matches expectations.
|
||||||
|
|
||||||
**Prevention**
|
**Prevention**
|
||||||
@@ -150,13 +150,13 @@ Every failure produces an artifact JSON file. Use it for deep triage:
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Inspect the artifact
|
# Inspect the artifact
|
||||||
cat reports/apophis/verify-<timestamp>.json | jq '.failures[0]'
|
cat reports/apophis/failure-<timestamp>.json | jq '.failures[0]'
|
||||||
|
|
||||||
# Replay the exact failure
|
# Replay the exact failure
|
||||||
apophis replay --artifact reports/apophis/verify-<timestamp>.json
|
apophis replay --artifact reports/apophis/failure-<timestamp>.json
|
||||||
|
|
||||||
# Filter by error category
|
# Filter by error category
|
||||||
cat reports/apophis/verify-<timestamp>.json | jq '.failures | map(select(.category == "runtime"))'
|
cat reports/apophis/failure-<timestamp>.json | jq '.failures | map(select(.category == "runtime"))'
|
||||||
```
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|||||||
+42
-36
@@ -2,15 +2,7 @@
|
|||||||
|
|
||||||
Deterministic contract verification for CI and local development.
|
Deterministic contract verification for CI and local development.
|
||||||
|
|
||||||
## What Verify Does
|
APOPHIS implements the invariant-driven approach from [Invariant-Driven Automated Testing](https://arxiv.org/abs/2602.23922) (Malhado Ribeiro, 2021): encode intended behavior as executable formulas, then verify them automatically with property-based generation and deterministic replay.
|
||||||
|
|
||||||
`apophis verify` runs behavioral contracts against your Fastify routes:
|
|
||||||
|
|
||||||
1. Discovers routes from your Fastify app
|
|
||||||
2. Filters routes by profile config and CLI flags
|
|
||||||
3. Generates test data from JSON Schema
|
|
||||||
4. Executes routes and checks `x-ensures` contracts
|
|
||||||
5. Reports pass/fail with deterministic seed and replay command
|
|
||||||
|
|
||||||
## When to Use It
|
## When to Use It
|
||||||
|
|
||||||
@@ -79,6 +71,8 @@ apophis verify --routes "POST /users/*"
|
|||||||
apophis verify --profile quick
|
apophis verify --profile quick
|
||||||
```
|
```
|
||||||
|
|
||||||
|
`*` and `?` wildcards are supported in `--routes`.
|
||||||
|
|
||||||
## `--changed` Flag
|
## `--changed` Flag
|
||||||
|
|
||||||
Run only routes modified in the current git branch:
|
Run only routes modified in the current git branch:
|
||||||
@@ -87,7 +81,7 @@ Run only routes modified in the current git branch:
|
|||||||
apophis verify --profile ci --changed
|
apophis verify --profile ci --changed
|
||||||
```
|
```
|
||||||
|
|
||||||
If no routes changed, exits 0 with a message.
|
If no routes changed, exits 2 with a message.
|
||||||
|
|
||||||
## Failure Output Format
|
## Failure Output Format
|
||||||
|
|
||||||
@@ -126,6 +120,8 @@ Next
|
|||||||
apophis replay --artifact reports/apophis/failure-2026-04-28T12-30-22Z.json
|
apophis replay --artifact reports/apophis/failure-2026-04-28T12-30-22Z.json
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Nondeterminism warnings appear in output when the same seed produces different results across runs. This indicates stateful behavior in your application that contracts cannot control.
|
||||||
|
|
||||||
## Machine Output for CI
|
## Machine Output for CI
|
||||||
|
|
||||||
Use concise formats to reduce log volume in large verify runs:
|
Use concise formats to reduce log volume in large verify runs:
|
||||||
@@ -137,6 +133,7 @@ Use concise formats to reduce log volume in large verify runs:
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Extract only failed routes from full ndjson
|
# Extract only failed routes from full ndjson
|
||||||
|
# Note: route.failed events are only emitted for failures, not passed routes
|
||||||
apophis verify --profile quick --format ndjson | jq 'select(.type == "route.failed")'
|
apophis verify --profile quick --format ndjson | jq 'select(.type == "route.failed")'
|
||||||
|
|
||||||
# Write artifact to disk and parse the file instead of stdout
|
# Write artifact to disk and parse the file instead of stdout
|
||||||
@@ -149,7 +146,7 @@ apophis verify --profile quick --format json --artifact-dir reports/apophis
|
|||||||
|---|---|
|
|---|---|
|
||||||
| 0 | All contracts passed |
|
| 0 | All contracts passed |
|
||||||
| 1 | One or more behavioral contracts failed |
|
| 1 | One or more behavioral contracts failed |
|
||||||
| 2 | Config error or no routes matched |
|
| 2 | Config error, no routes matched, no contracts found, or not a git repo |
|
||||||
| 3 | Internal APOPHIS error |
|
| 3 | Internal APOPHIS error |
|
||||||
| 130 | Interrupted (SIGINT) |
|
| 130 | Interrupted (SIGINT) |
|
||||||
|
|
||||||
@@ -158,42 +155,51 @@ apophis verify --profile quick --format json --artifact-dir reports/apophis
|
|||||||
```javascript
|
```javascript
|
||||||
// apophis.config.js
|
// apophis.config.js
|
||||||
export default {
|
export default {
|
||||||
mode: 'verify',
|
|
||||||
profile: 'quick',
|
profile: 'quick',
|
||||||
profiles: {
|
profiles: {
|
||||||
quick: {
|
quick: {
|
||||||
name: 'quick',
|
|
||||||
mode: 'verify',
|
mode: 'verify',
|
||||||
preset: 'safe-ci',
|
preset: 'safe-ci',
|
||||||
routes: ['POST /users']
|
routes: ['POST /users']
|
||||||
},
|
|
||||||
ci: {
|
|
||||||
name: 'ci',
|
|
||||||
mode: 'verify',
|
|
||||||
preset: 'safe-ci',
|
|
||||||
routes: []
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
presets: {
|
presets: {
|
||||||
'safe-ci': {
|
'safe-ci': {
|
||||||
name: 'safe-ci',
|
runs: 10,
|
||||||
depth: 'quick',
|
timeout: 5000
|
||||||
timeout: 5000,
|
|
||||||
parallel: false,
|
|
||||||
chaos: false,
|
|
||||||
observe: false
|
|
||||||
}
|
|
||||||
},
|
|
||||||
environments: {
|
|
||||||
local: {
|
|
||||||
name: 'local',
|
|
||||||
allowVerify: true,
|
|
||||||
allowObserve: true,
|
|
||||||
allowQualify: false,
|
|
||||||
allowChaos: false,
|
|
||||||
allowBlocking: true,
|
|
||||||
requireSink: false
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
```
|
```
|
||||||
|
|
||||||
|
For the full config schema, see [CLI Reference](cli.md).
|
||||||
|
|
||||||
|
## Workspace Support
|
||||||
|
|
||||||
|
Run verify across all packages in a monorepo workspace:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
apophis verify --workspace --profile quick --format json
|
||||||
|
```
|
||||||
|
|
||||||
|
Output includes per-package pass/fail summaries. Fails if any package fails.
|
||||||
|
|
||||||
|
## Test Budget
|
||||||
|
|
||||||
|
The `runs` field in your preset controls how many property-based tests execute per route. Default is 50. Lower for faster CI feedback, higher for deeper exploration:
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
profiles: {
|
||||||
|
quick: {
|
||||||
|
mode: 'verify',
|
||||||
|
preset: 'safe-ci',
|
||||||
|
routes: ['POST /users']
|
||||||
|
}
|
||||||
|
},
|
||||||
|
presets: {
|
||||||
|
'safe-ci': {
|
||||||
|
runs: 10,
|
||||||
|
timeout: 5000
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import crypto from 'crypto'
|
||||||
import type { FastifyInstance } from 'fastify'
|
import type { FastifyInstance } from 'fastify'
|
||||||
|
|
||||||
export async function userRoutes(fastify: FastifyInstance) {
|
export async function userRoutes(fastify: FastifyInstance) {
|
||||||
@@ -30,7 +31,7 @@ export async function userRoutes(fastify: FastifyInstance) {
|
|||||||
},
|
},
|
||||||
}, async (req, reply) => {
|
}, async (req, reply) => {
|
||||||
const { name, email } = req.body as { name: string; email: string }
|
const { name, email } = req.body as { name: string; email: string }
|
||||||
const id = `user-${Date.now()}`
|
const id = `user-${crypto.createHash('sha256').update(email).digest('hex').slice(0, 8)}`
|
||||||
const user = { id, name, email }
|
const user = { id, name, email }
|
||||||
fastify.db.users.set(id, user)
|
fastify.db.users.set(id, user)
|
||||||
reply.status(201)
|
reply.status(201)
|
||||||
|
|||||||
+2170
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,6 @@
|
|||||||
|
|
||||||
|
Mercedes Rodriguez
|
||||||
|
#BOAF9148155679Z
|
||||||
|
1500 - 00251743R
|
||||||
|
2000 - 00361Z903R
|
||||||
|
|
||||||
+2
-2
@@ -1,5 +1,5 @@
|
|||||||
{
|
{
|
||||||
"name": "apophis-fastify",
|
"name": "@apophis/fastify",
|
||||||
"version": "2.0.0",
|
"version": "2.0.0",
|
||||||
"description": "Contract-driven API testing plugin for Fastify with property-based testing, timeout enforcement, redirect capture, and deterministic concurrency",
|
"description": "Contract-driven API testing plugin for Fastify with property-based testing, timeout enforcement, redirect capture, and deterministic concurrency",
|
||||||
"main": "dist/index.js",
|
"main": "dist/index.js",
|
||||||
@@ -50,7 +50,7 @@
|
|||||||
"benchmark:cli": "npm run build && node scripts/bench/cli.mjs",
|
"benchmark:cli": "npm run build && node scripts/bench/cli.mjs",
|
||||||
"benchmark:hot": "npm run build && node scripts/bench/hot-paths.mjs",
|
"benchmark:hot": "npm run build && node scripts/bench/hot-paths.mjs",
|
||||||
"profile:qualify": "npm run build && mkdir -p .profiles && node --cpu-prof --cpu-prof-dir=.profiles --cpu-prof-name=qualify.cpuprofile dist/cli/index.js qualify --cwd src/cli/__fixtures__/protocol-lab --profile oauth-nightly --seed 42 --quiet",
|
"profile:qualify": "npm run build && mkdir -p .profiles && node --cpu-prof --cpu-prof-dir=.profiles --cpu-prof-name=qualify.cpuprofile dist/cli/index.js qualify --cwd src/cli/__fixtures__/protocol-lab --profile oauth-nightly --seed 42 --quiet",
|
||||||
"profile:qualify:quick": "npm run build && mkdir -p .profiles && node --cpu-prof --cpu-prof-dir=.profiles --cpu-prof-name=qualify-quick.cpuprofile dist/cli/index.js qualify --cwd src/cli/__fixtures__/protocol-lab --profile oauth-nightly --generation-profile quick --seed 42 --quiet",
|
"profile:qualify:quick": "npm run build && mkdir -p .profiles && node --cpu-prof --cpu-prof-dir=.profiles --cpu-prof-name=qualify-quick.cpuprofile dist/cli/index.js qualify --cwd src/cli/__fixtures__/protocol-lab --profile oauth-nightly --seed 42 --quiet",
|
||||||
"clean": "rm -rf dist",
|
"clean": "rm -rf dist",
|
||||||
"apophis:verify": "apophis verify --profile quick",
|
"apophis:verify": "apophis verify --profile quick",
|
||||||
"apophis:doctor": "apophis doctor"
|
"apophis:doctor": "apophis doctor"
|
||||||
|
|||||||
+1
-18
@@ -8,30 +8,13 @@ const __dirname = fileURLToPath(new URL('.', import.meta.url))
|
|||||||
const repoRoot = resolve(__dirname, '..', '..')
|
const repoRoot = resolve(__dirname, '..', '..')
|
||||||
|
|
||||||
const options = getBenchOptions()
|
const options = getBenchOptions()
|
||||||
const generationProfiles = (process.env.BENCH_GENERATION_PROFILES ?? 'default,quick,standard,thorough')
|
|
||||||
.split(',')
|
|
||||||
.map((value) => value.trim())
|
|
||||||
.filter(Boolean)
|
|
||||||
|
|
||||||
function withGenerationProfile(baseArgs, profile) {
|
|
||||||
if (profile === 'default') {
|
|
||||||
return baseArgs
|
|
||||||
}
|
|
||||||
return [...baseArgs, '--generation-profile', profile]
|
|
||||||
}
|
|
||||||
|
|
||||||
const scenarios = [
|
const scenarios = [
|
||||||
{ name: 'cli.help', args: ['--help'] },
|
{ name: 'cli.help', args: ['--help'] },
|
||||||
{ name: 'cli.version', args: ['--version'] },
|
{ name: 'cli.version', args: ['--version'] },
|
||||||
{ name: 'cli.doctor', args: ['doctor', '--cwd', 'src/cli/__fixtures__/tiny-fastify', '--quiet'] },
|
{ name: 'cli.doctor', args: ['doctor', '--cwd', 'src/cli/__fixtures__/tiny-fastify', '--quiet'] },
|
||||||
{ name: 'cli.observe.check', args: ['observe', '--cwd', 'src/cli/__fixtures__/observe-config', '--profile', 'staging-observe', '--check-config', '--quiet'] },
|
{ name: 'cli.observe.check', args: ['observe', '--cwd', 'src/cli/__fixtures__/observe-config', '--profile', 'staging-observe', '--check-config', '--quiet'] },
|
||||||
...generationProfiles.map((profile) => ({
|
{ name: 'cli.qualify', args: ['qualify', '--cwd', 'src/cli/__fixtures__/protocol-lab', '--profile', 'oauth-nightly', '--seed', '42', '--quiet'] },
|
||||||
name: `cli.qualify.profile[${profile}]`,
|
|
||||||
args: withGenerationProfile(
|
|
||||||
['qualify', '--cwd', 'src/cli/__fixtures__/protocol-lab', '--profile', 'oauth-nightly', '--seed', '42', '--quiet'],
|
|
||||||
profile,
|
|
||||||
),
|
|
||||||
})),
|
|
||||||
]
|
]
|
||||||
|
|
||||||
async function run() {
|
async function run() {
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
/**
|
/**
|
||||||
* APOPHIS configuration for broken-behavior fixture.
|
* APOPHIS configuration for broken-behavior fixture.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export default {
|
export default {
|
||||||
mode: "verify",
|
mode: "verify",
|
||||||
profiles: {
|
profiles: {
|
||||||
@@ -15,7 +14,6 @@ export default {
|
|||||||
presets: {
|
presets: {
|
||||||
"safe-ci": {
|
"safe-ci": {
|
||||||
name: "safe-ci",
|
name: "safe-ci",
|
||||||
depth: "quick",
|
|
||||||
timeout: 5000,
|
timeout: 5000,
|
||||||
parallel: false,
|
parallel: false,
|
||||||
chaos: false,
|
chaos: false,
|
||||||
|
|||||||
@@ -2,11 +2,9 @@
|
|||||||
* LEGACY APOPHIS configuration (old-style, for migration tests).
|
* LEGACY APOPHIS configuration (old-style, for migration tests).
|
||||||
* This uses deprecated field names that should be detected by `apophis migrate`.
|
* This uses deprecated field names that should be detected by `apophis migrate`.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export default {
|
export default {
|
||||||
// Deprecated: 'mode' used to be 'testMode'
|
// Deprecated: 'mode' used to be 'testMode'
|
||||||
testMode: "verify",
|
testMode: "verify",
|
||||||
|
|
||||||
// Deprecated: 'profiles' used to be 'testProfiles'
|
// Deprecated: 'profiles' used to be 'testProfiles'
|
||||||
testProfiles: {
|
testProfiles: {
|
||||||
quick: {
|
quick: {
|
||||||
@@ -17,7 +15,6 @@ export default {
|
|||||||
routeFilter: ["GET /legacy"],
|
routeFilter: ["GET /legacy"],
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|
||||||
// Deprecated: 'presets' used to be 'testPresets'
|
// Deprecated: 'presets' used to be 'testPresets'
|
||||||
testPresets: {
|
testPresets: {
|
||||||
"safe-ci": {
|
"safe-ci": {
|
||||||
@@ -28,7 +25,6 @@ export default {
|
|||||||
maxDuration: 5000,
|
maxDuration: 5000,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|
||||||
// Deprecated: 'environments' used to be 'envPolicies'
|
// Deprecated: 'environments' used to be 'envPolicies'
|
||||||
envPolicies: {
|
envPolicies: {
|
||||||
local: {
|
local: {
|
||||||
|
|||||||
@@ -2,7 +2,6 @@
|
|||||||
* Root-level APOPHIS config for monorepo.
|
* Root-level APOPHIS config for monorepo.
|
||||||
* Packages can override with their own configs.
|
* Packages can override with their own configs.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export default {
|
export default {
|
||||||
mode: "verify",
|
mode: "verify",
|
||||||
profiles: {
|
profiles: {
|
||||||
@@ -20,7 +19,6 @@ export default {
|
|||||||
presets: {
|
presets: {
|
||||||
"safe-ci": {
|
"safe-ci": {
|
||||||
name: "safe-ci",
|
name: "safe-ci",
|
||||||
depth: "quick",
|
|
||||||
timeout: 5000,
|
timeout: 5000,
|
||||||
parallel: false,
|
parallel: false,
|
||||||
chaos: false,
|
chaos: false,
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
/**
|
/**
|
||||||
* APOPHIS configuration for observe-config fixture.
|
* APOPHIS configuration for observe-config fixture.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export default {
|
export default {
|
||||||
mode: "observe",
|
mode: "observe",
|
||||||
profiles: {
|
profiles: {
|
||||||
@@ -15,7 +14,6 @@ export default {
|
|||||||
presets: {
|
presets: {
|
||||||
"observe-safe": {
|
"observe-safe": {
|
||||||
name: "observe-safe",
|
name: "observe-safe",
|
||||||
depth: "quick",
|
|
||||||
timeout: 5000,
|
timeout: 5000,
|
||||||
parallel: false,
|
parallel: false,
|
||||||
chaos: false,
|
chaos: false,
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
/**
|
/**
|
||||||
* APOPHIS configuration for protocol-lab fixture.
|
* APOPHIS configuration for protocol-lab fixture.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export default {
|
export default {
|
||||||
mode: "qualify",
|
mode: "qualify",
|
||||||
profiles: {
|
profiles: {
|
||||||
@@ -15,7 +14,6 @@ export default {
|
|||||||
presets: {
|
presets: {
|
||||||
deep: {
|
deep: {
|
||||||
name: "deep",
|
name: "deep",
|
||||||
depth: "deep",
|
|
||||||
timeout: 30000,
|
timeout: 30000,
|
||||||
parallel: false,
|
parallel: false,
|
||||||
chaos: true,
|
chaos: true,
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
/**
|
/**
|
||||||
* APOPHIS configuration for tiny-fastify fixture.
|
* APOPHIS configuration for tiny-fastify fixture.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export default {
|
export default {
|
||||||
mode: "verify",
|
mode: "verify",
|
||||||
profiles: {
|
profiles: {
|
||||||
@@ -15,7 +14,6 @@ export default {
|
|||||||
presets: {
|
presets: {
|
||||||
"safe-ci": {
|
"safe-ci": {
|
||||||
name: "safe-ci",
|
name: "safe-ci",
|
||||||
depth: "quick",
|
|
||||||
timeout: 5000,
|
timeout: 5000,
|
||||||
parallel: false,
|
parallel: false,
|
||||||
chaos: false,
|
chaos: false,
|
||||||
|
|||||||
@@ -11,7 +11,6 @@ export default {
|
|||||||
presets: {
|
presets: {
|
||||||
"safe-ci": {
|
"safe-ci": {
|
||||||
name: "safe-ci",
|
name: "safe-ci",
|
||||||
depth: "quick",
|
|
||||||
timeout: 5000,
|
timeout: 5000,
|
||||||
parallel: false,
|
parallel: false,
|
||||||
chaos: false,
|
chaos: false,
|
||||||
|
|||||||
@@ -11,7 +11,6 @@ export default {
|
|||||||
presets: {
|
presets: {
|
||||||
"safe-ci": {
|
"safe-ci": {
|
||||||
name: "safe-ci",
|
name: "safe-ci",
|
||||||
depth: "quick",
|
|
||||||
timeout: 5000,
|
timeout: 5000,
|
||||||
parallel: false,
|
parallel: false,
|
||||||
chaos: false,
|
chaos: false,
|
||||||
|
|||||||
@@ -11,7 +11,6 @@ export default {
|
|||||||
presets: {
|
presets: {
|
||||||
"safe-ci": {
|
"safe-ci": {
|
||||||
name: "safe-ci",
|
name: "safe-ci",
|
||||||
depth: "quick",
|
|
||||||
timeout: 5000,
|
timeout: 5000,
|
||||||
parallel: false,
|
parallel: false,
|
||||||
chaos: false,
|
chaos: false,
|
||||||
|
|||||||
@@ -116,8 +116,8 @@ export async function checkRouteDiscovery(options: RouteCheckOptions): Promise<R
|
|||||||
name: 'route-discovery',
|
name: 'route-discovery',
|
||||||
status: 'fail',
|
status: 'fail',
|
||||||
message: `App file ${appFile} does not export a valid object.`,
|
message: `App file ${appFile} does not export a valid object.`,
|
||||||
detail: 'Ensure the app file exports a Fastify instance as default.',
|
detail: 'Ensure the app file exports a Fastify instance or a factory function.',
|
||||||
remediation: 'Export your Fastify instance as default: export default app;',
|
remediation: 'Export your Fastify instance: export default app; or export const createApp = () => app; or module.exports = app;',
|
||||||
mode: 'all',
|
mode: 'all',
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -26,6 +26,16 @@ import { runDocsChecks } from './checks/docs.js';
|
|||||||
|
|
||||||
import { renderJson } from '../../renderers/json.js';
|
import { renderJson } from '../../renderers/json.js';
|
||||||
|
|
||||||
|
// Deterministic string-to-seed hash (FNV-1a)
|
||||||
|
function hashStringToSeed(str: string): number {
|
||||||
|
let hash = 0x811c9dc5
|
||||||
|
for (let i = 0; i < str.length; i++) {
|
||||||
|
hash ^= str.charCodeAt(i)
|
||||||
|
hash = Math.imul(hash, 0x01000193)
|
||||||
|
}
|
||||||
|
return Math.abs(hash >>> 0)
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Types
|
// Types
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -203,7 +213,7 @@ async function runPackageChecks(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 6. Determinism trust signal
|
// 6. Determinism trust signal
|
||||||
const testSeed = Math.floor(Math.random() * 0x7fffffff);
|
const testSeed = hashStringToSeed(packageName + cwd);
|
||||||
checks.push({
|
checks.push({
|
||||||
name: 'determinism',
|
name: 'determinism',
|
||||||
status: 'pass',
|
status: 'pass',
|
||||||
|
|||||||
@@ -17,7 +17,6 @@ export interface ScaffoldResult {
|
|||||||
export function safeCiScaffold(): ScaffoldResult {
|
export function safeCiScaffold(): ScaffoldResult {
|
||||||
const preset: PresetDefinition = {
|
const preset: PresetDefinition = {
|
||||||
name: 'safe-ci',
|
name: 'safe-ci',
|
||||||
depth: 'quick',
|
|
||||||
timeout: 5000,
|
timeout: 5000,
|
||||||
parallel: false,
|
parallel: false,
|
||||||
chaos: false,
|
chaos: false,
|
||||||
@@ -95,7 +94,6 @@ If \`apophis verify\` says "No behavioral contracts found", it means your routes
|
|||||||
export function platformObserveScaffold(): ScaffoldResult {
|
export function platformObserveScaffold(): ScaffoldResult {
|
||||||
const preset: PresetDefinition = {
|
const preset: PresetDefinition = {
|
||||||
name: 'platform-observe',
|
name: 'platform-observe',
|
||||||
depth: 'standard',
|
|
||||||
timeout: 10000,
|
timeout: 10000,
|
||||||
parallel: true,
|
parallel: true,
|
||||||
chaos: false,
|
chaos: false,
|
||||||
@@ -180,7 +178,6 @@ This project was scaffolded with \`apophis init --preset platform-observe\`.
|
|||||||
export function llmSafeScaffold(): ScaffoldResult {
|
export function llmSafeScaffold(): ScaffoldResult {
|
||||||
const preset: PresetDefinition = {
|
const preset: PresetDefinition = {
|
||||||
name: 'llm-safe',
|
name: 'llm-safe',
|
||||||
depth: 'quick',
|
|
||||||
timeout: 3000,
|
timeout: 3000,
|
||||||
parallel: false,
|
parallel: false,
|
||||||
chaos: false,
|
chaos: false,
|
||||||
@@ -258,7 +255,6 @@ If \`apophis verify\` says "No behavioral contracts found", it means your routes
|
|||||||
export function protocolLabScaffold(): ScaffoldResult {
|
export function protocolLabScaffold(): ScaffoldResult {
|
||||||
const preset: PresetDefinition = {
|
const preset: PresetDefinition = {
|
||||||
name: 'protocol-lab',
|
name: 'protocol-lab',
|
||||||
depth: 'deep',
|
|
||||||
timeout: 15000,
|
timeout: 15000,
|
||||||
parallel: false,
|
parallel: false,
|
||||||
chaos: true,
|
chaos: true,
|
||||||
|
|||||||
@@ -19,7 +19,7 @@
|
|||||||
import type { CliContext } from '../../core/context.js'
|
import type { CliContext } from '../../core/context.js'
|
||||||
import { loadConfig } from '../../core/config-loader.js'
|
import { loadConfig } from '../../core/config-loader.js'
|
||||||
import { PolicyEngine, detectEnvironment } from '../../core/policy-engine.js'
|
import { PolicyEngine, detectEnvironment } from '../../core/policy-engine.js'
|
||||||
import { resolveGenerationProfileOverride, GenerationProfileResolutionError } from '../../core/generation-profile.js'
|
|
||||||
import { SUCCESS, BEHAVIORAL_FAILURE, USAGE_ERROR, INTERNAL_ERROR } from '../../core/exit-codes.js'
|
import { SUCCESS, BEHAVIORAL_FAILURE, USAGE_ERROR, INTERNAL_ERROR } from '../../core/exit-codes.js'
|
||||||
import type { CommandResult, Artifact, FailureRecord } from '../../core/types.js'
|
import type { CommandResult, Artifact, FailureRecord } from '../../core/types.js'
|
||||||
import { classifyError, ErrorTaxonomy } from '../../core/error-taxonomy.js'
|
import { classifyError, ErrorTaxonomy } from '../../core/error-taxonomy.js'
|
||||||
@@ -54,13 +54,6 @@ function isReplayCompatibleRoute(route: string): boolean {
|
|||||||
return ROUTE_IDENTITY_PATTERN.test(route)
|
return ROUTE_IDENTITY_PATTERN.test(route)
|
||||||
}
|
}
|
||||||
|
|
||||||
function coerceDepth(value: unknown): TestConfig['depth'] {
|
|
||||||
if (value === 'quick' || value === 'standard' || value === 'thorough') {
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
return 'standard'
|
|
||||||
}
|
|
||||||
|
|
||||||
function coerceTimeout(value: unknown): number | undefined {
|
function coerceTimeout(value: unknown): number | undefined {
|
||||||
return typeof value === 'number' ? value : undefined
|
return typeof value === 'number' ? value : undefined
|
||||||
}
|
}
|
||||||
@@ -71,7 +64,6 @@ function coerceTimeout(value: unknown): number | undefined {
|
|||||||
|
|
||||||
export interface QualifyOptions {
|
export interface QualifyOptions {
|
||||||
profile?: string
|
profile?: string
|
||||||
generationProfile?: string
|
|
||||||
seed?: number
|
seed?: number
|
||||||
config?: string
|
config?: string
|
||||||
cwd?: string
|
cwd?: string
|
||||||
@@ -529,7 +521,6 @@ export async function qualifyCommand(
|
|||||||
): Promise<CommandResult> {
|
): Promise<CommandResult> {
|
||||||
const {
|
const {
|
||||||
profile,
|
profile,
|
||||||
generationProfile,
|
|
||||||
seed: explicitSeed,
|
seed: explicitSeed,
|
||||||
config: configPath,
|
config: configPath,
|
||||||
cwd,
|
cwd,
|
||||||
@@ -558,7 +549,6 @@ export async function qualifyCommand(
|
|||||||
}
|
}
|
||||||
|
|
||||||
const config = loadResult.config
|
const config = loadResult.config
|
||||||
const resolvedGenerationProfile = resolveGenerationProfileOverride(generationProfile, config)
|
|
||||||
|
|
||||||
// 2. Run policy engine checks
|
// 2. Run policy engine checks
|
||||||
const policyEngine = new PolicyEngine({
|
const policyEngine = new PolicyEngine({
|
||||||
@@ -600,12 +590,9 @@ export async function qualifyCommand(
|
|||||||
// 6. Build stateful config
|
// 6. Build stateful config
|
||||||
const presetName = profileDef?.preset
|
const presetName = profileDef?.preset
|
||||||
const preset = presetName ? config.presets?.[presetName] : undefined
|
const preset = presetName ? config.presets?.[presetName] : undefined
|
||||||
const presetDepth = coerceDepth((preset as { depth?: unknown } | undefined)?.depth)
|
|
||||||
const presetTimeout = coerceTimeout((preset as { timeout?: unknown } | undefined)?.timeout)
|
const presetTimeout = coerceTimeout((preset as { timeout?: unknown } | undefined)?.timeout)
|
||||||
const statefulConfig: TestConfig | undefined = gates.stateful
|
const statefulConfig: TestConfig | undefined = gates.stateful
|
||||||
? {
|
? {
|
||||||
depth: presetDepth,
|
|
||||||
generationProfile: resolvedGenerationProfile,
|
|
||||||
seed,
|
seed,
|
||||||
timeout: presetTimeout,
|
timeout: presetTimeout,
|
||||||
routes: profileDef?.routes,
|
routes: profileDef?.routes,
|
||||||
@@ -642,7 +629,7 @@ export async function qualifyCommand(
|
|||||||
}
|
}
|
||||||
return {
|
return {
|
||||||
exitCode: USAGE_ERROR,
|
exitCode: USAGE_ERROR,
|
||||||
message: 'No Fastify app found. Ensure app.js exports a Fastify instance.',
|
message: 'No Fastify app found. Ensure app.js exports a Fastify instance or a factory function.\n\nSupported patterns:\n export default app\n export const createApp = () => app\n module.exports = app',
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -752,12 +739,6 @@ export async function qualifyCommand(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (error instanceof GenerationProfileResolutionError) {
|
|
||||||
return {
|
|
||||||
exitCode: USAGE_ERROR,
|
|
||||||
message: error.message,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const message = error instanceof Error ? error.message : String(error)
|
const message = error instanceof Error ? error.message : String(error)
|
||||||
return {
|
return {
|
||||||
exitCode: INTERNAL_ERROR,
|
exitCode: INTERNAL_ERROR,
|
||||||
@@ -780,7 +761,6 @@ export async function handleQualify(
|
|||||||
): Promise<number> {
|
): Promise<number> {
|
||||||
const options: QualifyOptions = {
|
const options: QualifyOptions = {
|
||||||
profile: ctx.options.profile || undefined,
|
profile: ctx.options.profile || undefined,
|
||||||
generationProfile: ctx.options.generationProfile,
|
|
||||||
seed: undefined,
|
seed: undefined,
|
||||||
config: ctx.options.config || undefined,
|
config: ctx.options.config || undefined,
|
||||||
cwd: ctx.cwd,
|
cwd: ctx.cwd,
|
||||||
@@ -798,11 +778,6 @@ export async function handleQualify(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const generationProfileIdx = args.indexOf('--generation-profile')
|
|
||||||
if (generationProfileIdx !== -1 && args[generationProfileIdx + 1]) {
|
|
||||||
options.generationProfile = args[generationProfileIdx + 1]
|
|
||||||
}
|
|
||||||
|
|
||||||
const result = await qualifyCommand(options, ctx)
|
const result = await qualifyCommand(options, ctx)
|
||||||
const format = options.format || ctx.options.format || 'human'
|
const format = options.format || ctx.options.format || 'human'
|
||||||
const machineMode = format === 'json' || format === 'ndjson' || format === 'json-summary' || format === 'ndjson-summary'
|
const machineMode = format === 'json' || format === 'ndjson' || format === 'json-summary' || format === 'ndjson-summary'
|
||||||
|
|||||||
@@ -11,12 +11,22 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
import { runStatefulTests } from '../../../test/stateful-runner.js'
|
import { runStatefulTests } from '../../../test/stateful-runner.js'
|
||||||
|
import { CleanupManager } from '../../../infrastructure/cleanup-manager.js'
|
||||||
import type {
|
import type {
|
||||||
TestConfig,
|
TestConfig,
|
||||||
TestSuite,
|
TestSuite,
|
||||||
|
ScopeRegistry,
|
||||||
} from '../../../types.js'
|
} from '../../../types.js'
|
||||||
import type { QualifyRunnerDeps, StepTrace } from './runner.js'
|
import type { QualifyRunnerDeps, StepTrace } from './runner.js'
|
||||||
|
|
||||||
|
const minimalScopeRegistry: ScopeRegistry = {
|
||||||
|
scopes: new Map(),
|
||||||
|
defaultScope: { headers: {} },
|
||||||
|
register() {},
|
||||||
|
deriveFromRequest() { return { headers: {} } },
|
||||||
|
getHeaders() { return {} },
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Run stateful tests with the given config.
|
* Run stateful tests with the given config.
|
||||||
* Wraps the existing stateful runner.
|
* Wraps the existing stateful runner.
|
||||||
@@ -27,13 +37,15 @@ export async function runStatefulWithTraces(
|
|||||||
): Promise<{ result: TestSuite; traces: StepTrace[] }> {
|
): Promise<{ result: TestSuite; traces: StepTrace[] }> {
|
||||||
const started = Date.now()
|
const started = Date.now()
|
||||||
|
|
||||||
|
const cleanupManager = new CleanupManager(deps.fastify as any, minimalScopeRegistry, false)
|
||||||
|
|
||||||
const result = await runStatefulTests(
|
const result = await runStatefulTests(
|
||||||
deps.fastify,
|
deps.fastify,
|
||||||
config,
|
config,
|
||||||
undefined, // cleanupManager — injected if needed by caller
|
cleanupManager,
|
||||||
undefined, // scopeRegistry
|
minimalScopeRegistry,
|
||||||
deps.extensionRegistry,
|
deps.extensionRegistry,
|
||||||
undefined, // pluginContractRegistry
|
undefined, // pluginContractRegistry — will be passed from runner when available
|
||||||
undefined, // outboundContractRegistry
|
undefined, // outboundContractRegistry
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -22,7 +22,7 @@
|
|||||||
import type { CliContext } from '../../core/context.js'
|
import type { CliContext } from '../../core/context.js'
|
||||||
import { loadConfig, findWorkspacePackages } from '../../core/config-loader.js'
|
import { loadConfig, findWorkspacePackages } from '../../core/config-loader.js'
|
||||||
import { PolicyEngine, detectEnvironment } from '../../core/policy-engine.js'
|
import { PolicyEngine, detectEnvironment } from '../../core/policy-engine.js'
|
||||||
import { resolveGenerationProfileOverride, GenerationProfileResolutionError } from '../../core/generation-profile.js'
|
|
||||||
import { SUCCESS, BEHAVIORAL_FAILURE, USAGE_ERROR, INTERNAL_ERROR } from '../../core/exit-codes.js'
|
import { SUCCESS, BEHAVIORAL_FAILURE, USAGE_ERROR, INTERNAL_ERROR } from '../../core/exit-codes.js'
|
||||||
import type { CommandResult, Artifact, FailureRecord, RouteResult, WorkspaceRun, WorkspaceResult } from '../../core/types.js'
|
import type { CommandResult, Artifact, FailureRecord, RouteResult, WorkspaceRun, WorkspaceResult } from '../../core/types.js'
|
||||||
import { classifyError, ErrorTaxonomy } from '../../core/error-taxonomy.js'
|
import { classifyError, ErrorTaxonomy } from '../../core/error-taxonomy.js'
|
||||||
@@ -54,7 +54,6 @@ function isReplayCompatibleRoute(route: string): boolean {
|
|||||||
|
|
||||||
export interface VerifyOptions {
|
export interface VerifyOptions {
|
||||||
profile?: string
|
profile?: string
|
||||||
generationProfile?: string
|
|
||||||
routes?: string
|
routes?: string
|
||||||
seed?: number
|
seed?: number
|
||||||
changed?: boolean
|
changed?: boolean
|
||||||
@@ -381,7 +380,6 @@ export async function verifyCommand(
|
|||||||
): Promise<CommandResult> {
|
): Promise<CommandResult> {
|
||||||
const {
|
const {
|
||||||
profile,
|
profile,
|
||||||
generationProfile,
|
|
||||||
routes: routesFlag,
|
routes: routesFlag,
|
||||||
seed: explicitSeed,
|
seed: explicitSeed,
|
||||||
changed,
|
changed,
|
||||||
@@ -412,7 +410,6 @@ export async function verifyCommand(
|
|||||||
}
|
}
|
||||||
|
|
||||||
const config = loadResult.config
|
const config = loadResult.config
|
||||||
const resolvedGenerationProfile = resolveGenerationProfileOverride(generationProfile, config)
|
|
||||||
|
|
||||||
// 2a. Resolve profile — if explicitly requested but missing, list available ones
|
// 2a. Resolve profile — if explicitly requested but missing, list available ones
|
||||||
if (profile && !config.profiles?.[profile]) {
|
if (profile && !config.profiles?.[profile]) {
|
||||||
@@ -468,7 +465,7 @@ export async function verifyCommand(
|
|||||||
const errorMessage = err instanceof Error ? err.message : String(err)
|
const errorMessage = err instanceof Error ? err.message : String(err)
|
||||||
return {
|
return {
|
||||||
exitCode: USAGE_ERROR,
|
exitCode: USAGE_ERROR,
|
||||||
message: `No Fastify app found. Ensure app.js exports a Fastify instance.\n\nError: ${errorMessage}\n\nNext:\n Run \`apophis init\` to scaffold a working app.js and config.`,
|
message: `No Fastify app found. Ensure app.js exports a Fastify instance or a factory function.\n\nSupported patterns:\n export default app\n export const createApp = () => app\n module.exports = app\n\nError: ${errorMessage}\n\nNext:\n Run \`apophis init\` to scaffold a working app.js and config.`,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -476,7 +473,6 @@ export async function verifyCommand(
|
|||||||
const runResult = await runVerify({
|
const runResult = await runVerify({
|
||||||
fastify: fastify as any,
|
fastify: fastify as any,
|
||||||
seed,
|
seed,
|
||||||
generationProfile: resolvedGenerationProfile,
|
|
||||||
timeout: typeof config.presets?.[loadResult.presetName || '']?.timeout === 'number'
|
timeout: typeof config.presets?.[loadResult.presetName || '']?.timeout === 'number'
|
||||||
? (config.presets[loadResult.presetName || ''] as { timeout?: number }).timeout
|
? (config.presets[loadResult.presetName || ''] as { timeout?: number }).timeout
|
||||||
: undefined,
|
: undefined,
|
||||||
@@ -552,12 +548,7 @@ export async function verifyCommand(
|
|||||||
message: `Config validation failed: ${message}`,
|
message: `Config validation failed: ${message}`,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (error instanceof GenerationProfileResolutionError) {
|
|
||||||
return {
|
|
||||||
exitCode: USAGE_ERROR,
|
|
||||||
message,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return {
|
return {
|
||||||
exitCode: INTERNAL_ERROR,
|
exitCode: INTERNAL_ERROR,
|
||||||
message: `Internal error in verify command: ${message}`,
|
message: `Internal error in verify command: ${message}`,
|
||||||
@@ -579,7 +570,6 @@ export async function handleVerify(
|
|||||||
): Promise<number> {
|
): Promise<number> {
|
||||||
const options: VerifyOptions = {
|
const options: VerifyOptions = {
|
||||||
profile: ctx.options.profile || undefined,
|
profile: ctx.options.profile || undefined,
|
||||||
generationProfile: ctx.options.generationProfile,
|
|
||||||
routes: undefined,
|
routes: undefined,
|
||||||
seed: undefined,
|
seed: undefined,
|
||||||
changed: false,
|
changed: false,
|
||||||
@@ -611,11 +601,6 @@ export async function handleVerify(
|
|||||||
options.changed = true
|
options.changed = true
|
||||||
}
|
}
|
||||||
|
|
||||||
const generationProfileIdx = args.indexOf('--generation-profile')
|
|
||||||
if (generationProfileIdx !== -1 && args[generationProfileIdx + 1]) {
|
|
||||||
options.generationProfile = args[generationProfileIdx + 1]
|
|
||||||
}
|
|
||||||
|
|
||||||
const workspaceMode = args.includes('--workspace')
|
const workspaceMode = args.includes('--workspace')
|
||||||
|
|
||||||
if (workspaceMode) {
|
if (workspaceMode) {
|
||||||
|
|||||||
@@ -53,7 +53,6 @@ export interface VerifyRunResult {
|
|||||||
export interface VerifyRunnerDeps {
|
export interface VerifyRunnerDeps {
|
||||||
fastify: FastifyInjectInstance
|
fastify: FastifyInjectInstance
|
||||||
seed: number
|
seed: number
|
||||||
generationProfile?: 'quick' | 'standard' | 'thorough'
|
|
||||||
timeout?: number
|
timeout?: number
|
||||||
routeFilters?: string[]
|
routeFilters?: string[]
|
||||||
changed?: boolean
|
changed?: boolean
|
||||||
|
|||||||
@@ -30,7 +30,6 @@ export interface Config {
|
|||||||
environments?: Record<string, EnvironmentPolicy>;
|
environments?: Record<string, EnvironmentPolicy>;
|
||||||
profiles?: Record<string, ProfileDefinition>;
|
profiles?: Record<string, ProfileDefinition>;
|
||||||
presets?: Record<string, PresetDefinition>;
|
presets?: Record<string, PresetDefinition>;
|
||||||
generationProfiles?: Record<string, 'quick' | 'standard' | 'thorough' | { base: 'quick' | 'standard' | 'thorough' }>;
|
|
||||||
[key: string]: unknown;
|
[key: string]: unknown;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -107,11 +106,6 @@ const CONFIG_SCHEMA: Record<string, SchemaField> = {
|
|||||||
optional: true,
|
optional: true,
|
||||||
properties: {},
|
properties: {},
|
||||||
},
|
},
|
||||||
generationProfiles: {
|
|
||||||
type: 'object',
|
|
||||||
optional: true,
|
|
||||||
properties: {},
|
|
||||||
},
|
|
||||||
packs: {
|
packs: {
|
||||||
type: 'array',
|
type: 'array',
|
||||||
optional: true,
|
optional: true,
|
||||||
@@ -151,7 +145,6 @@ const PROFILE_SCHEMA: Record<string, SchemaField> = {
|
|||||||
// Schema for PresetDefinition values (inside presets.<name>)
|
// Schema for PresetDefinition values (inside presets.<name>)
|
||||||
const PRESET_SCHEMA: Record<string, SchemaField> = {
|
const PRESET_SCHEMA: Record<string, SchemaField> = {
|
||||||
name: { type: 'string', optional: false },
|
name: { type: 'string', optional: false },
|
||||||
depth: { type: 'string', optional: true, enumValues: ['quick', 'standard', 'deep'] },
|
|
||||||
timeout: { type: 'number', optional: true, min: 0 },
|
timeout: { type: 'number', optional: true, min: 0 },
|
||||||
parallel: { type: 'boolean', optional: true },
|
parallel: { type: 'boolean', optional: true },
|
||||||
chaos: { type: 'boolean', optional: true },
|
chaos: { type: 'boolean', optional: true },
|
||||||
@@ -160,12 +153,9 @@ const PRESET_SCHEMA: Record<string, SchemaField> = {
|
|||||||
sampling: { type: 'number', optional: true },
|
sampling: { type: 'number', optional: true },
|
||||||
blocking: { type: 'boolean', optional: true },
|
blocking: { type: 'boolean', optional: true },
|
||||||
sinks: { type: 'object', optional: true },
|
sinks: { type: 'object', optional: true },
|
||||||
|
runs: { type: 'number', optional: true, min: 1 },
|
||||||
};
|
};
|
||||||
|
|
||||||
const GENERATION_PROFILE_ALIAS_SCHEMA: Record<string, SchemaField> = {
|
|
||||||
base: { type: 'string', optional: false, enumValues: ['quick', 'standard', 'thorough'] },
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Config discovery
|
// Config discovery
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -259,7 +249,6 @@ function getDynamicContainerSchema(path: string): Record<string, SchemaField> |
|
|||||||
if (path === 'profiles') return PROFILE_SCHEMA;
|
if (path === 'profiles') return PROFILE_SCHEMA;
|
||||||
if (path === 'presets') return PRESET_SCHEMA;
|
if (path === 'presets') return PRESET_SCHEMA;
|
||||||
if (path === 'environments') return ENVIRONMENT_POLICY_SCHEMA;
|
if (path === 'environments') return ENVIRONMENT_POLICY_SCHEMA;
|
||||||
if (path === 'generationProfiles') return GENERATION_PROFILE_ALIAS_SCHEMA;
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -267,7 +256,7 @@ function getDynamicContainerSchema(path: string): Record<string, SchemaField> |
|
|||||||
* Check if a path is inside a dynamic container (e.g., profiles.foo, presets.bar).
|
* Check if a path is inside a dynamic container (e.g., profiles.foo, presets.bar).
|
||||||
*/
|
*/
|
||||||
function isInsideDynamicContainer(path: string): boolean {
|
function isInsideDynamicContainer(path: string): boolean {
|
||||||
return path.startsWith('profiles.') || path.startsWith('presets.') || path.startsWith('environments.') || path.startsWith('generationProfiles.');
|
return path.startsWith('profiles.') || path.startsWith('presets.') || path.startsWith('environments.');
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -379,18 +368,11 @@ export function validateConfigAgainstSchema(
|
|||||||
|
|
||||||
// Handle dynamic containers: profiles, presets, environments
|
// Handle dynamic containers: profiles, presets, environments
|
||||||
// The keys are user-defined names; their values have specific schemas
|
// The keys are user-defined names; their values have specific schemas
|
||||||
const isDynamicContainer = path === 'profiles' || path === 'presets' || path === 'environments' || path === 'generationProfiles';
|
const isDynamicContainer = path === 'profiles' || path === 'presets' || path === 'environments';
|
||||||
if (!fieldSchema && isDynamicContainer) {
|
if (!fieldSchema && isDynamicContainer) {
|
||||||
const childSchema = getDynamicContainerSchema(path);
|
const childSchema = getDynamicContainerSchema(path);
|
||||||
const fieldValue = obj[key];
|
const fieldValue = obj[key];
|
||||||
if (path === 'generationProfiles' && typeof fieldValue === 'string') {
|
if (childSchema && fieldValue !== null && typeof fieldValue === 'object') {
|
||||||
validateType(
|
|
||||||
fieldValue,
|
|
||||||
{ type: 'string', optional: false, enumValues: ['quick', 'standard', 'thorough'] },
|
|
||||||
currentPath,
|
|
||||||
key,
|
|
||||||
);
|
|
||||||
} else if (childSchema && fieldValue !== null && typeof fieldValue === 'object') {
|
|
||||||
// Validate the dynamic container value against its specific schema
|
// Validate the dynamic container value against its specific schema
|
||||||
validateConfigAgainstSchema(fieldValue, childSchema, currentPath);
|
validateConfigAgainstSchema(fieldValue, childSchema, currentPath);
|
||||||
} else if (childSchema) {
|
} else if (childSchema) {
|
||||||
@@ -633,19 +615,6 @@ export function validateConfigSemantics(config: Config): void {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (preset.depth !== undefined) {
|
|
||||||
const validDepths = ['quick', 'standard', 'deep'];
|
|
||||||
const depthValue = preset.depth;
|
|
||||||
if (typeof depthValue === 'string' && !validDepths.includes(depthValue as string)) {
|
|
||||||
throw new ConfigValidationError(
|
|
||||||
`Preset "${presetName}" has invalid depth: "${depthValue}"`,
|
|
||||||
`presets.${presetName}.depth`,
|
|
||||||
'depth',
|
|
||||||
depthValue,
|
|
||||||
`Must be one of: ${validDepths.join(', ')}.`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -101,10 +101,6 @@ export function createContext(options: Record<string, unknown> = {}): CliContext
|
|||||||
? options.color
|
? options.color
|
||||||
: 'auto';
|
: 'auto';
|
||||||
|
|
||||||
const generationProfile = typeof options.generationProfile === 'string'
|
|
||||||
? options.generationProfile
|
|
||||||
: undefined;
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
cwd,
|
cwd,
|
||||||
env: {
|
env: {
|
||||||
@@ -119,7 +115,6 @@ export function createContext(options: Record<string, unknown> = {}): CliContext
|
|||||||
options: {
|
options: {
|
||||||
config: typeof options.config === 'string' ? options.config : undefined,
|
config: typeof options.config === 'string' ? options.config : undefined,
|
||||||
profile: typeof options.profile === 'string' ? options.profile : undefined,
|
profile: typeof options.profile === 'string' ? options.profile : undefined,
|
||||||
generationProfile,
|
|
||||||
format,
|
format,
|
||||||
color,
|
color,
|
||||||
quiet: options.quiet === true,
|
quiet: options.quiet === true,
|
||||||
|
|||||||
@@ -1,51 +0,0 @@
|
|||||||
import type { Config } from './config-loader.js'
|
|
||||||
|
|
||||||
export type ResolvedGenerationProfile = 'quick' | 'standard' | 'thorough'
|
|
||||||
|
|
||||||
export class GenerationProfileResolutionError extends Error {
|
|
||||||
constructor(message: string) {
|
|
||||||
super(message)
|
|
||||||
this.name = 'GenerationProfileResolutionError'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function isBuiltInProfile(value: string): value is ResolvedGenerationProfile {
|
|
||||||
return value === 'quick' || value === 'standard' || value === 'thorough'
|
|
||||||
}
|
|
||||||
|
|
||||||
export function resolveGenerationProfileOverride(
|
|
||||||
rawProfile: string | undefined,
|
|
||||||
config: Config,
|
|
||||||
): ResolvedGenerationProfile | undefined {
|
|
||||||
if (!rawProfile) {
|
|
||||||
return undefined
|
|
||||||
}
|
|
||||||
|
|
||||||
if (isBuiltInProfile(rawProfile)) {
|
|
||||||
return rawProfile
|
|
||||||
}
|
|
||||||
|
|
||||||
const aliases = config.generationProfiles
|
|
||||||
if (!aliases) {
|
|
||||||
throw new GenerationProfileResolutionError(
|
|
||||||
`Unknown generation profile "${rawProfile}". Use one of: quick, standard, thorough, or define an alias in config.generationProfiles.`,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
const alias = aliases[rawProfile]
|
|
||||||
if (!alias) {
|
|
||||||
const available = Object.keys(aliases).join(', ') || 'none'
|
|
||||||
throw new GenerationProfileResolutionError(
|
|
||||||
`Unknown generation profile "${rawProfile}". Built-ins: quick, standard, thorough. Config aliases: ${available}.`,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
const target = typeof alias === 'string' ? alias : alias.base
|
|
||||||
if (!isBuiltInProfile(target)) {
|
|
||||||
throw new GenerationProfileResolutionError(
|
|
||||||
`Invalid generation profile alias "${rawProfile}". Alias must resolve to quick, standard, or thorough.`,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return target
|
|
||||||
}
|
|
||||||
@@ -22,7 +22,6 @@ const HELP_HEADER = `
|
|||||||
${pc.dim('Global Options:')}
|
${pc.dim('Global Options:')}
|
||||||
--config <path> Config file path
|
--config <path> Config file path
|
||||||
--profile <name> Profile name from config
|
--profile <name> Profile name from config
|
||||||
--generation-profile <name> Generation budget profile (built-in or config alias)
|
|
||||||
--cwd <path> Working directory override
|
--cwd <path> Working directory override
|
||||||
--format <mode> Output format: human | json | ndjson (default: human)
|
--format <mode> Output format: human | json | ndjson (default: human)
|
||||||
--color <mode> Color mode: auto | always | never (default: auto)
|
--color <mode> Color mode: auto | always | never (default: auto)
|
||||||
@@ -71,7 +70,6 @@ function getCommandHelp(command: string): string {
|
|||||||
|
|
||||||
${pc.dim('Options:')}
|
${pc.dim('Options:')}
|
||||||
--profile <name> Profile name from config
|
--profile <name> Profile name from config
|
||||||
--generation-profile <name> Generation budget profile (built-in or config alias)
|
|
||||||
--routes <filter> Route filter pattern
|
--routes <filter> Route filter pattern
|
||||||
--seed <number> Deterministic seed
|
--seed <number> Deterministic seed
|
||||||
--changed Filter to git-modified routes
|
--changed Filter to git-modified routes
|
||||||
@@ -103,7 +101,6 @@ function getCommandHelp(command: string): string {
|
|||||||
|
|
||||||
${pc.dim('Options:')}
|
${pc.dim('Options:')}
|
||||||
--profile <name> Profile name from config
|
--profile <name> Profile name from config
|
||||||
--generation-profile <name> Generation budget profile (built-in or config alias)
|
|
||||||
--seed <number> Deterministic seed
|
--seed <number> Deterministic seed
|
||||||
|
|
||||||
${pc.dim('Examples:')}
|
${pc.dim('Examples:')}
|
||||||
@@ -225,7 +222,6 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
|
|||||||
// Global flags
|
// Global flags
|
||||||
cli.option('--config <path>', 'Config file path');
|
cli.option('--config <path>', 'Config file path');
|
||||||
cli.option('--profile <name>', 'Profile name from config');
|
cli.option('--profile <name>', 'Profile name from config');
|
||||||
cli.option('--generation-profile <name>', 'Generation budget profile (built-in or config alias)');
|
|
||||||
cli.option('--cwd <path>', 'Working directory override');
|
cli.option('--cwd <path>', 'Working directory override');
|
||||||
cli.option('--format <mode>', 'Output format: human | json | ndjson', { default: 'human' });
|
cli.option('--format <mode>', 'Output format: human | json | ndjson', { default: 'human' });
|
||||||
cli.option('--color <mode>', 'Color mode: auto | always | never', { default: 'auto' });
|
cli.option('--color <mode>', 'Color mode: auto | always | never', { default: 'auto' });
|
||||||
@@ -270,7 +266,6 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
|
|||||||
break;
|
break;
|
||||||
case 'verify':
|
case 'verify':
|
||||||
cmd.option('--profile <name>', 'Profile name from config');
|
cmd.option('--profile <name>', 'Profile name from config');
|
||||||
cmd.option('--generation-profile <name>', 'Generation budget profile (built-in or config alias)');
|
|
||||||
cmd.option('--routes <filter>', 'Route filter pattern');
|
cmd.option('--routes <filter>', 'Route filter pattern');
|
||||||
cmd.option('--seed <number>', 'Deterministic seed');
|
cmd.option('--seed <number>', 'Deterministic seed');
|
||||||
cmd.option('--changed', 'Filter to git-modified routes');
|
cmd.option('--changed', 'Filter to git-modified routes');
|
||||||
@@ -281,7 +276,6 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
|
|||||||
break;
|
break;
|
||||||
case 'qualify':
|
case 'qualify':
|
||||||
cmd.option('--profile <name>', 'Profile name from config');
|
cmd.option('--profile <name>', 'Profile name from config');
|
||||||
cmd.option('--generation-profile <name>', 'Generation budget profile (built-in or config alias)');
|
|
||||||
cmd.option('--seed <number>', 'Deterministic seed');
|
cmd.option('--seed <number>', 'Deterministic seed');
|
||||||
break;
|
break;
|
||||||
case 'replay':
|
case 'replay':
|
||||||
@@ -373,16 +367,15 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
|
|||||||
// Handle unknown flags
|
// Handle unknown flags
|
||||||
const knownGlobalFlags = new Set([
|
const knownGlobalFlags = new Set([
|
||||||
'--config', '--profile', '--cwd', '--format', '--color',
|
'--config', '--profile', '--cwd', '--format', '--color',
|
||||||
'--generation-profile',
|
|
||||||
'--quiet', '--verbose', '--artifact-dir', '--workspace',
|
'--quiet', '--verbose', '--artifact-dir', '--workspace',
|
||||||
'-v', '--version', '-h', '--help',
|
'-v', '--version', '-h', '--help',
|
||||||
]);
|
]);
|
||||||
|
|
||||||
const commandSpecificFlags: Record<string, Set<string>> = {
|
const commandSpecificFlags: Record<string, Set<string>> = {
|
||||||
init: new Set(['--preset', '--force', '--noninteractive']),
|
init: new Set(['--preset', '--force', '--noninteractive']),
|
||||||
verify: new Set(['--profile', '--generation-profile', '--routes', '--seed', '--changed', '--workspace']),
|
verify: new Set(['--profile', '--routes', '--seed', '--changed', '--workspace']),
|
||||||
observe: new Set(['--profile', '--check-config', '--workspace']),
|
observe: new Set(['--profile', '--check-config', '--workspace']),
|
||||||
qualify: new Set(['--profile', '--generation-profile', '--seed', '--workspace']),
|
qualify: new Set(['--profile', '--seed', '--workspace']),
|
||||||
replay: new Set(['--artifact']),
|
replay: new Set(['--artifact']),
|
||||||
doctor: new Set(['--mode', '--strict', '--workspace']),
|
doctor: new Set(['--mode', '--strict', '--workspace']),
|
||||||
migrate: new Set(['--check', '--dry-run', '--write']),
|
migrate: new Set(['--check', '--dry-run', '--write']),
|
||||||
|
|||||||
@@ -29,7 +29,6 @@ export interface CliContext {
|
|||||||
options: {
|
options: {
|
||||||
config: string | undefined;
|
config: string | undefined;
|
||||||
profile: string | undefined;
|
profile: string | undefined;
|
||||||
generationProfile?: string;
|
|
||||||
format: OutputFormat;
|
format: OutputFormat;
|
||||||
color: ColorMode;
|
color: ColorMode;
|
||||||
quiet: boolean;
|
quiet: boolean;
|
||||||
@@ -132,7 +131,6 @@ export interface ProfileDefinition {
|
|||||||
* required: ["name"],
|
* required: ["name"],
|
||||||
* properties: {
|
* properties: {
|
||||||
* name: { type: "string" },
|
* name: { type: "string" },
|
||||||
* depth: { type: "string", enum: ["quick", "standard", "deep"] },
|
|
||||||
* timeout: { type: "number" },
|
* timeout: { type: "number" },
|
||||||
* parallel: { type: "boolean" },
|
* parallel: { type: "boolean" },
|
||||||
* chaos: { type: "boolean" },
|
* chaos: { type: "boolean" },
|
||||||
@@ -143,7 +141,6 @@ export interface ProfileDefinition {
|
|||||||
*/
|
*/
|
||||||
export interface PresetDefinition {
|
export interface PresetDefinition {
|
||||||
name: string;
|
name: string;
|
||||||
depth?: "quick" | "standard" | "deep";
|
|
||||||
timeout?: number;
|
timeout?: number;
|
||||||
parallel?: boolean;
|
parallel?: boolean;
|
||||||
chaos?: boolean;
|
chaos?: boolean;
|
||||||
|
|||||||
@@ -183,29 +183,6 @@ function generateNextSteps(failure: FailureRecord): string {
|
|||||||
// Progress and summary rendering
|
// Progress and summary rendering
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
/**
|
|
||||||
* Render progress for a running command.
|
|
||||||
* Safe for CI (no spinners, just text updates).
|
|
||||||
*/
|
|
||||||
export function renderProgress(
|
|
||||||
current: number,
|
|
||||||
total: number,
|
|
||||||
label: string,
|
|
||||||
ctx: OutputContext,
|
|
||||||
): string {
|
|
||||||
const c = getColorizer(ctx);
|
|
||||||
const pct = total > 0 ? Math.round((current / total) * 100) : 0;
|
|
||||||
|
|
||||||
if (ctx.isCI || !ctx.isTTY) {
|
|
||||||
// CI mode: simple text, no spinner
|
|
||||||
return `${label} [${current}/${total}] ${pct}%`;
|
|
||||||
}
|
|
||||||
|
|
||||||
// TTY mode: with color
|
|
||||||
const bar = renderProgressBar(current, total, 20, ctx);
|
|
||||||
return `${c.dim(label)} ${bar} ${c.bold(`${pct}%`)}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Render a simple ASCII progress bar.
|
* Render a simple ASCII progress bar.
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -87,9 +87,3 @@ export const discoverRoutes = (instance: { routes?: Array<{ method: string; url:
|
|||||||
// Fastify 5 fallback: routes registered before plugin
|
// Fastify 5 fallback: routes registered before plugin
|
||||||
return discoverRoutesFallback(instance)
|
return discoverRoutesFallback(instance)
|
||||||
}
|
}
|
||||||
/**
|
|
||||||
* Clear captured routes for an instance (useful for testing).
|
|
||||||
*/
|
|
||||||
export const clearCapturedRoutes = (instance: object): void => {
|
|
||||||
capturedRoutes.delete(instance)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -143,46 +143,6 @@ export class PluginContractRegistry {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
// Built-in Plugin Contracts
|
|
||||||
// ============================================================================
|
|
||||||
export const BUILTIN_PLUGIN_CONTRACTS: Record<string, PluginContractSpec> = {
|
|
||||||
'@fastify/auth': {
|
|
||||||
appliesTo: '**',
|
|
||||||
hooks: {
|
|
||||||
onRequest: {
|
|
||||||
requires: ['request_headers(this).authorization != null'],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
'@fastify/compress': {
|
|
||||||
appliesTo: '**',
|
|
||||||
hooks: {
|
|
||||||
onSend: {
|
|
||||||
ensures: ['response_headers(this).content-encoding != null'],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
'@fastify/cors': {
|
|
||||||
appliesTo: '**',
|
|
||||||
hooks: {
|
|
||||||
onRequest: {
|
|
||||||
ensures: ['response_headers(this).access-control-allow-origin != null'],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
'@fastify/rate-limit': {
|
|
||||||
appliesTo: '**',
|
|
||||||
hooks: {
|
|
||||||
onRequest: {
|
|
||||||
ensures: [
|
|
||||||
'response_headers(this).x-ratelimit-limit != null',
|
|
||||||
'response_headers(this).x-ratelimit-remaining != null',
|
|
||||||
],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
// ============================================================================
|
|
||||||
// Factory
|
// Factory
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
export function createPluginContractRegistry(): PluginContractRegistry {
|
export function createPluginContractRegistry(): PluginContractRegistry {
|
||||||
|
|||||||
@@ -7,13 +7,9 @@ import type { Arbitrary } from 'fast-check'
|
|||||||
import * as fc from 'fast-check'
|
import * as fc from 'fast-check'
|
||||||
import { CONTENT_TYPE } from '../infrastructure/http-executor.js'
|
import { CONTENT_TYPE } from '../infrastructure/http-executor.js'
|
||||||
|
|
||||||
export type GenerationProfile = 'quick' | 'standard' | 'thorough'
|
|
||||||
|
|
||||||
export interface SchemaToArbOptions {
|
export interface SchemaToArbOptions {
|
||||||
/** 'request' skips readOnly, 'response' skips writeOnly */
|
/** 'request' skips readOnly, 'response' skips writeOnly */
|
||||||
readonly context: 'request' | 'response'
|
readonly context: 'request' | 'response'
|
||||||
/** Generation budget profile: quick favors speed, thorough favors breadth */
|
|
||||||
readonly generationProfile?: GenerationProfile
|
|
||||||
}
|
}
|
||||||
|
|
||||||
interface ContextCache {
|
interface ContextCache {
|
||||||
@@ -29,45 +25,32 @@ const patternRegexCache = new Map<string, RegExp>()
|
|||||||
const STABLE_SCHEMA_CACHE_LIMIT = 512
|
const STABLE_SCHEMA_CACHE_LIMIT = 512
|
||||||
const PATTERN_REGEX_CACHE_LIMIT = 256
|
const PATTERN_REGEX_CACHE_LIMIT = 256
|
||||||
|
|
||||||
function normalizeProfile(profile: GenerationProfile | undefined): GenerationProfile {
|
// Fixed defaults for generated data size (previously controlled by generationProfile tier)
|
||||||
return profile ?? 'standard'
|
const DEFAULT_STRING_MAX = 128
|
||||||
|
const DEFAULT_ARRAY_MAX = 10
|
||||||
|
const DEFAULT_ADDITIONAL_PROPS_MAX = 6
|
||||||
|
|
||||||
|
function defaultStringMaxLength(): number | undefined {
|
||||||
|
return DEFAULT_STRING_MAX
|
||||||
}
|
}
|
||||||
|
|
||||||
function defaultStringMaxLength(profile: GenerationProfile): number | undefined {
|
function defaultArrayMaxLength(): number | undefined {
|
||||||
if (profile === 'quick') return 48
|
return DEFAULT_ARRAY_MAX
|
||||||
if (profile === 'standard') return 128
|
|
||||||
return undefined
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function defaultArrayMaxLength(profile: GenerationProfile): number | undefined {
|
function additionalPropsMaxKeys(): number {
|
||||||
if (profile === 'quick') return 4
|
return DEFAULT_ADDITIONAL_PROPS_MAX
|
||||||
if (profile === 'standard') return 10
|
|
||||||
return undefined
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function additionalPropsMaxKeys(profile: GenerationProfile): number {
|
function buildFallbackAnyArb(): Arbitrary<unknown> {
|
||||||
if (profile === 'quick') return 3
|
|
||||||
if (profile === 'standard') return 6
|
|
||||||
return 10
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildFallbackAnyArb(profile: GenerationProfile): Arbitrary<unknown> {
|
|
||||||
if (profile === 'thorough') {
|
|
||||||
return fc.anything()
|
|
||||||
}
|
|
||||||
|
|
||||||
const stringMax = profile === 'quick' ? 24 : 64
|
|
||||||
const arrayMax = profile === 'quick' ? 3 : 6
|
|
||||||
const dictMax = profile === 'quick' ? 2 : 4
|
|
||||||
|
|
||||||
return fc.oneof(
|
return fc.oneof(
|
||||||
fc.constant(null),
|
fc.constant(null),
|
||||||
fc.boolean(),
|
fc.boolean(),
|
||||||
fc.integer(),
|
fc.integer(),
|
||||||
fc.double({ noNaN: true }),
|
fc.double({ noNaN: true }),
|
||||||
fc.string({ maxLength: stringMax }),
|
fc.string({ maxLength: 64 }),
|
||||||
fc.array(fc.string({ maxLength: 16 }), { maxLength: arrayMax }),
|
fc.array(fc.string({ maxLength: 16 }), { maxLength: 6 }),
|
||||||
fc.dictionary(fc.string({ maxLength: 16 }), fc.string({ maxLength: 24 }), { maxKeys: dictMax }),
|
fc.dictionary(fc.string({ maxLength: 16 }), fc.string({ maxLength: 24 }), { maxKeys: 4 }),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -106,7 +89,6 @@ const getObject = (schema: unknown, key: string): Record<string, unknown> | unde
|
|||||||
|
|
||||||
const buildStringArb = (
|
const buildStringArb = (
|
||||||
schema: Record<string, unknown>,
|
schema: Record<string, unknown>,
|
||||||
profile: GenerationProfile,
|
|
||||||
): Arbitrary<string> => {
|
): Arbitrary<string> => {
|
||||||
const minLength = getNumber(schema, 'minLength')
|
const minLength = getNumber(schema, 'minLength')
|
||||||
const maxLength = getNumber(schema, 'maxLength')
|
const maxLength = getNumber(schema, 'maxLength')
|
||||||
@@ -149,7 +131,7 @@ const buildStringArb = (
|
|||||||
if (minLength !== undefined) constraints.minLength = minLength
|
if (minLength !== undefined) constraints.minLength = minLength
|
||||||
if (maxLength !== undefined) constraints.maxLength = maxLength
|
if (maxLength !== undefined) constraints.maxLength = maxLength
|
||||||
else {
|
else {
|
||||||
const capped = defaultStringMaxLength(profile)
|
const capped = defaultStringMaxLength()
|
||||||
if (capped !== undefined) constraints.maxLength = capped
|
if (capped !== undefined) constraints.maxLength = capped
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -298,14 +280,13 @@ function getSchemaFingerprint(schema: Record<string, unknown>): string | undefin
|
|||||||
function getStableCachedArbitrary(
|
function getStableCachedArbitrary(
|
||||||
schema: Record<string, unknown>,
|
schema: Record<string, unknown>,
|
||||||
context: SchemaToArbOptions['context'],
|
context: SchemaToArbOptions['context'],
|
||||||
profile: GenerationProfile,
|
|
||||||
): Arbitrary<unknown> | undefined {
|
): Arbitrary<unknown> | undefined {
|
||||||
const fingerprint = getSchemaFingerprint(schema)
|
const fingerprint = getSchemaFingerprint(schema)
|
||||||
if (!fingerprint) {
|
if (!fingerprint) {
|
||||||
return undefined
|
return undefined
|
||||||
}
|
}
|
||||||
|
|
||||||
const key = `${context}:${profile}:${fingerprint}`
|
const key = `${context}:${fingerprint}`
|
||||||
const cached = stableSchemaArbitraryCache.get(key)
|
const cached = stableSchemaArbitraryCache.get(key)
|
||||||
if (!cached) {
|
if (!cached) {
|
||||||
return undefined
|
return undefined
|
||||||
@@ -319,7 +300,6 @@ function getStableCachedArbitrary(
|
|||||||
function setStableCachedArbitrary(
|
function setStableCachedArbitrary(
|
||||||
schema: Record<string, unknown>,
|
schema: Record<string, unknown>,
|
||||||
context: SchemaToArbOptions['context'],
|
context: SchemaToArbOptions['context'],
|
||||||
profile: GenerationProfile,
|
|
||||||
arbitrary: Arbitrary<unknown>,
|
arbitrary: Arbitrary<unknown>,
|
||||||
): void {
|
): void {
|
||||||
const fingerprint = getSchemaFingerprint(schema)
|
const fingerprint = getSchemaFingerprint(schema)
|
||||||
@@ -327,7 +307,7 @@ function setStableCachedArbitrary(
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
const key = `${context}:${profile}:${fingerprint}`
|
const key = `${context}:${fingerprint}`
|
||||||
if (stableSchemaArbitraryCache.has(key)) {
|
if (stableSchemaArbitraryCache.has(key)) {
|
||||||
stableSchemaArbitraryCache.delete(key)
|
stableSchemaArbitraryCache.delete(key)
|
||||||
}
|
}
|
||||||
@@ -353,19 +333,18 @@ const buildIntegerArb = (schema: Record<string, unknown>): Arbitrary<number> =>
|
|||||||
const buildArrayArb = (
|
const buildArrayArb = (
|
||||||
schema: Record<string, unknown>,
|
schema: Record<string, unknown>,
|
||||||
options: SchemaToArbOptions,
|
options: SchemaToArbOptions,
|
||||||
profile: GenerationProfile,
|
|
||||||
): Arbitrary<unknown[]> => {
|
): Arbitrary<unknown[]> => {
|
||||||
const itemsSchema = getObject(schema, 'items')
|
const itemsSchema = getObject(schema, 'items')
|
||||||
const itemArb = itemsSchema !== undefined
|
const itemArb = itemsSchema !== undefined
|
||||||
? convertSchemaInternal(itemsSchema, options, false)
|
? convertSchemaInternal(itemsSchema, options, false)
|
||||||
: buildFallbackAnyArb(profile)
|
: buildFallbackAnyArb()
|
||||||
const minItems = getNumber(schema, 'minItems')
|
const minItems = getNumber(schema, 'minItems')
|
||||||
const maxItems = getNumber(schema, 'maxItems')
|
const maxItems = getNumber(schema, 'maxItems')
|
||||||
const constraints: { minLength?: number; maxLength?: number } = {}
|
const constraints: { minLength?: number; maxLength?: number } = {}
|
||||||
if (minItems !== undefined) constraints.minLength = minItems
|
if (minItems !== undefined) constraints.minLength = minItems
|
||||||
if (maxItems !== undefined) constraints.maxLength = maxItems
|
if (maxItems !== undefined) constraints.maxLength = maxItems
|
||||||
else {
|
else {
|
||||||
const capped = defaultArrayMaxLength(profile)
|
const capped = defaultArrayMaxLength()
|
||||||
if (capped !== undefined) constraints.maxLength = capped
|
if (capped !== undefined) constraints.maxLength = capped
|
||||||
}
|
}
|
||||||
return fc.array(itemArb, constraints)
|
return fc.array(itemArb, constraints)
|
||||||
@@ -374,7 +353,6 @@ const buildArrayArb = (
|
|||||||
const buildObjectArb = (
|
const buildObjectArb = (
|
||||||
schema: Record<string, unknown>,
|
schema: Record<string, unknown>,
|
||||||
options: SchemaToArbOptions,
|
options: SchemaToArbOptions,
|
||||||
profile: GenerationProfile,
|
|
||||||
): Arbitrary<Record<string, unknown>> => {
|
): Arbitrary<Record<string, unknown>> => {
|
||||||
const properties = getObject(schema, 'properties') ?? {}
|
const properties = getObject(schema, 'properties') ?? {}
|
||||||
const required = new Set(getArray(schema, 'required') as string[] ?? [])
|
const required = new Set(getArray(schema, 'required') as string[] ?? [])
|
||||||
@@ -398,14 +376,14 @@ const buildObjectArb = (
|
|||||||
const baseArb = fc.record(arbs)
|
const baseArb = fc.record(arbs)
|
||||||
|
|
||||||
if (additionalProperties === true) {
|
if (additionalProperties === true) {
|
||||||
const extraValueArb = buildFallbackAnyArb(profile)
|
const extraValueArb = buildFallbackAnyArb()
|
||||||
const keyMaxLength = profile === 'quick' ? 16 : 32
|
const keyMaxLength = 32
|
||||||
return fc.tuple(
|
return fc.tuple(
|
||||||
baseArb,
|
baseArb,
|
||||||
fc.dictionary(
|
fc.dictionary(
|
||||||
fc.string({ maxLength: keyMaxLength }),
|
fc.string({ maxLength: keyMaxLength }),
|
||||||
extraValueArb,
|
extraValueArb,
|
||||||
{ maxKeys: additionalPropsMaxKeys(profile) },
|
{ maxKeys: additionalPropsMaxKeys() },
|
||||||
),
|
),
|
||||||
).map(([base, extra]) => ({
|
).map(([base, extra]) => ({
|
||||||
...base,
|
...base,
|
||||||
@@ -418,7 +396,6 @@ const buildObjectArb = (
|
|||||||
|
|
||||||
const buildMultipartArb = (
|
const buildMultipartArb = (
|
||||||
schema: Record<string, unknown>,
|
schema: Record<string, unknown>,
|
||||||
profile: GenerationProfile,
|
|
||||||
): Arbitrary<{ fields: Record<string, unknown>; files: Record<string, { originalname: string; mimetype: string; size: number; buffer: Buffer } | { originalname: string; mimetype: string; size: number; buffer: Buffer }[]> }> => {
|
): Arbitrary<{ fields: Record<string, unknown>; files: Record<string, { originalname: string; mimetype: string; size: number; buffer: Buffer } | { originalname: string; mimetype: string; size: number; buffer: Buffer }[]> }> => {
|
||||||
const fieldsSchema = getObject(schema, 'x-multipart-fields') ?? {}
|
const fieldsSchema = getObject(schema, 'x-multipart-fields') ?? {}
|
||||||
const filesSchema = getObject(schema, 'x-multipart-files') ?? {}
|
const filesSchema = getObject(schema, 'x-multipart-files') ?? {}
|
||||||
@@ -426,7 +403,7 @@ const buildMultipartArb = (
|
|||||||
const fieldArbs: Record<string, Arbitrary<unknown>> = {}
|
const fieldArbs: Record<string, Arbitrary<unknown>> = {}
|
||||||
for (const [key, propSchema] of Object.entries(fieldsSchema)) {
|
for (const [key, propSchema] of Object.entries(fieldsSchema)) {
|
||||||
if (isObject(propSchema)) {
|
if (isObject(propSchema)) {
|
||||||
fieldArbs[key] = convertSchemaInternal(propSchema, { context: 'request', generationProfile: profile }, false)
|
fieldArbs[key] = convertSchemaInternal(propSchema, { context: 'request' }, false)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -463,7 +440,6 @@ const convertSchemaInternal = (
|
|||||||
options: SchemaToArbOptions,
|
options: SchemaToArbOptions,
|
||||||
useStableCache: boolean,
|
useStableCache: boolean,
|
||||||
): Arbitrary<unknown> => {
|
): Arbitrary<unknown> => {
|
||||||
const profile = normalizeProfile(options.generationProfile)
|
|
||||||
const cacheKey = options.context
|
const cacheKey = options.context
|
||||||
const cachedBySchema = schemaArbitraryCache.get(schema)
|
const cachedBySchema = schemaArbitraryCache.get(schema)
|
||||||
const cached = cachedBySchema?.[cacheKey]
|
const cached = cachedBySchema?.[cacheKey]
|
||||||
@@ -472,7 +448,7 @@ const convertSchemaInternal = (
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (useStableCache) {
|
if (useStableCache) {
|
||||||
const stableCached = getStableCachedArbitrary(schema, cacheKey, profile)
|
const stableCached = getStableCachedArbitrary(schema, cacheKey)
|
||||||
if (stableCached) {
|
if (stableCached) {
|
||||||
const contextCache = cachedBySchema ?? {}
|
const contextCache = cachedBySchema ?? {}
|
||||||
contextCache[cacheKey] = stableCached
|
contextCache[cacheKey] = stableCached
|
||||||
@@ -489,11 +465,11 @@ const convertSchemaInternal = (
|
|||||||
let arb: Arbitrary<unknown>
|
let arb: Arbitrary<unknown>
|
||||||
|
|
||||||
if (contentType === CONTENT_TYPE.MULTIPART) {
|
if (contentType === CONTENT_TYPE.MULTIPART) {
|
||||||
arb = buildMultipartArb(schema, profile)
|
arb = buildMultipartArb(schema)
|
||||||
} else if (enumValues !== undefined && enumValues.length > 0) {
|
} else if (enumValues !== undefined && enumValues.length > 0) {
|
||||||
arb = fc.constantFrom(...enumValues)
|
arb = fc.constantFrom(...enumValues)
|
||||||
} else if (type === 'string') {
|
} else if (type === 'string') {
|
||||||
arb = buildStringArb(schema, profile)
|
arb = buildStringArb(schema)
|
||||||
} else if (type === 'integer') {
|
} else if (type === 'integer') {
|
||||||
arb = buildIntegerArb(schema)
|
arb = buildIntegerArb(schema)
|
||||||
} else if (type === 'number') {
|
} else if (type === 'number') {
|
||||||
@@ -501,11 +477,11 @@ const convertSchemaInternal = (
|
|||||||
} else if (type === 'boolean') {
|
} else if (type === 'boolean') {
|
||||||
arb = fc.boolean()
|
arb = fc.boolean()
|
||||||
} else if (type === 'array') {
|
} else if (type === 'array') {
|
||||||
arb = buildArrayArb(schema, options, profile)
|
arb = buildArrayArb(schema, options)
|
||||||
} else if (type === 'object') {
|
} else if (type === 'object') {
|
||||||
arb = buildObjectArb(schema, options, profile)
|
arb = buildObjectArb(schema, options)
|
||||||
} else {
|
} else {
|
||||||
arb = buildFallbackAnyArb(profile)
|
arb = buildFallbackAnyArb()
|
||||||
}
|
}
|
||||||
|
|
||||||
if (nullable === true) {
|
if (nullable === true) {
|
||||||
@@ -516,7 +492,7 @@ const convertSchemaInternal = (
|
|||||||
contextCache[cacheKey] = arb
|
contextCache[cacheKey] = arb
|
||||||
schemaArbitraryCache.set(schema, contextCache)
|
schemaArbitraryCache.set(schema, contextCache)
|
||||||
if (useStableCache) {
|
if (useStableCache) {
|
||||||
setStableCachedArbitrary(schema, cacheKey, profile, arb)
|
setStableCachedArbitrary(schema, cacheKey, arb)
|
||||||
}
|
}
|
||||||
|
|
||||||
return arb
|
return arb
|
||||||
|
|||||||
@@ -7,13 +7,13 @@
|
|||||||
*
|
*
|
||||||
* Inferred contracts are additive: they supplement, never replace, explicit x-ensures.
|
* Inferred contracts are additive: they supplement, never replace, explicit x-ensures.
|
||||||
*
|
*
|
||||||
* Supported inference:
|
* Supported inference (all wrapped with status-code guard):
|
||||||
* - required fields → response_body(this).field != null
|
* - required fields → response_code(this) == N => response_body(this).field != null
|
||||||
* - minimum (number/integer) → response_body(this).field >= N
|
* - minimum (number/integer) → response_code(this) == N => response_body(this).field >= N
|
||||||
* - maximum (number/integer) → response_body(this).field <= N
|
* - maximum (number/integer) → response_code(this) == N => response_body(this).field <= N
|
||||||
* - pattern (string) → response_body(this).field matches "..."
|
* - pattern (string) → response_code(this) == N => response_body(this).field matches "..."
|
||||||
* - const → response_body(this).field == value
|
* - const → response_code(this) == N => response_body(this).field == value
|
||||||
* - enum (small) → response_body(this).field == "a" || response_body(this).field == "b"
|
* - enum (small) → response_code(this) == N => response_body(this).field == "a" || ...
|
||||||
*
|
*
|
||||||
* Not inferred (leave to x-ensures for business logic):
|
* Not inferred (leave to x-ensures for business logic):
|
||||||
* - minLength/maxLength
|
* - minLength/maxLength
|
||||||
@@ -188,7 +188,12 @@ export function inferContractsFromRouteSchema(
|
|||||||
const code = parseInt(statusCode, 10)
|
const code = parseInt(statusCode, 10)
|
||||||
if (code >= 200 && code < 300) {
|
if (code >= 200 && code < 300) {
|
||||||
const inferred = inferContractsFromResponseSchema(statusSchema)
|
const inferred = inferContractsFromResponseSchema(statusSchema)
|
||||||
formulas.push(...inferred)
|
// Wrap each inferred contract with a status-code guard so it only
|
||||||
|
// applies when the response actually matches the schema it was
|
||||||
|
// inferred from. Prevents a 200-schema const from failing on a 404.
|
||||||
|
for (const formula of inferred) {
|
||||||
|
formulas.push(`response_code(this) == ${code} => ${formula}`)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -220,7 +220,6 @@ function setNestedValue(obj: Record<string, unknown>, path: string, value: unkno
|
|||||||
function createConditionalDependencyArbitrary(
|
function createConditionalDependencyArbitrary(
|
||||||
contract: ResolvedOutboundContract,
|
contract: ResolvedOutboundContract,
|
||||||
request: Record<string, unknown>,
|
request: Record<string, unknown>,
|
||||||
generationProfile: 'quick' | 'standard' | 'thorough',
|
|
||||||
): Arbitrary<DependencyResponseSample> {
|
): Arbitrary<DependencyResponseSample> {
|
||||||
const statuses = Object.keys(contract.response).map(Number)
|
const statuses = Object.keys(contract.response).map(Number)
|
||||||
if (statuses.length === 0) {
|
if (statuses.length === 0) {
|
||||||
@@ -229,7 +228,7 @@ function createConditionalDependencyArbitrary(
|
|||||||
return fc.integer({ min: 0, max: statuses.length - 1 }).chain((statusIndex) => {
|
return fc.integer({ min: 0, max: statuses.length - 1 }).chain((statusIndex) => {
|
||||||
const statusCode = statuses[statusIndex]!
|
const statusCode = statuses[statusIndex]!
|
||||||
const schema = contract.response[statusCode]
|
const schema = contract.response[statusCode]
|
||||||
const bodyArb = convertSchema(schema ?? {}, { context: 'response', generationProfile })
|
const bodyArb = convertSchema(schema ?? {}, { context: 'response' })
|
||||||
return bodyArb.map((rawBody) => ({
|
return bodyArb.map((rawBody) => ({
|
||||||
contractName: contract.name,
|
contractName: contract.name,
|
||||||
statusCode,
|
statusCode,
|
||||||
@@ -239,11 +238,10 @@ function createConditionalDependencyArbitrary(
|
|||||||
}
|
}
|
||||||
function createRequestArbitrary(
|
function createRequestArbitrary(
|
||||||
route: RouteContract,
|
route: RouteContract,
|
||||||
generationProfile: 'quick' | 'standard' | 'thorough',
|
|
||||||
): Arbitrary<Record<string, unknown>> {
|
): Arbitrary<Record<string, unknown>> {
|
||||||
const bodySchema = route.schema?.body as Record<string, unknown> | undefined
|
const bodySchema = route.schema?.body as Record<string, unknown> | undefined
|
||||||
const bodyArb = bodySchema !== undefined
|
const bodyArb = bodySchema !== undefined
|
||||||
? convertSchema(bodySchema, { context: 'request', generationProfile })
|
? convertSchema(bodySchema, { context: 'request' })
|
||||||
: fc.constant({})
|
: fc.constant({})
|
||||||
const pathParams = route.path.match(/:([a-zA-Z_][a-zA-Z0-9_]*)/g) ?? []
|
const pathParams = route.path.match(/:([a-zA-Z_][a-zA-Z0-9_]*)/g) ?? []
|
||||||
const pathParamArbs: Record<string, fc.Arbitrary<string>> = {}
|
const pathParamArbs: Record<string, fc.Arbitrary<string>> = {}
|
||||||
@@ -264,11 +262,10 @@ function createRequestArbitrary(
|
|||||||
function createConditionalDependenciesArbitrary(
|
function createConditionalDependenciesArbitrary(
|
||||||
contracts: ResolvedOutboundContract[],
|
contracts: ResolvedOutboundContract[],
|
||||||
request: Record<string, unknown>,
|
request: Record<string, unknown>,
|
||||||
generationProfile: 'quick' | 'standard' | 'thorough',
|
|
||||||
): Arbitrary<ReadonlyArray<DependencyResponseSample>> {
|
): Arbitrary<ReadonlyArray<DependencyResponseSample>> {
|
||||||
if (contracts.length === 0) return fc.constant([])
|
if (contracts.length === 0) return fc.constant([])
|
||||||
const arbs = contracts.map((contract) =>
|
const arbs = contracts.map((contract) =>
|
||||||
createConditionalDependencyArbitrary(contract, request, generationProfile)
|
createConditionalDependencyArbitrary(contract, request)
|
||||||
)
|
)
|
||||||
return fc.tuple(...arbs)
|
return fc.tuple(...arbs)
|
||||||
}
|
}
|
||||||
@@ -292,11 +289,10 @@ export function createTripleBoundaryArbitrary(
|
|||||||
route: RouteContract,
|
route: RouteContract,
|
||||||
contracts: ResolvedOutboundContract[],
|
contracts: ResolvedOutboundContract[],
|
||||||
chaosConfig: ChaosConfig,
|
chaosConfig: ChaosConfig,
|
||||||
generationProfile: 'quick' | 'standard' | 'thorough' = 'standard',
|
|
||||||
): Arbitrary<TripleBoundaryCommand> {
|
): Arbitrary<TripleBoundaryCommand> {
|
||||||
const requestArb = createRequestArbitrary(route, generationProfile)
|
const requestArb = createRequestArbitrary(route)
|
||||||
return requestArb.chain((request) => {
|
return requestArb.chain((request) => {
|
||||||
const depArb = createConditionalDependenciesArbitrary(contracts, request, generationProfile)
|
const depArb = createConditionalDependenciesArbitrary(contracts, request)
|
||||||
const chaosArb = createChaosEventArbitrary(route, contracts, chaosConfig)
|
const chaosArb = createChaosEventArbitrary(route, contracts, chaosConfig)
|
||||||
return fc.tuple(depArb, chaosArb).map(([dependencyResponses, chaosEvents]) => ({
|
return fc.tuple(depArb, chaosArb).map(([dependencyResponses, chaosEvents]) => ({
|
||||||
route,
|
route,
|
||||||
@@ -360,47 +356,3 @@ export function applyChaosToAllResponses(
|
|||||||
)
|
)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
// ============================================================================
|
|
||||||
// Formatting
|
|
||||||
// ============================================================================
|
|
||||||
export function formatTripleBoundaryCounterexample(result: TripleBoundaryResult): string {
|
|
||||||
const lines: string[] = []
|
|
||||||
lines.push('Triple-boundary counterexample:')
|
|
||||||
lines.push('')
|
|
||||||
lines.push(`Route: ${result.command.route.method} ${result.command.route.path}`)
|
|
||||||
lines.push('')
|
|
||||||
lines.push('Request:')
|
|
||||||
lines.push(JSON.stringify(result.command.request, null, 2))
|
|
||||||
lines.push('')
|
|
||||||
if (result.command.dependencyResponses.length > 0) {
|
|
||||||
lines.push('Dependency responses:')
|
|
||||||
for (const dep of result.command.dependencyResponses) {
|
|
||||||
lines.push(` ${dep.contractName}: ${dep.statusCode}`)
|
|
||||||
lines.push(` ${JSON.stringify(dep.body)}`)
|
|
||||||
}
|
|
||||||
lines.push('')
|
|
||||||
}
|
|
||||||
if (result.command.chaosEvents.length > 0) {
|
|
||||||
lines.push('Chaos events:')
|
|
||||||
for (const chaos of result.command.chaosEvents) {
|
|
||||||
if (chaos.type === 'none') continue
|
|
||||||
lines.push(` ${chaos.type}`)
|
|
||||||
if (chaos.contractName) lines.push(` Target: ${chaos.contractName}`)
|
|
||||||
if (chaos.delayMs) lines.push(` Delay: ${chaos.delayMs}ms`)
|
|
||||||
if (chaos.statusCode) lines.push(` Status: ${chaos.statusCode}`)
|
|
||||||
if (chaos.corruptionStrategy) lines.push(` Corruption: ${chaos.corruptionStrategy}`)
|
|
||||||
if (chaos.corruptionField) lines.push(` Field: ${chaos.corruptionField}`)
|
|
||||||
}
|
|
||||||
lines.push('')
|
|
||||||
}
|
|
||||||
if (result.failureBoundary) {
|
|
||||||
lines.push(`Failure boundary: ${result.failureBoundary}`)
|
|
||||||
}
|
|
||||||
if (result.failureDescription) {
|
|
||||||
lines.push(`Description: ${result.failureDescription}`)
|
|
||||||
}
|
|
||||||
if (result.error) {
|
|
||||||
lines.push(`Error: ${result.error}`)
|
|
||||||
}
|
|
||||||
return lines.join('\n')
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -285,7 +285,7 @@ function evaluateQuantified(
|
|||||||
bodyFn: (item: unknown) => boolean
|
bodyFn: (item: unknown) => boolean
|
||||||
): boolean {
|
): boolean {
|
||||||
if (!Array.isArray(collection)) {
|
if (!Array.isArray(collection)) {
|
||||||
throw new Error(`Quantified expression requires an array collection, got: ${typeof collection}`)
|
return false
|
||||||
}
|
}
|
||||||
if (quantifier === 'for') {
|
if (quantifier === 'for') {
|
||||||
return collection.every(bodyFn)
|
return collection.every(bodyFn)
|
||||||
@@ -299,7 +299,7 @@ async function evaluateQuantifiedAsync(
|
|||||||
bodyFn: (item: unknown) => Promise<boolean>
|
bodyFn: (item: unknown) => Promise<boolean>
|
||||||
): Promise<boolean> {
|
): Promise<boolean> {
|
||||||
if (!Array.isArray(collection)) {
|
if (!Array.isArray(collection)) {
|
||||||
throw new Error(`Quantified expression requires an array collection, got: ${typeof collection}`)
|
return false
|
||||||
}
|
}
|
||||||
if (quantifier === 'for') {
|
if (quantifier === 'for') {
|
||||||
for (const item of collection) {
|
for (const item of collection) {
|
||||||
|
|||||||
+20
-1
@@ -13,14 +13,33 @@ export default fp(apophisPlugin, {
|
|||||||
|
|
||||||
export * from './types.js'
|
export * from './types.js'
|
||||||
|
|
||||||
// Chaos-v3: Pure chaos application for property-based testing
|
// Quality engines
|
||||||
export {
|
export {
|
||||||
applyChaosToExecution,
|
applyChaosToExecution,
|
||||||
applyChaosToAllResponses,
|
applyChaosToAllResponses,
|
||||||
createChaosEventArbitrary,
|
createChaosEventArbitrary,
|
||||||
extractDelays,
|
extractDelays,
|
||||||
sleep,
|
sleep,
|
||||||
|
hasAppliedChaos,
|
||||||
|
formatChaosEvents,
|
||||||
type ChaosEvent,
|
type ChaosEvent,
|
||||||
type ChaosEventType,
|
type ChaosEventType,
|
||||||
type ChaosApplicationResult,
|
type ChaosApplicationResult,
|
||||||
} from './quality/chaos-v3.js'
|
} from './quality/chaos-v3.js'
|
||||||
|
|
||||||
|
export {
|
||||||
|
FlakeDetector,
|
||||||
|
type FlakeReport,
|
||||||
|
type FlakeRerun,
|
||||||
|
type FlakeOptions,
|
||||||
|
} from './quality/flake.js'
|
||||||
|
|
||||||
|
export {
|
||||||
|
runMutationTesting,
|
||||||
|
testMutation,
|
||||||
|
type Mutation,
|
||||||
|
type MutationType,
|
||||||
|
type MutationResult,
|
||||||
|
type MutationReport,
|
||||||
|
type MutationConfig,
|
||||||
|
} from './quality/mutation.js'
|
||||||
|
|||||||
@@ -29,7 +29,6 @@ export interface OutboundMockRuntime {
|
|||||||
interface OutboundMockOptions {
|
interface OutboundMockOptions {
|
||||||
readonly contracts: ResolvedOutboundContract[]
|
readonly contracts: ResolvedOutboundContract[]
|
||||||
readonly mode: 'example' | 'property'
|
readonly mode: 'example' | 'property'
|
||||||
readonly generationProfile?: 'quick' | 'standard' | 'thorough'
|
|
||||||
readonly overrides?: Record<string, {
|
readonly overrides?: Record<string, {
|
||||||
readonly forceStatus?: number
|
readonly forceStatus?: number
|
||||||
readonly headers?: Record<string, string>
|
readonly headers?: Record<string, string>
|
||||||
@@ -77,7 +76,7 @@ export function createOutboundMockRuntime(opts: OutboundMockOptions): OutboundMo
|
|||||||
const schema = contract.response[statusCode]
|
const schema = contract.response[statusCode]
|
||||||
if (!schema) return null
|
if (!schema) return null
|
||||||
// Generate base response from schema
|
// Generate base response from schema
|
||||||
const arb = convertSchema(schema, { context: 'response', generationProfile: opts.generationProfile })
|
const arb = convertSchema(schema, { context: 'response' })
|
||||||
const samples = fc.sample(arb, { numRuns: 1, seed: opts.seed + calls.length })
|
const samples = fc.sample(arb, { numRuns: 1, seed: opts.seed + calls.length })
|
||||||
let body = samples[0] ?? null
|
let body = samples[0] ?? null
|
||||||
if (typeof body !== 'object' || body === null) return body
|
if (typeof body !== 'object' || body === null) return body
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ import { parse } from '../formula/parser.js'
|
|||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
const normalizeTestConfig = (opts: TestConfig = {}): TestConfig => ({
|
const normalizeTestConfig = (opts: TestConfig = {}): TestConfig => ({
|
||||||
depth: opts.depth ?? 'standard',
|
runs: opts.runs,
|
||||||
scope: opts.scope,
|
scope: opts.scope,
|
||||||
seed: opts.seed,
|
seed: opts.seed,
|
||||||
timeout: opts.timeout,
|
timeout: opts.timeout,
|
||||||
|
|||||||
+1
-5
@@ -13,7 +13,7 @@ import { registerValidationHooks, storeRouteContract } from '../infrastructure/h
|
|||||||
import { extractContract } from '../domain/contract.js'
|
import { extractContract } from '../domain/contract.js'
|
||||||
import { createExtensionRegistry } from '../extension/registry.js'
|
import { createExtensionRegistry } from '../extension/registry.js'
|
||||||
import type { ApophisExtension } from '../extension/types.js'
|
import type { ApophisExtension } from '../extension/types.js'
|
||||||
import { createPluginContractRegistry, BUILTIN_PLUGIN_CONTRACTS } from '../domain/plugin-contracts.js'
|
import { createPluginContractRegistry } from '../domain/plugin-contracts.js'
|
||||||
import type { PluginContractRegistry } from '../domain/plugin-contracts.js'
|
import type { PluginContractRegistry } from '../domain/plugin-contracts.js'
|
||||||
import { OutboundContractRegistry } from '../domain/outbound-contracts.js'
|
import { OutboundContractRegistry } from '../domain/outbound-contracts.js'
|
||||||
import { createOutboundMockRuntime, type OutboundMockRuntime } from '../infrastructure/outbound-mock-runtime.js'
|
import { createOutboundMockRuntime, type OutboundMockRuntime } from '../infrastructure/outbound-mock-runtime.js'
|
||||||
@@ -75,10 +75,6 @@ export const apophisPlugin = async (fastify: FastifyInstance, opts: ApophisOptio
|
|||||||
// Initialize scope registry with explicit config or empty
|
// Initialize scope registry with explicit config or empty
|
||||||
const scope = new ScopeRegistry(opts.scopes ?? {})
|
const scope = new ScopeRegistry(opts.scopes ?? {})
|
||||||
const cleanupManager = new CleanupManager(fastify, scope, opts.cleanup ?? false)
|
const cleanupManager = new CleanupManager(fastify, scope, opts.cleanup ?? false)
|
||||||
// Register built-in plugin contracts
|
|
||||||
for (const [name, spec] of Object.entries(BUILTIN_PLUGIN_CONTRACTS)) {
|
|
||||||
pluginContractRegistry.register(name, spec)
|
|
||||||
}
|
|
||||||
// Register user-provided plugin contracts
|
// Register user-provided plugin contracts
|
||||||
if (opts.pluginContracts) {
|
if (opts.pluginContracts) {
|
||||||
for (const [name, spec] of Object.entries(opts.pluginContracts)) {
|
for (const [name, spec] of Object.entries(opts.pluginContracts)) {
|
||||||
|
|||||||
@@ -80,7 +80,6 @@ export function oauth21ProfilePack(opts: PackOptions = {}): Partial<Config> {
|
|||||||
presets: {
|
presets: {
|
||||||
'protocol-lab': {
|
'protocol-lab': {
|
||||||
name: 'protocol-lab',
|
name: 'protocol-lab',
|
||||||
depth: 'deep',
|
|
||||||
timeout: opts.timeout ?? 15000,
|
timeout: opts.timeout ?? 15000,
|
||||||
parallel: false,
|
parallel: false,
|
||||||
chaos: true,
|
chaos: true,
|
||||||
@@ -88,7 +87,6 @@ export function oauth21ProfilePack(opts: PackOptions = {}): Partial<Config> {
|
|||||||
},
|
},
|
||||||
'safe-ci': {
|
'safe-ci': {
|
||||||
name: 'safe-ci',
|
name: 'safe-ci',
|
||||||
depth: 'quick',
|
|
||||||
timeout: opts.timeout ?? 5000,
|
timeout: opts.timeout ?? 5000,
|
||||||
parallel: false,
|
parallel: false,
|
||||||
chaos: false,
|
chaos: false,
|
||||||
@@ -118,7 +116,6 @@ export function rfc8628DeviceAuthorizationPack(opts: PackOptions = {}): Partial<
|
|||||||
presets: {
|
presets: {
|
||||||
'protocol-lab': {
|
'protocol-lab': {
|
||||||
name: 'protocol-lab',
|
name: 'protocol-lab',
|
||||||
depth: 'deep',
|
|
||||||
timeout: opts.timeout ?? 20000,
|
timeout: opts.timeout ?? 20000,
|
||||||
parallel: false,
|
parallel: false,
|
||||||
chaos: true,
|
chaos: true,
|
||||||
@@ -148,7 +145,6 @@ export function rfc8693TokenExchangePack(opts: PackOptions = {}): Partial<Config
|
|||||||
presets: {
|
presets: {
|
||||||
'protocol-lab': {
|
'protocol-lab': {
|
||||||
name: 'protocol-lab',
|
name: 'protocol-lab',
|
||||||
depth: 'deep',
|
|
||||||
timeout: opts.timeout ?? 15000,
|
timeout: opts.timeout ?? 15000,
|
||||||
parallel: false,
|
parallel: false,
|
||||||
chaos: true,
|
chaos: true,
|
||||||
|
|||||||
+20
-14
@@ -7,7 +7,7 @@
|
|||||||
* If the test suite passes, the mutation "survives" — indicating a gap in coverage.
|
* If the test suite passes, the mutation "survives" — indicating a gap in coverage.
|
||||||
*
|
*
|
||||||
* Usage:
|
* Usage:
|
||||||
* const report = await runMutationTesting(fastify, { depth: 'quick' })
|
* const report = await runMutationTesting(fastify, { runs: 10 })
|
||||||
* console.log(`Mutation score: ${report.score}%`)
|
* console.log(`Mutation score: ${report.score}%`)
|
||||||
*/
|
*/
|
||||||
import type { FastifyInstance } from 'fastify'
|
import type { FastifyInstance } from 'fastify'
|
||||||
@@ -44,7 +44,7 @@ export interface MutationReport {
|
|||||||
readonly weakContracts: string[] // contracts that survived all mutations
|
readonly weakContracts: string[] // contracts that survived all mutations
|
||||||
}
|
}
|
||||||
export interface MutationConfig {
|
export interface MutationConfig {
|
||||||
readonly depth?: TestConfig['depth']
|
readonly runs?: number
|
||||||
readonly seed?: number
|
readonly seed?: number
|
||||||
/** Max mutations per contract (default: 5) */
|
/** Max mutations per contract (default: 5) */
|
||||||
readonly maxMutationsPerContract?: number
|
readonly maxMutationsPerContract?: number
|
||||||
@@ -214,7 +214,7 @@ export async function runMutationTesting(
|
|||||||
const suite = await runPetitTestsWithMutation(
|
const suite = await runPetitTestsWithMutation(
|
||||||
fastify as unknown as FastifyInjectInstance,
|
fastify as unknown as FastifyInjectInstance,
|
||||||
{
|
{
|
||||||
depth: config.depth ?? 'quick',
|
runs: config.runs ?? 10,
|
||||||
seed: config.seed,
|
seed: config.seed,
|
||||||
},
|
},
|
||||||
mutatedContract
|
mutatedContract
|
||||||
@@ -256,20 +256,26 @@ export async function runMutationTesting(
|
|||||||
}
|
}
|
||||||
/**
|
/**
|
||||||
* Run petit tests with a mutated contract.
|
* Run petit tests with a mutated contract.
|
||||||
* This is a simplified version that tests a single mutated contract.
|
* Injects the mutated contract so the runner uses it instead of discovering from Fastify.
|
||||||
*/
|
*/
|
||||||
async function runPetitTestsWithMutation(
|
async function runPetitTestsWithMutation(
|
||||||
fastify: FastifyInjectInstance,
|
fastify: FastifyInjectInstance,
|
||||||
config: { depth?: TestConfig['depth']; seed?: number },
|
config: { runs?: number; seed?: number },
|
||||||
mutatedContract: RouteContract
|
mutatedContract: RouteContract
|
||||||
): Promise<TestSuite> {
|
): Promise<TestSuite> {
|
||||||
// For now, run the full suite - the mutated contract will be discovered
|
return runPetitTests(
|
||||||
// In a real implementation, you'd inject the mutated contract into the discovery
|
fastify,
|
||||||
return runPetitTests(fastify, {
|
{
|
||||||
depth: config.depth ?? 'quick',
|
runs: config.runs ?? 10,
|
||||||
seed: config.seed,
|
seed: config.seed,
|
||||||
routes: [`${mutatedContract.method} ${mutatedContract.path}`],
|
routes: [`${mutatedContract.method} ${mutatedContract.path}`],
|
||||||
})
|
},
|
||||||
|
undefined,
|
||||||
|
undefined,
|
||||||
|
undefined,
|
||||||
|
undefined,
|
||||||
|
[mutatedContract]
|
||||||
|
)
|
||||||
}
|
}
|
||||||
/**
|
/**
|
||||||
* Quick mutation test for a single contract formula.
|
* Quick mutation test for a single contract formula.
|
||||||
@@ -279,14 +285,14 @@ export async function testMutation(
|
|||||||
fastify: FastifyInstance,
|
fastify: FastifyInstance,
|
||||||
contract: RouteContract,
|
contract: RouteContract,
|
||||||
mutation: Mutation,
|
mutation: Mutation,
|
||||||
config: Pick<MutationConfig, 'depth' | 'seed'> = {}
|
config: Pick<MutationConfig, 'runs' | 'seed'> = {}
|
||||||
): Promise<boolean> {
|
): Promise<boolean> {
|
||||||
const mutatedContract = applyMutation(contract, mutation)
|
const mutatedContract = applyMutation(contract, mutation)
|
||||||
try {
|
try {
|
||||||
const suite = await runPetitTestsWithMutation(
|
const suite = await runPetitTestsWithMutation(
|
||||||
fastify as unknown as FastifyInjectInstance,
|
fastify as unknown as FastifyInjectInstance,
|
||||||
{
|
{
|
||||||
depth: config.depth ?? 'quick',
|
runs: config.runs ?? 10,
|
||||||
seed: config.seed,
|
seed: config.seed,
|
||||||
},
|
},
|
||||||
mutatedContract
|
mutatedContract
|
||||||
|
|||||||
@@ -57,7 +57,7 @@ async function expectLoadConfigError(
|
|||||||
test('schema: accepts minimal valid configs', () => {
|
test('schema: accepts minimal valid configs', () => {
|
||||||
validateConfigAgainstSchema({}, CONFIG_SCHEMA);
|
validateConfigAgainstSchema({}, CONFIG_SCHEMA);
|
||||||
validateConfigAgainstSchema({ mode: 'verify', routes: ['GET /users'], seed: 42 }, CONFIG_SCHEMA);
|
validateConfigAgainstSchema({ mode: 'verify', routes: ['GET /users'], seed: 42 }, CONFIG_SCHEMA);
|
||||||
validateConfigAgainstSchema({ presets: { quick: { depth: 'quick', timeout: 5000 } } }, CONFIG_SCHEMA);
|
validateConfigAgainstSchema({ presets: { quick: { runs: 10, timeout: 5000 } } }, CONFIG_SCHEMA);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('schema: rejects unknown keys with guidance', () => {
|
test('schema: rejects unknown keys with guidance', () => {
|
||||||
@@ -131,11 +131,7 @@ test('schema: rejects enum and numeric range violations with clear guidance', ()
|
|||||||
path: 'profiles.default.mode',
|
path: 'profiles.default.mode',
|
||||||
expectedGuidance: ['verify', 'observe', 'qualify'],
|
expectedGuidance: ['verify', 'observe', 'qualify'],
|
||||||
},
|
},
|
||||||
{
|
|
||||||
value: { presets: { quick: { depth: 'super-deep' } } },
|
|
||||||
path: 'presets.quick.depth',
|
|
||||||
expectedGuidance: ['quick', 'standard', 'deep'],
|
|
||||||
},
|
|
||||||
] as const;
|
] as const;
|
||||||
|
|
||||||
for (const c of enumCases) {
|
for (const c of enumCases) {
|
||||||
@@ -215,9 +211,9 @@ test('semantic: validates cross-reference and value rules', () => {
|
|||||||
{ value: { seed: 3.14 }, path: 'seed', guidance: 'integer' },
|
{ value: { seed: 3.14 }, path: 'seed', guidance: 'integer' },
|
||||||
{ value: { presets: { quick: { timeout: -100 } } }, path: 'presets.quick.timeout', guidance: 'non-negative' },
|
{ value: { presets: { quick: { timeout: -100 } } }, path: 'presets.quick.timeout', guidance: 'non-negative' },
|
||||||
{
|
{
|
||||||
value: { presets: { quick: { depth: 'super-deep' } } },
|
value: { presets: { quick: { timeout: -100 } } },
|
||||||
path: 'presets.quick.depth',
|
path: 'presets.quick.timeout',
|
||||||
guidance: 'quick, standard, deep',
|
guidance: 'non-negative',
|
||||||
},
|
},
|
||||||
] as const;
|
] as const;
|
||||||
|
|
||||||
@@ -232,7 +228,7 @@ test('semantic: validates cross-reference and value rules', () => {
|
|||||||
{ routes: ['GET /users', 'POST /items'] },
|
{ routes: ['GET /users', 'POST /items'] },
|
||||||
{ seed: -42 },
|
{ seed: -42 },
|
||||||
{ seed: 0 },
|
{ seed: 0 },
|
||||||
{ presets: { quick: { timeout: 0, depth: 'standard' } } },
|
{ presets: { quick: { timeout: 0 } } },
|
||||||
];
|
];
|
||||||
|
|
||||||
for (const value of acceptCases) {
|
for (const value of acceptCases) {
|
||||||
@@ -322,7 +318,7 @@ test('loadConfig: resolves profile and preset and applies profile overrides', as
|
|||||||
},
|
},
|
||||||
presets: {
|
presets: {
|
||||||
quick: {
|
quick: {
|
||||||
depth: 'quick',
|
runs: 10,
|
||||||
timeout: 5000,
|
timeout: 5000,
|
||||||
parallel: false,
|
parallel: false,
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -23,12 +23,10 @@
|
|||||||
* - Property and state model-based testing focused on confidence
|
* - Property and state model-based testing focused on confidence
|
||||||
* - Iterative small steps with rapid feedback loops
|
* - Iterative small steps with rapid feedback loops
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { test } from 'node:test';
|
import { test } from 'node:test';
|
||||||
import assert from 'node:assert';
|
import assert from 'node:assert';
|
||||||
import { writeFileSync, readFileSync } from 'node:fs';
|
import { writeFileSync, readFileSync } from 'node:fs';
|
||||||
import { resolve } from 'node:path';
|
import { resolve } from 'node:path';
|
||||||
|
|
||||||
import {
|
import {
|
||||||
migrateCommand,
|
migrateCommand,
|
||||||
detectAllLegacyPatterns,
|
detectAllLegacyPatterns,
|
||||||
@@ -36,31 +34,25 @@ import {
|
|||||||
type MigrateOptions,
|
type MigrateOptions,
|
||||||
type MigrationItem,
|
type MigrationItem,
|
||||||
} from '../../cli/commands/migrate/index.js';
|
} from '../../cli/commands/migrate/index.js';
|
||||||
|
|
||||||
import {
|
import {
|
||||||
rewriteConfigFile,
|
rewriteConfigFile,
|
||||||
detectLegacyConfigFields,
|
detectLegacyConfigFields,
|
||||||
detectLegacyFieldsNoEquivalent,
|
detectLegacyFieldsNoEquivalent,
|
||||||
detectMixedLegacyModernFields,
|
detectMixedLegacyModernFields,
|
||||||
} from '../../cli/commands/migrate/rewriters/config-rewriter.js';
|
} from '../../cli/commands/migrate/rewriters/config-rewriter.js';
|
||||||
|
|
||||||
import {
|
import {
|
||||||
rewriteRouteAnnotations,
|
rewriteRouteAnnotations,
|
||||||
detectLegacyRouteAnnotations,
|
detectLegacyRouteAnnotations,
|
||||||
detectAmbiguousRoutePatterns,
|
detectAmbiguousRoutePatterns,
|
||||||
} from '../../cli/commands/migrate/rewriters/route-rewriter.js';
|
} from '../../cli/commands/migrate/rewriters/route-rewriter.js';
|
||||||
|
|
||||||
import {
|
import {
|
||||||
rewriteCodePatterns,
|
rewriteCodePatterns,
|
||||||
detectLegacyCodePatterns,
|
detectLegacyCodePatterns,
|
||||||
detectAmbiguousCodePatterns,
|
detectAmbiguousCodePatterns,
|
||||||
} from '../../cli/commands/migrate/rewriters/code-rewriter.js';
|
} from '../../cli/commands/migrate/rewriters/code-rewriter.js';
|
||||||
|
|
||||||
import { createTempDir, cleanup, makeCtx } from './helpers.js';
|
import { createTempDir, cleanup, makeCtx } from './helpers.js';
|
||||||
|
|
||||||
test('migrate --check detects broad legacy config field set', async () => {
|
test('migrate --check detects broad legacy config field set', async () => {
|
||||||
const dir = createTempDir();
|
const dir = createTempDir();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const legacyConfig = `export default {
|
const legacyConfig = `export default {
|
||||||
testMode: "verify",
|
testMode: "verify",
|
||||||
@@ -82,12 +74,9 @@ test('migrate --check detects broad legacy config field set', async () => {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
};`;
|
};`;
|
||||||
|
|
||||||
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
|
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
|
||||||
|
|
||||||
const ctx = makeCtx({ cwd: dir });
|
const ctx = makeCtx({ cwd: dir });
|
||||||
const result = await migrateCommand({ check: true }, ctx);
|
const result = await migrateCommand({ check: true }, ctx);
|
||||||
|
|
||||||
assert.strictEqual(result.exitCode, 1, 'Should exit 1 when legacy patterns are found');
|
assert.strictEqual(result.exitCode, 1, 'Should exit 1 when legacy patterns are found');
|
||||||
const legacyNames = result.items.map((item) => item.legacy);
|
const legacyNames = result.items.map((item) => item.legacy);
|
||||||
assert.ok(legacyNames.includes('testMode'), 'Should detect testMode');
|
assert.ok(legacyNames.includes('testMode'), 'Should detect testMode');
|
||||||
@@ -103,29 +92,23 @@ test('migrate --check detects broad legacy config field set', async () => {
|
|||||||
cleanup(dir);
|
cleanup(dir);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Test 1: Mixed legacy and modern config detection
|
// Test 1: Mixed legacy and modern config detection
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
test('migrate detects mixed legacy and modern config fields', async () => {
|
test('migrate detects mixed legacy and modern config fields', async () => {
|
||||||
const dir = createTempDir();
|
const dir = createTempDir();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Config with both legacy and modern fields present
|
// Config with both legacy and modern fields present
|
||||||
const mixedConfig = `export default {
|
const mixedConfig = `export default {
|
||||||
// Legacy field
|
// Legacy field
|
||||||
testMode: "verify",
|
testMode: "verify",
|
||||||
|
|
||||||
// Modern field (conflicts with legacy)
|
// Modern field (conflicts with legacy)
|
||||||
mode: "observe",
|
mode: "observe",
|
||||||
|
|
||||||
profiles: {
|
profiles: {
|
||||||
quick: {
|
quick: {
|
||||||
preset: "safe-ci",
|
preset: "safe-ci",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|
||||||
// Legacy container
|
// Legacy container
|
||||||
testProfiles: {
|
testProfiles: {
|
||||||
old: {
|
old: {
|
||||||
@@ -133,22 +116,17 @@ test('migrate detects mixed legacy and modern config fields', async () => {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
};`;
|
};`;
|
||||||
|
|
||||||
writeFileSync(resolve(dir, 'apophis.config.js'), mixedConfig);
|
writeFileSync(resolve(dir, 'apophis.config.js'), mixedConfig);
|
||||||
|
|
||||||
const ctx = makeCtx({ cwd: dir });
|
const ctx = makeCtx({ cwd: dir });
|
||||||
const result = await migrateCommand({ check: true }, ctx);
|
const result = await migrateCommand({ check: true }, ctx);
|
||||||
|
|
||||||
// Should detect legacy patterns
|
// Should detect legacy patterns
|
||||||
assert.strictEqual(result.exitCode, 1, 'Should exit 1 when legacy patterns found');
|
assert.strictEqual(result.exitCode, 1, 'Should exit 1 when legacy patterns found');
|
||||||
assert.ok(result.items.length > 0, 'Should detect legacy items');
|
assert.ok(result.items.length > 0, 'Should detect legacy items');
|
||||||
|
|
||||||
// Check that mixed fields are reported
|
// Check that mixed fields are reported
|
||||||
const legacyNames = result.items.map((item) => item.legacy);
|
const legacyNames = result.items.map((item) => item.legacy);
|
||||||
assert.ok(legacyNames.includes('testMode'), 'Should detect testMode');
|
assert.ok(legacyNames.includes('testMode'), 'Should detect testMode');
|
||||||
assert.ok(legacyNames.includes('testProfiles'), 'Should detect testProfiles');
|
assert.ok(legacyNames.includes('testProfiles'), 'Should detect testProfiles');
|
||||||
assert.ok(legacyNames.includes('usesPreset'), 'Should detect usesPreset');
|
assert.ok(legacyNames.includes('usesPreset'), 'Should detect usesPreset');
|
||||||
|
|
||||||
// Verify guidance mentions the conflict
|
// Verify guidance mentions the conflict
|
||||||
const testModeItem = result.items.find((item) => item.legacy === 'testMode');
|
const testModeItem = result.items.find((item) => item.legacy === 'testMode');
|
||||||
assert.ok(testModeItem, 'Should have testMode item');
|
assert.ok(testModeItem, 'Should have testMode item');
|
||||||
@@ -157,19 +135,15 @@ test('migrate detects mixed legacy and modern config fields', async () => {
|
|||||||
cleanup(dir);
|
cleanup(dir);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Test 2: Dry-run shows exact rewrites
|
// Test 2: Dry-run shows exact rewrites
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
test('migrate dry-run shows exact file path, line number, legacy text, replacement text', async () => {
|
test('migrate dry-run shows exact file path, line number, legacy text, replacement text', async () => {
|
||||||
const dir = createTempDir();
|
const dir = createTempDir();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const legacyConfig = `export default {
|
const legacyConfig = `export default {
|
||||||
// Line 2
|
// Line 2
|
||||||
testMode: "verify",
|
testMode: "verify",
|
||||||
|
|
||||||
profiles: {
|
profiles: {
|
||||||
quick: {
|
quick: {
|
||||||
// Line 7
|
// Line 7
|
||||||
@@ -177,36 +151,27 @@ test('migrate dry-run shows exact file path, line number, legacy text, replaceme
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
};`;
|
};`;
|
||||||
|
|
||||||
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
|
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
|
||||||
|
|
||||||
const ctx = makeCtx({ cwd: dir });
|
const ctx = makeCtx({ cwd: dir });
|
||||||
const result = await migrateCommand({ dryRun: true }, ctx);
|
const result = await migrateCommand({ dryRun: true }, ctx);
|
||||||
|
|
||||||
assert.strictEqual(result.exitCode, 1, 'Should exit 1 when legacy patterns found');
|
assert.strictEqual(result.exitCode, 1, 'Should exit 1 when legacy patterns found');
|
||||||
assert.ok(result.message, 'Should have output message');
|
assert.ok(result.message, 'Should have output message');
|
||||||
|
|
||||||
// Verify dry-run output contains exact details
|
// Verify dry-run output contains exact details
|
||||||
assert.ok(result.message.includes('Dry run'), 'Should indicate dry run');
|
assert.ok(result.message.includes('Dry run'), 'Should indicate dry run');
|
||||||
assert.ok(result.message.includes('testMode'), 'Should show legacy text');
|
assert.ok(result.message.includes('testMode'), 'Should show legacy text');
|
||||||
assert.ok(result.message.includes('mode'), 'Should show replacement text');
|
assert.ok(result.message.includes('mode'), 'Should show replacement text');
|
||||||
assert.ok(result.message.includes('usesPreset'), 'Should show usesPreset');
|
assert.ok(result.message.includes('usesPreset'), 'Should show usesPreset');
|
||||||
assert.ok(result.message.includes('preset'), 'Should show preset replacement');
|
assert.ok(result.message.includes('preset'), 'Should show preset replacement');
|
||||||
|
|
||||||
// Verify file path is shown
|
// Verify file path is shown
|
||||||
assert.ok(result.message.includes('apophis.config.js'), 'Should show file path');
|
assert.ok(result.message.includes('apophis.config.js'), 'Should show file path');
|
||||||
|
|
||||||
// Verify line numbers are shown
|
// Verify line numbers are shown
|
||||||
assert.ok(result.message.includes(':2') || result.message.includes(': 2'), 'Should show line number');
|
assert.ok(result.message.includes(':2') || result.message.includes(': 2'), 'Should show line number');
|
||||||
|
|
||||||
// Verify total count
|
// Verify total count
|
||||||
assert.ok(result.message.includes('Total:'), 'Should show total count');
|
assert.ok(result.message.includes('Total:'), 'Should show total count');
|
||||||
assert.ok(result.message.includes('3'), 'Should show correct total (3 items)');
|
assert.ok(result.message.includes('3'), 'Should show correct total (3 items)');
|
||||||
|
|
||||||
// Verify files would be modified
|
// Verify files would be modified
|
||||||
assert.ok(result.filesWouldBeModified, 'Should list files that would be modified');
|
assert.ok(result.filesWouldBeModified, 'Should list files that would be modified');
|
||||||
assert.strictEqual(result.filesWouldBeModified.length, 1, 'Should show 1 file would be modified');
|
assert.strictEqual(result.filesWouldBeModified.length, 1, 'Should show 1 file would be modified');
|
||||||
|
|
||||||
// Verify file was NOT modified
|
// Verify file was NOT modified
|
||||||
const content = readFileSync(resolve(dir, 'apophis.config.js'), 'utf-8');
|
const content = readFileSync(resolve(dir, 'apophis.config.js'), 'utf-8');
|
||||||
assert.ok(content.includes('testMode'), 'File should still have testMode');
|
assert.ok(content.includes('testMode'), 'File should still have testMode');
|
||||||
@@ -215,14 +180,11 @@ test('migrate dry-run shows exact file path, line number, legacy text, replaceme
|
|||||||
cleanup(dir);
|
cleanup(dir);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Test 3: Write performs rewrites correctly
|
// Test 3: Write performs rewrites correctly
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
test('migrate write performs rewrites correctly', async () => {
|
test('migrate write performs rewrites correctly', async () => {
|
||||||
const dir = createTempDir();
|
const dir = createTempDir();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const legacyConfig = `export default {
|
const legacyConfig = `export default {
|
||||||
testMode: "verify",
|
testMode: "verify",
|
||||||
@@ -232,16 +194,12 @@ test('migrate write performs rewrites correctly', async () => {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
};`;
|
};`;
|
||||||
|
|
||||||
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
|
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
|
||||||
|
|
||||||
const ctx = makeCtx({ cwd: dir });
|
const ctx = makeCtx({ cwd: dir });
|
||||||
const result = await migrateCommand({ write: true }, ctx);
|
const result = await migrateCommand({ write: true }, ctx);
|
||||||
|
|
||||||
assert.strictEqual(result.exitCode, 1, 'Should exit 1 when rewrites performed');
|
assert.strictEqual(result.exitCode, 1, 'Should exit 1 when rewrites performed');
|
||||||
assert.ok(result.completed.length > 0, 'Should have completed items');
|
assert.ok(result.completed.length > 0, 'Should have completed items');
|
||||||
assert.ok(result.filesModified && result.filesModified.length > 0, 'Should list modified files');
|
assert.ok(result.filesModified && result.filesModified.length > 0, 'Should list modified files');
|
||||||
|
|
||||||
// Verify file WAS modified
|
// Verify file WAS modified
|
||||||
const content = readFileSync(resolve(dir, 'apophis.config.js'), 'utf-8');
|
const content = readFileSync(resolve(dir, 'apophis.config.js'), 'utf-8');
|
||||||
assert.ok(!content.includes('testMode'), 'File should not have testMode');
|
assert.ok(!content.includes('testMode'), 'File should not have testMode');
|
||||||
@@ -254,35 +212,26 @@ test('migrate write performs rewrites correctly', async () => {
|
|||||||
cleanup(dir);
|
cleanup(dir);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Test 4: Ambiguous rewrite stops and shows context
|
// Test 4: Ambiguous rewrite stops and shows context
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
test('migrate ambiguous rewrite stops and shows surrounding context', async () => {
|
test('migrate ambiguous rewrite stops and shows surrounding context', async () => {
|
||||||
const dir = createTempDir();
|
const dir = createTempDir();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Create a file with an ambiguous code pattern
|
// Create a file with an ambiguous code pattern
|
||||||
const ambiguousCode = `import Fastify from 'fastify';
|
const ambiguousCode = `import Fastify from 'fastify';
|
||||||
const app = Fastify();
|
const app = Fastify();
|
||||||
|
|
||||||
// This is ambiguous: what does oldApi() mean here?
|
// This is ambiguous: what does oldApi() mean here?
|
||||||
app.register(oldApi());
|
app.register(oldApi());
|
||||||
|
|
||||||
export default app;`;
|
export default app;`;
|
||||||
|
|
||||||
writeFileSync(resolve(dir, 'app.js'), ambiguousCode);
|
writeFileSync(resolve(dir, 'app.js'), ambiguousCode);
|
||||||
|
|
||||||
// Also create a config file so migration has something to work with
|
// Also create a config file so migration has something to work with
|
||||||
const config = `export default {
|
const config = `export default {
|
||||||
mode: "verify",
|
mode: "verify",
|
||||||
};`;
|
};`;
|
||||||
writeFileSync(resolve(dir, 'apophis.config.js'), config);
|
writeFileSync(resolve(dir, 'apophis.config.js'), config);
|
||||||
|
|
||||||
const ctx = makeCtx({ cwd: dir });
|
const ctx = makeCtx({ cwd: dir });
|
||||||
const result = await migrateCommand({ write: true }, ctx);
|
const result = await migrateCommand({ write: true }, ctx);
|
||||||
|
|
||||||
// Should stop with exit code 2 (USAGE_ERROR) because ambiguous patterns found
|
// Should stop with exit code 2 (USAGE_ERROR) because ambiguous patterns found
|
||||||
assert.strictEqual(result.exitCode, 2, 'Should exit 2 when ambiguous patterns found in write mode');
|
assert.strictEqual(result.exitCode, 2, 'Should exit 2 when ambiguous patterns found in write mode');
|
||||||
assert.ok(result.remaining.length > 0, 'Should have remaining items');
|
assert.ok(result.remaining.length > 0, 'Should have remaining items');
|
||||||
@@ -290,21 +239,17 @@ export default app;`;
|
|||||||
assert.ok(result.message.includes('Ambiguous'), 'Should mention ambiguous patterns');
|
assert.ok(result.message.includes('Ambiguous'), 'Should mention ambiguous patterns');
|
||||||
assert.ok(result.message.includes('oldApi()'), 'Should show the ambiguous pattern');
|
assert.ok(result.message.includes('oldApi()'), 'Should show the ambiguous pattern');
|
||||||
assert.ok(result.message.includes('manual choice'), 'Should mention manual choice');
|
assert.ok(result.message.includes('manual choice'), 'Should mention manual choice');
|
||||||
|
|
||||||
// Verify context is shown (surrounding lines)
|
// Verify context is shown (surrounding lines)
|
||||||
assert.ok(result.message.includes('app.register'), 'Should show surrounding context');
|
assert.ok(result.message.includes('app.register'), 'Should show surrounding context');
|
||||||
} finally {
|
} finally {
|
||||||
cleanup(dir);
|
cleanup(dir);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Test 5: Legacy field with no equivalent emits guidance
|
// Test 5: Legacy field with no equivalent emits guidance
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
test('migrate legacy field with no direct equivalent emits human guidance', async () => {
|
test('migrate legacy field with no direct equivalent emits human guidance', async () => {
|
||||||
const dir = createTempDir();
|
const dir = createTempDir();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Config with a legacy field that has no direct equivalent
|
// Config with a legacy field that has no direct equivalent
|
||||||
const legacyConfig = `export default {
|
const legacyConfig = `export default {
|
||||||
@@ -317,16 +262,12 @@ test('migrate legacy field with no direct equivalent emits human guidance', asyn
|
|||||||
// This field is deprecated with no direct equivalent
|
// This field is deprecated with no direct equivalent
|
||||||
legacyField: true,
|
legacyField: true,
|
||||||
};`;
|
};`;
|
||||||
|
|
||||||
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
|
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
|
||||||
|
|
||||||
const ctx = makeCtx({ cwd: dir });
|
const ctx = makeCtx({ cwd: dir });
|
||||||
const result = await migrateCommand({ check: true }, ctx);
|
const result = await migrateCommand({ check: true }, ctx);
|
||||||
|
|
||||||
// Should detect the legacy field with no equivalent
|
// Should detect the legacy field with no equivalent
|
||||||
assert.strictEqual(result.exitCode, 1, 'Should exit 1 when legacy patterns found');
|
assert.strictEqual(result.exitCode, 1, 'Should exit 1 when legacy patterns found');
|
||||||
assert.ok(result.items.length > 0, 'Should detect legacy items');
|
assert.ok(result.items.length > 0, 'Should detect legacy items');
|
||||||
|
|
||||||
const legacyFieldItem = result.items.find((item) => item.legacy === 'legacyField');
|
const legacyFieldItem = result.items.find((item) => item.legacy === 'legacyField');
|
||||||
assert.ok(legacyFieldItem, 'Should detect legacyField');
|
assert.ok(legacyFieldItem, 'Should detect legacyField');
|
||||||
assert.ok(legacyFieldItem.guidance, 'Should have guidance for legacyField');
|
assert.ok(legacyFieldItem.guidance, 'Should have guidance for legacyField');
|
||||||
@@ -343,14 +284,11 @@ test('migrate legacy field with no direct equivalent emits human guidance', asyn
|
|||||||
cleanup(dir);
|
cleanup(dir);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Test 6: Partial migration reports completed and remaining
|
// Test 6: Partial migration reports completed and remaining
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
test('migrate partial migration reports completed and remaining items', async () => {
|
test('migrate partial migration reports completed and remaining items', async () => {
|
||||||
const dir = createTempDir();
|
const dir = createTempDir();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const legacyConfig = `export default {
|
const legacyConfig = `export default {
|
||||||
testMode: "verify",
|
testMode: "verify",
|
||||||
@@ -360,12 +298,9 @@ test('migrate partial migration reports completed and remaining items', async ()
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
};`;
|
};`;
|
||||||
|
|
||||||
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
|
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
|
||||||
|
|
||||||
const ctx = makeCtx({ cwd: dir });
|
const ctx = makeCtx({ cwd: dir });
|
||||||
const result = await migrateCommand({ write: true }, ctx);
|
const result = await migrateCommand({ write: true }, ctx);
|
||||||
|
|
||||||
assert.ok(result.completed.length > 0, 'Should have completed items');
|
assert.ok(result.completed.length > 0, 'Should have completed items');
|
||||||
assert.ok(result.message, 'Should have output message');
|
assert.ok(result.message, 'Should have output message');
|
||||||
assert.ok(result.message.includes('Completed'), 'Should mention completed');
|
assert.ok(result.message.includes('Completed'), 'Should mention completed');
|
||||||
@@ -374,20 +309,16 @@ test('migrate partial migration reports completed and remaining items', async ()
|
|||||||
cleanup(dir);
|
cleanup(dir);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Test 7: Preserves comments/formatting where feasible
|
// Test 7: Preserves comments/formatting where feasible
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
test('migrate preserves comments and formatting where feasible', async () => {
|
test('migrate preserves comments and formatting where feasible', async () => {
|
||||||
const dir = createTempDir();
|
const dir = createTempDir();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Config with specific formatting (comments, indentation)
|
// Config with specific formatting (comments, indentation)
|
||||||
const legacyConfig = `export default {
|
const legacyConfig = `export default {
|
||||||
// This is a comment about testMode
|
// This is a comment about testMode
|
||||||
testMode: "verify",
|
testMode: "verify",
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Block comment about testProfiles
|
* Block comment about testProfiles
|
||||||
*/
|
*/
|
||||||
@@ -398,19 +329,14 @@ test('migrate preserves comments and formatting where feasible', async () => {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
};`;
|
};`;
|
||||||
|
|
||||||
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
|
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
|
||||||
|
|
||||||
const ctx = makeCtx({ cwd: dir });
|
const ctx = makeCtx({ cwd: dir });
|
||||||
const result = await migrateCommand({ write: true }, ctx);
|
const result = await migrateCommand({ write: true }, ctx);
|
||||||
|
|
||||||
const content = readFileSync(resolve(dir, 'apophis.config.js'), 'utf-8');
|
const content = readFileSync(resolve(dir, 'apophis.config.js'), 'utf-8');
|
||||||
|
|
||||||
// Verify comments are preserved
|
// Verify comments are preserved
|
||||||
assert.ok(content.includes('// This is a comment about testMode'), 'Should preserve line comment');
|
assert.ok(content.includes('// This is a comment about testMode'), 'Should preserve line comment');
|
||||||
assert.ok(content.includes('Block comment about testProfiles'), 'Should preserve block comment');
|
assert.ok(content.includes('Block comment about testProfiles'), 'Should preserve block comment');
|
||||||
assert.ok(content.includes('// Inline comment'), 'Should preserve inline comment');
|
assert.ok(content.includes('// Inline comment'), 'Should preserve inline comment');
|
||||||
|
|
||||||
// Verify replacements were made
|
// Verify replacements were made
|
||||||
assert.ok(content.includes('mode:'), 'Should have mode');
|
assert.ok(content.includes('mode:'), 'Should have mode');
|
||||||
assert.ok(content.includes('profiles:'), 'Should have profiles');
|
assert.ok(content.includes('profiles:'), 'Should have profiles');
|
||||||
@@ -419,14 +345,11 @@ test('migrate preserves comments and formatting where feasible', async () => {
|
|||||||
cleanup(dir);
|
cleanup(dir);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Test 8: Migrate exits 0 when config is already modern
|
// Test 8: Migrate exits 0 when config is already modern
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
test('migrate exits 0 when config is already modern', async () => {
|
test('migrate exits 0 when config is already modern', async () => {
|
||||||
const dir = createTempDir();
|
const dir = createTempDir();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const modernConfig = `export default {
|
const modernConfig = `export default {
|
||||||
mode: "verify",
|
mode: "verify",
|
||||||
@@ -438,7 +361,7 @@ test('migrate exits 0 when config is already modern', async () => {
|
|||||||
},
|
},
|
||||||
presets: {
|
presets: {
|
||||||
"safe-ci": {
|
"safe-ci": {
|
||||||
depth: "quick",
|
,
|
||||||
timeout: 5000,
|
timeout: 5000,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -448,12 +371,9 @@ test('migrate exits 0 when config is already modern', async () => {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
};`;
|
};`;
|
||||||
|
|
||||||
writeFileSync(resolve(dir, 'apophis.config.js'), modernConfig);
|
writeFileSync(resolve(dir, 'apophis.config.js'), modernConfig);
|
||||||
|
|
||||||
const ctx = makeCtx({ cwd: dir });
|
const ctx = makeCtx({ cwd: dir });
|
||||||
const result = await migrateCommand({ check: true }, ctx);
|
const result = await migrateCommand({ check: true }, ctx);
|
||||||
|
|
||||||
assert.strictEqual(result.exitCode, 0, 'Should exit 0 for modern config');
|
assert.strictEqual(result.exitCode, 0, 'Should exit 0 for modern config');
|
||||||
assert.strictEqual(result.items.length, 0, 'Should have no items');
|
assert.strictEqual(result.items.length, 0, 'Should have no items');
|
||||||
assert.ok(result.message, 'Should have message');
|
assert.ok(result.message, 'Should have message');
|
||||||
@@ -462,35 +382,25 @@ test('migrate exits 0 when config is already modern', async () => {
|
|||||||
cleanup(dir);
|
cleanup(dir);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Test 9: Migrate exits 2 when ambiguous in write mode
|
// Test 9: Migrate exits 2 when ambiguous in write mode
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
test('migrate exits 2 when ambiguous patterns found in write mode', async () => {
|
test('migrate exits 2 when ambiguous patterns found in write mode', async () => {
|
||||||
const dir = createTempDir();
|
const dir = createTempDir();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const config = `export default {
|
const config = `export default {
|
||||||
mode: "verify",
|
mode: "verify",
|
||||||
};`;
|
};`;
|
||||||
|
|
||||||
writeFileSync(resolve(dir, 'apophis.config.js'), config);
|
writeFileSync(resolve(dir, 'apophis.config.js'), config);
|
||||||
|
|
||||||
// Create app with an ambiguous pattern
|
// Create app with an ambiguous pattern
|
||||||
const code = `import Fastify from 'fastify';
|
const code = `import Fastify from 'fastify';
|
||||||
const app = Fastify();
|
const app = Fastify();
|
||||||
|
|
||||||
// Ambiguous pattern
|
// Ambiguous pattern
|
||||||
app.register(oldApi());
|
app.register(oldApi());
|
||||||
|
|
||||||
export default app;`;
|
export default app;`;
|
||||||
|
|
||||||
writeFileSync(resolve(dir, 'app.js'), code);
|
writeFileSync(resolve(dir, 'app.js'), code);
|
||||||
|
|
||||||
const ctx = makeCtx({ cwd: dir });
|
const ctx = makeCtx({ cwd: dir });
|
||||||
const result = await migrateCommand({ write: true }, ctx);
|
const result = await migrateCommand({ write: true }, ctx);
|
||||||
|
|
||||||
// Should exit 2 because ambiguous patterns found
|
// Should exit 2 because ambiguous patterns found
|
||||||
assert.strictEqual(result.exitCode, 2, 'Should exit 2 when ambiguous patterns found in write mode');
|
assert.strictEqual(result.exitCode, 2, 'Should exit 2 when ambiguous patterns found in write mode');
|
||||||
assert.ok(result.remaining.length > 0, 'Should have remaining ambiguous items');
|
assert.ok(result.remaining.length > 0, 'Should have remaining ambiguous items');
|
||||||
@@ -499,14 +409,11 @@ export default app;`;
|
|||||||
cleanup(dir);
|
cleanup(dir);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Test 10: Migrate emits guidance for each legacy field
|
// Test 10: Migrate emits guidance for each legacy field
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
test('migrate emits guidance for each legacy field', async () => {
|
test('migrate emits guidance for each legacy field', async () => {
|
||||||
const dir = createTempDir();
|
const dir = createTempDir();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const legacyConfig = `export default {
|
const legacyConfig = `export default {
|
||||||
testMode: "verify",
|
testMode: "verify",
|
||||||
@@ -516,14 +423,10 @@ test('migrate emits guidance for each legacy field', async () => {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
};`;
|
};`;
|
||||||
|
|
||||||
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
|
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
|
||||||
|
|
||||||
const ctx = makeCtx({ cwd: dir });
|
const ctx = makeCtx({ cwd: dir });
|
||||||
const result = await migrateCommand({ check: true }, ctx);
|
const result = await migrateCommand({ check: true }, ctx);
|
||||||
|
|
||||||
assert.ok(result.items.length > 0, 'Should have items');
|
assert.ok(result.items.length > 0, 'Should have items');
|
||||||
|
|
||||||
for (const item of result.items) {
|
for (const item of result.items) {
|
||||||
assert.ok(item.guidance, `Item ${item.legacy} should have guidance`);
|
assert.ok(item.guidance, `Item ${item.legacy} should have guidance`);
|
||||||
assert.ok(
|
assert.ok(
|
||||||
@@ -535,14 +438,11 @@ test('migrate emits guidance for each legacy field', async () => {
|
|||||||
cleanup(dir);
|
cleanup(dir);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Test 11: Config rewriter replaces legacy fields
|
// Test 11: Config rewriter replaces legacy fields
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
test('config rewriter replaces legacy fields', () => {
|
test('config rewriter replaces legacy fields', () => {
|
||||||
const dir = createTempDir();
|
const dir = createTempDir();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const content = `export default {
|
const content = `export default {
|
||||||
testMode: "verify",
|
testMode: "verify",
|
||||||
@@ -552,17 +452,13 @@ test('config rewriter replaces legacy fields', () => {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
};`;
|
};`;
|
||||||
|
|
||||||
writeFileSync(resolve(dir, 'test.config.js'), content);
|
writeFileSync(resolve(dir, 'test.config.js'), content);
|
||||||
|
|
||||||
const items = detectLegacyConfigFields(content, 'test.config.js');
|
const items = detectLegacyConfigFields(content, 'test.config.js');
|
||||||
assert.strictEqual(items.length, 3, 'Should detect 3 legacy fields');
|
assert.strictEqual(items.length, 3, 'Should detect 3 legacy fields');
|
||||||
|
|
||||||
const result = rewriteConfigFile(
|
const result = rewriteConfigFile(
|
||||||
resolve(dir, 'test.config.js'),
|
resolve(dir, 'test.config.js'),
|
||||||
items,
|
items,
|
||||||
);
|
);
|
||||||
|
|
||||||
assert.strictEqual(result.modified, true, 'Should modify content');
|
assert.strictEqual(result.modified, true, 'Should modify content');
|
||||||
assert.ok(result.content.includes('mode:'), 'Should have mode');
|
assert.ok(result.content.includes('mode:'), 'Should have mode');
|
||||||
assert.ok(result.content.includes('profiles:'), 'Should have profiles');
|
assert.ok(result.content.includes('profiles:'), 'Should have profiles');
|
||||||
@@ -572,35 +468,28 @@ test('config rewriter replaces legacy fields', () => {
|
|||||||
cleanup(dir);
|
cleanup(dir);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Test 12: Route rewriter detects x-validate-runtime annotation
|
// Test 12: Route rewriter detects x-validate-runtime annotation
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
test('route rewriter detects x-validate-runtime annotation', () => {
|
test('route rewriter detects x-validate-runtime annotation', () => {
|
||||||
const dir = createTempDir();
|
const dir = createTempDir();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const content = `export default {
|
const content = `export default {
|
||||||
schema: {
|
schema: {
|
||||||
'x-validate-runtime': true,
|
'x-validate-runtime': true,
|
||||||
},
|
},
|
||||||
};`;
|
};`;
|
||||||
|
|
||||||
writeFileSync(resolve(dir, 'test.routes.js'), content);
|
writeFileSync(resolve(dir, 'test.routes.js'), content);
|
||||||
|
|
||||||
const items = detectLegacyRouteAnnotations(content, 'test.routes.js');
|
const items = detectLegacyRouteAnnotations(content, 'test.routes.js');
|
||||||
assert.strictEqual(items.length, 1, 'Should detect 1 legacy annotation');
|
assert.strictEqual(items.length, 1, 'Should detect 1 legacy annotation');
|
||||||
const firstItem = items[0];
|
const firstItem = items[0];
|
||||||
assert.ok(firstItem, 'Expected one migration item');
|
assert.ok(firstItem, 'Expected one migration item');
|
||||||
assert.strictEqual(firstItem.legacy, 'x-validate-runtime');
|
assert.strictEqual(firstItem.legacy, 'x-validate-runtime');
|
||||||
assert.strictEqual(firstItem.replacement, 'runtime');
|
assert.strictEqual(firstItem.replacement, 'runtime');
|
||||||
|
|
||||||
const result = rewriteRouteAnnotations(
|
const result = rewriteRouteAnnotations(
|
||||||
resolve(dir, 'test.routes.js'),
|
resolve(dir, 'test.routes.js'),
|
||||||
items,
|
items,
|
||||||
);
|
);
|
||||||
|
|
||||||
assert.strictEqual(result.modified, true, 'Should modify content');
|
assert.strictEqual(result.modified, true, 'Should modify content');
|
||||||
assert.ok(result.content.includes("'runtime'"), 'Should have runtime');
|
assert.ok(result.content.includes("'runtime'"), 'Should have runtime');
|
||||||
assert.ok(!result.content.includes('x-validate-runtime'), 'Should not have legacy annotation');
|
assert.ok(!result.content.includes('x-validate-runtime'), 'Should not have legacy annotation');
|
||||||
@@ -608,34 +497,25 @@ test('route rewriter detects x-validate-runtime annotation', () => {
|
|||||||
cleanup(dir);
|
cleanup(dir);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Test 13: Code rewriter detects legacy patterns
|
// Test 13: Code rewriter detects legacy patterns
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
test('code rewriter detects legacy patterns', () => {
|
test('code rewriter detects legacy patterns', () => {
|
||||||
const dir = createTempDir();
|
const dir = createTempDir();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const content = `import Fastify from 'fastify';
|
const content = `import Fastify from 'fastify';
|
||||||
const app = Fastify();
|
const app = Fastify();
|
||||||
|
|
||||||
app.register(contract());
|
app.register(contract());
|
||||||
app.register(stateful());
|
app.register(stateful());
|
||||||
app.register(scenario());
|
app.register(scenario());
|
||||||
|
|
||||||
export default app;`;
|
export default app;`;
|
||||||
|
|
||||||
writeFileSync(resolve(dir, 'test.app.js'), content);
|
writeFileSync(resolve(dir, 'test.app.js'), content);
|
||||||
|
|
||||||
const items = detectLegacyCodePatterns(content, 'test.app.js');
|
const items = detectLegacyCodePatterns(content, 'test.app.js');
|
||||||
assert.strictEqual(items.length, 3, 'Should detect 3 legacy patterns');
|
assert.strictEqual(items.length, 3, 'Should detect 3 legacy patterns');
|
||||||
|
|
||||||
const result = rewriteCodePatterns(
|
const result = rewriteCodePatterns(
|
||||||
resolve(dir, 'test.app.js'),
|
resolve(dir, 'test.app.js'),
|
||||||
items,
|
items,
|
||||||
);
|
);
|
||||||
|
|
||||||
assert.strictEqual(result.modified, true, 'Should modify content');
|
assert.strictEqual(result.modified, true, 'Should modify content');
|
||||||
assert.ok(result.content.includes("verify({ kind: 'contract' })"), 'Should have verify');
|
assert.ok(result.content.includes("verify({ kind: 'contract' })"), 'Should have verify');
|
||||||
assert.ok(result.content.includes("qualify({ kind: 'stateful' })"), 'Should have qualify stateful');
|
assert.ok(result.content.includes("qualify({ kind: 'stateful' })"), 'Should have qualify stateful');
|
||||||
@@ -644,28 +524,21 @@ export default app;`;
|
|||||||
cleanup(dir);
|
cleanup(dir);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Test 14: Dry-run default mode (safe by default)
|
// Test 14: Dry-run default mode (safe by default)
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
test('migrate defaults to dry-run mode (safe by default)', async () => {
|
test('migrate defaults to dry-run mode (safe by default)', async () => {
|
||||||
const dir = createTempDir();
|
const dir = createTempDir();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const legacyConfig = `export default {
|
const legacyConfig = `export default {
|
||||||
testMode: "verify",
|
testMode: "verify",
|
||||||
};`;
|
};`;
|
||||||
|
|
||||||
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
|
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
|
||||||
|
|
||||||
const ctx = makeCtx({ cwd: dir });
|
const ctx = makeCtx({ cwd: dir });
|
||||||
// No mode specified — should default to dry-run
|
// No mode specified — should default to dry-run
|
||||||
const result = await migrateCommand({}, ctx);
|
const result = await migrateCommand({}, ctx);
|
||||||
|
|
||||||
assert.strictEqual(result.exitCode, 1, 'Should exit 1 in dry-run mode');
|
assert.strictEqual(result.exitCode, 1, 'Should exit 1 in dry-run mode');
|
||||||
assert.ok(result.message?.includes('Dry run'), 'Should indicate dry run');
|
assert.ok(result.message?.includes('Dry run'), 'Should indicate dry run');
|
||||||
|
|
||||||
// Verify file was NOT modified
|
// Verify file was NOT modified
|
||||||
const content = readFileSync(resolve(dir, 'apophis.config.js'), 'utf-8');
|
const content = readFileSync(resolve(dir, 'apophis.config.js'), 'utf-8');
|
||||||
assert.ok(content.includes('testMode'), 'File should still have testMode');
|
assert.ok(content.includes('testMode'), 'File should still have testMode');
|
||||||
@@ -673,38 +546,30 @@ test('migrate defaults to dry-run mode (safe by default)', async () => {
|
|||||||
cleanup(dir);
|
cleanup(dir);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Test 15: Mixed legacy/modern field detection at rewriter level
|
// Test 15: Mixed legacy/modern field detection at rewriter level
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
test('config rewriter detects mixed legacy and modern fields', () => {
|
test('config rewriter detects mixed legacy and modern fields', () => {
|
||||||
const dir = createTempDir();
|
const dir = createTempDir();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const content = `export default {
|
const content = `export default {
|
||||||
// Both legacy and modern present
|
// Both legacy and modern present
|
||||||
testMode: "verify",
|
testMode: "verify",
|
||||||
mode: "observe",
|
mode: "observe",
|
||||||
|
|
||||||
testProfiles: {
|
testProfiles: {
|
||||||
quick: {
|
quick: {
|
||||||
usesPreset: "safe-ci",
|
usesPreset: "safe-ci",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|
||||||
profiles: {
|
profiles: {
|
||||||
modern: {
|
modern: {
|
||||||
preset: "safe-ci",
|
preset: "safe-ci",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
};`;
|
};`;
|
||||||
|
|
||||||
writeFileSync(resolve(dir, 'test.config.js'), content);
|
writeFileSync(resolve(dir, 'test.config.js'), content);
|
||||||
|
|
||||||
const mixedReports = detectMixedLegacyModernFields(content, 'test.config.js');
|
const mixedReports = detectMixedLegacyModernFields(content, 'test.config.js');
|
||||||
assert.ok(mixedReports.length > 0, 'Should detect mixed fields');
|
assert.ok(mixedReports.length > 0, 'Should detect mixed fields');
|
||||||
|
|
||||||
const testModeReport = mixedReports.find((r) => r.legacy === 'testMode');
|
const testModeReport = mixedReports.find((r) => r.legacy === 'testMode');
|
||||||
assert.ok(testModeReport, 'Should report testMode as mixed');
|
assert.ok(testModeReport, 'Should report testMode as mixed');
|
||||||
assert.ok(testModeReport.guidance.includes('testMode'), 'Guidance should mention testMode');
|
assert.ok(testModeReport.guidance.includes('testMode'), 'Guidance should mention testMode');
|
||||||
@@ -713,14 +578,11 @@ test('config rewriter detects mixed legacy and modern fields', () => {
|
|||||||
cleanup(dir);
|
cleanup(dir);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Test 16: Ambiguous route pattern detection
|
// Test 16: Ambiguous route pattern detection
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
test('route rewriter detects ambiguous route patterns with context', () => {
|
test('route rewriter detects ambiguous route patterns with context', () => {
|
||||||
const dir = createTempDir();
|
const dir = createTempDir();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const content = `export default {
|
const content = `export default {
|
||||||
schema: {
|
schema: {
|
||||||
@@ -728,12 +590,9 @@ test('route rewriter detects ambiguous route patterns with context', () => {
|
|||||||
'x-validate': true,
|
'x-validate': true,
|
||||||
},
|
},
|
||||||
};`;
|
};`;
|
||||||
|
|
||||||
writeFileSync(resolve(dir, 'test.routes.js'), content);
|
writeFileSync(resolve(dir, 'test.routes.js'), content);
|
||||||
|
|
||||||
const items = detectAmbiguousRoutePatterns(content, 'test.routes.js');
|
const items = detectAmbiguousRoutePatterns(content, 'test.routes.js');
|
||||||
assert.strictEqual(items.length, 1, 'Should detect 1 ambiguous pattern');
|
assert.strictEqual(items.length, 1, 'Should detect 1 ambiguous pattern');
|
||||||
|
|
||||||
const firstItem = items[0];
|
const firstItem = items[0];
|
||||||
assert.ok(firstItem, 'Expected one migration item');
|
assert.ok(firstItem, 'Expected one migration item');
|
||||||
assert.strictEqual(firstItem.legacy, 'x-validate');
|
assert.strictEqual(firstItem.legacy, 'x-validate');
|
||||||
@@ -744,28 +603,20 @@ test('route rewriter detects ambiguous route patterns with context', () => {
|
|||||||
cleanup(dir);
|
cleanup(dir);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Test 17: Ambiguous code pattern detection with context
|
// Test 17: Ambiguous code pattern detection with context
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
test('code rewriter detects ambiguous code patterns with surrounding context', () => {
|
test('code rewriter detects ambiguous code patterns with surrounding context', () => {
|
||||||
const dir = createTempDir();
|
const dir = createTempDir();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const content = `import Fastify from 'fastify';
|
const content = `import Fastify from 'fastify';
|
||||||
const app = Fastify();
|
const app = Fastify();
|
||||||
|
|
||||||
// Ambiguous pattern
|
// Ambiguous pattern
|
||||||
app.register(oldApi());
|
app.register(oldApi());
|
||||||
|
|
||||||
export default app;`;
|
export default app;`;
|
||||||
|
|
||||||
writeFileSync(resolve(dir, 'test.app.js'), content);
|
writeFileSync(resolve(dir, 'test.app.js'), content);
|
||||||
|
|
||||||
const items = detectAmbiguousCodePatterns(content, 'test.app.js');
|
const items = detectAmbiguousCodePatterns(content, 'test.app.js');
|
||||||
assert.strictEqual(items.length, 1, 'Should detect 1 ambiguous pattern');
|
assert.strictEqual(items.length, 1, 'Should detect 1 ambiguous pattern');
|
||||||
|
|
||||||
const firstItem = items[0];
|
const firstItem = items[0];
|
||||||
assert.ok(firstItem, 'Expected one migration item');
|
assert.ok(firstItem, 'Expected one migration item');
|
||||||
assert.strictEqual(firstItem.legacy, 'oldApi()');
|
assert.strictEqual(firstItem.legacy, 'oldApi()');
|
||||||
@@ -777,42 +628,32 @@ export default app;`;
|
|||||||
cleanup(dir);
|
cleanup(dir);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Test 18: Legacy fixture detection
|
// Test 18: Legacy fixture detection
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
test('migrate detects legacy patterns in fixture config', async () => {
|
test('migrate detects legacy patterns in fixture config', async () => {
|
||||||
const ctx = makeCtx({ cwd: 'src/cli/__fixtures__/legacy-config' });
|
const ctx = makeCtx({ cwd: 'src/cli/__fixtures__/legacy-config' });
|
||||||
const result = await migrateCommand({ check: true }, ctx);
|
const result = await migrateCommand({ check: true }, ctx);
|
||||||
|
|
||||||
assert.strictEqual(result.exitCode, 1, 'Should detect legacy patterns in fixture');
|
assert.strictEqual(result.exitCode, 1, 'Should detect legacy patterns in fixture');
|
||||||
assert.ok(result.items.length > 0, 'Should find legacy items');
|
assert.ok(result.items.length > 0, 'Should find legacy items');
|
||||||
|
|
||||||
const legacyNames = result.items.map((item) => item.legacy);
|
const legacyNames = result.items.map((item) => item.legacy);
|
||||||
assert.ok(legacyNames.includes('testMode'), 'Should detect testMode in fixture');
|
assert.ok(legacyNames.includes('testMode'), 'Should detect testMode in fixture');
|
||||||
assert.ok(legacyNames.includes('testProfiles'), 'Should detect testProfiles in fixture');
|
assert.ok(legacyNames.includes('testProfiles'), 'Should detect testProfiles in fixture');
|
||||||
assert.ok(legacyNames.includes('testPresets'), 'Should detect testPresets in fixture');
|
assert.ok(legacyNames.includes('testPresets'), 'Should detect testPresets in fixture');
|
||||||
assert.ok(legacyNames.includes('envPolicies'), 'Should detect envPolicies in fixture');
|
assert.ok(legacyNames.includes('envPolicies'), 'Should detect envPolicies in fixture');
|
||||||
});
|
});
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Test 19: JSON output format
|
// Test 19: JSON output format
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
test('migrate outputs JSON format with all fields', async () => {
|
test('migrate outputs JSON format with all fields', async () => {
|
||||||
const dir = createTempDir();
|
const dir = createTempDir();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const legacyConfig = `export default {
|
const legacyConfig = `export default {
|
||||||
testMode: "verify",
|
testMode: "verify",
|
||||||
};`;
|
};`;
|
||||||
|
|
||||||
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
|
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
|
||||||
|
|
||||||
const ctx = makeCtx({ cwd: dir, options: { ...makeCtx().options, format: 'json' } });
|
const ctx = makeCtx({ cwd: dir, options: { ...makeCtx().options, format: 'json' } });
|
||||||
const result = await migrateCommand({ check: true }, ctx);
|
const result = await migrateCommand({ check: true }, ctx);
|
||||||
|
|
||||||
assert.strictEqual(result.exitCode, 1, 'Should exit 1');
|
assert.strictEqual(result.exitCode, 1, 'Should exit 1');
|
||||||
assert.ok(result.items.length > 0, 'Should have items');
|
assert.ok(result.items.length > 0, 'Should have items');
|
||||||
assert.ok(result.totalRewrites, 'Should have totalRewrites');
|
assert.ok(result.totalRewrites, 'Should have totalRewrites');
|
||||||
@@ -821,18 +662,14 @@ test('migrate outputs JSON format with all fields', async () => {
|
|||||||
cleanup(dir);
|
cleanup(dir);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Test 20: No files found returns usage error
|
// Test 20: No files found returns usage error
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
test('migrate returns usage error when no files found', async () => {
|
test('migrate returns usage error when no files found', async () => {
|
||||||
const dir = createTempDir();
|
const dir = createTempDir();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const ctx = makeCtx({ cwd: dir });
|
const ctx = makeCtx({ cwd: dir });
|
||||||
const result = await migrateCommand({ check: true }, ctx);
|
const result = await migrateCommand({ check: true }, ctx);
|
||||||
|
|
||||||
assert.strictEqual(result.exitCode, 2, 'Should exit 2 when no files found');
|
assert.strictEqual(result.exitCode, 2, 'Should exit 2 when no files found');
|
||||||
assert.ok(result.message?.includes('No config or app files found'), 'Should mention no files found');
|
assert.ok(result.message?.includes('No config or app files found'), 'Should mention no files found');
|
||||||
} finally {
|
} finally {
|
||||||
|
|||||||
@@ -152,7 +152,7 @@ test('contract runner supports cross-operation APOSTL ensures', async () => {
|
|||||||
await fastify.register(apophisPlugin, {})
|
await fastify.register(apophisPlugin, {})
|
||||||
registerItemApi(fastify)
|
registerItemApi(fastify)
|
||||||
await fastify.ready()
|
await fastify.ready()
|
||||||
const result = await fastify.apophis.contract({ depth: 'quick', seed: 7 })
|
const result = await fastify.apophis.contract({ runs: 10, seed: 7 })
|
||||||
const failures = result.tests.filter((entry: TestResult) => !entry.ok)
|
const failures = result.tests.filter((entry: TestResult) => !entry.ok)
|
||||||
assert.strictEqual(failures.length, 0)
|
assert.strictEqual(failures.length, 0)
|
||||||
} finally {
|
} finally {
|
||||||
@@ -169,7 +169,7 @@ test('stateful runner supports cross-operation APOSTL ensures', async () => {
|
|||||||
await fastify.register(apophisPlugin, {})
|
await fastify.register(apophisPlugin, {})
|
||||||
registerItemApi(fastify)
|
registerItemApi(fastify)
|
||||||
await fastify.ready()
|
await fastify.ready()
|
||||||
const result = await fastify.apophis.stateful({ depth: 'quick', seed: 11 })
|
const result = await fastify.apophis.stateful({ runs: 10, seed: 11 })
|
||||||
const failures = result.tests.filter((entry: TestResult) => !entry.ok)
|
const failures = result.tests.filter((entry: TestResult) => !entry.ok)
|
||||||
assert.strictEqual(failures.length, 0)
|
assert.strictEqual(failures.length, 0)
|
||||||
} finally {
|
} finally {
|
||||||
@@ -217,7 +217,7 @@ test('contract runner applies chaos injection when configured', async () => {
|
|||||||
}, async () => ({ ok: true }))
|
}, async () => ({ ok: true }))
|
||||||
await fastify.ready()
|
await fastify.ready()
|
||||||
const result = await fastify.apophis.contract({
|
const result = await fastify.apophis.contract({
|
||||||
depth: 'quick',
|
runs: 10,
|
||||||
seed: 3,
|
seed: 3,
|
||||||
chaos: {
|
chaos: {
|
||||||
probability: 1,
|
probability: 1,
|
||||||
@@ -241,7 +241,7 @@ test('contract runner supports previous(...) with response_body(this) path place
|
|||||||
await fastify.register(apophisPlugin, {})
|
await fastify.register(apophisPlugin, {})
|
||||||
registerPlanApi(fastify)
|
registerPlanApi(fastify)
|
||||||
await fastify.ready()
|
await fastify.ready()
|
||||||
const result = await fastify.apophis.contract({ depth: 'quick', seed: 17 })
|
const result = await fastify.apophis.contract({ runs: 10, seed: 17 })
|
||||||
const failures = result.tests.filter((entry: TestResult) => !entry.ok)
|
const failures = result.tests.filter((entry: TestResult) => !entry.ok)
|
||||||
assert.strictEqual(failures.length, 0)
|
assert.strictEqual(failures.length, 0)
|
||||||
} finally {
|
} finally {
|
||||||
@@ -258,7 +258,7 @@ test('stateful runner supports previous(...) with response_body(this) path place
|
|||||||
await fastify.register(apophisPlugin, {})
|
await fastify.register(apophisPlugin, {})
|
||||||
registerPlanApi(fastify)
|
registerPlanApi(fastify)
|
||||||
await fastify.ready()
|
await fastify.ready()
|
||||||
const result = await fastify.apophis.stateful({ depth: 'quick', seed: 19 })
|
const result = await fastify.apophis.stateful({ runs: 10, seed: 19 })
|
||||||
const failures = result.tests.filter((entry: TestResult) => !entry.ok)
|
const failures = result.tests.filter((entry: TestResult) => !entry.ok)
|
||||||
assert.strictEqual(failures.length, 0)
|
assert.strictEqual(failures.length, 0)
|
||||||
} finally {
|
} finally {
|
||||||
@@ -406,7 +406,7 @@ test('contract runner validates hypermedia links with route_exists', async () =>
|
|||||||
})
|
})
|
||||||
registerHypermediaApi(fastify)
|
registerHypermediaApi(fastify)
|
||||||
await fastify.ready()
|
await fastify.ready()
|
||||||
const result = await fastify.apophis.contract({ depth: 'quick', seed: 23 })
|
const result = await fastify.apophis.contract({ runs: 10, seed: 23 })
|
||||||
const failures = result.tests.filter((entry: TestResult) => !entry.ok)
|
const failures = result.tests.filter((entry: TestResult) => !entry.ok)
|
||||||
if (failures.length > 0) {
|
if (failures.length > 0) {
|
||||||
console.log('Contract failures:', failures.map((f: TestResult) => ({
|
console.log('Contract failures:', failures.map((f: TestResult) => ({
|
||||||
@@ -441,7 +441,7 @@ test('stateful runner validates cross-route relationships', async () => {
|
|||||||
})
|
})
|
||||||
registerHypermediaApi(fastify)
|
registerHypermediaApi(fastify)
|
||||||
await fastify.ready()
|
await fastify.ready()
|
||||||
const result = await fastify.apophis.stateful({ depth: 'quick', seed: 29 })
|
const result = await fastify.apophis.stateful({ runs: 10, seed: 29 })
|
||||||
const failures = result.tests.filter((entry: TestResult) => !entry.ok)
|
const failures = result.tests.filter((entry: TestResult) => !entry.ok)
|
||||||
if (failures.length > 0) {
|
if (failures.length > 0) {
|
||||||
console.log('Stateful failures:', failures.map((f: TestResult) => ({
|
console.log('Stateful failures:', failures.map((f: TestResult) => ({
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ test('APOPHIS_DEBUG=1 logs requests and responses', async () => {
|
|||||||
originalDebug(msg, _obj)
|
originalDebug(msg, _obj)
|
||||||
}
|
}
|
||||||
|
|
||||||
const result = await fastify.apophis.contract({ depth: 'quick' })
|
const result = await fastify.apophis.contract({ runs: 10 })
|
||||||
assert.ok(result.tests.length > 0, 'should have tests')
|
assert.ok(result.tests.length > 0, 'should have tests')
|
||||||
|
|
||||||
// Should have logged at least one request and one response
|
// Should have logged at least one request and one response
|
||||||
@@ -80,7 +80,7 @@ test('APOPHIS_DEBUG=0 does not log requests', async () => {
|
|||||||
originalDebug(msg, _obj)
|
originalDebug(msg, _obj)
|
||||||
}
|
}
|
||||||
|
|
||||||
const result = await fastify.apophis.contract({ depth: 'quick' })
|
const result = await fastify.apophis.contract({ runs: 10 })
|
||||||
assert.ok(result.tests.length > 0, 'should have tests')
|
assert.ok(result.tests.length > 0, 'should have tests')
|
||||||
|
|
||||||
// Should not have any request/response logs
|
// Should not have any request/response logs
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ test('example: minimal API compiles and runs', async () => {
|
|||||||
|
|
||||||
await fastify.ready()
|
await fastify.ready()
|
||||||
|
|
||||||
const result = await fastify.apophis.contract({ depth: 'quick' })
|
const result = await fastify.apophis.contract({ runs: 10 })
|
||||||
assert.ok(result.tests.length > 0, 'should have test results')
|
assert.ok(result.tests.length > 0, 'should have test results')
|
||||||
console.log('Minimal example:', result.summary)
|
console.log('Minimal example:', result.summary)
|
||||||
} finally {
|
} finally {
|
||||||
@@ -111,7 +111,7 @@ test('example: CRUD API with contracts compiles and runs', async () => {
|
|||||||
|
|
||||||
await fastify.ready()
|
await fastify.ready()
|
||||||
|
|
||||||
const result = await fastify.apophis.contract({ depth: 'quick' })
|
const result = await fastify.apophis.contract({ runs: 10 })
|
||||||
assert.ok(result.tests.length > 0, 'should have test results')
|
assert.ok(result.tests.length > 0, 'should have test results')
|
||||||
console.log('CRUD example:', result.summary)
|
console.log('CRUD example:', result.summary)
|
||||||
} finally {
|
} finally {
|
||||||
@@ -149,7 +149,7 @@ test('example: prefix registration works', async () => {
|
|||||||
const itemContract = contracts.find(c => c.path === '/api/v1/items')
|
const itemContract = contracts.find(c => c.path === '/api/v1/items')
|
||||||
assert.ok(itemContract, 'should discover prefixed route')
|
assert.ok(itemContract, 'should discover prefixed route')
|
||||||
|
|
||||||
const result = await fastify.apophis.contract({ depth: 'quick' })
|
const result = await fastify.apophis.contract({ runs: 10 })
|
||||||
assert.ok(result.tests.length > 0, 'should run tests on prefixed routes')
|
assert.ok(result.tests.length > 0, 'should run tests on prefixed routes')
|
||||||
} finally {
|
} finally {
|
||||||
await fastify.close()
|
await fastify.close()
|
||||||
|
|||||||
@@ -523,11 +523,35 @@ test('evaluate: returns error for missing previous context', () => {
|
|||||||
assert.strictEqual(result.success, false)
|
assert.strictEqual(result.success, false)
|
||||||
assert.ok((result as { success: false; error: string }).error.includes('No previous context'))
|
assert.ok((result as { success: false; error: string }).error.includes('No previous context'))
|
||||||
})
|
})
|
||||||
test('evaluate: returns error for non-array in quantified expression', () => {
|
test('evaluate: returns false for non-array in quantified expression', () => {
|
||||||
const ast = parse('for x in response_code(this): x == 1')
|
const ast = parse('for x in response_code(this): x == 1')
|
||||||
const result = evaluate(ast.ast, makeContext())
|
const result = evaluate(ast.ast, makeContext())
|
||||||
assert.strictEqual(result.success, false)
|
assert.strictEqual(result.success, true)
|
||||||
assert.ok((result as { success: false; error: string }).error.includes('array collection'))
|
assert.strictEqual((result as { success: true; value: unknown }).value, false)
|
||||||
|
})
|
||||||
|
test('evaluate: returns false for undefined collection in quantified expression', () => {
|
||||||
|
const ast = parse('for x in response_body(this): x == 1')
|
||||||
|
const result = evaluate(ast.ast, makeContext({ response: { body: undefined, headers: {}, statusCode: 200, responseTime: 0 } }))
|
||||||
|
assert.strictEqual(result.success, true)
|
||||||
|
assert.strictEqual((result as { success: true; value: unknown }).value, false)
|
||||||
|
})
|
||||||
|
test('evaluate: returns false for error object collection in quantified expression', () => {
|
||||||
|
const ast = parse('for x in response_body(this): x == 1')
|
||||||
|
const result = evaluate(ast.ast, makeContext({ response: { body: { error: 'Chaos error: forced 503' }, headers: {}, statusCode: 200, responseTime: 0 } }))
|
||||||
|
assert.strictEqual(result.success, true)
|
||||||
|
assert.strictEqual((result as { success: true; value: unknown }).value, false)
|
||||||
|
})
|
||||||
|
test('evaluate: quantified nested in conditional handles undefined gracefully', () => {
|
||||||
|
const ast = parse('if status:503 then for x in response_body(this): x == 1 else true')
|
||||||
|
const result = evaluate(ast.ast, makeContext({ response: { statusCode: 503, body: undefined, headers: {}, responseTime: 0 } }))
|
||||||
|
assert.strictEqual(result.success, true)
|
||||||
|
assert.strictEqual((result as { success: true; value: unknown }).value, false)
|
||||||
|
})
|
||||||
|
test('evaluate: exists returns false for non-array collection', () => {
|
||||||
|
const ast = parse('exists x in response_body(this): x == 1')
|
||||||
|
const result = evaluate(ast.ast, makeContext({ response: { body: { error: 'fail' }, headers: {}, statusCode: 200, responseTime: 0 } }))
|
||||||
|
assert.strictEqual(result.success, true)
|
||||||
|
assert.strictEqual((result as { success: true; value: unknown }).value, false)
|
||||||
})
|
})
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
// Unit Tests: Substitutor
|
// Unit Tests: Substitutor
|
||||||
|
|||||||
@@ -243,7 +243,7 @@ test('petit-runner executes tests against real API', async () => {
|
|||||||
]
|
]
|
||||||
const fastifyWithRoutes = Object.assign(fastify, { routes: mockRoutes })
|
const fastifyWithRoutes = Object.assign(fastify, { routes: mockRoutes })
|
||||||
const result = await runPetitTests(fastifyWithRoutes as any, {
|
const result = await runPetitTests(fastifyWithRoutes as any, {
|
||||||
depth: 'quick',
|
runs: 10,
|
||||||
scope: undefined,
|
scope: undefined,
|
||||||
seed: undefined
|
seed: undefined
|
||||||
})
|
})
|
||||||
@@ -367,7 +367,7 @@ test('full integration: plugin + routes + test execution', async () => {
|
|||||||
const createUserContract = contracts.find(c => c.path === '/users' && c.method === 'POST')
|
const createUserContract = contracts.find(c => c.path === '/users' && c.method === 'POST')
|
||||||
assert.ok(createUserContract, 'create user contract should exist')
|
assert.ok(createUserContract, 'create user contract should exist')
|
||||||
assert.strictEqual(createUserContract.category, 'constructor')
|
assert.strictEqual(createUserContract.category, 'constructor')
|
||||||
const testResult = await fastify.apophis.contract({ depth: 'quick' })
|
const testResult = await fastify.apophis.contract({ runs: 10 })
|
||||||
assert.ok(Array.isArray(testResult.tests), 'tests should be an array')
|
assert.ok(Array.isArray(testResult.tests), 'tests should be an array')
|
||||||
assert.ok(testResult.tests.length > 0, 'tests should not be empty')
|
assert.ok(testResult.tests.length > 0, 'tests should not be empty')
|
||||||
await fastify.apophis.cleanup()
|
await fastify.apophis.cleanup()
|
||||||
@@ -439,7 +439,7 @@ test('mode filtering: stateful mode only runs constructor/mutator routes', async
|
|||||||
}, async () => ({ status: 'ok' }))
|
}, async () => ({ status: 'ok' }))
|
||||||
await fastify.ready()
|
await fastify.ready()
|
||||||
// Run in stateful mode
|
// Run in stateful mode
|
||||||
const result = await fastify.apophis.contract({ depth: 'quick' })
|
const result = await fastify.apophis.contract({ runs: 10 })
|
||||||
// In stateful mode, utility routes should be excluded
|
// In stateful mode, utility routes should be excluded
|
||||||
// The test should only run constructor and mutator routes
|
// The test should only run constructor and mutator routes
|
||||||
assert.ok(Array.isArray(result.tests), 'tests should be an array')
|
assert.ok(Array.isArray(result.tests), 'tests should be an array')
|
||||||
@@ -474,7 +474,7 @@ test('failing contract produces ContractViolation with suggestion', async () =>
|
|||||||
return { status: 'created' } // Returns 200, not 201
|
return { status: 'created' } // Returns 200, not 201
|
||||||
})
|
})
|
||||||
await fastify.ready()
|
await fastify.ready()
|
||||||
const result = await fastify.apophis.contract({ depth: 'quick' })
|
const result = await fastify.apophis.contract({ runs: 10 })
|
||||||
// Find the failing test
|
// Find the failing test
|
||||||
const failingTests = result.tests.filter(t => !t.ok)
|
const failingTests = result.tests.filter(t => !t.ok)
|
||||||
assert.ok(failingTests.length > 0, 'should have at least one failing test')
|
assert.ok(failingTests.length > 0, 'should have at least one failing test')
|
||||||
@@ -647,7 +647,7 @@ test('integration: contract routes option limits tested routes', async () => {
|
|||||||
}, async () => ({ ok: true }))
|
}, async () => ({ ok: true }))
|
||||||
await fastify.ready()
|
await fastify.ready()
|
||||||
const result = await fastify.apophis.contract({
|
const result = await fastify.apophis.contract({
|
||||||
depth: 'quick',
|
runs: 10,
|
||||||
routes: ['GET /included'],
|
routes: ['GET /included'],
|
||||||
})
|
})
|
||||||
const includedTests = result.tests.filter(t => t.name.includes('GET /included'))
|
const includedTests = result.tests.filter(t => t.name.includes('GET /included'))
|
||||||
@@ -673,7 +673,7 @@ test('integration: contract variants are tagged and run in declared order', asyn
|
|||||||
}, async () => ({ ok: true }))
|
}, async () => ({ ok: true }))
|
||||||
await fastify.ready()
|
await fastify.ready()
|
||||||
const result = await fastify.apophis.contract({
|
const result = await fastify.apophis.contract({
|
||||||
depth: 'quick',
|
runs: 10,
|
||||||
variants: [
|
variants: [
|
||||||
{ name: 'json', headers: { accept: 'application/json' } },
|
{ name: 'json', headers: { accept: 'application/json' } },
|
||||||
{ name: 'xml', headers: { accept: 'application/xml' } },
|
{ name: 'xml', headers: { accept: 'application/xml' } },
|
||||||
@@ -715,7 +715,7 @@ test('integration: variant headers override scope headers', async () => {
|
|||||||
}, async () => ({ ok: true }))
|
}, async () => ({ ok: true }))
|
||||||
await fastify.ready()
|
await fastify.ready()
|
||||||
const result = await fastify.apophis.contract({
|
const result = await fastify.apophis.contract({
|
||||||
depth: 'quick',
|
runs: 10,
|
||||||
variants: [
|
variants: [
|
||||||
{ name: 'xml', headers: { accept: 'application/xml' } },
|
{ name: 'xml', headers: { accept: 'application/xml' } },
|
||||||
],
|
],
|
||||||
@@ -744,7 +744,7 @@ test('integration: route-level x-variants are extracted and executed', async ()
|
|||||||
}, async () => ({ ok: true }))
|
}, async () => ({ ok: true }))
|
||||||
await fastify.ready()
|
await fastify.ready()
|
||||||
// No call-site variants; route-level variants should drive execution
|
// No call-site variants; route-level variants should drive execution
|
||||||
const result = await fastify.apophis.contract({ depth: 'quick' })
|
const result = await fastify.apophis.contract({ runs: 10 })
|
||||||
const jsonTests = result.tests.filter((t) => t.name.includes('[variant:json]'))
|
const jsonTests = result.tests.filter((t) => t.name.includes('[variant:json]'))
|
||||||
const xmlTests = result.tests.filter((t) => t.name.includes('[variant:xml]'))
|
const xmlTests = result.tests.filter((t) => t.name.includes('[variant:xml]'))
|
||||||
assert.ok(jsonTests.length > 0, 'route json variant should produce tests')
|
assert.ok(jsonTests.length > 0, 'route json variant should produce tests')
|
||||||
@@ -753,3 +753,40 @@ test('integration: route-level x-variants are extracted and executed', async ()
|
|||||||
await fastify.close()
|
await fastify.close()
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
test('integration: inferred contracts are guarded by status code', async () => {
|
||||||
|
const fastify = Fastify() as unknown as TestFastifyInstance
|
||||||
|
try {
|
||||||
|
await fastify.register(swagger, {})
|
||||||
|
await fastify.register(apophisPlugin, {})
|
||||||
|
fastify.get('/status-guarded', {
|
||||||
|
schema: {
|
||||||
|
'x-category': 'observer',
|
||||||
|
response: {
|
||||||
|
200: {
|
||||||
|
type: 'object',
|
||||||
|
properties: { status: { type: 'string', const: 'success' } },
|
||||||
|
required: ['status']
|
||||||
|
},
|
||||||
|
404: {
|
||||||
|
type: 'object',
|
||||||
|
properties: { error: { type: 'string' } },
|
||||||
|
required: ['error']
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} as Record<string, unknown>
|
||||||
|
}, async (request, reply) => {
|
||||||
|
// Return 404 to verify the 200-schema const doesn't fail
|
||||||
|
reply.status(404)
|
||||||
|
return { error: 'not found' }
|
||||||
|
})
|
||||||
|
await fastify.ready()
|
||||||
|
const result = await fastify.apophis.contract({ runs: 10 })
|
||||||
|
// Should pass because the inferred const contract is guarded:
|
||||||
|
// response_code(this) == 200 => response_body(this).status == "success"
|
||||||
|
// The 404 response doesn't trigger the antecedent, so the implication holds.
|
||||||
|
assert.strictEqual(result.summary.failed, 0, 'inferred 200-schema const should not fail on 404')
|
||||||
|
} finally {
|
||||||
|
await fastify.close()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|||||||
+31
-13
@@ -7,7 +7,7 @@
|
|||||||
import { convertSchema } from '../domain/schema-to-arbitrary.js'
|
import { convertSchema } from '../domain/schema-to-arbitrary.js'
|
||||||
import { lookupCache, storeCache } from '../incremental/cache.js'
|
import { lookupCache, storeCache } from '../incremental/cache.js'
|
||||||
import type { ApiCommand } from '../domain/stateful.js'
|
import type { ApiCommand } from '../domain/stateful.js'
|
||||||
import type { DepthConfig, RouteContract } from '../types.js'
|
import type { RouteContract, RunConfig } from '../types.js'
|
||||||
import * as fc from 'fast-check'
|
import * as fc from 'fast-check'
|
||||||
|
|
||||||
const buildCommand = (route: RouteContract, data: unknown): ApiCommand => ({
|
const buildCommand = (route: RouteContract, data: unknown): ApiCommand => ({
|
||||||
@@ -19,11 +19,10 @@ const buildCommand = (route: RouteContract, data: unknown): ApiCommand => ({
|
|||||||
|
|
||||||
export const generateCommands = (
|
export const generateCommands = (
|
||||||
routes: RouteContract[],
|
routes: RouteContract[],
|
||||||
depth: DepthConfig,
|
runConfig: RunConfig,
|
||||||
seed?: number,
|
seed?: number,
|
||||||
generationProfile: 'quick' | 'standard' | 'thorough' = 'standard',
|
|
||||||
): { commands: ApiCommand[][], cacheHits: number, cacheMisses: number } => {
|
): { commands: ApiCommand[][], cacheHits: number, cacheMisses: number } => {
|
||||||
const commandsPerRoute = Math.max(1, Math.floor(depth.contractRuns / Math.max(routes.length, 1)))
|
const commandsPerRoute = Math.max(1, Math.floor(runConfig.contractRuns / Math.max(routes.length, 1)))
|
||||||
let cacheHits = 0
|
let cacheHits = 0
|
||||||
let cacheMisses = 0
|
let cacheMisses = 0
|
||||||
const allCommands = routes.map((route) => {
|
const allCommands = routes.map((route) => {
|
||||||
@@ -40,7 +39,7 @@ export const generateCommands = (
|
|||||||
cacheMisses++
|
cacheMisses++
|
||||||
const bodySchema = route.schema?.body as Record<string, unknown> | undefined
|
const bodySchema = route.schema?.body as Record<string, unknown> | undefined
|
||||||
const bodyArb = bodySchema !== undefined
|
const bodyArb = bodySchema !== undefined
|
||||||
? convertSchema(bodySchema, { context: 'request', generationProfile })
|
? convertSchema(bodySchema, { context: 'request' })
|
||||||
: fc.constant({})
|
: fc.constant({})
|
||||||
const pathParams = route.path.match(/:([a-zA-Z_][a-zA-Z0-9_]*)/g) ?? []
|
const pathParams = route.path.match(/:([a-zA-Z_][a-zA-Z0-9_]*)/g) ?? []
|
||||||
const pathParamArbs: Record<string, fc.Arbitrary<string>> = {}
|
const pathParamArbs: Record<string, fc.Arbitrary<string>> = {}
|
||||||
@@ -82,7 +81,7 @@ import { buildPetitSuite, filterPetitRoutes } from './route-filter.js'
|
|||||||
import { executePetitCommandStep } from './petit-command-step.js'
|
import { executePetitCommandStep } from './petit-command-step.js'
|
||||||
import { runTripleBoundaryPropertyTest } from './triple-boundary-runner.js'
|
import { runTripleBoundaryPropertyTest } from './triple-boundary-runner.js'
|
||||||
import { makeTrackedResource } from '../domain/state-operations.js'
|
import { makeTrackedResource } from '../domain/state-operations.js'
|
||||||
import { resolveDepth, resolveGenerationProfile } from '../types.js'
|
import { resolveRuns } from '../types.js'
|
||||||
import type {
|
import type {
|
||||||
EvalContext,
|
EvalContext,
|
||||||
FastifyInjectInstance,
|
FastifyInjectInstance,
|
||||||
@@ -105,17 +104,37 @@ export const runPetitTests = async (
|
|||||||
config: TestConfig,
|
config: TestConfig,
|
||||||
scopeRegistry?: ScopeRegistry,
|
scopeRegistry?: ScopeRegistry,
|
||||||
extensionRegistry?: ExtensionRegistry,
|
extensionRegistry?: ExtensionRegistry,
|
||||||
_pluginContractRegistry?: import('../domain/plugin-contracts.js').PluginContractRegistry,
|
pluginContractRegistry?: import('../domain/plugin-contracts.js').PluginContractRegistry,
|
||||||
outboundContractRegistry?: OutboundContractRegistry
|
outboundContractRegistry?: OutboundContractRegistry,
|
||||||
|
overrideContracts?: RouteContract[]
|
||||||
): Promise<TestSuite> => {
|
): Promise<TestSuite> => {
|
||||||
const startTime = Date.now()
|
const startTime = Date.now()
|
||||||
if (extensionRegistry) await extensionRegistry.runSuiteStartHooks(config)
|
if (extensionRegistry) await extensionRegistry.runSuiteStartHooks(config)
|
||||||
|
|
||||||
const allRoutes = discoverRoutes(fastify)
|
const allRoutes = overrideContracts ?? discoverRoutes(fastify)
|
||||||
const { routes, skippedRoutes } = filterPetitRoutes(allRoutes, config)
|
const { routes, skippedRoutes } = filterPetitRoutes(allRoutes, config)
|
||||||
const depth = resolveDepth(config.depth ?? 'standard')
|
|
||||||
const generationProfile = config.generationProfile ?? resolveGenerationProfile(config.depth)
|
// Merge plugin contracts into route contracts
|
||||||
const { commands: commandGroups, cacheHits, cacheMisses } = generateCommands(routes, depth, config.seed, generationProfile)
|
if (pluginContractRegistry) {
|
||||||
|
for (const route of routes) {
|
||||||
|
const composed = pluginContractRegistry.composeContracts(route)
|
||||||
|
for (const phase of Object.values(composed.phases)) {
|
||||||
|
for (const req of phase.requires) {
|
||||||
|
if (!route.requires.includes(req.formula)) {
|
||||||
|
route.requires.push(req.formula)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (const ens of phase.ensures) {
|
||||||
|
if (!route.ensures.includes(ens.formula)) {
|
||||||
|
route.ensures.push(ens.formula)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const runConfig = resolveRuns(config.runs)
|
||||||
|
const { commands: commandGroups, cacheHits, cacheMisses } = generateCommands(routes, runConfig, config.seed)
|
||||||
const allCommands = commandGroups.flat()
|
const allCommands = commandGroups.flat()
|
||||||
const rng = config.seed !== undefined ? new SeededRng(config.seed) : undefined
|
const rng = config.seed !== undefined ? new SeededRng(config.seed) : undefined
|
||||||
|
|
||||||
@@ -161,7 +180,6 @@ export const runPetitTests = async (
|
|||||||
? createOutboundMockRuntime({
|
? createOutboundMockRuntime({
|
||||||
contracts: outboundContractRegistry.resolve(Array.from(outboundNames)),
|
contracts: outboundContractRegistry.resolve(Array.from(outboundNames)),
|
||||||
mode: config.outboundMocks?.mode ?? 'example',
|
mode: config.outboundMocks?.mode ?? 'example',
|
||||||
generationProfile,
|
|
||||||
overrides: config.outboundMocks?.overrides,
|
overrides: config.outboundMocks?.overrides,
|
||||||
unmatched: config.outboundMocks?.unmatched ?? 'error',
|
unmatched: config.outboundMocks?.unmatched ?? 'error',
|
||||||
seed: config.seed !== undefined ? hashCombine(config.seed, 0x6d6f636b) : Math.floor(Math.random() * 0xffffffff),
|
seed: config.seed !== undefined ? hashCombine(config.seed, 0x6d6f636b) : Math.floor(Math.random() * 0xffffffff),
|
||||||
|
|||||||
@@ -0,0 +1,425 @@
|
|||||||
|
/**
|
||||||
|
* Quality Engine Tests — Flake Detection, Mutation Testing, Chaos
|
||||||
|
*/
|
||||||
|
import { test } from 'node:test'
|
||||||
|
import assert from 'node:assert'
|
||||||
|
import Fastify from 'fastify'
|
||||||
|
import swagger from '@fastify/swagger'
|
||||||
|
import apophisPlugin from '../index.js'
|
||||||
|
import { FlakeDetector } from '../quality/flake.js'
|
||||||
|
import { runMutationTesting, testMutation, type Mutation } from '../quality/mutation.js'
|
||||||
|
import * as fc from 'fast-check'
|
||||||
|
import {
|
||||||
|
applyChaosToExecution,
|
||||||
|
applyChaosToAllResponses,
|
||||||
|
applyChaosToDependencyResponse,
|
||||||
|
createChaosEventArbitrary,
|
||||||
|
extractDelays,
|
||||||
|
sleep,
|
||||||
|
hasAppliedChaos,
|
||||||
|
formatChaosEvents,
|
||||||
|
type ChaosEvent,
|
||||||
|
} from '../quality/chaos-v3.js'
|
||||||
|
import type { EvalContext, TestResult, RouteContract } from '../types.js'
|
||||||
|
|
||||||
|
// Quality engines require test environment
|
||||||
|
process.env.NODE_ENV = 'test'
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Helpers
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
function makeCtx(overrides?: Partial<EvalContext['response']>): EvalContext {
|
||||||
|
return {
|
||||||
|
request: { body: {}, headers: {}, query: {}, params: {} },
|
||||||
|
response: { body: { id: '1' }, headers: {}, statusCode: 200, ...overrides },
|
||||||
|
} as EvalContext
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeTestResult(ok: boolean): TestResult {
|
||||||
|
return {
|
||||||
|
ok,
|
||||||
|
name: 'test',
|
||||||
|
id: 1,
|
||||||
|
diagnostics: ok ? {} : { error: 'failed' },
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Chaos Tests
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
test('applyChaosToExecution: no chaos when events are empty', () => {
|
||||||
|
const ctx = makeCtx()
|
||||||
|
const result = applyChaosToExecution(ctx, [])
|
||||||
|
assert.strictEqual(result.applied, false)
|
||||||
|
assert.strictEqual(result.ctx.response.statusCode, 200)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('applyChaosToExecution: inbound error changes status code', () => {
|
||||||
|
const ctx = makeCtx()
|
||||||
|
const result = applyChaosToExecution(ctx, [
|
||||||
|
{ type: 'inbound-error', target: 'inbound', statusCode: 503 },
|
||||||
|
])
|
||||||
|
assert.strictEqual(result.applied, true)
|
||||||
|
assert.strictEqual(result.ctx.response.statusCode, 503)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('applyChaosToExecution: inbound dropout simulates gateway timeout', () => {
|
||||||
|
const ctx = makeCtx()
|
||||||
|
const result = applyChaosToExecution(ctx, [
|
||||||
|
{ type: 'inbound-dropout', target: 'inbound' },
|
||||||
|
])
|
||||||
|
assert.strictEqual(result.ctx.response.statusCode, 504)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('applyChaosToExecution: inbound corruption truncates response body', () => {
|
||||||
|
const ctx = makeCtx({ body: { id: '1', name: 'Alice', email: 'a@b.com' } })
|
||||||
|
const result = applyChaosToExecution(ctx, [
|
||||||
|
{ type: 'inbound-corruption', target: 'inbound', corruptionStrategy: 'truncate' },
|
||||||
|
])
|
||||||
|
const body = result.ctx.response.body as Record<string, unknown>
|
||||||
|
assert.ok(Object.keys(body).length <= 2)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('applyChaosToExecution: inbound corruption with field-corrupt', () => {
|
||||||
|
const ctx = makeCtx({ body: { id: '1', name: 'Alice' } })
|
||||||
|
const result = applyChaosToExecution(ctx, [
|
||||||
|
{ type: 'inbound-corruption', target: 'inbound', corruptionStrategy: 'field-corrupt', corruptionField: 'name' },
|
||||||
|
])
|
||||||
|
const body = result.ctx.response.body as Record<string, unknown>
|
||||||
|
assert.strictEqual(body.name, null)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('applyChaosToExecution: inbound corruption malformed', () => {
|
||||||
|
const ctx = makeCtx({ body: { id: '1' } })
|
||||||
|
const result = applyChaosToExecution(ctx, [
|
||||||
|
{ type: 'inbound-corruption', target: 'inbound', corruptionStrategy: 'malformed' },
|
||||||
|
])
|
||||||
|
assert.strictEqual(result.ctx.response.body, '{"broken":')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('applyChaosToExecution: ignores none events', () => {
|
||||||
|
const ctx = makeCtx()
|
||||||
|
const result = applyChaosToExecution(ctx, [
|
||||||
|
{ type: 'none', target: 'inbound' },
|
||||||
|
])
|
||||||
|
assert.strictEqual(result.applied, false)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('applyChaosToExecution: only applies first non-delay event', () => {
|
||||||
|
const ctx = makeCtx()
|
||||||
|
const result = applyChaosToExecution(ctx, [
|
||||||
|
{ type: 'inbound-error', target: 'inbound', statusCode: 503 },
|
||||||
|
{ type: 'inbound-dropout', target: 'inbound' },
|
||||||
|
])
|
||||||
|
assert.strictEqual(result.ctx.response.statusCode, 503)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('applyChaosToDependencyResponse: outbound error changes status', () => {
|
||||||
|
const response = { contractName: 'api', statusCode: 200, body: { ok: true } }
|
||||||
|
const result = applyChaosToDependencyResponse(response, [
|
||||||
|
{ type: 'outbound-error', target: 'outbound', contractName: 'api', statusCode: 503 },
|
||||||
|
])
|
||||||
|
assert.strictEqual(result.statusCode, 503)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('applyChaosToDependencyResponse: ignores events for other contracts', () => {
|
||||||
|
const response = { contractName: 'api', statusCode: 200, body: {} }
|
||||||
|
const result = applyChaosToDependencyResponse(response, [
|
||||||
|
{ type: 'outbound-error', target: 'outbound', contractName: 'other', statusCode: 503 },
|
||||||
|
])
|
||||||
|
assert.strictEqual(result.statusCode, 200)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('applyChaosToAllResponses: applies chaos to multiple responses', () => {
|
||||||
|
const responses = [
|
||||||
|
{ contractName: 'a', statusCode: 200, body: {} },
|
||||||
|
{ contractName: 'b', statusCode: 200, body: {} },
|
||||||
|
]
|
||||||
|
const events = [
|
||||||
|
{ type: 'outbound-error', target: 'outbound', contractName: 'a', statusCode: 503 },
|
||||||
|
{ type: 'outbound-error', target: 'outbound', contractName: 'b', statusCode: 504 },
|
||||||
|
] as ChaosEvent[]
|
||||||
|
const result = applyChaosToAllResponses(responses, events)
|
||||||
|
assert.strictEqual(result[0]!.statusCode, 503)
|
||||||
|
assert.strictEqual(result[1]!.statusCode, 504)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('extractDelays: computes total delay', () => {
|
||||||
|
const events = [
|
||||||
|
{ type: 'inbound-delay', target: 'inbound', delayMs: 100 },
|
||||||
|
{ type: 'outbound-delay', target: 'outbound', delayMs: 200 },
|
||||||
|
] as ChaosEvent[]
|
||||||
|
const result = extractDelays(events)
|
||||||
|
assert.strictEqual(result.totalMs, 300)
|
||||||
|
assert.strictEqual(result.events.length, 2)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('sleep: resolves after specified ms', async () => {
|
||||||
|
const start = Date.now()
|
||||||
|
await sleep(10)
|
||||||
|
const elapsed = Date.now() - start
|
||||||
|
assert.ok(elapsed >= 8 && elapsed < 100)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('hasAppliedChaos: detects applied chaos', () => {
|
||||||
|
assert.strictEqual(hasAppliedChaos([{ type: 'none', target: 'inbound' }]), false)
|
||||||
|
assert.strictEqual(hasAppliedChaos([{ type: 'inbound-error', target: 'inbound' }]), true)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('formatChaosEvents: formats events for diagnostics', () => {
|
||||||
|
const events = [
|
||||||
|
{ type: 'inbound-error', target: 'inbound', statusCode: 503 },
|
||||||
|
] as ChaosEvent[]
|
||||||
|
const formatted = formatChaosEvents(events)
|
||||||
|
assert.ok(formatted.includes('inbound-error'))
|
||||||
|
assert.ok(formatted.includes('503'))
|
||||||
|
})
|
||||||
|
|
||||||
|
test('formatChaosEvents: returns "No chaos applied" for empty events', () => {
|
||||||
|
assert.strictEqual(formatChaosEvents([]), 'No chaos applied')
|
||||||
|
assert.strictEqual(formatChaosEvents([{ type: 'none', target: 'inbound' }]), 'No chaos applied')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('createChaosEventArbitrary: generates deterministic events with seed', () => {
|
||||||
|
const arb = createChaosEventArbitrary(
|
||||||
|
{ probability: 1, delay: { probability: 1, minMs: 100, maxMs: 200 } },
|
||||||
|
[]
|
||||||
|
)
|
||||||
|
const sample1 = fc.sample(arb, { seed: 42, numRuns: 5 })
|
||||||
|
const sample2 = fc.sample(arb, { seed: 42, numRuns: 5 })
|
||||||
|
assert.deepStrictEqual(sample1, sample2)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('createChaosEventArbitrary: returns empty array when no config', () => {
|
||||||
|
const arb = createChaosEventArbitrary(undefined, [])
|
||||||
|
const sample = fc.sample(arb, { numRuns: 10 })
|
||||||
|
assert.ok(sample.every((events) => events.length === 0 || events.every((e) => e.type === 'none')))
|
||||||
|
})
|
||||||
|
|
||||||
|
test('createChaosEventArbitrary: generates outbound events for contracts', () => {
|
||||||
|
const arb = createChaosEventArbitrary(
|
||||||
|
{
|
||||||
|
probability: 1,
|
||||||
|
error: { probability: 1, statusCode: 503 },
|
||||||
|
outbound: [
|
||||||
|
{
|
||||||
|
target: 'api',
|
||||||
|
error: {
|
||||||
|
probability: 1,
|
||||||
|
responses: [{ statusCode: 500 }],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
['api']
|
||||||
|
)
|
||||||
|
const sample = fc.sample(arb, { numRuns: 20 })
|
||||||
|
const hasOutbound = sample.some((events) =>
|
||||||
|
events.some((e) => e.target === 'outbound')
|
||||||
|
)
|
||||||
|
assert.ok(hasOutbound, 'expected some outbound events')
|
||||||
|
})
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Flake Detection Tests
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
test('FlakeDetector: deterministic failure returns high confidence', async () => {
|
||||||
|
const detector = new FlakeDetector({ sameSeedReruns: 1, seedVariations: 2 })
|
||||||
|
const report = await detector.detectFlake(
|
||||||
|
makeTestResult(false),
|
||||||
|
async () => ({ passed: false }),
|
||||||
|
42
|
||||||
|
)
|
||||||
|
assert.strictEqual(report.isFlaky, false)
|
||||||
|
assert.strictEqual(report.confidence, 'high')
|
||||||
|
assert.strictEqual(report.reruns.length, 3)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('FlakeDetector: passing same-seed rerun flags flaky', async () => {
|
||||||
|
const detector = new FlakeDetector({ sameSeedReruns: 1, seedVariations: 0 })
|
||||||
|
let callCount = 0
|
||||||
|
const report = await detector.detectFlake(
|
||||||
|
makeTestResult(false),
|
||||||
|
async () => {
|
||||||
|
callCount++
|
||||||
|
return { passed: callCount === 1 }
|
||||||
|
},
|
||||||
|
42
|
||||||
|
)
|
||||||
|
assert.strictEqual(report.isFlaky, true)
|
||||||
|
assert.strictEqual(report.confidence, 'low')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('FlakeDetector: medium confidence when some pass', async () => {
|
||||||
|
const detector = new FlakeDetector({ sameSeedReruns: 1, seedVariations: 2 })
|
||||||
|
let callCount = 0
|
||||||
|
const report = await detector.detectFlake(
|
||||||
|
makeTestResult(false),
|
||||||
|
async () => {
|
||||||
|
callCount++
|
||||||
|
return { passed: callCount % 2 === 0 }
|
||||||
|
},
|
||||||
|
42
|
||||||
|
)
|
||||||
|
assert.strictEqual(report.isFlaky, true)
|
||||||
|
assert.strictEqual(report.confidence, 'medium')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('FlakeDetector: uses Date.now() when no seed provided', async () => {
|
||||||
|
const detector = new FlakeDetector({ sameSeedReruns: 0, seedVariations: 1 })
|
||||||
|
const report = await detector.detectFlake(
|
||||||
|
makeTestResult(false),
|
||||||
|
async () => ({ passed: false })
|
||||||
|
)
|
||||||
|
assert.strictEqual(report.reruns.length, 1)
|
||||||
|
assert.ok(report.reruns[0]!.seed > 0)
|
||||||
|
})
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Mutation Testing Tests
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
function makeContract(ensures: string[], requires: string[] = []): RouteContract {
|
||||||
|
return {
|
||||||
|
path: '/items',
|
||||||
|
method: 'POST',
|
||||||
|
category: 'constructor',
|
||||||
|
requires,
|
||||||
|
ensures,
|
||||||
|
invariants: [],
|
||||||
|
regexPatterns: {},
|
||||||
|
validateRuntime: false,
|
||||||
|
schema: {
|
||||||
|
body: {
|
||||||
|
type: 'object',
|
||||||
|
properties: { name: { type: 'string' } },
|
||||||
|
},
|
||||||
|
response: {
|
||||||
|
201: {
|
||||||
|
type: 'object',
|
||||||
|
properties: { id: { type: 'string' } },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
test('testMutation: detects flipped operator mutation', async () => {
|
||||||
|
const fastify = Fastify()
|
||||||
|
await fastify.register(swagger)
|
||||||
|
await fastify.register(apophisPlugin)
|
||||||
|
fastify.post('/items', {
|
||||||
|
schema: {
|
||||||
|
body: { type: 'object', properties: { name: { type: 'string' } } },
|
||||||
|
response: { 201: { type: 'object', properties: { id: { type: 'string' } } } },
|
||||||
|
'x-ensures': ['response_code(this) == 201'],
|
||||||
|
},
|
||||||
|
}, async (request, reply) => {
|
||||||
|
reply.status(201)
|
||||||
|
return { id: '1' }
|
||||||
|
})
|
||||||
|
await fastify.ready()
|
||||||
|
|
||||||
|
const mutation: Mutation = {
|
||||||
|
id: 'm1',
|
||||||
|
route: 'POST /items',
|
||||||
|
original: 'response_code(this) == 201',
|
||||||
|
mutated: 'response_code(this) != 201',
|
||||||
|
type: 'flip-operator',
|
||||||
|
}
|
||||||
|
|
||||||
|
const killed = await testMutation(fastify, makeContract(['response_code(this) == 201']), mutation)
|
||||||
|
assert.strictEqual(killed, true)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('runMutationTesting: produces report with score', async () => {
|
||||||
|
const fastify = Fastify()
|
||||||
|
await fastify.register(swagger)
|
||||||
|
await fastify.register(apophisPlugin)
|
||||||
|
fastify.post('/items', {
|
||||||
|
schema: {
|
||||||
|
body: { type: 'object', properties: { name: { type: 'string' } } },
|
||||||
|
response: { 201: { type: 'object', properties: { id: { type: 'string' } } } },
|
||||||
|
'x-ensures': ['response_code(this) == 201'],
|
||||||
|
},
|
||||||
|
}, async (request, reply) => {
|
||||||
|
reply.status(201)
|
||||||
|
return { id: '1' }
|
||||||
|
})
|
||||||
|
await fastify.ready()
|
||||||
|
|
||||||
|
const report = await runMutationTesting(fastify, { runs: 5, seed: 42, maxMutationsPerContract: 3 })
|
||||||
|
assert.ok(typeof report.score === 'number')
|
||||||
|
assert.ok(report.score >= 0 && report.score <= 100)
|
||||||
|
assert.ok(report.mutations.length > 0)
|
||||||
|
assert.ok(report.durationMs > 0)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('runMutationTesting: skips routes without contracts', async () => {
|
||||||
|
const fastify = Fastify()
|
||||||
|
await fastify.register(swagger)
|
||||||
|
await fastify.register(apophisPlugin)
|
||||||
|
fastify.get('/health', {}, async () => 'ok')
|
||||||
|
await fastify.ready()
|
||||||
|
|
||||||
|
const report = await runMutationTesting(fastify, { runs: 5, seed: 42 })
|
||||||
|
assert.strictEqual(report.mutations.length, 0)
|
||||||
|
assert.strictEqual(report.score, 0)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('runMutationTesting: filters by route', async () => {
|
||||||
|
const fastify = Fastify()
|
||||||
|
await fastify.register(swagger)
|
||||||
|
await fastify.register(apophisPlugin)
|
||||||
|
fastify.post('/items', {
|
||||||
|
schema: {
|
||||||
|
body: { type: 'object', properties: { name: { type: 'string' } } },
|
||||||
|
response: { 201: { type: 'object', properties: { id: { type: 'string' } } } },
|
||||||
|
'x-ensures': ['response_code(this) == 201'],
|
||||||
|
},
|
||||||
|
}, async (request, reply) => {
|
||||||
|
reply.status(201)
|
||||||
|
return { id: '1' }
|
||||||
|
})
|
||||||
|
fastify.post('/other', {
|
||||||
|
schema: {
|
||||||
|
body: { type: 'object', properties: { name: { type: 'string' } } },
|
||||||
|
response: { 201: { type: 'object', properties: { id: { type: 'string' } } } },
|
||||||
|
'x-ensures': ['response_code(this) == 201'],
|
||||||
|
},
|
||||||
|
}, async (request, reply) => {
|
||||||
|
reply.status(201)
|
||||||
|
return { id: '2' }
|
||||||
|
})
|
||||||
|
await fastify.ready()
|
||||||
|
|
||||||
|
const report = await runMutationTesting(fastify, { runs: 5, seed: 42, routes: ['/items'] })
|
||||||
|
const itemsMutations = report.mutations.filter((m) => m.mutation.route.includes('/items'))
|
||||||
|
const otherMutations = report.mutations.filter((m) => m.mutation.route.includes('/other'))
|
||||||
|
assert.ok(itemsMutations.length > 0)
|
||||||
|
assert.strictEqual(otherMutations.length, 0)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('runMutationTesting: identifies weak contracts', async () => {
|
||||||
|
const fastify = Fastify()
|
||||||
|
await fastify.register(swagger)
|
||||||
|
await fastify.register(apophisPlugin)
|
||||||
|
fastify.post('/items', {
|
||||||
|
schema: {
|
||||||
|
body: { type: 'object', properties: { name: { type: 'string' } } },
|
||||||
|
response: { 201: { type: 'object', properties: { id: { type: 'string' } } } },
|
||||||
|
'x-ensures': ['response_code(this) == 201', 'response_body(this).id != null'],
|
||||||
|
},
|
||||||
|
}, async (request, reply) => {
|
||||||
|
reply.status(201)
|
||||||
|
return { id: '1' }
|
||||||
|
})
|
||||||
|
await fastify.ready()
|
||||||
|
|
||||||
|
const report = await runMutationTesting(fastify, { runs: 5, seed: 42 })
|
||||||
|
// All mutations should be killed since the handler always returns 201 with id
|
||||||
|
assert.strictEqual(report.weakContracts.length, 0)
|
||||||
|
})
|
||||||
@@ -7,7 +7,7 @@ import apophisPlugin from '../index.js'
|
|||||||
import type { TestResult } from '../types.js'
|
import type { TestResult } from '../types.js'
|
||||||
type TestFastifyInstance = FastifyInstance & {
|
type TestFastifyInstance = FastifyInstance & {
|
||||||
apophis: {
|
apophis: {
|
||||||
contract: (opts?: { depth?: string; scope?: string; seed?: number }) => Promise<any>
|
contract: (opts?: { runs?: number; scope?: string; seed?: number }) => Promise<any>
|
||||||
spec: () => Record<string, unknown>
|
spec: () => Record<string, unknown>
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -44,19 +44,19 @@ test('scope isolation: routes with x-scope are filtered by scope parameter', asy
|
|||||||
}, async () => ({ user: true }))
|
}, async () => ({ user: true }))
|
||||||
await fastify.ready()
|
await fastify.ready()
|
||||||
// Test with no scope - should discover all 3 routes
|
// Test with no scope - should discover all 3 routes
|
||||||
const allResult = await fastify.apophis.contract({ depth: 'quick', scope: undefined })
|
const allResult = await fastify.apophis.contract({ runs: 10, scope: undefined })
|
||||||
const allPaths = new Set(allResult.tests.map((t: TestResult) => t.name.split(' ')[1]))
|
const allPaths = new Set(allResult.tests.map((t: TestResult) => t.name.split(' ')[1]))
|
||||||
assert.ok(allPaths.has('/public'), 'public route should be in all scope')
|
assert.ok(allPaths.has('/public'), 'public route should be in all scope')
|
||||||
assert.ok(allPaths.has('/admin'), 'admin route should be in all scope')
|
assert.ok(allPaths.has('/admin'), 'admin route should be in all scope')
|
||||||
assert.ok(allPaths.has('/user'), 'user route should be in all scope')
|
assert.ok(allPaths.has('/user'), 'user route should be in all scope')
|
||||||
// Test with admin scope - should only get public + admin
|
// Test with admin scope - should only get public + admin
|
||||||
const adminResult = await fastify.apophis.contract({ depth: 'quick', scope: 'admin' })
|
const adminResult = await fastify.apophis.contract({ runs: 10, scope: 'admin' })
|
||||||
const adminPaths = new Set(adminResult.tests.map((t: TestResult) => t.name.split(' ')[1]))
|
const adminPaths = new Set(adminResult.tests.map((t: TestResult) => t.name.split(' ')[1]))
|
||||||
assert.ok(adminPaths.has('/public'), 'public route should be in admin scope')
|
assert.ok(adminPaths.has('/public'), 'public route should be in admin scope')
|
||||||
assert.ok(adminPaths.has('/admin'), 'admin route should be in admin scope')
|
assert.ok(adminPaths.has('/admin'), 'admin route should be in admin scope')
|
||||||
assert.ok(!adminPaths.has('/user'), 'user route should NOT be in admin scope')
|
assert.ok(!adminPaths.has('/user'), 'user route should NOT be in admin scope')
|
||||||
// Test with user scope - should only get public + user
|
// Test with user scope - should only get public + user
|
||||||
const userResult = await fastify.apophis.contract({ depth: 'quick', scope: 'user' })
|
const userResult = await fastify.apophis.contract({ runs: 10, scope: 'user' })
|
||||||
const userPaths = new Set(userResult.tests.map((t: TestResult) => t.name.split(' ')[1]))
|
const userPaths = new Set(userResult.tests.map((t: TestResult) => t.name.split(' ')[1]))
|
||||||
assert.ok(userPaths.has('/public'), 'public route should be in user scope')
|
assert.ok(userPaths.has('/public'), 'public route should be in user scope')
|
||||||
assert.ok(!userPaths.has('/admin'), 'admin route should NOT be in user scope')
|
assert.ok(!userPaths.has('/admin'), 'admin route should NOT be in user scope')
|
||||||
@@ -88,7 +88,7 @@ test('scope isolation: scope headers are passed to requests', async () => {
|
|||||||
headers: { 'x-custom-header': 'test-value' },
|
headers: { 'x-custom-header': 'test-value' },
|
||||||
metadata: {}
|
metadata: {}
|
||||||
})
|
})
|
||||||
await fastify.apophis.contract({ depth: 'quick', scope: 'test' })
|
await fastify.apophis.contract({ runs: 10, scope: 'test' })
|
||||||
assert.strictEqual(receivedHeaders['x-custom-header'], 'test-value', 'scope header should be passed to request')
|
assert.strictEqual(receivedHeaders['x-custom-header'], 'test-value', 'scope header should be passed to request')
|
||||||
} finally {
|
} finally {
|
||||||
await fastify.close()
|
await fastify.close()
|
||||||
@@ -130,7 +130,7 @@ test('scope isolation: non-matching scope returns empty test suite', async () =>
|
|||||||
}, async () => ({ ok: true }))
|
}, async () => ({ ok: true }))
|
||||||
await fastify.ready()
|
await fastify.ready()
|
||||||
// Test with non-matching scope
|
// Test with non-matching scope
|
||||||
const result = await fastify.apophis.contract({ depth: 'quick', scope: 'other' })
|
const result = await fastify.apophis.contract({ runs: 10, scope: 'other' })
|
||||||
assert.strictEqual(result.tests.length, 0, 'no tests should run for non-matching scope')
|
assert.strictEqual(result.tests.length, 0, 'no tests should run for non-matching scope')
|
||||||
assert.strictEqual(result.summary.passed, 0, 'no tests should pass')
|
assert.strictEqual(result.summary.passed, 0, 'no tests should pass')
|
||||||
assert.strictEqual(result.summary.failed, 0, 'no tests should fail')
|
assert.strictEqual(result.summary.failed, 0, 'no tests should fail')
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ test('serverless: fastify.ready() without listen works', async () => {
|
|||||||
await fastify.ready()
|
await fastify.ready()
|
||||||
|
|
||||||
// Should be able to run tests
|
// Should be able to run tests
|
||||||
const result = await fastify.apophis.contract({ depth: 'quick' })
|
const result = await fastify.apophis.contract({ runs: 10 })
|
||||||
assert.ok(result.tests.length > 0, 'should have tests')
|
assert.ok(result.tests.length > 0, 'should have tests')
|
||||||
|
|
||||||
// Should be able to get spec
|
// Should be able to get spec
|
||||||
@@ -115,7 +115,7 @@ test('serverless: multiple ready() calls are safe', async () => {
|
|||||||
// Second ready() should be safe (idempotent)
|
// Second ready() should be safe (idempotent)
|
||||||
await fastify.ready()
|
await fastify.ready()
|
||||||
|
|
||||||
const result = await fastify.apophis.contract({ depth: 'quick' })
|
const result = await fastify.apophis.contract({ runs: 10 })
|
||||||
assert.ok(result.tests.length > 0, 'should still work after multiple ready() calls')
|
assert.ok(result.tests.length > 0, 'should still work after multiple ready() calls')
|
||||||
} finally {
|
} finally {
|
||||||
await fastify.close()
|
await fastify.close()
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ test('stateful runner handles empty routes', async () => {
|
|||||||
|
|
||||||
const result = await runStatefulTests(fastify as any, {
|
const result = await runStatefulTests(fastify as any, {
|
||||||
|
|
||||||
depth: 'quick',
|
runs: 10,
|
||||||
scope: undefined,
|
scope: undefined,
|
||||||
seed: 42,
|
seed: 42,
|
||||||
})
|
})
|
||||||
@@ -62,7 +62,7 @@ test('stateful runner executes commands', async () => {
|
|||||||
|
|
||||||
const result = await runStatefulTests(fastify as any, {
|
const result = await runStatefulTests(fastify as any, {
|
||||||
|
|
||||||
depth: 'quick',
|
runs: 10,
|
||||||
scope: undefined,
|
scope: undefined,
|
||||||
seed: 42,
|
seed: 42,
|
||||||
})
|
})
|
||||||
@@ -103,7 +103,7 @@ test('stateful runner detects status code violations', async () => {
|
|||||||
|
|
||||||
const result = await runStatefulTests(fastify as any, {
|
const result = await runStatefulTests(fastify as any, {
|
||||||
|
|
||||||
depth: 'quick',
|
runs: 10,
|
||||||
scope: undefined,
|
scope: undefined,
|
||||||
seed: 42,
|
seed: 42,
|
||||||
})
|
})
|
||||||
@@ -139,7 +139,7 @@ test('stateful runner evaluates APOSTL formulas', async () => {
|
|||||||
|
|
||||||
const result = await runStatefulTests(fastify as any, {
|
const result = await runStatefulTests(fastify as any, {
|
||||||
|
|
||||||
depth: 'quick',
|
runs: 10,
|
||||||
scope: undefined,
|
scope: undefined,
|
||||||
seed: 42,
|
seed: 42,
|
||||||
})
|
})
|
||||||
@@ -189,7 +189,7 @@ test('stateful runner tracks resource state', async () => {
|
|||||||
|
|
||||||
const result = await runStatefulTests(fastify as any, {
|
const result = await runStatefulTests(fastify as any, {
|
||||||
|
|
||||||
depth: 'quick',
|
runs: 10,
|
||||||
scope: undefined,
|
scope: undefined,
|
||||||
seed: 42,
|
seed: 42,
|
||||||
})
|
})
|
||||||
@@ -264,7 +264,7 @@ test('stateful runner substitutes path params from resource state', async () =>
|
|||||||
await fastify.ready()
|
await fastify.ready()
|
||||||
|
|
||||||
const result = await runStatefulTests(fastify as any, {
|
const result = await runStatefulTests(fastify as any, {
|
||||||
depth: 'quick',
|
runs: 10,
|
||||||
scope: undefined,
|
scope: undefined,
|
||||||
seed: 42,
|
seed: 42,
|
||||||
})
|
})
|
||||||
@@ -308,7 +308,7 @@ test('stateful runner supports config-level variants', async () => {
|
|||||||
await fastify.ready()
|
await fastify.ready()
|
||||||
|
|
||||||
const result = await runStatefulTests(fastify as any, {
|
const result = await runStatefulTests(fastify as any, {
|
||||||
depth: 'quick',
|
runs: 10,
|
||||||
scope: undefined,
|
scope: undefined,
|
||||||
seed: 42,
|
seed: 42,
|
||||||
variants: [
|
variants: [
|
||||||
@@ -359,7 +359,7 @@ test('stateful runner supports route-level x-variants', async () => {
|
|||||||
await fastify.ready()
|
await fastify.ready()
|
||||||
|
|
||||||
const result = await runStatefulTests(fastify as any, {
|
const result = await runStatefulTests(fastify as any, {
|
||||||
depth: 'quick',
|
runs: 10,
|
||||||
scope: undefined,
|
scope: undefined,
|
||||||
seed: 42,
|
seed: 42,
|
||||||
})
|
})
|
||||||
|
|||||||
+27
-10
@@ -6,7 +6,7 @@
|
|||||||
* generate → execute → validate → update → check-invariants
|
* generate → execute → validate → update → check-invariants
|
||||||
*/
|
*/
|
||||||
import type { ExtensionRegistry } from '../extension/types.js'
|
import type { ExtensionRegistry } from '../extension/types.js'
|
||||||
import { resolveDepth, resolveGenerationProfile } from '../types.js'
|
import { resolveRuns } from '../types.js'
|
||||||
import { discoverRoutes } from '../domain/discovery.js'
|
import { discoverRoutes } from '../domain/discovery.js'
|
||||||
import { convertSchema } from '../domain/schema-to-arbitrary.js'
|
import { convertSchema } from '../domain/schema-to-arbitrary.js'
|
||||||
import { SeededRng } from '../infrastructure/seeded-rng.js'
|
import { SeededRng } from '../infrastructure/seeded-rng.js'
|
||||||
@@ -22,7 +22,7 @@ import type { OutboundContractRegistry } from '../domain/outbound-contracts.js'
|
|||||||
import * as fc from 'fast-check'
|
import * as fc from 'fast-check'
|
||||||
import type { ModelState } from '../domain/stateful.js'
|
import type { ModelState } from '../domain/stateful.js'
|
||||||
import type { CleanupManager } from '../infrastructure/cleanup-manager.js'
|
import type { CleanupManager } from '../infrastructure/cleanup-manager.js'
|
||||||
import type { DepthConfig, EvalContext, FastifyInjectInstance, RouteContract, ScopeRegistry, TestConfig, TestResult, TestSuite } from '../types.js'
|
import type { EvalContext, FastifyInjectInstance, RouteContract, ScopeRegistry, TestConfig, TestResult, TestSuite } from '../types.js'
|
||||||
|
|
||||||
// Pure: hash helpers for deterministic sub-seeds
|
// Pure: hash helpers for deterministic sub-seeds
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -61,7 +61,6 @@ class ApiOperation implements StatefulApiOperation {
|
|||||||
// ============================================================================
|
// ============================================================================
|
||||||
const createCommandArbitrary = (
|
const createCommandArbitrary = (
|
||||||
routes: RouteContract[],
|
routes: RouteContract[],
|
||||||
generationProfile: 'quick' | 'standard' | 'thorough',
|
|
||||||
): { arb: fc.Arbitrary<ApiOperation>, cacheHits: number, cacheMisses: number } => {
|
): { arb: fc.Arbitrary<ApiOperation>, cacheHits: number, cacheMisses: number } => {
|
||||||
let cacheHits = 0
|
let cacheHits = 0
|
||||||
let cacheMisses = 0
|
let cacheMisses = 0
|
||||||
@@ -74,7 +73,7 @@ const createCommandArbitrary = (
|
|||||||
cacheMisses++
|
cacheMisses++
|
||||||
const bodySchema = route.schema?.body as Record<string, unknown> | undefined
|
const bodySchema = route.schema?.body as Record<string, unknown> | undefined
|
||||||
const arb = bodySchema !== undefined
|
const arb = bodySchema !== undefined
|
||||||
? convertSchema(bodySchema, { context: 'request', generationProfile })
|
? convertSchema(bodySchema, { context: 'request' })
|
||||||
: fc.constant({})
|
: fc.constant({})
|
||||||
return arb.map((params) => new ApiOperation(route, params as Record<string, unknown>))
|
return arb.map((params) => new ApiOperation(route, params as Record<string, unknown>))
|
||||||
})
|
})
|
||||||
@@ -93,8 +92,7 @@ export const runStatefulTests = async (
|
|||||||
outboundContractRegistry?: OutboundContractRegistry
|
outboundContractRegistry?: OutboundContractRegistry
|
||||||
): Promise<TestSuite> => {
|
): Promise<TestSuite> => {
|
||||||
const startTime = Date.now()
|
const startTime = Date.now()
|
||||||
const depth = resolveDepth(config.depth ?? 'standard')
|
const runConfig = resolveRuns(config.runs)
|
||||||
const generationProfile = config.generationProfile ?? resolveGenerationProfile(config.depth)
|
|
||||||
if (extensionRegistry) {
|
if (extensionRegistry) {
|
||||||
await extensionRegistry.runSuiteStartHooks(config)
|
await extensionRegistry.runSuiteStartHooks(config)
|
||||||
}
|
}
|
||||||
@@ -102,6 +100,26 @@ export const runStatefulTests = async (
|
|||||||
// Skip HEAD routes — auto-generated by Fastify for GET routes, no response body
|
// Skip HEAD routes — auto-generated by Fastify for GET routes, no response body
|
||||||
const filteredRoutes = allRoutes.filter((r) => r.category !== 'utility' && r.method !== 'HEAD')
|
const filteredRoutes = allRoutes.filter((r) => r.category !== 'utility' && r.method !== 'HEAD')
|
||||||
const routes = filterByScope(filteredRoutes, config.scope)
|
const routes = filterByScope(filteredRoutes, config.scope)
|
||||||
|
|
||||||
|
// Merge plugin contracts into route contracts
|
||||||
|
if (pluginContractRegistry) {
|
||||||
|
for (const route of routes) {
|
||||||
|
const composed = pluginContractRegistry.composeContracts(route)
|
||||||
|
for (const phase of Object.values(composed.phases)) {
|
||||||
|
for (const req of phase.requires) {
|
||||||
|
if (!route.requires.includes(req.formula)) {
|
||||||
|
route.requires.push(req.formula)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (const ens of phase.ensures) {
|
||||||
|
if (!route.ensures.includes(ens.formula)) {
|
||||||
|
route.ensures.push(ens.formula)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (routes.length === 0) {
|
if (routes.length === 0) {
|
||||||
return {
|
return {
|
||||||
tests: [],
|
tests: [],
|
||||||
@@ -131,7 +149,7 @@ export const runStatefulTests = async (
|
|||||||
variantName ? `[variant:${variantName}] ${name}` : name
|
variantName ? `[variant:${variantName}] ${name}` : name
|
||||||
// Get scope headers for test requests
|
// Get scope headers for test requests
|
||||||
const baseScopeHeaders = scopeRegistry?.getHeaders(config.scope ?? null) ?? {}
|
const baseScopeHeaders = scopeRegistry?.getHeaders(config.scope ?? null) ?? {}
|
||||||
const { arb: commandArb, cacheHits, cacheMisses } = createCommandArbitrary(routes, generationProfile)
|
const { arb: commandArb, cacheHits, cacheMisses } = createCommandArbitrary(routes)
|
||||||
let allResults: TestResult[] = []
|
let allResults: TestResult[] = []
|
||||||
let globalTestId = 0
|
let globalTestId = 0
|
||||||
// Create seeded RNG for reproducible path param selection
|
// Create seeded RNG for reproducible path param selection
|
||||||
@@ -158,7 +176,6 @@ export const runStatefulTests = async (
|
|||||||
suiteMockRuntime = createOutboundMockRuntime({
|
suiteMockRuntime = createOutboundMockRuntime({
|
||||||
contracts: allResolved,
|
contracts: allResolved,
|
||||||
mode: config.outboundMocks?.mode ?? 'example',
|
mode: config.outboundMocks?.mode ?? 'example',
|
||||||
generationProfile,
|
|
||||||
overrides: config.outboundMocks?.overrides,
|
overrides: config.outboundMocks?.overrides,
|
||||||
unmatched: config.outboundMocks?.unmatched ?? 'error',
|
unmatched: config.outboundMocks?.unmatched ?? 'error',
|
||||||
seed: outboundSeed,
|
seed: outboundSeed,
|
||||||
@@ -166,7 +183,7 @@ export const runStatefulTests = async (
|
|||||||
suiteMockRuntime.install()
|
suiteMockRuntime.install()
|
||||||
}
|
}
|
||||||
// Run property-based stateful tests per variant
|
// Run property-based stateful tests per variant
|
||||||
const numRuns = depth.statefulRuns
|
const numRuns = runConfig.statefulRuns
|
||||||
const seed = config.seed
|
const seed = config.seed
|
||||||
let counterexampleOutput: string | undefined
|
let counterexampleOutput: string | undefined
|
||||||
const hashString = (s: string): number => {
|
const hashString = (s: string): number => {
|
||||||
@@ -256,7 +273,7 @@ export const runStatefulTests = async (
|
|||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
const prop = fc.asyncProperty(
|
const prop = fc.asyncProperty(
|
||||||
fc.array(commandArb, { minLength: 1, maxLength: depth.maxCommands }),
|
fc.array(commandArb, { minLength: 1, maxLength: runConfig.maxCommands }),
|
||||||
async (cmds) => {
|
async (cmds) => {
|
||||||
await runSequence(cmds)
|
await runSequence(cmds)
|
||||||
return true
|
return true
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ import type {
|
|||||||
TestConfig,
|
TestConfig,
|
||||||
TestResult,
|
TestResult,
|
||||||
} from '../types.js'
|
} from '../types.js'
|
||||||
import { resolveDepth, resolveGenerationProfile } from '../types.js'
|
import { resolveRuns } from '../types.js'
|
||||||
|
|
||||||
export const runTripleBoundaryPropertyTest = async (
|
export const runTripleBoundaryPropertyTest = async (
|
||||||
route: RouteContract,
|
route: RouteContract,
|
||||||
@@ -39,10 +39,9 @@ export const runTripleBoundaryPropertyTest = async (
|
|||||||
if (!config.chaos) return []
|
if (!config.chaos) return []
|
||||||
|
|
||||||
const results: TestResult[] = []
|
const results: TestResult[] = []
|
||||||
const generationProfile = config.generationProfile ?? resolveGenerationProfile(config.depth)
|
const arbitrary = createTripleBoundaryArbitrary(route, contracts, config.chaos)
|
||||||
const arbitrary = createTripleBoundaryArbitrary(route, contracts, config.chaos, generationProfile)
|
const runConfig = resolveRuns(config.runs)
|
||||||
const depth = resolveDepth(config.depth ?? 'standard')
|
const numRuns = Math.max(10, Math.floor(runConfig.contractRuns / 2))
|
||||||
const numRuns = Math.max(10, Math.floor(depth.contractRuns / 2))
|
|
||||||
|
|
||||||
const property = fc.asyncProperty(arbitrary, async (cmd) => {
|
const property = fc.asyncProperty(arbitrary, async (cmd) => {
|
||||||
const testId = testIdBase + results.length + 1
|
const testId = testIdBase + results.length + 1
|
||||||
|
|||||||
+2
-5
@@ -39,18 +39,15 @@ export type {
|
|||||||
|
|
||||||
// Formula and test configuration types
|
// Formula and test configuration types
|
||||||
export type {
|
export type {
|
||||||
TestDepth,
|
|
||||||
TestConfig,
|
TestConfig,
|
||||||
ResolvedOutboundContract,
|
ResolvedOutboundContract,
|
||||||
OutboundChaosConfig,
|
OutboundChaosConfig,
|
||||||
ChaosConfig,
|
ChaosConfig,
|
||||||
DepthConfig,
|
RunConfig,
|
||||||
} from './types/formula.js'
|
} from './types/formula.js'
|
||||||
|
|
||||||
export {
|
export {
|
||||||
DEPTH_CONFIGS,
|
resolveRuns,
|
||||||
resolveDepth,
|
|
||||||
resolveGenerationProfile,
|
|
||||||
} from './types/formula.js'
|
} from './types/formula.js'
|
||||||
|
|
||||||
// Extension types
|
// Extension types
|
||||||
|
|||||||
+1
-1
@@ -201,7 +201,7 @@ export interface ApophisTestDecorations {
|
|||||||
|
|
||||||
// Forward declarations to avoid circular deps — these are defined in sibling modules
|
// Forward declarations to avoid circular deps — these are defined in sibling modules
|
||||||
export interface TestConfig {
|
export interface TestConfig {
|
||||||
readonly depth?: import('./formula.js').TestDepth
|
readonly runs?: number
|
||||||
readonly scope?: string
|
readonly scope?: string
|
||||||
readonly seed?: number
|
readonly seed?: number
|
||||||
readonly timeout?: number
|
readonly timeout?: number
|
||||||
|
|||||||
+10
-34
@@ -7,11 +7,8 @@
|
|||||||
// Test: Configuration
|
// Test: Configuration
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
|
|
||||||
export type TestDepth = 'quick' | 'standard' | 'thorough' | { runs: number }
|
|
||||||
|
|
||||||
export interface TestConfig {
|
export interface TestConfig {
|
||||||
readonly depth?: TestDepth
|
readonly runs?: number
|
||||||
readonly generationProfile?: GenerationProfile
|
|
||||||
readonly scope?: string
|
readonly scope?: string
|
||||||
readonly seed?: number
|
readonly seed?: number
|
||||||
readonly timeout?: number
|
readonly timeout?: number
|
||||||
@@ -204,49 +201,28 @@ export interface ChaosConfig {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
// Depth Configuration
|
// Test run configuration
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
|
|
||||||
export interface DepthConfig {
|
export interface RunConfig {
|
||||||
readonly contractRuns: number
|
readonly contractRuns: number
|
||||||
readonly propertyRuns: number
|
readonly propertyRuns: number
|
||||||
readonly statefulRuns: number
|
readonly statefulRuns: number
|
||||||
readonly maxCommands: number
|
readonly maxCommands: number
|
||||||
}
|
}
|
||||||
|
|
||||||
export type GenerationProfile = 'quick' | 'standard' | 'thorough'
|
const DEFAULT_RUNS = 50
|
||||||
|
|
||||||
export const DEPTH_CONFIGS: Record<'quick' | 'standard' | 'thorough', DepthConfig> = {
|
export function resolveRuns(runs: number | undefined): RunConfig {
|
||||||
quick: { contractRuns: 10, propertyRuns: 50, statefulRuns: 5, maxCommands: 10 },
|
const r = runs ?? DEFAULT_RUNS
|
||||||
standard: { contractRuns: 50, propertyRuns: 100, statefulRuns: 20, maxCommands: 30 },
|
|
||||||
thorough: { contractRuns: 200, propertyRuns: 1000, statefulRuns: 100, maxCommands: 50 }
|
|
||||||
}
|
|
||||||
|
|
||||||
export function resolveDepth(depth: TestDepth): DepthConfig {
|
|
||||||
if (typeof depth === 'string') {
|
|
||||||
return DEPTH_CONFIGS[depth]
|
|
||||||
}
|
|
||||||
return {
|
return {
|
||||||
contractRuns: depth.runs,
|
contractRuns: r,
|
||||||
propertyRuns: depth.runs,
|
propertyRuns: r * 2,
|
||||||
statefulRuns: Math.max(1, Math.floor(depth.runs / 10)),
|
statefulRuns: Math.max(1, Math.floor(r / 10)),
|
||||||
maxCommands: Math.max(5, Math.floor(depth.runs / 5)),
|
maxCommands: Math.max(5, Math.floor(r / 2)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export function resolveGenerationProfile(depth: TestDepth | undefined): GenerationProfile {
|
|
||||||
if (depth === undefined) {
|
|
||||||
return 'standard'
|
|
||||||
}
|
|
||||||
if (typeof depth === 'string') {
|
|
||||||
return depth
|
|
||||||
}
|
|
||||||
|
|
||||||
if (depth.runs <= 25) return 'quick'
|
|
||||||
if (depth.runs >= 250) return 'thorough'
|
|
||||||
return 'standard'
|
|
||||||
}
|
|
||||||
|
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
// Test: Results
|
// Test: Results
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
|
|||||||
Reference in New Issue
Block a user