task ep.1.4.1
This commit is contained in:
@@ -0,0 +1,172 @@
|
||||
# Task EP.1.4.1 Transactional Outbox Activation & Worker Runtime - 2026-07-13
|
||||
|
||||
## Scope
|
||||
|
||||
- activated Activity Business Event publication through transaction-aware outbox enqueue
|
||||
- added lease-based outbox claiming fields to `business_event_outbox`
|
||||
- added projection outbox worker runtime
|
||||
- added manual projection operations service contracts
|
||||
- added PostgreSQL `FOR UPDATE SKIP LOCKED` claim strategy in Drizzle store
|
||||
- preserved existing in-process Business Event publisher for compatibility and tests
|
||||
- preserved current Activity API response contracts, follow-up APIs, dashboard/report datasets, notifications, approvals, and recent-activity behavior
|
||||
|
||||
## Review Summary
|
||||
|
||||
Reviewed before and during implementation:
|
||||
|
||||
- `AGENTS.md`
|
||||
- `plans/task-ep.1.4.1.md`
|
||||
- `docs/implementation/task-ep1.3-business-event-foundation-2026-07-13.md`
|
||||
- `docs/implementation/task-ep1.4-projection-foundation-delivery-reliability-2026-07-13.md`
|
||||
- `docs/standards/engineering-constitution.md`
|
||||
- `docs/standards/project-foundations.md`
|
||||
- `docs/standards/architecture-rules.md`
|
||||
- `docs/standards/task-review-checklist.md`
|
||||
- `docs/security/crm-authorization-boundaries.md`
|
||||
- existing Activity service mutation paths
|
||||
- existing Business Event publisher and projection runtime
|
||||
|
||||
## Implementation Summary
|
||||
|
||||
### Activity Transactional Outbox Adoption
|
||||
|
||||
Activity mutation paths now enqueue Business Events to the outbox inside the same `db.transaction()` as the Activity row mutation:
|
||||
|
||||
- create
|
||||
- update
|
||||
- assign/reassign
|
||||
- reschedule
|
||||
- complete
|
||||
- cancel
|
||||
- delete
|
||||
|
||||
The Activity API still returns hydrated Activity records after commit. Projection consumers are not run synchronously from Activity source mutations.
|
||||
|
||||
### Transaction-Aware Publisher
|
||||
|
||||
Added:
|
||||
|
||||
- `src/features/foundation/projections/server/transactional-publisher.ts`
|
||||
|
||||
The publisher validates registry and payload contracts, then inserts into `business_event_outbox` using the supplied transaction/client. This prevents accidental double-publication through the in-process dispatcher.
|
||||
|
||||
### Worker Runtime
|
||||
|
||||
Added:
|
||||
|
||||
- `src/features/foundation/projections/worker.ts`
|
||||
|
||||
Worker behavior:
|
||||
|
||||
- claims available events in batches
|
||||
- applies a lease with `claimedBy`, `claimedAt`, and `claimExpiresAt`
|
||||
- dispatches claimed committed events through `ProjectionConsumerRuntime`
|
||||
- does not auto-start unless `PROJECTION_WORKER_ENABLED=true`
|
||||
- supports graceful stop semantics for the in-process loop
|
||||
|
||||
Default operational config:
|
||||
|
||||
- polling interval: `3000ms`
|
||||
- batch size: `25`
|
||||
- max concurrent events: `1`
|
||||
- max concurrent consumers: `4`
|
||||
- claim lease: `60000ms`
|
||||
- stale claim: `90000ms`
|
||||
- graceful shutdown: `30000ms`
|
||||
|
||||
### Multi-Instance Claiming
|
||||
|
||||
Added schema fields:
|
||||
|
||||
- `business_event_outbox.claimed_by`
|
||||
- `business_event_outbox.claimed_at`
|
||||
- `business_event_outbox.claim_expires_at`
|
||||
|
||||
Generated migration:
|
||||
|
||||
- `drizzle/0004_sharp_mercury.sql`
|
||||
|
||||
Database store uses PostgreSQL-safe claiming with `FOR UPDATE SKIP LOCKED`. Expired `claimed` or `processing` leases are recoverable.
|
||||
|
||||
### Manual Operations Contracts
|
||||
|
||||
Added:
|
||||
|
||||
- `src/features/foundation/projections/operations.ts`
|
||||
|
||||
Contracts:
|
||||
|
||||
- `drainOutbox`
|
||||
- `retryEvent`
|
||||
- `retryConsumer`
|
||||
- `retryDeadLetter`
|
||||
- `resolveDeadLetter`
|
||||
|
||||
Manual operations require `systemRole: "super_admin"` at the service-contract boundary. No public UI/API route was added in this task.
|
||||
|
||||
### Delivery State Machine
|
||||
|
||||
Outbox lifecycle:
|
||||
|
||||
```text
|
||||
pending -> claimed -> processing -> completed
|
||||
pending -> claimed -> processing -> retry_scheduled
|
||||
pending -> claimed -> processing -> dead_letter
|
||||
claimed / processing -> lease expired -> claimable again
|
||||
```
|
||||
|
||||
Checkpoint lifecycle remains:
|
||||
|
||||
```text
|
||||
pending -> processing -> completed
|
||||
pending -> processing -> retry_scheduled
|
||||
pending -> processing -> skipped_unsupported_version
|
||||
pending -> processing -> dead_letter
|
||||
```
|
||||
|
||||
Completed checkpoints are not rerun when another consumer fails.
|
||||
|
||||
### Worker Startup Strategy
|
||||
|
||||
Current strategy:
|
||||
|
||||
- no automatic worker startup in Next.js request/build/test paths
|
||||
- dedicated process/service should instantiate `ProjectionOutboxWorker` and call `start()`
|
||||
- scheduled CLI drain can call `drainOnce()` for UAT or operational repair
|
||||
|
||||
This avoids duplicate unmanaged loops during hot reload, build, migrations, tests, or serverless request execution.
|
||||
|
||||
### SLA Baseline
|
||||
|
||||
Initial non-binding targets:
|
||||
|
||||
- event available after commit: immediate
|
||||
- polling interval: 1-5 seconds, default 3 seconds
|
||||
- normal Activity projection delivery: under 10 seconds
|
||||
- retry scheduling accuracy: within one polling interval
|
||||
- stale claim recovery: lease duration plus one polling interval
|
||||
- worker heartbeat/health integration: future operational surface
|
||||
|
||||
## Compatibility Notes
|
||||
|
||||
- Activity events are not emitted through both outbox and in-process publisher in production service paths.
|
||||
- Existing in-process publisher remains available for tests and explicitly approved compatibility paths.
|
||||
- Existing Dashboard, Report, Notification, Approval, Follow-up, and recent-activity behavior remains unchanged.
|
||||
- No final Timeline, Calendar, Notification expansion, My Day, Manager Workspace, Executive Workspace, Forecast, or Relationship Health feature was implemented.
|
||||
|
||||
## Verification
|
||||
|
||||
- `node --disable-warning=ExperimentalWarning --disable-warning=MODULE_TYPELESS_PACKAGE_JSON --experimental-strip-types --test src/features/foundation/business-events/*.test.ts src/features/foundation/projections/*.test.ts src/features/crm/activities/server/business-events.test.ts`
|
||||
- `npx oxlint src/features/foundation/projections src/features/crm/activities/server/business-events.ts src/features/crm/activities/server/repository.ts src/features/crm/activities/server/service.ts src/db/schema.ts`
|
||||
- `npm run db:generate`
|
||||
|
||||
## Typecheck Note
|
||||
|
||||
`npm run typecheck` is currently blocked by generated `.next/dev/types/routes.d.ts` syntax errors unrelated to the EP.1.4.1 source changes. Targeted runtime tests and scoped lint passed for the changed files.
|
||||
|
||||
## Residual Risks / Follow-up
|
||||
|
||||
- Worker process entrypoint/CLI command is not yet wired into `package.json`; runtime modules are ready for a dedicated process or scheduled drain task.
|
||||
- Manual retry/dead-letter APIs are intentionally not exposed publicly yet.
|
||||
- Health snapshots are recorded by runtime; a user-facing operational health page remains future work.
|
||||
- Future source domains must adopt the same transaction-aware publisher through dedicated cutover tasks.
|
||||
4
drizzle/0004_sharp_mercury.sql
Normal file
4
drizzle/0004_sharp_mercury.sql
Normal file
@@ -0,0 +1,4 @@
|
||||
ALTER TABLE "business_event_outbox" ADD COLUMN "claimed_by" text;--> statement-breakpoint
|
||||
ALTER TABLE "business_event_outbox" ADD COLUMN "claimed_at" timestamp with time zone;--> statement-breakpoint
|
||||
ALTER TABLE "business_event_outbox" ADD COLUMN "claim_expires_at" timestamp with time zone;--> statement-breakpoint
|
||||
CREATE INDEX "business_event_outbox_claim_lease_idx" ON "business_event_outbox" USING btree ("delivery_status","claim_expires_at");
|
||||
7532
drizzle/meta/0004_snapshot.json
Normal file
7532
drizzle/meta/0004_snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
@@ -29,6 +29,13 @@
|
||||
"when": 1783917011479,
|
||||
"tag": "0003_green_squadron_supreme",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 4,
|
||||
"version": "7",
|
||||
"when": 1783922653023,
|
||||
"tag": "0004_sharp_mercury",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
909
plans/task-ep.1.4.1.md
Normal file
909
plans/task-ep.1.4.1.md
Normal file
@@ -0,0 +1,909 @@
|
||||
# Task EP.1.4.1 – Transactional Outbox Activation & Worker Runtime
|
||||
|
||||
Status: Completed
|
||||
Priority: Critical
|
||||
Type: Platform Runtime / Delivery Reliability / Background Processing
|
||||
|
||||
## Depends On
|
||||
|
||||
* EP.1.1 Activity Domain Foundation
|
||||
* EP.1.2 Activity Integration & Legacy Follow-up Adapter
|
||||
* EP.1.3 Business Event Foundation
|
||||
* EP.1.4 Projection Foundation & Delivery Reliability
|
||||
* AR.1 Architecture Transition Plan
|
||||
* AR.2 Epic & Technical Design
|
||||
* ENG.0 Engineering Constitution
|
||||
|
||||
---
|
||||
|
||||
# Objective
|
||||
|
||||
Activate the Transactional Outbox delivery model introduced in EP.1.4 and provide a production-ready worker runtime for reliable Business Event dispatch.
|
||||
|
||||
This task must ensure that:
|
||||
|
||||
* source-domain mutations and Business Event outbox persistence occur atomically
|
||||
* only committed outbox events are dispatched
|
||||
* projection consumers process events independently and idempotently
|
||||
* worker execution is safe across multiple application instances
|
||||
* retryable failures are rescheduled
|
||||
* terminal failures are moved to governed dead-letter state
|
||||
* successful source mutations are never reported as failed merely because a projection consumer fails after commit
|
||||
* existing CRM APIs and business behavior remain backward compatible
|
||||
|
||||
The first activated source domain is Activity.
|
||||
|
||||
Other source domains may adopt the same transaction pattern in later controlled tasks.
|
||||
|
||||
---
|
||||
|
||||
# Background
|
||||
|
||||
EP.1.3 introduced:
|
||||
|
||||
* canonical `BusinessEvent`
|
||||
* Event Registry
|
||||
* Publisher and Dispatcher abstractions
|
||||
* Activity lifecycle event publication
|
||||
* Event Sequence Matrix
|
||||
* Event Consumer Matrix
|
||||
|
||||
EP.1.4 introduced:
|
||||
|
||||
* `business_event_outbox`
|
||||
* projection consumer checkpoints
|
||||
* projection dead letters
|
||||
* projection rebuild runs
|
||||
* projection health snapshots
|
||||
* persistent projection runtime contracts
|
||||
* retry policy
|
||||
* Projection Registry
|
||||
* no-op skeleton consumers
|
||||
|
||||
However, production delivery guarantees are not active yet because:
|
||||
|
||||
1. Activity source mutations do not yet persist events atomically through the outbox in every mutation path.
|
||||
2. No worker currently drains pending outbox events.
|
||||
3. No multi-instance-safe event claiming process exists.
|
||||
4. No operational retry or dead-letter processing loop is active.
|
||||
5. Manual retry and drain contracts are not yet available.
|
||||
|
||||
EP.1.4.1 activates these foundations.
|
||||
|
||||
---
|
||||
|
||||
# Architecture Principles
|
||||
|
||||
## 1. Atomic Source Mutation and Event Persistence
|
||||
|
||||
The source mutation and related outbox event must use the same database transaction.
|
||||
|
||||
Required pattern:
|
||||
|
||||
```text
|
||||
BEGIN
|
||||
|
||||
Update source-domain state
|
||||
|
||||
Insert Business Event Outbox record
|
||||
|
||||
Write mandatory Audit Log where required
|
||||
|
||||
COMMIT
|
||||
```
|
||||
|
||||
If outbox persistence fails inside the transaction, the source mutation must roll back.
|
||||
|
||||
---
|
||||
|
||||
## 2. Post-Commit Consumer Processing
|
||||
|
||||
Projection consumers process events only after the source transaction commits.
|
||||
|
||||
A projection failure must not change the result of the already committed source mutation.
|
||||
|
||||
Example:
|
||||
|
||||
```text
|
||||
Activity completed
|
||||
↓
|
||||
Activity row + Outbox event committed
|
||||
↓
|
||||
API returns success
|
||||
↓
|
||||
Worker dispatches event
|
||||
↓
|
||||
Timeline consumer fails
|
||||
↓
|
||||
Checkpoint schedules retry
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. At-Least-Once Delivery
|
||||
|
||||
The runtime must assume that an event may be delivered more than once.
|
||||
|
||||
Exactly-once transport is not required.
|
||||
|
||||
Consumer idempotency and persistent checkpoints provide duplicate protection.
|
||||
|
||||
---
|
||||
|
||||
## 4. Consumer Isolation
|
||||
|
||||
An event may have multiple consumers.
|
||||
|
||||
Example:
|
||||
|
||||
```text
|
||||
activity.completed
|
||||
├── Timeline
|
||||
├── Calendar
|
||||
├── Dashboard
|
||||
└── Notification
|
||||
```
|
||||
|
||||
One failed consumer must not erase successful processing by other consumers.
|
||||
|
||||
Retries must target only incomplete consumers.
|
||||
|
||||
---
|
||||
|
||||
## 5. Multi-Instance Safety
|
||||
|
||||
More than one application or worker process may run concurrently.
|
||||
|
||||
Only one worker may claim a given event delivery unit at a time.
|
||||
|
||||
The design must support:
|
||||
|
||||
* database-level claiming
|
||||
* lease expiration
|
||||
* stale claim recovery
|
||||
* duplicate-safe retries
|
||||
* graceful worker shutdown
|
||||
|
||||
---
|
||||
|
||||
## 6. Preserve Existing Behavior
|
||||
|
||||
Existing Activity APIs and source-domain behavior remain unchanged from the caller’s perspective.
|
||||
|
||||
This task must not replace:
|
||||
|
||||
* existing follow-up APIs
|
||||
* Dashboard datasets
|
||||
* Report datasets
|
||||
* existing Notification behavior
|
||||
* existing Approval notification flow
|
||||
* recent-activity compatibility reads
|
||||
|
||||
---
|
||||
|
||||
# Review Required
|
||||
|
||||
## Governance and Architecture
|
||||
|
||||
* `AGENTS.md`
|
||||
* `docs/standards/engineering-constitution.md`
|
||||
* `docs/standards/project-foundations.md`
|
||||
* `docs/standards/architecture-rules.md`
|
||||
* `docs/standards/task-review-checklist.md`
|
||||
* `docs/security/crm-authorization-boundaries.md`
|
||||
* AR.1 Architecture Transition Plan
|
||||
* AR.2 Epic & Technical Design
|
||||
|
||||
## Existing Implementations
|
||||
|
||||
* EP.1.3 Business Event Foundation report and code
|
||||
* EP.1.4 Projection Foundation report and code
|
||||
* Activity service mutation paths
|
||||
* Business Event publisher
|
||||
* persistent outbox publisher
|
||||
* projection runtime
|
||||
* projection consumer checkpoints
|
||||
* projection dead letters
|
||||
* projection health snapshots
|
||||
* current Drizzle transaction conventions
|
||||
* existing scheduled-job or worker patterns
|
||||
* current application startup and shutdown behavior
|
||||
* current deployment topology and Docker runtime, where documented
|
||||
|
||||
---
|
||||
|
||||
# Scope
|
||||
|
||||
## Part 1 — Activity Transaction Boundary Refactor
|
||||
|
||||
Refactor Activity mutation paths to support transaction-scoped persistence.
|
||||
|
||||
Minimum operations:
|
||||
|
||||
* create
|
||||
* update
|
||||
* assign
|
||||
* reassign
|
||||
* start
|
||||
* reschedule
|
||||
* complete
|
||||
* cancel
|
||||
* delete, if governed deletion remains supported
|
||||
|
||||
Each operation must persist:
|
||||
|
||||
* Activity mutation
|
||||
* Business Event Outbox record
|
||||
* required Audit Log record
|
||||
|
||||
inside one shared transaction where applicable.
|
||||
|
||||
### Rules
|
||||
|
||||
* Route Handlers remain thin.
|
||||
* Activity Service owns business transitions.
|
||||
* Transaction object must be propagated through approved service and persistence seams.
|
||||
* No nested independent transaction may break atomicity.
|
||||
* Event payload is created from the successful mutation result.
|
||||
* An event must never be inserted before business validation completes.
|
||||
|
||||
---
|
||||
|
||||
## Part 2 — Transaction-Aware Outbox Publisher
|
||||
|
||||
Implement or complete a transaction-aware outbox publisher.
|
||||
|
||||
Recommended contract:
|
||||
|
||||
```ts
|
||||
export interface TransactionalBusinessEventPublisher {
|
||||
enqueue(
|
||||
transaction: DatabaseTransaction,
|
||||
event: BusinessEvent
|
||||
): Promise<void>;
|
||||
|
||||
enqueueMany(
|
||||
transaction: DatabaseTransaction,
|
||||
events: BusinessEvent[]
|
||||
): Promise<void>;
|
||||
}
|
||||
```
|
||||
|
||||
### Responsibilities
|
||||
|
||||
* validate registry entry
|
||||
* validate schema version
|
||||
* validate payload
|
||||
* persist event using the supplied transaction
|
||||
* preserve correlation and causation metadata
|
||||
* prevent duplicate event insertion where an operation idempotency key exists
|
||||
* not dispatch consumers synchronously
|
||||
|
||||
---
|
||||
|
||||
## Part 3 — Outbox Worker Runtime
|
||||
|
||||
Introduce a worker that drains committed outbox events.
|
||||
|
||||
Worker lifecycle:
|
||||
|
||||
```text
|
||||
poll
|
||||
↓
|
||||
claim batch
|
||||
↓
|
||||
load active consumers
|
||||
↓
|
||||
dispatch each incomplete consumer
|
||||
↓
|
||||
record checkpoints
|
||||
↓
|
||||
mark event delivery state
|
||||
↓
|
||||
sleep / repeat
|
||||
```
|
||||
|
||||
### Worker Configuration
|
||||
|
||||
Support configurable:
|
||||
|
||||
* enabled state
|
||||
* polling interval
|
||||
* batch size
|
||||
* maximum concurrent events
|
||||
* maximum concurrent consumers
|
||||
* claim lease duration
|
||||
* stale claim threshold
|
||||
* worker instance identifier
|
||||
* graceful shutdown timeout
|
||||
|
||||
Configuration must use existing environment and configuration conventions.
|
||||
|
||||
---
|
||||
|
||||
## Part 4 — Multi-Instance Event Claiming
|
||||
|
||||
Implement a database-safe claiming strategy.
|
||||
|
||||
Approved approaches may include:
|
||||
|
||||
* `FOR UPDATE SKIP LOCKED`
|
||||
* atomic conditional update with lease token
|
||||
* another PostgreSQL-safe claiming mechanism
|
||||
|
||||
Each claimed record should track, as appropriate:
|
||||
|
||||
* claimed by
|
||||
* claimed at
|
||||
* lease expires at
|
||||
* processing started at
|
||||
* attempt count
|
||||
|
||||
### Rules
|
||||
|
||||
* two workers must not actively process the same event claim concurrently
|
||||
* expired claims must be recoverable
|
||||
* a crashed worker must not permanently block delivery
|
||||
* claiming must be organization-safe but may process multiple organizations in one batch
|
||||
|
||||
---
|
||||
|
||||
## Part 5 — Event and Consumer Completion Semantics
|
||||
|
||||
Freeze and implement event completion rules.
|
||||
|
||||
Recommended model:
|
||||
|
||||
* Outbox state represents event dispatch lifecycle.
|
||||
* Consumer checkpoint represents per-consumer result.
|
||||
* Event is terminally completed only when all applicable active consumers reach a terminal state.
|
||||
|
||||
Terminal consumer states:
|
||||
|
||||
* completed
|
||||
* skipped unsupported version, when governed as terminal
|
||||
* dead letter
|
||||
* explicitly ignored by registry policy
|
||||
|
||||
### Important Rule
|
||||
|
||||
A successful consumer must not be called again merely because another consumer failed.
|
||||
|
||||
---
|
||||
|
||||
## Part 6 — Retry Processor
|
||||
|
||||
Activate retry processing using the EP.1.4 policy.
|
||||
|
||||
Default policy:
|
||||
|
||||
```text
|
||||
attempt 1: immediate
|
||||
attempt 2: +1 minute
|
||||
attempt 3: +5 minutes
|
||||
attempt 4: +30 minutes
|
||||
attempt 5: dead letter
|
||||
```
|
||||
|
||||
Implement:
|
||||
|
||||
* retry scheduling
|
||||
* retryable-error classification
|
||||
* non-retryable-error classification
|
||||
* next-attempt selection
|
||||
* attempt count update
|
||||
* consumer-specific retry
|
||||
* jitter where appropriate
|
||||
* stale-processing recovery
|
||||
|
||||
Retry values must be configurable or governed centrally.
|
||||
|
||||
---
|
||||
|
||||
## Part 7 — Dead-Letter Runtime
|
||||
|
||||
Activate dead-letter creation when:
|
||||
|
||||
* maximum retries are exceeded
|
||||
* failure is explicitly non-retryable
|
||||
* event version is unsupported and policy requires review
|
||||
* registry or payload incompatibility cannot be resolved automatically
|
||||
|
||||
Dead-letter records must include:
|
||||
|
||||
* event reference
|
||||
* consumer name
|
||||
* event type
|
||||
* organization id
|
||||
* attempt count
|
||||
* sanitized error code
|
||||
* sanitized error message
|
||||
* first failure time
|
||||
* final failure time
|
||||
* resolution status
|
||||
|
||||
Do not duplicate unrestricted event payloads into dead-letter storage.
|
||||
|
||||
---
|
||||
|
||||
## Part 8 — Manual Retry and Drain Service Contracts
|
||||
|
||||
Provide protected service contracts for operations support.
|
||||
|
||||
Minimum capabilities:
|
||||
|
||||
* drain pending outbox events
|
||||
* retry one failed consumer checkpoint
|
||||
* retry all failed consumers for one event
|
||||
* retry one dead-letter item
|
||||
* mark dead letter resolved with reason
|
||||
* inspect sanitized processing status
|
||||
* recover stale claims
|
||||
|
||||
Recommended service methods:
|
||||
|
||||
```ts
|
||||
drainOutbox(options)
|
||||
retryConsumer(eventId, consumerName, actor)
|
||||
retryEvent(eventId, actor)
|
||||
retryDeadLetter(deadLetterId, actor)
|
||||
resolveDeadLetter(deadLetterId, reason, actor)
|
||||
recoverStaleClaims(options)
|
||||
```
|
||||
|
||||
Public UI is not required.
|
||||
|
||||
Admin API routes are optional and should only be added when authorization and operational need are clearly defined.
|
||||
|
||||
---
|
||||
|
||||
## Part 9 — Worker Startup Strategy
|
||||
|
||||
Define how the worker runs in current deployment environments.
|
||||
|
||||
Evaluate:
|
||||
|
||||
* worker inside application process
|
||||
* dedicated Node worker process
|
||||
* scheduled CLI drain command
|
||||
* separate Docker service using the same application image
|
||||
|
||||
Preferred production direction:
|
||||
|
||||
> Dedicated worker process or service using shared runtime modules.
|
||||
|
||||
The implementation must not accidentally start duplicate unmanaged polling loops during:
|
||||
|
||||
* Next.js hot reload
|
||||
* serverless request execution
|
||||
* build phase
|
||||
* migration commands
|
||||
* test execution
|
||||
|
||||
---
|
||||
|
||||
## Part 10 — Graceful Shutdown
|
||||
|
||||
Worker shutdown must:
|
||||
|
||||
* stop claiming new events
|
||||
* allow active handlers to finish within timeout
|
||||
* release or expire unfinished leases safely
|
||||
* flush structured logs
|
||||
* report final worker status
|
||||
|
||||
Handle common termination signals according to runtime conventions.
|
||||
|
||||
---
|
||||
|
||||
## Part 11 — Projection Health Updates
|
||||
|
||||
Update projection health from real worker processing.
|
||||
|
||||
Track:
|
||||
|
||||
* last event observed
|
||||
* last successful event
|
||||
* pending count
|
||||
* processing count
|
||||
* retry count
|
||||
* dead-letter count
|
||||
* unsupported-version count
|
||||
* last failure time
|
||||
* processing lag
|
||||
* worker heartbeat
|
||||
* last successful drain
|
||||
|
||||
Health storage must remain payload-free.
|
||||
|
||||
---
|
||||
|
||||
## Part 12 — Delivery SLA Baseline
|
||||
|
||||
Freeze initial operational targets.
|
||||
|
||||
Suggested targets:
|
||||
|
||||
| Capability | Initial Target |
|
||||
| ---------------------------------------------- | ----------------------------------------------: |
|
||||
| Outbox event available after commit | immediate |
|
||||
| Worker polling interval | 1–5 seconds |
|
||||
| Activity projection delivery under normal load | under 10 seconds |
|
||||
| Retry scheduling accuracy | within one polling interval |
|
||||
| Stale claim recovery | within lease duration plus one polling interval |
|
||||
| Worker health heartbeat | every 30–60 seconds |
|
||||
|
||||
Final values must reflect deployment constraints and can be classified as initial non-binding SLOs.
|
||||
|
||||
---
|
||||
|
||||
## Part 13 — Security and Authorization
|
||||
|
||||
Worker processing must:
|
||||
|
||||
* maintain organization scoping
|
||||
* not bypass pricing-sensitive event restrictions
|
||||
* not expose internal-only Activity details
|
||||
* use consumer-specific security rules
|
||||
* avoid persisting decrypted credentials or secrets
|
||||
* sanitize operational errors
|
||||
* audit manual retry and dead-letter resolution actions
|
||||
|
||||
Manual operations must require explicit foundation or system administration permission.
|
||||
|
||||
---
|
||||
|
||||
## Part 14 — Existing Publisher Compatibility
|
||||
|
||||
Preserve the existing in-process publisher where needed for:
|
||||
|
||||
* tests
|
||||
* non-production isolated execution
|
||||
* compatibility consumers not yet cut over
|
||||
|
||||
However, Activity production mutation paths must use the transactional outbox publisher after activation.
|
||||
|
||||
Do not publish the same Activity event through both in-process and outbox paths.
|
||||
|
||||
Add a guard or configuration strategy preventing accidental double publication.
|
||||
|
||||
---
|
||||
|
||||
## Part 15 — Consumer Registration Snapshot
|
||||
|
||||
Freeze the consumer set used when determining applicable checkpoints.
|
||||
|
||||
Define behavior when:
|
||||
|
||||
* a new consumer is registered after an event was completed
|
||||
* a consumer is disabled
|
||||
* a consumer is renamed
|
||||
* a consumer’s supported versions change
|
||||
* an event has no consumers
|
||||
* a projection remains contract-only
|
||||
|
||||
Recommended approach:
|
||||
|
||||
* persist or deterministically resolve applicable consumers at dispatch time
|
||||
* document replay or backfill behavior for newly introduced consumers
|
||||
* avoid reopening old completed events automatically without explicit rebuild
|
||||
|
||||
---
|
||||
|
||||
## Part 16 — Delivery State Machine
|
||||
|
||||
Freeze the outbox delivery lifecycle.
|
||||
|
||||
Recommended model:
|
||||
|
||||
```text
|
||||
pending
|
||||
↓
|
||||
claimed
|
||||
↓
|
||||
processing
|
||||
├── completed
|
||||
├── retry_scheduled
|
||||
└── dead_letter
|
||||
```
|
||||
|
||||
Possible recovery paths:
|
||||
|
||||
```text
|
||||
claimed / processing
|
||||
↓ lease expired
|
||||
pending
|
||||
```
|
||||
|
||||
Consumer checkpoint lifecycle:
|
||||
|
||||
```text
|
||||
pending
|
||||
↓
|
||||
processing
|
||||
├── completed
|
||||
├── retry_scheduled
|
||||
├── skipped_unsupported_version
|
||||
└── dead_letter
|
||||
```
|
||||
|
||||
Implement valid transitions and reject invalid direct state jumps.
|
||||
|
||||
---
|
||||
|
||||
## Part 17 — End-to-End Reliability Scenario
|
||||
|
||||
Implement an end-to-end validation path for at least one Activity operation.
|
||||
|
||||
Required scenario:
|
||||
|
||||
```text
|
||||
Complete Activity
|
||||
↓
|
||||
Activity mutation persisted
|
||||
↓
|
||||
activity.completed Outbox event persisted in same transaction
|
||||
↓
|
||||
API returns success
|
||||
↓
|
||||
Worker claims event
|
||||
↓
|
||||
Skeleton consumers execute
|
||||
↓
|
||||
Consumer checkpoints become completed
|
||||
↓
|
||||
Duplicate dispatch is skipped
|
||||
```
|
||||
|
||||
Failure variation:
|
||||
|
||||
```text
|
||||
One consumer fails
|
||||
↓
|
||||
Activity remains completed
|
||||
↓
|
||||
Other consumers remain completed
|
||||
↓
|
||||
Failed consumer schedules retry
|
||||
↓
|
||||
Retry completes or moves to dead letter
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
# Deliverables
|
||||
|
||||
## 1. Activity Transactional Outbox Adoption
|
||||
|
||||
Atomic source mutation, audit, and event enqueue.
|
||||
|
||||
## 2. Transaction-Aware Outbox Publisher
|
||||
|
||||
Shared transaction-scoped persistence contract.
|
||||
|
||||
## 3. Outbox Worker Runtime
|
||||
|
||||
Polling, claiming, dispatch, and completion processing.
|
||||
|
||||
## 4. Multi-Instance Claiming Strategy
|
||||
|
||||
Lease-based or lock-based safe batch processing.
|
||||
|
||||
## 5. Per-Consumer Dispatch Semantics
|
||||
|
||||
Independent checkpoints and isolated retries.
|
||||
|
||||
## 6. Retry Processor
|
||||
|
||||
Configured retry and backoff implementation.
|
||||
|
||||
## 7. Dead-Letter Runtime
|
||||
|
||||
Terminal failure persistence and recovery contract.
|
||||
|
||||
## 8. Manual Retry and Drain Services
|
||||
|
||||
Protected operational service interfaces.
|
||||
|
||||
## 9. Worker Startup and Deployment Strategy
|
||||
|
||||
Current development, UAT, and production execution model.
|
||||
|
||||
## 10. Graceful Shutdown Handling
|
||||
|
||||
Safe worker termination behavior.
|
||||
|
||||
## 11. Projection Health Integration
|
||||
|
||||
Real worker health and lag metrics.
|
||||
|
||||
## 12. Delivery SLA Baseline
|
||||
|
||||
Initial delivery and recovery targets.
|
||||
|
||||
## 13. Security and Operations Matrix
|
||||
|
||||
Worker, retry, dead-letter, and administrative access rules.
|
||||
|
||||
## 14. Existing Publisher Compatibility Strategy
|
||||
|
||||
No-double-publication and transition rules.
|
||||
|
||||
## 15. Consumer Registration Strategy
|
||||
|
||||
Rules for new, disabled, renamed, and version-changed consumers.
|
||||
|
||||
## 16. Delivery State Machine
|
||||
|
||||
Outbox and checkpoint lifecycle rules.
|
||||
|
||||
## 17. End-to-End Reliability Tests
|
||||
|
||||
Atomicity, worker delivery, retry, duplicate, and recovery scenarios.
|
||||
|
||||
## 18. EP.1.5 Readiness Report
|
||||
|
||||
Confirm that Timeline can rely on durable incremental Activity events.
|
||||
|
||||
---
|
||||
|
||||
# Constraints
|
||||
|
||||
Must preserve:
|
||||
|
||||
* existing Activity API contracts
|
||||
* Activity lifecycle business behavior
|
||||
* legacy follow-up APIs and storage
|
||||
* recent-activity compatibility layer
|
||||
* existing Dashboard and Report consumers
|
||||
* existing Approval and Notification behavior
|
||||
* existing Audit Log behavior
|
||||
* existing CRM security and pricing boundaries
|
||||
* EP.1.3 Event Contract and Registry
|
||||
* EP.1.4 Projection Registry and checkpoint model
|
||||
|
||||
Do not implement:
|
||||
|
||||
* final Timeline projection
|
||||
* Timeline UI
|
||||
* final Calendar projection
|
||||
* Calendar UI
|
||||
* Notification event expansion
|
||||
* My Day
|
||||
* Manager Workspace
|
||||
* Executive Workspace
|
||||
* Relationship Health
|
||||
* Forecast replacement
|
||||
* external Kafka, RabbitMQ, or Redis Streams
|
||||
* legacy follow-up migration
|
||||
* follow-up dual-write
|
||||
* Dashboard source replacement
|
||||
|
||||
Do not start unmanaged worker loops inside request handlers or Client Components.
|
||||
|
||||
No breaking API changes.
|
||||
|
||||
---
|
||||
|
||||
# Compatibility Strategy
|
||||
|
||||
Activity is the first source domain activated with transactional outbox delivery.
|
||||
|
||||
Other source domains remain on their current event or notification paths until dedicated cutover tasks are approved.
|
||||
|
||||
During transition:
|
||||
|
||||
* Activity uses transactional outbox.
|
||||
* Existing Approval notifications remain unchanged.
|
||||
* Existing in-process publisher remains available for tests and approved compatibility use.
|
||||
* No event may be emitted through both outbox and in-process production paths.
|
||||
* Existing projection skeletons remain non-user-facing.
|
||||
|
||||
---
|
||||
|
||||
# Testing Requirements
|
||||
|
||||
## Atomicity Tests
|
||||
|
||||
* Activity mutation and outbox insert commit together
|
||||
* outbox insert failure rolls back Activity mutation
|
||||
* validation failure creates neither mutation nor event
|
||||
* audit failure behavior follows existing transaction governance
|
||||
* multiple events enqueue in deterministic sequence
|
||||
|
||||
## Worker Claim Tests
|
||||
|
||||
* one worker claims one event
|
||||
* two workers do not process the same active lease
|
||||
* stale lease becomes recoverable
|
||||
* batch size is honored
|
||||
* claim order is deterministic where required
|
||||
* graceful shutdown stops new claims
|
||||
|
||||
## Consumer Tests
|
||||
|
||||
* all active consumers receive the event
|
||||
* one failure does not invalidate successful consumers
|
||||
* completed checkpoint skips duplicate execution
|
||||
* unsupported version follows policy
|
||||
* no-consumer event terminates safely
|
||||
* disabled consumer behavior follows registry rule
|
||||
|
||||
## Retry Tests
|
||||
|
||||
* retryable failure schedules next attempt
|
||||
* non-retryable failure creates dead letter
|
||||
* maximum attempts create dead letter
|
||||
* retry affects only incomplete consumer
|
||||
* manual retry does not rerun completed consumers
|
||||
* dead-letter retry preserves original event id
|
||||
|
||||
## Restart and Recovery Tests
|
||||
|
||||
* pending event survives process restart simulation
|
||||
* retry-scheduled checkpoint survives restart
|
||||
* stale processing claim is recovered
|
||||
* duplicate delivery after restart creates no duplicate effect
|
||||
|
||||
## Security Tests
|
||||
|
||||
* organization isolation
|
||||
* sensitive payload remains minimized
|
||||
* dead-letter error is sanitized
|
||||
* manual retry requires authorization
|
||||
* internal-only Activity does not leak through operational status APIs
|
||||
|
||||
## Compatibility Tests
|
||||
|
||||
* Activity API response remains unchanged
|
||||
* existing recent-activity reads remain unchanged
|
||||
* existing follow-up APIs remain unchanged
|
||||
* Dashboard and Reports remain unchanged
|
||||
* Approval notifications remain unchanged
|
||||
* no duplicate Activity Business Event publication occurs
|
||||
|
||||
## End-to-End Tests
|
||||
|
||||
* Activity create through outbox to consumer completion
|
||||
* Activity complete through outbox to consumer completion
|
||||
* consumer failure followed by successful retry
|
||||
* consumer failure reaching dead letter
|
||||
* duplicate dispatch skipped
|
||||
* stale claim recovery
|
||||
|
||||
---
|
||||
|
||||
# Acceptance Criteria
|
||||
|
||||
* Activity source mutations persist outbox events atomically.
|
||||
* Event persistence failure inside the transaction rolls back the Activity mutation.
|
||||
* Worker dispatch occurs only after transaction commit.
|
||||
* Activity APIs return success even when a non-owning consumer later fails.
|
||||
* Outbox events are claimed safely in multi-instance execution.
|
||||
* Consumer checkpoints are persistent and idempotent.
|
||||
* Successful consumers are not rerun because another consumer failed.
|
||||
* Retry scheduling and backoff operate according to policy.
|
||||
* Terminal failures create sanitized dead-letter records.
|
||||
* Manual retry and stale-claim recovery service contracts exist.
|
||||
* Worker can shut down gracefully.
|
||||
* Projection health reflects actual worker state.
|
||||
* No Activity event is double-published through in-process and outbox publishers.
|
||||
* Existing Follow-up, Dashboard, Report, Approval, Notification, and recent-activity behavior remains unchanged.
|
||||
* End-to-end atomicity, dispatch, duplicate, retry, and recovery tests pass.
|
||||
* TypeScript, migration checks, targeted lint, and worker/runtime tests pass.
|
||||
|
||||
---
|
||||
|
||||
# Success Criteria
|
||||
|
||||
ALLA OS has an active, production-ready Transactional Outbox delivery path for Activity Business Events.
|
||||
|
||||
Source mutations and event persistence are atomic.
|
||||
|
||||
Projection delivery is post-commit, independently retryable, idempotent, observable, and safe across process restarts and multiple worker instances.
|
||||
|
||||
Future Timeline and Calendar projections can rely on durable incremental Activity events without coupling projection failures to source-domain mutation results.
|
||||
|
||||
Ready for:
|
||||
|
||||
# EP.1.5 – Timeline Projection Foundation
|
||||
@@ -467,6 +467,9 @@ export const businessEventOutbox = pgTable(
|
||||
deliveryStatus: text('delivery_status').default('pending').notNull(),
|
||||
attemptCount: integer('attempt_count').default(0).notNull(),
|
||||
availableAt: timestamp('available_at', { withTimezone: true }).defaultNow().notNull(),
|
||||
claimedBy: text('claimed_by'),
|
||||
claimedAt: timestamp('claimed_at', { withTimezone: true }),
|
||||
claimExpiresAt: timestamp('claim_expires_at', { withTimezone: true }),
|
||||
processingStartedAt: timestamp('processing_started_at', { withTimezone: true }),
|
||||
dispatchedAt: timestamp('dispatched_at', { withTimezone: true }),
|
||||
failedAt: timestamp('failed_at', { withTimezone: true }),
|
||||
@@ -482,6 +485,10 @@ export const businessEventOutbox = pgTable(
|
||||
table.deliveryStatus,
|
||||
table.availableAt
|
||||
),
|
||||
claimLeaseIdx: index('business_event_outbox_claim_lease_idx').on(
|
||||
table.deliveryStatus,
|
||||
table.claimExpiresAt
|
||||
),
|
||||
organizationEventIdx: index('business_event_outbox_org_event_idx').on(
|
||||
table.organizationId,
|
||||
table.eventType
|
||||
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
type BusinessEvent
|
||||
} from '../../../foundation/business-events/index.ts';
|
||||
import type { ActivityRecord } from '../api/types.ts';
|
||||
import type { ActivityRow } from './repository';
|
||||
|
||||
export type ActivityBusinessEventType =
|
||||
| 'activity.created'
|
||||
@@ -25,6 +26,83 @@ interface PublishActivityBusinessEventInput {
|
||||
previousActivity?: ActivityRecord | null;
|
||||
}
|
||||
|
||||
interface EnqueueActivityBusinessEventInput extends PublishActivityBusinessEventInput {
|
||||
client: Parameters<
|
||||
import('../../../foundation/projections/server/transactional-publisher.ts').TransactionalBusinessEventPublisher['enqueue']
|
||||
>[0];
|
||||
}
|
||||
|
||||
function toIso(value: Date | null): string | null {
|
||||
return value?.toISOString() ?? null;
|
||||
}
|
||||
|
||||
function toMetadata(value: unknown): Record<string, unknown> | null {
|
||||
return value && typeof value === 'object' && !Array.isArray(value)
|
||||
? (value as Record<string, unknown>)
|
||||
: null;
|
||||
}
|
||||
|
||||
export function activityRowToBusinessEventRecord(row: ActivityRow): ActivityRecord {
|
||||
const metadata = toMetadata(row.metadata);
|
||||
const scheduledStartAt = toIso(row.scheduledStartAt);
|
||||
const dueAt = toIso(row.dueAt);
|
||||
const completedAt = toIso(row.completedAt);
|
||||
const cancelledAt = toIso(row.cancelledAt);
|
||||
const skippedAt = toIso(row.skippedAt);
|
||||
|
||||
return {
|
||||
id: row.id,
|
||||
organizationId: row.organizationId,
|
||||
primaryEntityType: row.primaryEntityType as ActivityRecord['primaryEntityType'],
|
||||
primaryEntityId: row.primaryEntityId,
|
||||
primaryRecordLabel: row.primaryRecordLabel,
|
||||
relatedRecords: Array.isArray(row.relatedRecords)
|
||||
? (row.relatedRecords as ActivityRecord['relatedRecords'])
|
||||
: [],
|
||||
customerId: row.customerId,
|
||||
contactId: row.contactId,
|
||||
leadId: row.leadId,
|
||||
opportunityId: row.opportunityId,
|
||||
quotationId: row.quotationId,
|
||||
activityType: row.activityType as ActivityRecord['activityType'],
|
||||
subject: row.subject,
|
||||
description: row.description,
|
||||
ownerId: row.ownerId,
|
||||
assignedToId: row.assignedToId,
|
||||
scheduledStartAt,
|
||||
scheduledEndAt: toIso(row.scheduledEndAt),
|
||||
dueAt,
|
||||
completedAt,
|
||||
cancelledAt,
|
||||
skippedAt,
|
||||
status: row.status as ActivityRecord['status'],
|
||||
priority: row.priority as ActivityRecord['priority'],
|
||||
outcomeSummary: row.outcomeSummary,
|
||||
cancellationReason: row.cancellationReason,
|
||||
skipReason: row.skipReason,
|
||||
nextAction: row.nextAction,
|
||||
branchId: row.branchId,
|
||||
productType: row.productType,
|
||||
isInternalOnly: row.isInternalOnly,
|
||||
isPricingSensitive: row.isPricingSensitive,
|
||||
parentActivityId: row.parentActivityId,
|
||||
metadata,
|
||||
createdBy: row.createdBy,
|
||||
updatedBy: row.updatedBy,
|
||||
createdAt: row.createdAt.toISOString(),
|
||||
updatedAt: row.updatedAt.toISOString(),
|
||||
deletedAt: toIso(row.deletedAt),
|
||||
activityCode: typeof metadata?.activityCode === 'string' ? metadata.activityCode : null,
|
||||
effectiveStatus: row.status as ActivityRecord['effectiveStatus'],
|
||||
ownerName: null,
|
||||
assignedToName: null,
|
||||
createdByName: null,
|
||||
updatedByName: null,
|
||||
canViewPricingSensitiveContent: false,
|
||||
isContentRedacted: row.isPricingSensitive || row.isInternalOnly
|
||||
};
|
||||
}
|
||||
|
||||
function compactRelatedRecords(activity: ActivityRecord) {
|
||||
return [
|
||||
activity.customerId ? { type: 'customer', id: activity.customerId } : null,
|
||||
@@ -112,3 +190,15 @@ export async function publishActivityBusinessEvent(
|
||||
|
||||
return event;
|
||||
}
|
||||
|
||||
export async function enqueueActivityBusinessEvent(
|
||||
input: EnqueueActivityBusinessEventInput
|
||||
): Promise<BusinessEvent> {
|
||||
const { transactionalBusinessEventPublisher } = await import(
|
||||
'../../../foundation/projections/server/transactional-publisher.ts'
|
||||
);
|
||||
const event = createActivityBusinessEvent(input);
|
||||
await transactionalBusinessEventPublisher.enqueue(input.client, event);
|
||||
|
||||
return event;
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import type { CrmActivityEntityType } from '@/features/crm/activity/types';
|
||||
|
||||
export type ActivityRow = typeof crmActivities.$inferSelect;
|
||||
export type InsertActivityRow = typeof crmActivities.$inferInsert;
|
||||
type ActivityDbClient = Pick<typeof db, 'insert' | 'update' | 'select'>;
|
||||
|
||||
export interface ActivityListQuery {
|
||||
organizationId: string;
|
||||
@@ -24,8 +25,11 @@ export interface ActivityContextQuery {
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
export async function insertActivity(values: InsertActivityRow): Promise<ActivityRow> {
|
||||
const [created] = await db.insert(crmActivities).values(values).returning();
|
||||
export async function insertActivity(
|
||||
values: InsertActivityRow,
|
||||
client: ActivityDbClient = db
|
||||
): Promise<ActivityRow> {
|
||||
const [created] = await client.insert(crmActivities).values(values).returning();
|
||||
|
||||
return created;
|
||||
}
|
||||
@@ -33,9 +37,10 @@ export async function insertActivity(values: InsertActivityRow): Promise<Activit
|
||||
export async function updateActivityRow(
|
||||
id: string,
|
||||
organizationId: string,
|
||||
values: Partial<InsertActivityRow>
|
||||
values: Partial<InsertActivityRow>,
|
||||
client: ActivityDbClient = db
|
||||
): Promise<ActivityRow> {
|
||||
const [updated] = await db
|
||||
const [updated] = await client
|
||||
.update(crmActivities)
|
||||
.set(values)
|
||||
.where(and(eq(crmActivities.id, id), eq(crmActivities.organizationId, organizationId)))
|
||||
|
||||
@@ -46,7 +46,10 @@ import type {
|
||||
} from '../api/types';
|
||||
import { getActivityRow, insertActivity, listActivityRows, updateActivityRow } from './repository';
|
||||
import { hydrateActivityRecords } from './read-model';
|
||||
import { publishActivityBusinessEvent } from './business-events';
|
||||
import {
|
||||
activityRowToBusinessEventRecord,
|
||||
enqueueActivityBusinessEvent
|
||||
} from './business-events';
|
||||
|
||||
type ActivityAccessContext = CrmSecurityContext;
|
||||
|
||||
@@ -608,7 +611,9 @@ export async function createActivity(
|
||||
}
|
||||
};
|
||||
|
||||
const created = await insertActivity({
|
||||
const created = await db.transaction(async (tx) => {
|
||||
const createdRow = await insertActivity(
|
||||
{
|
||||
id: crypto.randomUUID(),
|
||||
organizationId,
|
||||
primaryEntityType: values.primaryEntityType,
|
||||
@@ -645,27 +650,35 @@ export async function createActivity(
|
||||
metadata,
|
||||
createdBy: userId,
|
||||
updatedBy: userId
|
||||
});
|
||||
|
||||
const activity = await getActivityById(created.id, organizationId, context);
|
||||
},
|
||||
tx
|
||||
);
|
||||
const eventActivity = activityRowToBusinessEventRecord(createdRow);
|
||||
const correlationId = crypto.randomUUID();
|
||||
|
||||
await publishActivityBusinessEvent({
|
||||
await enqueueActivityBusinessEvent({
|
||||
client: tx,
|
||||
eventType: 'activity.created',
|
||||
activity,
|
||||
activity: eventActivity,
|
||||
actorUserId: userId,
|
||||
correlationId
|
||||
});
|
||||
|
||||
if (activity.assignedToId && activity.assignedToId !== activity.ownerId) {
|
||||
await publishActivityBusinessEvent({
|
||||
if (eventActivity.assignedToId && eventActivity.assignedToId !== eventActivity.ownerId) {
|
||||
await enqueueActivityBusinessEvent({
|
||||
client: tx,
|
||||
eventType: 'activity.assigned',
|
||||
activity,
|
||||
activity: eventActivity,
|
||||
actorUserId: userId,
|
||||
correlationId
|
||||
});
|
||||
}
|
||||
|
||||
return createdRow;
|
||||
});
|
||||
|
||||
const activity = await getActivityById(created.id, organizationId, context);
|
||||
|
||||
return activity;
|
||||
}
|
||||
|
||||
@@ -688,7 +701,8 @@ export async function updateActivity(
|
||||
await assertActivityTypeValue(organizationId, values.activityType);
|
||||
const nextStatus = values.status;
|
||||
|
||||
await updateActivityRow(id, organizationId, {
|
||||
const activityEventRecord = await db.transaction(async (tx) => {
|
||||
const updatedRow = await updateActivityRow(id, organizationId, {
|
||||
primaryEntityType: values.primaryEntityType,
|
||||
primaryEntityId: values.primaryEntityId,
|
||||
primaryRecordLabel: values.primaryRecordLabel,
|
||||
@@ -729,15 +743,16 @@ export async function updateActivity(
|
||||
skippedAt: nextStatus === 'skipped' ? new Date() : null,
|
||||
updatedAt: new Date(),
|
||||
updatedBy: userId
|
||||
});
|
||||
}, tx);
|
||||
|
||||
const activity = await getActivityById(id, organizationId, context);
|
||||
const nextActivity = activityRowToBusinessEventRecord(updatedRow);
|
||||
const correlationId = crypto.randomUUID();
|
||||
|
||||
if (current.assignedToId !== activity.assignedToId) {
|
||||
await publishActivityBusinessEvent({
|
||||
if (current.assignedToId !== nextActivity.assignedToId) {
|
||||
await enqueueActivityBusinessEvent({
|
||||
client: tx,
|
||||
eventType: current.assignedToId ? 'activity.reassigned' : 'activity.assigned',
|
||||
activity,
|
||||
activity: nextActivity,
|
||||
previousActivity: current,
|
||||
actorUserId: userId,
|
||||
correlationId
|
||||
@@ -745,49 +760,58 @@ export async function updateActivity(
|
||||
}
|
||||
|
||||
if (
|
||||
current.scheduledStartAt !== activity.scheduledStartAt ||
|
||||
current.scheduledEndAt !== activity.scheduledEndAt ||
|
||||
current.dueAt !== activity.dueAt
|
||||
current.scheduledStartAt !== nextActivity.scheduledStartAt ||
|
||||
current.scheduledEndAt !== nextActivity.scheduledEndAt ||
|
||||
current.dueAt !== nextActivity.dueAt
|
||||
) {
|
||||
await publishActivityBusinessEvent({
|
||||
await enqueueActivityBusinessEvent({
|
||||
client: tx,
|
||||
eventType: 'activity.rescheduled',
|
||||
activity,
|
||||
activity: nextActivity,
|
||||
previousActivity: current,
|
||||
actorUserId: userId,
|
||||
correlationId
|
||||
});
|
||||
}
|
||||
|
||||
if (current.status !== 'in_progress' && activity.status === 'in_progress') {
|
||||
await publishActivityBusinessEvent({
|
||||
if (current.status !== 'in_progress' && nextActivity.status === 'in_progress') {
|
||||
await enqueueActivityBusinessEvent({
|
||||
client: tx,
|
||||
eventType: 'activity.started',
|
||||
activity,
|
||||
activity: nextActivity,
|
||||
previousActivity: current,
|
||||
actorUserId: userId,
|
||||
correlationId
|
||||
});
|
||||
}
|
||||
|
||||
if (current.status !== 'completed' && activity.status === 'completed') {
|
||||
await publishActivityBusinessEvent({
|
||||
if (current.status !== 'completed' && nextActivity.status === 'completed') {
|
||||
await enqueueActivityBusinessEvent({
|
||||
client: tx,
|
||||
eventType: 'activity.completed',
|
||||
activity,
|
||||
activity: nextActivity,
|
||||
previousActivity: current,
|
||||
actorUserId: userId,
|
||||
correlationId
|
||||
});
|
||||
}
|
||||
|
||||
if (current.status !== 'cancelled' && activity.status === 'cancelled') {
|
||||
await publishActivityBusinessEvent({
|
||||
if (current.status !== 'cancelled' && nextActivity.status === 'cancelled') {
|
||||
await enqueueActivityBusinessEvent({
|
||||
client: tx,
|
||||
eventType: 'activity.cancelled',
|
||||
activity,
|
||||
activity: nextActivity,
|
||||
previousActivity: current,
|
||||
actorUserId: userId,
|
||||
correlationId
|
||||
});
|
||||
}
|
||||
|
||||
return nextActivity;
|
||||
});
|
||||
|
||||
const activity = await getActivityById(activityEventRecord.id, organizationId, context);
|
||||
|
||||
return activity;
|
||||
}
|
||||
|
||||
@@ -802,23 +826,27 @@ export async function completeActivity(
|
||||
const current = await getHydratedActivityOrThrow(id, organizationId, context);
|
||||
assertTransitionAllowed(current.status, 'completed');
|
||||
|
||||
await updateActivityRow(id, organizationId, {
|
||||
const completedEventRecord = await db.transaction(async (tx) => {
|
||||
const updatedRow = await updateActivityRow(id, organizationId, {
|
||||
status: 'completed',
|
||||
outcomeSummary: payload.outcomeSummary.trim(),
|
||||
nextAction: payload.nextAction?.trim() || null,
|
||||
completedAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
updatedBy: userId
|
||||
});
|
||||
|
||||
const activity = await getActivityById(id, organizationId, context);
|
||||
await publishActivityBusinessEvent({
|
||||
}, tx);
|
||||
const nextActivity = activityRowToBusinessEventRecord(updatedRow);
|
||||
await enqueueActivityBusinessEvent({
|
||||
client: tx,
|
||||
eventType: 'activity.completed',
|
||||
activity,
|
||||
activity: nextActivity,
|
||||
previousActivity: current,
|
||||
actorUserId: userId
|
||||
});
|
||||
return nextActivity;
|
||||
});
|
||||
|
||||
const activity = await getActivityById(completedEventRecord.id, organizationId, context);
|
||||
return activity;
|
||||
}
|
||||
|
||||
@@ -833,22 +861,26 @@ export async function cancelActivity(
|
||||
const current = await getHydratedActivityOrThrow(id, organizationId, context);
|
||||
assertTransitionAllowed(current.status, 'cancelled');
|
||||
|
||||
await updateActivityRow(id, organizationId, {
|
||||
const cancelledEventRecord = await db.transaction(async (tx) => {
|
||||
const updatedRow = await updateActivityRow(id, organizationId, {
|
||||
status: 'cancelled',
|
||||
cancellationReason: payload.cancellationReason.trim(),
|
||||
cancelledAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
updatedBy: userId
|
||||
});
|
||||
|
||||
const activity = await getActivityById(id, organizationId, context);
|
||||
await publishActivityBusinessEvent({
|
||||
}, tx);
|
||||
const nextActivity = activityRowToBusinessEventRecord(updatedRow);
|
||||
await enqueueActivityBusinessEvent({
|
||||
client: tx,
|
||||
eventType: 'activity.cancelled',
|
||||
activity,
|
||||
activity: nextActivity,
|
||||
previousActivity: current,
|
||||
actorUserId: userId
|
||||
});
|
||||
return nextActivity;
|
||||
});
|
||||
|
||||
const activity = await getActivityById(cancelledEventRecord.id, organizationId, context);
|
||||
return activity;
|
||||
}
|
||||
|
||||
@@ -866,22 +898,26 @@ export async function assignActivity(
|
||||
await assertMembershipUserBelongsToOrganization(payload.assignedToId, organizationId);
|
||||
}
|
||||
|
||||
await updateActivityRow(id, organizationId, {
|
||||
const assignedEventRecord = await db.transaction(async (tx) => {
|
||||
const updatedRow = await updateActivityRow(id, organizationId, {
|
||||
assignedToId: payload.assignedToId,
|
||||
updatedAt: new Date(),
|
||||
updatedBy: userId
|
||||
});
|
||||
|
||||
const activity = await getActivityById(id, organizationId, context);
|
||||
if (current.assignedToId !== activity.assignedToId) {
|
||||
await publishActivityBusinessEvent({
|
||||
}, tx);
|
||||
const nextActivity = activityRowToBusinessEventRecord(updatedRow);
|
||||
if (current.assignedToId !== nextActivity.assignedToId) {
|
||||
await enqueueActivityBusinessEvent({
|
||||
client: tx,
|
||||
eventType: current.assignedToId ? 'activity.reassigned' : 'activity.assigned',
|
||||
activity,
|
||||
activity: nextActivity,
|
||||
previousActivity: current,
|
||||
actorUserId: userId
|
||||
});
|
||||
}
|
||||
return nextActivity;
|
||||
});
|
||||
|
||||
const activity = await getActivityById(assignedEventRecord.id, organizationId, context);
|
||||
return activity;
|
||||
}
|
||||
|
||||
@@ -902,7 +938,8 @@ export async function rescheduleActivity(
|
||||
const nextStatus = current.status === 'draft' ? 'scheduled' : current.status;
|
||||
assertTransitionAllowed(current.status, nextStatus);
|
||||
|
||||
await updateActivityRow(id, organizationId, {
|
||||
const rescheduledEventRecord = await db.transaction(async (tx) => {
|
||||
const updatedRow = await updateActivityRow(id, organizationId, {
|
||||
scheduledStartAt:
|
||||
payload.scheduledStartAt === undefined
|
||||
? toDateOrNull(current.scheduledStartAt)
|
||||
@@ -915,16 +952,19 @@ export async function rescheduleActivity(
|
||||
status: nextStatus,
|
||||
updatedAt: new Date(),
|
||||
updatedBy: userId
|
||||
});
|
||||
|
||||
const activity = await getActivityById(id, organizationId, context);
|
||||
await publishActivityBusinessEvent({
|
||||
}, tx);
|
||||
const nextActivity = activityRowToBusinessEventRecord(updatedRow);
|
||||
await enqueueActivityBusinessEvent({
|
||||
client: tx,
|
||||
eventType: 'activity.rescheduled',
|
||||
activity,
|
||||
activity: nextActivity,
|
||||
previousActivity: current,
|
||||
actorUserId: userId
|
||||
});
|
||||
return nextActivity;
|
||||
});
|
||||
|
||||
const activity = await getActivityById(rescheduledEventRecord.id, organizationId, context);
|
||||
return activity;
|
||||
}
|
||||
|
||||
@@ -937,15 +977,18 @@ export async function deleteActivity(
|
||||
assertActivityPermission(context, CRM_ACTIVITY_PERMISSIONS.delete);
|
||||
const current = await getHydratedActivityOrThrow(id, organizationId, context);
|
||||
|
||||
await db.transaction(async (tx) => {
|
||||
await updateActivityRow(id, organizationId, {
|
||||
deletedAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
updatedBy: userId
|
||||
});
|
||||
}, tx);
|
||||
|
||||
await publishActivityBusinessEvent({
|
||||
await enqueueActivityBusinessEvent({
|
||||
client: tx,
|
||||
eventType: 'activity.deleted',
|
||||
activity: current,
|
||||
actorUserId: userId
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
export * from './consumers.ts';
|
||||
export * from './matrices.ts';
|
||||
export * from './memory-store.ts';
|
||||
export * from './operations.ts';
|
||||
export * from './publisher.ts';
|
||||
export * from './rebuild.ts';
|
||||
export * from './registry.ts';
|
||||
export * from './retry-policy.ts';
|
||||
export * from './runtime.ts';
|
||||
export * from './types.ts';
|
||||
export * from './worker.ts';
|
||||
|
||||
@@ -2,6 +2,7 @@ import type { BusinessEvent } from '../business-events/types.ts';
|
||||
import type {
|
||||
BeginProjectionCheckpointInput,
|
||||
BusinessEventDeliveryRecord,
|
||||
ClaimBusinessEventDeliveriesInput,
|
||||
CompleteProjectionCheckpointInput,
|
||||
CreateProjectionDeadLetterInput,
|
||||
FailProjectionCheckpointInput,
|
||||
@@ -31,12 +32,47 @@ export class InMemoryProjectionRuntimeStore implements ProjectionRuntimeStore {
|
||||
status: 'pending',
|
||||
attemptCount: 0,
|
||||
availableAt: new Date().toISOString(),
|
||||
claimedBy: null,
|
||||
claimedAt: null,
|
||||
claimExpiresAt: null,
|
||||
event
|
||||
};
|
||||
this.deliveries.set(event.eventId, record);
|
||||
return record;
|
||||
}
|
||||
|
||||
async claimAvailableEvents(
|
||||
input: ClaimBusinessEventDeliveriesInput
|
||||
): Promise<BusinessEventDeliveryRecord[]> {
|
||||
const claimable = [...this.deliveries.values()]
|
||||
.filter((delivery) => {
|
||||
if (delivery.status === 'pending' || delivery.status === 'retry_scheduled') {
|
||||
return new Date(delivery.availableAt).getTime() <= input.now.getTime();
|
||||
}
|
||||
|
||||
if (delivery.status === 'claimed' || delivery.status === 'processing') {
|
||||
return (
|
||||
delivery.claimExpiresAt !== null &&
|
||||
delivery.claimExpiresAt !== undefined &&
|
||||
new Date(delivery.claimExpiresAt).getTime() <= input.now.getTime()
|
||||
);
|
||||
}
|
||||
|
||||
return false;
|
||||
})
|
||||
.toSorted((a, b) => a.availableAt.localeCompare(b.availableAt))
|
||||
.slice(0, input.batchSize);
|
||||
|
||||
for (const delivery of claimable) {
|
||||
delivery.status = 'claimed';
|
||||
delivery.claimedBy = input.workerId;
|
||||
delivery.claimedAt = input.now.toISOString();
|
||||
delivery.claimExpiresAt = input.leaseExpiresAt.toISOString();
|
||||
}
|
||||
|
||||
return claimable;
|
||||
}
|
||||
|
||||
async markDeliveryProcessing(eventId: string): Promise<void> {
|
||||
const delivery = this.requireDelivery(eventId);
|
||||
delivery.status = 'processing';
|
||||
|
||||
101
src/features/foundation/projections/operations.ts
Normal file
101
src/features/foundation/projections/operations.ts
Normal file
@@ -0,0 +1,101 @@
|
||||
import { ProjectionConsumerRuntime } from './runtime.ts';
|
||||
import { ProjectionOutboxWorker, type ProjectionWorkerDrainResult } from './worker.ts';
|
||||
import type { BusinessEvent } from '../business-events/types.ts';
|
||||
import type { ProjectionConsumer, ProjectionRuntimeStore } from './types.ts';
|
||||
|
||||
export interface ProjectionManualActor {
|
||||
userId: string;
|
||||
organizationId: string;
|
||||
systemRole?: string;
|
||||
}
|
||||
|
||||
export interface ProjectionManualRetryResult {
|
||||
status: 'accepted' | 'completed';
|
||||
eventId?: string;
|
||||
consumerName?: string;
|
||||
message: string;
|
||||
}
|
||||
|
||||
export class ProjectionOperationsService {
|
||||
private readonly store: ProjectionRuntimeStore;
|
||||
private readonly consumers: ProjectionConsumer[];
|
||||
|
||||
constructor(options: { store: ProjectionRuntimeStore; consumers: ProjectionConsumer[] }) {
|
||||
this.store = options.store;
|
||||
this.consumers = options.consumers;
|
||||
}
|
||||
|
||||
async drainOutbox(worker: ProjectionOutboxWorker): Promise<ProjectionWorkerDrainResult> {
|
||||
return worker.drainOnce();
|
||||
}
|
||||
|
||||
async retryEvent(event: BusinessEvent, actor: ProjectionManualActor): Promise<ProjectionManualRetryResult> {
|
||||
assertProjectionOperator(actor);
|
||||
const runtime = new ProjectionConsumerRuntime({
|
||||
store: this.store,
|
||||
consumers: this.consumers
|
||||
});
|
||||
await runtime.processEvent(event);
|
||||
|
||||
return {
|
||||
status: 'completed',
|
||||
eventId: event.eventId,
|
||||
message: 'Retry completed for incomplete projection consumers'
|
||||
};
|
||||
}
|
||||
|
||||
async retryConsumer(
|
||||
event: BusinessEvent,
|
||||
consumerName: string,
|
||||
actor: ProjectionManualActor
|
||||
): Promise<ProjectionManualRetryResult> {
|
||||
assertProjectionOperator(actor);
|
||||
const consumer = this.consumers.find((item) => item.consumerName === consumerName);
|
||||
if (!consumer) {
|
||||
throw new Error(`Unknown projection consumer: ${consumerName}`);
|
||||
}
|
||||
|
||||
const runtime = new ProjectionConsumerRuntime({
|
||||
store: this.store,
|
||||
consumers: [consumer]
|
||||
});
|
||||
await runtime.processEvent(event);
|
||||
|
||||
return {
|
||||
status: 'completed',
|
||||
eventId: event.eventId,
|
||||
consumerName,
|
||||
message: 'Retry completed for selected projection consumer'
|
||||
};
|
||||
}
|
||||
|
||||
async retryDeadLetter(deadLetterId: string, actor: ProjectionManualActor): Promise<ProjectionManualRetryResult> {
|
||||
assertProjectionOperator(actor);
|
||||
return {
|
||||
status: 'accepted',
|
||||
message: `Dead-letter retry accepted for ${deadLetterId}`
|
||||
};
|
||||
}
|
||||
|
||||
async resolveDeadLetter(
|
||||
deadLetterId: string,
|
||||
reason: string,
|
||||
actor: ProjectionManualActor
|
||||
): Promise<ProjectionManualRetryResult> {
|
||||
assertProjectionOperator(actor);
|
||||
if (!reason.trim()) {
|
||||
throw new Error('Dead-letter resolution reason is required');
|
||||
}
|
||||
|
||||
return {
|
||||
status: 'accepted',
|
||||
message: `Dead-letter ${deadLetterId} resolved`
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function assertProjectionOperator(actor: ProjectionManualActor): void {
|
||||
if (actor.systemRole !== 'super_admin') {
|
||||
throw new Error('Projection operation requires system administration access');
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import { ProjectionOutboxBusinessEventPublisher } from './publisher.ts';
|
||||
import { ProjectionConsumerError } from './retry-policy.ts';
|
||||
import { ProjectionConsumerRuntime } from './runtime.ts';
|
||||
import type { ProjectionConsumer } from './types.ts';
|
||||
import { ProjectionOutboxWorker } from './worker.ts';
|
||||
|
||||
function activityEvent(overrides: Record<string, unknown> = {}) {
|
||||
return createBusinessEvent({
|
||||
@@ -135,7 +136,7 @@ test('consumer failure schedules retry without throwing or blocking unrelated co
|
||||
const runtime = new ProjectionConsumerRuntime({ store, consumers: [failing, healthy] });
|
||||
const result = await runtime.processEvent(event);
|
||||
|
||||
assert.equal(result.status, 'failed');
|
||||
assert.equal(result.status, 'retry_scheduled');
|
||||
assert.deepEqual(handled, ['healthy']);
|
||||
assert.equal(
|
||||
[...store.checkpoints.values()].some((checkpoint) => checkpoint.status === 'retry_scheduled'),
|
||||
@@ -188,3 +189,57 @@ test('non-retryable failure creates dead letter', async () => {
|
||||
assert.equal(store.deadLetters.length, 1);
|
||||
assert.equal(store.deadLetters[0]?.failure.code, 'projection_invariant_failed');
|
||||
});
|
||||
|
||||
test('worker claims available events and drains through runtime', async () => {
|
||||
const store = new InMemoryProjectionRuntimeStore();
|
||||
const event = activityEvent();
|
||||
await store.persistEvent(event);
|
||||
|
||||
const worker = new ProjectionOutboxWorker({
|
||||
store,
|
||||
consumers: INITIAL_PROJECTION_CONSUMERS,
|
||||
config: {
|
||||
enabled: false,
|
||||
workerId: 'worker-a',
|
||||
batchSize: 10,
|
||||
claimLeaseMs: 60_000
|
||||
}
|
||||
});
|
||||
|
||||
const result = await worker.drainOnce();
|
||||
|
||||
assert.equal(result.claimedCount, 1);
|
||||
assert.equal(result.processedCount, 1);
|
||||
assert.equal(store.deliveries.get(event.eventId)?.claimedBy, 'worker-a');
|
||||
assert.equal(store.deliveries.get(event.eventId)?.status, 'completed');
|
||||
});
|
||||
|
||||
test('worker claim honors active lease and recovers stale lease', async () => {
|
||||
const store = new InMemoryProjectionRuntimeStore();
|
||||
const event = activityEvent();
|
||||
await store.persistEvent(event);
|
||||
|
||||
const firstClaim = await store.claimAvailableEvents({
|
||||
workerId: 'worker-a',
|
||||
batchSize: 1,
|
||||
leaseExpiresAt: new Date('2099-07-13T00:01:00.000Z'),
|
||||
now: new Date('2099-07-13T00:00:00.000Z')
|
||||
});
|
||||
const blockedClaim = await store.claimAvailableEvents({
|
||||
workerId: 'worker-b',
|
||||
batchSize: 1,
|
||||
leaseExpiresAt: new Date('2099-07-13T00:01:30.000Z'),
|
||||
now: new Date('2099-07-13T00:00:30.000Z')
|
||||
});
|
||||
const recoveredClaim = await store.claimAvailableEvents({
|
||||
workerId: 'worker-b',
|
||||
batchSize: 1,
|
||||
leaseExpiresAt: new Date('2099-07-13T00:03:00.000Z'),
|
||||
now: new Date('2099-07-13T00:02:00.000Z')
|
||||
});
|
||||
|
||||
assert.equal(firstClaim.length, 1);
|
||||
assert.equal(blockedClaim.length, 0);
|
||||
assert.equal(recoveredClaim.length, 1);
|
||||
assert.equal(recoveredClaim[0]?.claimedBy, 'worker-b');
|
||||
});
|
||||
|
||||
@@ -88,7 +88,12 @@ export class ProjectionConsumerRuntime {
|
||||
},
|
||||
this.now()
|
||||
);
|
||||
return { eventId: event.eventId, eventType: event.eventType, status: 'failed', consumerResults };
|
||||
return {
|
||||
eventId: event.eventId,
|
||||
eventType: event.eventType,
|
||||
status: 'retry_scheduled',
|
||||
consumerResults
|
||||
};
|
||||
}
|
||||
|
||||
await this.store.markDeliveryCompleted(event.eventId, this.now());
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { and, eq } from 'drizzle-orm';
|
||||
import { and, eq, sql } from 'drizzle-orm';
|
||||
import {
|
||||
businessEventOutbox,
|
||||
projectionConsumerCheckpoints,
|
||||
@@ -10,6 +10,7 @@ import type { BusinessEvent } from '../../business-events/types.ts';
|
||||
import type {
|
||||
BeginProjectionCheckpointInput,
|
||||
BusinessEventDeliveryRecord,
|
||||
ClaimBusinessEventDeliveriesInput,
|
||||
CompleteProjectionCheckpointInput,
|
||||
CreateProjectionDeadLetterInput,
|
||||
FailProjectionCheckpointInput,
|
||||
@@ -47,6 +48,43 @@ export class DrizzleProjectionRuntimeStore implements ProjectionRuntimeStore {
|
||||
return mapDelivery(created);
|
||||
}
|
||||
|
||||
async claimAvailableEvents(
|
||||
input: ClaimBusinessEventDeliveriesInput
|
||||
): Promise<BusinessEventDeliveryRecord[]> {
|
||||
const result = await db.execute(sql`
|
||||
WITH claimable AS (
|
||||
SELECT id
|
||||
FROM business_event_outbox
|
||||
WHERE
|
||||
(
|
||||
delivery_status IN ('pending', 'retry_scheduled')
|
||||
AND available_at <= ${input.now}
|
||||
)
|
||||
OR (
|
||||
delivery_status IN ('claimed', 'processing')
|
||||
AND claim_expires_at IS NOT NULL
|
||||
AND claim_expires_at <= ${input.now}
|
||||
)
|
||||
ORDER BY available_at ASC, created_at ASC, id ASC
|
||||
LIMIT ${input.batchSize}
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
UPDATE business_event_outbox
|
||||
SET
|
||||
delivery_status = 'claimed',
|
||||
claimed_by = ${input.workerId},
|
||||
claimed_at = ${input.now},
|
||||
claim_expires_at = ${input.leaseExpiresAt},
|
||||
updated_at = ${input.now}
|
||||
FROM claimable
|
||||
WHERE business_event_outbox.id = claimable.id
|
||||
RETURNING business_event_outbox.*
|
||||
`);
|
||||
|
||||
const rows = Array.isArray(result) ? result : result.rows;
|
||||
return rows.map((row) => mapDelivery(row as typeof businessEventOutbox.$inferSelect));
|
||||
}
|
||||
|
||||
async markDeliveryProcessing(eventId: string, now: Date): Promise<void> {
|
||||
const existing = await this.getDelivery(eventId);
|
||||
await db
|
||||
@@ -65,7 +103,7 @@ export class DrizzleProjectionRuntimeStore implements ProjectionRuntimeStore {
|
||||
}
|
||||
|
||||
async markDeliveryFailed(eventId: string, error: ProjectionFailure, now: Date): Promise<void> {
|
||||
await updateDeliveryStatus(eventId, 'failed', now, {
|
||||
await updateDeliveryStatus(eventId, 'retry_scheduled', now, {
|
||||
failedAt: now,
|
||||
lastErrorCode: error.code,
|
||||
lastErrorMessage: error.message
|
||||
@@ -253,7 +291,7 @@ export class DrizzleProjectionRuntimeStore implements ProjectionRuntimeStore {
|
||||
|
||||
async function updateDeliveryStatus(
|
||||
eventId: string,
|
||||
deliveryStatus: 'completed' | 'failed' | 'dead_letter',
|
||||
deliveryStatus: 'completed' | 'retry_scheduled' | 'dead_letter',
|
||||
now: Date,
|
||||
values: Partial<typeof businessEventOutbox.$inferInsert>
|
||||
): Promise<void> {
|
||||
@@ -272,6 +310,9 @@ function mapDelivery(row: typeof businessEventOutbox.$inferSelect): BusinessEven
|
||||
status: row.deliveryStatus as BusinessEventDeliveryRecord['status'],
|
||||
attemptCount: row.attemptCount,
|
||||
availableAt: row.availableAt.toISOString(),
|
||||
claimedBy: row.claimedBy,
|
||||
claimedAt: row.claimedAt?.toISOString() ?? null,
|
||||
claimExpiresAt: row.claimExpiresAt?.toISOString() ?? null,
|
||||
event: row.eventEnvelope as BusinessEvent
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import { businessEventOutbox } from '../../../../db/schema.ts';
|
||||
import { db } from '../../../../lib/db.ts';
|
||||
import {
|
||||
assertBusinessEventPayload,
|
||||
assertRegisteredBusinessEvent
|
||||
} from '../../business-events/registry.ts';
|
||||
import type { BusinessEvent } from '../../business-events/types.ts';
|
||||
|
||||
type OutboxDatabaseClient = Pick<typeof db, 'insert'>;
|
||||
|
||||
export interface TransactionalBusinessEventPublisher {
|
||||
enqueue(client: OutboxDatabaseClient, event: BusinessEvent): Promise<void>;
|
||||
enqueueMany(client: OutboxDatabaseClient, events: BusinessEvent[]): Promise<void>;
|
||||
}
|
||||
|
||||
export class DrizzleTransactionalBusinessEventPublisher
|
||||
implements TransactionalBusinessEventPublisher
|
||||
{
|
||||
async enqueue(client: OutboxDatabaseClient, event: BusinessEvent): Promise<void> {
|
||||
assertRegisteredBusinessEvent(event.eventType);
|
||||
assertBusinessEventPayload(event.eventType, event.payload);
|
||||
|
||||
await client.insert(businessEventOutbox).values({
|
||||
id: event.eventId,
|
||||
organizationId: event.organizationId,
|
||||
eventType: event.eventType,
|
||||
schemaVersion: event.schemaVersion,
|
||||
entityType: event.entityType,
|
||||
entityId: event.entityId,
|
||||
correlationId: event.correlationId,
|
||||
causationId: event.causationId ?? null,
|
||||
eventEnvelope: event,
|
||||
deliveryStatus: 'pending',
|
||||
occurredAt: new Date(event.occurredAt)
|
||||
});
|
||||
}
|
||||
|
||||
async enqueueMany(client: OutboxDatabaseClient, events: BusinessEvent[]): Promise<void> {
|
||||
for (const event of events) {
|
||||
await this.enqueue(client, event);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const transactionalBusinessEventPublisher =
|
||||
new DrizzleTransactionalBusinessEventPublisher();
|
||||
@@ -2,8 +2,10 @@ import type { BusinessEvent } from '../business-events/types.ts';
|
||||
|
||||
export type BusinessEventDeliveryStatus =
|
||||
| 'pending'
|
||||
| 'claimed'
|
||||
| 'processing'
|
||||
| 'completed'
|
||||
| 'retry_scheduled'
|
||||
| 'failed'
|
||||
| 'dead_letter';
|
||||
|
||||
@@ -66,6 +68,9 @@ export interface BusinessEventDeliveryRecord {
|
||||
status: BusinessEventDeliveryStatus;
|
||||
attemptCount: number;
|
||||
availableAt: string;
|
||||
claimedBy?: string | null;
|
||||
claimedAt?: string | null;
|
||||
claimExpiresAt?: string | null;
|
||||
event: BusinessEvent;
|
||||
}
|
||||
|
||||
@@ -86,6 +91,7 @@ export interface ProjectionCheckpointRecord {
|
||||
|
||||
export interface ProjectionRuntimeStore {
|
||||
persistEvent(event: BusinessEvent): Promise<BusinessEventDeliveryRecord>;
|
||||
claimAvailableEvents(input: ClaimBusinessEventDeliveriesInput): Promise<BusinessEventDeliveryRecord[]>;
|
||||
markDeliveryProcessing(eventId: string, now: Date): Promise<void>;
|
||||
markDeliveryCompleted(eventId: string, now: Date): Promise<void>;
|
||||
markDeliveryFailed(eventId: string, error: ProjectionFailure, now: Date): Promise<void>;
|
||||
@@ -99,6 +105,13 @@ export interface ProjectionRuntimeStore {
|
||||
recordHealthSnapshot(input: ProjectionHealthSnapshotInput): Promise<void>;
|
||||
}
|
||||
|
||||
export interface ClaimBusinessEventDeliveriesInput {
|
||||
workerId: string;
|
||||
batchSize: number;
|
||||
leaseExpiresAt: Date;
|
||||
now: Date;
|
||||
}
|
||||
|
||||
export interface BeginProjectionCheckpointInput {
|
||||
organizationId: string;
|
||||
consumerName: string;
|
||||
|
||||
128
src/features/foundation/projections/worker.ts
Normal file
128
src/features/foundation/projections/worker.ts
Normal file
@@ -0,0 +1,128 @@
|
||||
import { INITIAL_PROJECTION_CONSUMERS } from './consumers.ts';
|
||||
import { ProjectionConsumerRuntime } from './runtime.ts';
|
||||
import type {
|
||||
BusinessEventDeliveryRecord,
|
||||
ProjectionConsumer,
|
||||
ProjectionRuntimeResult,
|
||||
ProjectionRuntimeStore
|
||||
} from './types.ts';
|
||||
|
||||
export interface ProjectionWorkerConfig {
|
||||
enabled: boolean;
|
||||
pollingIntervalMs: number;
|
||||
batchSize: number;
|
||||
maxConcurrentEvents: number;
|
||||
maxConcurrentConsumers: number;
|
||||
claimLeaseMs: number;
|
||||
staleClaimMs: number;
|
||||
workerId: string;
|
||||
gracefulShutdownMs: number;
|
||||
}
|
||||
|
||||
export interface ProjectionWorkerDrainResult {
|
||||
workerId: string;
|
||||
claimedCount: number;
|
||||
processedCount: number;
|
||||
results: ProjectionRuntimeResult[];
|
||||
}
|
||||
|
||||
export const DEFAULT_PROJECTION_WORKER_CONFIG: ProjectionWorkerConfig = {
|
||||
enabled: process.env.PROJECTION_WORKER_ENABLED === 'true',
|
||||
pollingIntervalMs: Number(process.env.PROJECTION_WORKER_POLLING_INTERVAL_MS ?? 3000),
|
||||
batchSize: Number(process.env.PROJECTION_WORKER_BATCH_SIZE ?? 25),
|
||||
maxConcurrentEvents: Number(process.env.PROJECTION_WORKER_MAX_CONCURRENT_EVENTS ?? 1),
|
||||
maxConcurrentConsumers: Number(process.env.PROJECTION_WORKER_MAX_CONCURRENT_CONSUMERS ?? 4),
|
||||
claimLeaseMs: Number(process.env.PROJECTION_WORKER_CLAIM_LEASE_MS ?? 60_000),
|
||||
staleClaimMs: Number(process.env.PROJECTION_WORKER_STALE_CLAIM_MS ?? 90_000),
|
||||
workerId:
|
||||
process.env.PROJECTION_WORKER_ID ??
|
||||
`projection-worker-${process.pid}-${crypto.randomUUID().slice(0, 8)}`,
|
||||
gracefulShutdownMs: Number(process.env.PROJECTION_WORKER_GRACEFUL_SHUTDOWN_MS ?? 30_000)
|
||||
};
|
||||
|
||||
interface ProjectionOutboxWorkerOptions {
|
||||
store: ProjectionRuntimeStore;
|
||||
consumers?: ProjectionConsumer[];
|
||||
config?: Partial<ProjectionWorkerConfig>;
|
||||
now?: () => Date;
|
||||
}
|
||||
|
||||
export class ProjectionOutboxWorker {
|
||||
private readonly store: ProjectionRuntimeStore;
|
||||
private readonly runtime: ProjectionConsumerRuntime;
|
||||
private readonly config: ProjectionWorkerConfig;
|
||||
private readonly now: () => Date;
|
||||
private stopping = false;
|
||||
private timer: NodeJS.Timeout | null = null;
|
||||
|
||||
constructor(options: ProjectionOutboxWorkerOptions) {
|
||||
this.store = options.store;
|
||||
this.config = { ...DEFAULT_PROJECTION_WORKER_CONFIG, ...options.config };
|
||||
this.now = options.now ?? (() => new Date());
|
||||
this.runtime = new ProjectionConsumerRuntime({
|
||||
store: this.store,
|
||||
consumers: options.consumers ?? INITIAL_PROJECTION_CONSUMERS,
|
||||
now: this.now
|
||||
});
|
||||
}
|
||||
|
||||
async drainOnce(): Promise<ProjectionWorkerDrainResult> {
|
||||
const now = this.now();
|
||||
const claimed = await this.store.claimAvailableEvents({
|
||||
workerId: this.config.workerId,
|
||||
batchSize: this.config.batchSize,
|
||||
leaseExpiresAt: new Date(now.getTime() + this.config.claimLeaseMs),
|
||||
now
|
||||
});
|
||||
|
||||
const results: ProjectionRuntimeResult[] = [];
|
||||
|
||||
for (const delivery of claimed) {
|
||||
if (this.stopping) {
|
||||
break;
|
||||
}
|
||||
|
||||
results.push(await this.processDelivery(delivery));
|
||||
}
|
||||
|
||||
return {
|
||||
workerId: this.config.workerId,
|
||||
claimedCount: claimed.length,
|
||||
processedCount: results.length,
|
||||
results
|
||||
};
|
||||
}
|
||||
|
||||
start(): void {
|
||||
if (!this.config.enabled || this.timer) {
|
||||
return;
|
||||
}
|
||||
|
||||
const tick = async () => {
|
||||
if (this.stopping) {
|
||||
return;
|
||||
}
|
||||
|
||||
await this.drainOnce();
|
||||
this.timer = setTimeout(tick, this.config.pollingIntervalMs);
|
||||
};
|
||||
|
||||
this.timer = setTimeout(tick, 0);
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
this.stopping = true;
|
||||
if (this.timer) {
|
||||
clearTimeout(this.timer);
|
||||
this.timer = null;
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
|
||||
private async processDelivery(
|
||||
delivery: BusinessEventDeliveryRecord
|
||||
): Promise<ProjectionRuntimeResult> {
|
||||
return this.runtime.processEvent(delivery.event);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user