task-d.7.5
This commit is contained in:
66
docs/implementation/task-d75-csv-commit-import-engine.md
Normal file
66
docs/implementation/task-d75-csv-commit-import-engine.md
Normal file
@@ -0,0 +1,66 @@
|
|||||||
|
# Task D.7.5 CSV Commit Import Engine
|
||||||
|
|
||||||
|
Date: 2026-07-03
|
||||||
|
Status: implemented
|
||||||
|
|
||||||
|
## Scope Delivered
|
||||||
|
|
||||||
|
- Added protected commit API:
|
||||||
|
- `POST /api/setup/import/commit`
|
||||||
|
- Added setup CSV commit service:
|
||||||
|
- `src/features/setup/server/csv-commit.service.ts`
|
||||||
|
- `src/features/setup/server/csv/commit-builder.ts`
|
||||||
|
- `src/features/setup/server/csv/upsert-executor.ts`
|
||||||
|
- `src/features/setup/server/csv/import-report.ts`
|
||||||
|
|
||||||
|
## Behavior
|
||||||
|
|
||||||
|
- Commit requires `previewHash`.
|
||||||
|
- Commit re-runs D.7.4 preview on the submitted files.
|
||||||
|
- Commit rejects mismatched `previewHash`.
|
||||||
|
- Commit rejects preview results with blocking errors.
|
||||||
|
- Commit follows dependency order from `setup/templates/templates.json` through the D.7.4 preview ordering.
|
||||||
|
- `dryRun=true` returns planned actions and does not start transaction writes.
|
||||||
|
- Non-dry-run commits execute in a single database transaction.
|
||||||
|
- Supported rows use idempotent business-key upsert behavior.
|
||||||
|
- Password updates only run for existing users when `password` is supplied and `reset_password=true`; otherwise password input is ignored with a report warning.
|
||||||
|
- Import reports never return password hashes.
|
||||||
|
|
||||||
|
## Supported Templates
|
||||||
|
|
||||||
|
The commit executor supports the templates registered by D.7.3/D.7.4:
|
||||||
|
|
||||||
|
- `users.csv`
|
||||||
|
- `organizations.csv`
|
||||||
|
- `memberships.csv`
|
||||||
|
- `crm-role-assignments.csv`
|
||||||
|
- `master-options.csv`
|
||||||
|
- `branches.csv`
|
||||||
|
- `product-types.csv`
|
||||||
|
- `document-sequences.csv`
|
||||||
|
- `approval-workflows.csv`
|
||||||
|
- `approval-steps.csv`
|
||||||
|
- `approval-matrix.csv`
|
||||||
|
- `customers.csv`
|
||||||
|
- `contacts.csv`
|
||||||
|
- `contact-shares.csv`
|
||||||
|
- `leads.csv`
|
||||||
|
- `opportunities.csv`
|
||||||
|
- `opportunity-parties.csv`
|
||||||
|
- `quotations.csv`
|
||||||
|
- `quotation-items.csv`
|
||||||
|
- `quotation-parties.csv`
|
||||||
|
- `quotation-topics.csv`
|
||||||
|
- `quotation-topic-items.csv`
|
||||||
|
|
||||||
|
## Safety Notes
|
||||||
|
|
||||||
|
- D.7.5 reuses D.7.4 parsing, template detection, validation, dependency resolution, preview actions, and `previewHash`.
|
||||||
|
- Setup import writes directly with Drizzle for this slice because no safe setup import service contract exists for every supported template.
|
||||||
|
- The import report includes `AUDIT_GAP` warnings for direct Drizzle writes; a future service wrapper can add per-row audit semantics without changing the preview contract.
|
||||||
|
- Storage, attachments, approval execution, PDFs, setup state persistence, seed manifest persistence, and destructive reset remain out of scope.
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
|
||||||
|
- `npm run typecheck` passed.
|
||||||
|
- `npx oxlint src/features/setup/server/csv src/features/setup/server/csv-commit.service.ts src/app/api/setup/import/commit/route.ts` passed.
|
||||||
493
plans/task-d.7.5.md
Normal file
493
plans/task-d.7.5.md
Normal file
@@ -0,0 +1,493 @@
|
|||||||
|
# Task D.7.5 – CSV Commit Import Engine
|
||||||
|
|
||||||
|
## Objective
|
||||||
|
|
||||||
|
Implement the CSV Commit Import Engine for ALLA OS Initial Setup.
|
||||||
|
|
||||||
|
This task must take a previously validated CSV preview result from D.7.4, verify its `previewHash`, and commit supported CSV imports safely using transactions and idempotent upsert behavior.
|
||||||
|
|
||||||
|
D.7.5 must reuse the D.7.4 preview/validation engine and must not duplicate validation logic.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Review Before Implementation
|
||||||
|
|
||||||
|
Review:
|
||||||
|
|
||||||
|
- AGENTS.md
|
||||||
|
- plans/task-d.7.md
|
||||||
|
- plans/task-d.7.1.md
|
||||||
|
- plans/task-d.7.2.md
|
||||||
|
- plans/task-d.7.3.md
|
||||||
|
- plans/task-d.7.4.md
|
||||||
|
- docs/setup/seed-matrix.md
|
||||||
|
- docs/setup/csv-template-spec.md
|
||||||
|
- docs/setup/import-mapping.md
|
||||||
|
- docs/setup/setup-wizard-blueprint.md
|
||||||
|
- docs/setup/verification-matrix.md
|
||||||
|
- setup/templates/*
|
||||||
|
- setup/templates/templates.json
|
||||||
|
- src/features/setup/server/csv-preview.service.ts
|
||||||
|
- src/features/setup/server/csv/**
|
||||||
|
- src/app/api/setup/import/preview/route.ts
|
||||||
|
- src/db/schema.ts
|
||||||
|
- src/db/seeds/**/*
|
||||||
|
- Existing CRM/foundation services for organizations, users, master options, document sequences, approval, customers, contacts, leads, opportunities, and quotations.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Implementation Scope
|
||||||
|
|
||||||
|
Create:
|
||||||
|
|
||||||
|
src/features/setup/server/csv-commit.service.ts
|
||||||
|
|
||||||
|
Supporting modules if needed:
|
||||||
|
|
||||||
|
src/features/setup/server/csv/commit-builder.ts
|
||||||
|
src/features/setup/server/csv/upsert-executor.ts
|
||||||
|
src/features/setup/server/csv/import-report.ts
|
||||||
|
|
||||||
|
Create API:
|
||||||
|
|
||||||
|
POST /api/setup/import/commit
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Core Rule
|
||||||
|
|
||||||
|
D.7.5 must use the exact same parsing, template detection, validation, dependency resolution, and preview report behavior from D.7.4.
|
||||||
|
|
||||||
|
Do not create a second validation engine.
|
||||||
|
|
||||||
|
Commit flow:
|
||||||
|
|
||||||
|
1. Receive CSV files and `previewHash`.
|
||||||
|
2. Re-run preview validation using D.7.4 engine.
|
||||||
|
3. Compare new preview hash with submitted `previewHash`.
|
||||||
|
4. Reject commit if hashes do not match.
|
||||||
|
5. Reject commit if preview contains blocking errors.
|
||||||
|
6. Commit valid files in dependency/import order.
|
||||||
|
7. Return structured import report.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Input
|
||||||
|
|
||||||
|
Accept `multipart/form-data`.
|
||||||
|
|
||||||
|
Required:
|
||||||
|
|
||||||
|
- CSV files
|
||||||
|
- `previewHash`
|
||||||
|
|
||||||
|
Optional:
|
||||||
|
|
||||||
|
- `mode`: `strict | permissive`
|
||||||
|
- `organizationId`
|
||||||
|
- `organizationCode`
|
||||||
|
- `dryRun`
|
||||||
|
- `allowOptionCreate`
|
||||||
|
- `skipWarnings`
|
||||||
|
- `replaceExisting`
|
||||||
|
|
||||||
|
Default behavior:
|
||||||
|
|
||||||
|
- `mode = strict`
|
||||||
|
- `dryRun = false`
|
||||||
|
- `allowOptionCreate = false`
|
||||||
|
- `skipWarnings = false`
|
||||||
|
- `replaceExisting = false`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Commit Rules
|
||||||
|
|
||||||
|
### General
|
||||||
|
|
||||||
|
- Commit only rows classified as `create` or `update`.
|
||||||
|
- Do not commit rows classified as `error`.
|
||||||
|
- In strict mode, any blocking error prevents the whole commit.
|
||||||
|
- Warnings do not block commit unless `skipWarnings = false` and the warning is configured as blocking.
|
||||||
|
- Unsupported templates must never commit.
|
||||||
|
- Unknown templates must never commit.
|
||||||
|
|
||||||
|
### Transaction
|
||||||
|
|
||||||
|
- Use one database transaction for all DB-only writes in the import batch.
|
||||||
|
- If any DB write fails, rollback the whole batch.
|
||||||
|
- Storage writes are out of scope for D.7.5.
|
||||||
|
- Attachments are still unsupported/deferred.
|
||||||
|
|
||||||
|
### Upsert
|
||||||
|
|
||||||
|
Every importer must be idempotent.
|
||||||
|
|
||||||
|
Re-running the same CSV should produce either:
|
||||||
|
|
||||||
|
- `updated`
|
||||||
|
- `skipped`
|
||||||
|
- or no-op
|
||||||
|
|
||||||
|
Never create duplicate records.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Supported Commit Templates
|
||||||
|
|
||||||
|
Implement commit for templates currently supported by D.7.3/D.7.4:
|
||||||
|
|
||||||
|
Foundation / Organization:
|
||||||
|
|
||||||
|
- users.csv
|
||||||
|
- organizations.csv
|
||||||
|
- memberships.csv
|
||||||
|
- crm-role-assignments.csv
|
||||||
|
- master-options.csv
|
||||||
|
- branches.csv
|
||||||
|
- product-types.csv
|
||||||
|
- document-sequences.csv
|
||||||
|
- approval-workflows.csv
|
||||||
|
- approval-steps.csv
|
||||||
|
- approval-matrix.csv
|
||||||
|
|
||||||
|
Business:
|
||||||
|
|
||||||
|
- customers.csv
|
||||||
|
- contacts.csv
|
||||||
|
- contact-shares.csv
|
||||||
|
- leads.csv
|
||||||
|
- opportunities.csv
|
||||||
|
- opportunity-parties.csv
|
||||||
|
- quotations.csv
|
||||||
|
- quotation-items.csv
|
||||||
|
- quotation-parties.csv
|
||||||
|
- quotation-topics.csv
|
||||||
|
- quotation-topic-items.csv
|
||||||
|
|
||||||
|
If a template cannot yet be safely committed because the service contract is unclear, mark it as blocked with a clear reason. Do not fake success.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Required Import Order
|
||||||
|
|
||||||
|
Use import order from `setup/templates/templates.json`.
|
||||||
|
|
||||||
|
Commit order must follow resolved dependency order, not upload order.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Importer Responsibilities
|
||||||
|
|
||||||
|
Each importer must define:
|
||||||
|
|
||||||
|
- template name
|
||||||
|
- target table or feature service
|
||||||
|
- business key
|
||||||
|
- create behavior
|
||||||
|
- update behavior
|
||||||
|
- skip behavior
|
||||||
|
- conflict behavior
|
||||||
|
- reference resolution
|
||||||
|
- audit behavior if existing service supports it
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Data Mapping
|
||||||
|
|
||||||
|
Follow:
|
||||||
|
|
||||||
|
docs/setup/import-mapping.md
|
||||||
|
|
||||||
|
Important examples:
|
||||||
|
|
||||||
|
branches.csv
|
||||||
|
|
||||||
|
maps to:
|
||||||
|
|
||||||
|
ms_options
|
||||||
|
category = crm_branch
|
||||||
|
|
||||||
|
product-types.csv
|
||||||
|
|
||||||
|
maps to:
|
||||||
|
|
||||||
|
ms_options
|
||||||
|
category = crm_product_type
|
||||||
|
|
||||||
|
document-sequences.csv
|
||||||
|
|
||||||
|
resolves:
|
||||||
|
|
||||||
|
organization_code
|
||||||
|
branch_code
|
||||||
|
product_type_code
|
||||||
|
document_type
|
||||||
|
period
|
||||||
|
prefix
|
||||||
|
|
||||||
|
quotation-topic-items.csv
|
||||||
|
|
||||||
|
uses `topic_key` from uploaded batch to resolve committed quotation topic ids.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Security
|
||||||
|
|
||||||
|
- API must be protected.
|
||||||
|
- Require `super_admin` for setup-wide import.
|
||||||
|
- Organization admin may be supported later, but D.7.5 can restrict to `super_admin`.
|
||||||
|
- Do not return raw SQL, stack traces, password hashes, or secrets.
|
||||||
|
- Raw `password` from users.csv must be hashed before storage.
|
||||||
|
- Never log raw passwords.
|
||||||
|
- Empty password must not overwrite existing password.
|
||||||
|
- Existing user password can be reset only when explicit reset flag exists.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Password Rules
|
||||||
|
|
||||||
|
For `users.csv`:
|
||||||
|
|
||||||
|
- New credential user requires either:
|
||||||
|
- `password`, or
|
||||||
|
- future invitation flow support.
|
||||||
|
- Existing user:
|
||||||
|
- empty password = keep existing password
|
||||||
|
- password + reset_password=true = update password hash
|
||||||
|
- password + reset_password=false = ignore password with warning
|
||||||
|
- Never return password or hash in report.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Report Contract
|
||||||
|
|
||||||
|
Return:
|
||||||
|
|
||||||
|
ImportReport
|
||||||
|
|
||||||
|
Fields:
|
||||||
|
|
||||||
|
- status: PASS | WARNING | FAIL
|
||||||
|
- generatedAt
|
||||||
|
- previewHash
|
||||||
|
- committedHash
|
||||||
|
- dryRun
|
||||||
|
- summary
|
||||||
|
- files
|
||||||
|
- errors
|
||||||
|
- warnings
|
||||||
|
|
||||||
|
File report:
|
||||||
|
|
||||||
|
- fileName
|
||||||
|
- template
|
||||||
|
- status
|
||||||
|
- imported
|
||||||
|
- updated
|
||||||
|
- skipped
|
||||||
|
- errors
|
||||||
|
- warnings
|
||||||
|
|
||||||
|
Row report:
|
||||||
|
|
||||||
|
- row
|
||||||
|
- action: create | update | skip | error
|
||||||
|
- businessKey
|
||||||
|
- entityId if safe
|
||||||
|
- message
|
||||||
|
- warnings
|
||||||
|
- errors
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
{
|
||||||
|
"status": "PASS",
|
||||||
|
"previewHash": "...",
|
||||||
|
"committedHash": "...",
|
||||||
|
"summary": {
|
||||||
|
"imported": 10,
|
||||||
|
"updated": 4,
|
||||||
|
"skipped": 2,
|
||||||
|
"errors": 0,
|
||||||
|
"warnings": 1
|
||||||
|
},
|
||||||
|
"files": []
|
||||||
|
}
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Error Contract
|
||||||
|
|
||||||
|
Every error must include:
|
||||||
|
|
||||||
|
- file
|
||||||
|
- row
|
||||||
|
- column
|
||||||
|
- code
|
||||||
|
- message
|
||||||
|
- suggestedFix
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
{
|
||||||
|
"file": "customers.csv",
|
||||||
|
"row": 12,
|
||||||
|
"column": "customer_code",
|
||||||
|
"code": "UPSERT_FAILED",
|
||||||
|
"message": "Customer could not be imported.",
|
||||||
|
"suggestedFix": "Review the customer row and retry after fixing related references."
|
||||||
|
}
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Dry Run
|
||||||
|
|
||||||
|
Support `dryRun=true`.
|
||||||
|
|
||||||
|
Dry run must:
|
||||||
|
|
||||||
|
- re-run preview
|
||||||
|
- build commit plan
|
||||||
|
- return planned actions
|
||||||
|
- not write database
|
||||||
|
- not run transaction writes
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Audit
|
||||||
|
|
||||||
|
If existing feature services automatically write audit logs, allow them.
|
||||||
|
|
||||||
|
If importing directly with Drizzle because no safe service exists, do not invent audit semantics in D.7.5.
|
||||||
|
|
||||||
|
Instead:
|
||||||
|
|
||||||
|
- include audit gap warning in report
|
||||||
|
- document follow-up
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Out Of Scope
|
||||||
|
|
||||||
|
- CSV preview validation logic duplication
|
||||||
|
- CSV template creation
|
||||||
|
- Setup Wizard UI
|
||||||
|
- Seed manifest persistence
|
||||||
|
- Setup state persistence
|
||||||
|
- Rollback after successful commit
|
||||||
|
- Storage import
|
||||||
|
- Attachment import
|
||||||
|
- Email settings import
|
||||||
|
- Unsupported templates
|
||||||
|
- Destructive reset
|
||||||
|
- PDF generation
|
||||||
|
- Approval workflow execution
|
||||||
|
- Document artifact creation
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Deliverables
|
||||||
|
|
||||||
|
Required:
|
||||||
|
|
||||||
|
src/features/setup/server/csv-commit.service.ts
|
||||||
|
src/features/setup/server/csv/commit-builder.ts
|
||||||
|
src/features/setup/server/csv/upsert-executor.ts
|
||||||
|
src/features/setup/server/csv/import-report.ts
|
||||||
|
src/app/api/setup/import/commit/route.ts
|
||||||
|
|
||||||
|
Optional:
|
||||||
|
|
||||||
|
src/features/setup/server/csv/importers/*.ts
|
||||||
|
|
||||||
|
Recommended importer organization:
|
||||||
|
|
||||||
|
src/features/setup/server/csv/importers/
|
||||||
|
users.importer.ts
|
||||||
|
organizations.importer.ts
|
||||||
|
memberships.importer.ts
|
||||||
|
crm-role-assignments.importer.ts
|
||||||
|
master-options.importer.ts
|
||||||
|
document-sequences.importer.ts
|
||||||
|
approval.importer.ts
|
||||||
|
customers.importer.ts
|
||||||
|
contacts.importer.ts
|
||||||
|
leads.importer.ts
|
||||||
|
opportunities.importer.ts
|
||||||
|
quotations.importer.ts
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Acceptance Criteria
|
||||||
|
|
||||||
|
✓ POST /api/setup/import/commit exists.
|
||||||
|
|
||||||
|
✓ Commit requires `previewHash`.
|
||||||
|
|
||||||
|
✓ Commit re-runs D.7.4 preview engine.
|
||||||
|
|
||||||
|
✓ Commit rejects mismatched preview hash.
|
||||||
|
|
||||||
|
✓ Commit rejects preview with blocking errors.
|
||||||
|
|
||||||
|
✓ Commit follows dependency/import order from templates.json.
|
||||||
|
|
||||||
|
✓ Commit is transaction-safe for DB writes.
|
||||||
|
|
||||||
|
✓ Re-running the same valid CSV does not duplicate records.
|
||||||
|
|
||||||
|
✓ users.csv password handling is safe.
|
||||||
|
|
||||||
|
✓ branches.csv imports into ms_options category crm_branch.
|
||||||
|
|
||||||
|
✓ product-types.csv imports into ms_options category crm_product_type.
|
||||||
|
|
||||||
|
✓ quotation-topic-items.csv resolves topic_key from committed/imported quotation topics.
|
||||||
|
|
||||||
|
✓ Unsupported templates do not commit.
|
||||||
|
|
||||||
|
✓ dryRun returns planned actions without writing DB.
|
||||||
|
|
||||||
|
✓ Structured import report is returned.
|
||||||
|
|
||||||
|
✓ No storage/binary attachment import is implemented.
|
||||||
|
|
||||||
|
✓ Typecheck passes or unrelated failures are documented.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Suggested Verification
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
npm run typecheck
|
||||||
|
|
||||||
|
Run scoped lint:
|
||||||
|
|
||||||
|
npx oxlint src/features/setup/server/csv src/features/setup/server/csv-commit.service.ts src/app/api/setup/import/commit
|
||||||
|
|
||||||
|
Manual test flow:
|
||||||
|
|
||||||
|
1. Call POST /api/setup/import/preview with valid CSV batch.
|
||||||
|
2. Copy previewHash.
|
||||||
|
3. Call POST /api/setup/import/commit with same files and previewHash.
|
||||||
|
4. Confirm rows are created/updated.
|
||||||
|
5. Re-run same commit.
|
||||||
|
6. Confirm no duplicates.
|
||||||
|
7. Modify CSV after preview.
|
||||||
|
8. Confirm commit rejects previewHash mismatch.
|
||||||
|
9. Submit CSV with blocking errors.
|
||||||
|
10. Confirm commit fails without partial writes.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Follow-up
|
||||||
|
|
||||||
|
After this task:
|
||||||
|
|
||||||
|
D.7.6 – Dashboard Setup Wizard UI
|
||||||
|
|
||||||
|
D.7.8 – Seed Manifest and Setup State Persistence
|
||||||
|
|
||||||
|
D.7.7 – Scoped Reset and Reseed Engine
|
||||||
51
src/app/api/setup/import/commit/route.ts
Normal file
51
src/app/api/setup/import/commit/route.ts
Normal file
@@ -0,0 +1,51 @@
|
|||||||
|
import { NextRequest, NextResponse } from 'next/server';
|
||||||
|
import { commitSetupCsvImport } from '@/features/setup/server/csv-commit.service';
|
||||||
|
import { AuthError, requireSystemRole } from '@/lib/auth/session';
|
||||||
|
|
||||||
|
function isFile(value: FormDataEntryValue): value is File {
|
||||||
|
return typeof value === 'object' && 'arrayBuffer' in value && 'name' in value;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseBoolean(value: FormDataEntryValue | null): boolean {
|
||||||
|
if (typeof value !== 'string') return false;
|
||||||
|
return value.trim().toLowerCase() === 'true';
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function POST(request: NextRequest) {
|
||||||
|
try {
|
||||||
|
const session = await requireSystemRole('super_admin');
|
||||||
|
const formData = await request.formData();
|
||||||
|
const previewHash = String(formData.get('previewHash') ?? '');
|
||||||
|
const dryRun = parseBoolean(formData.get('dryRun'));
|
||||||
|
const uploadedFiles = Array.from(formData.values()).filter(isFile);
|
||||||
|
|
||||||
|
if (uploadedFiles.length === 0) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ message: 'Upload at least one setup CSV file using multipart/form-data.' },
|
||||||
|
{ status: 400 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const files = await Promise.all(
|
||||||
|
uploadedFiles.map(async (file) => ({
|
||||||
|
fileName: file.name,
|
||||||
|
bytes: new Uint8Array(await file.arrayBuffer())
|
||||||
|
}))
|
||||||
|
);
|
||||||
|
|
||||||
|
const report = await commitSetupCsvImport({
|
||||||
|
files,
|
||||||
|
previewHash,
|
||||||
|
dryRun,
|
||||||
|
actorId: session.user.id
|
||||||
|
});
|
||||||
|
|
||||||
|
return NextResponse.json(report, { status: report.status === 'FAIL' ? 400 : 200 });
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof AuthError) {
|
||||||
|
return NextResponse.json({ message: error.message }, { status: error.status });
|
||||||
|
}
|
||||||
|
|
||||||
|
return NextResponse.json({ message: 'Unable commit setup CSV import' }, { status: 500 });
|
||||||
|
}
|
||||||
|
}
|
||||||
131
src/features/setup/server/csv-commit.service.ts
Normal file
131
src/features/setup/server/csv-commit.service.ts
Normal file
@@ -0,0 +1,131 @@
|
|||||||
|
import 'server-only';
|
||||||
|
|
||||||
|
import { buildCsvCommitPlan } from './csv/commit-builder';
|
||||||
|
import { buildImportReport, createImportError, type ImportReport } from './csv/import-report';
|
||||||
|
import { CsvCommitExecutionError, executeCsvCommitPlan } from './csv/upsert-executor';
|
||||||
|
import { previewSetupCsvImport } from './csv-preview.service';
|
||||||
|
import type { CsvPreviewFileInput } from './csv/types';
|
||||||
|
|
||||||
|
export interface CommitSetupCsvImportInput {
|
||||||
|
files: CsvPreviewFileInput[];
|
||||||
|
previewHash: string;
|
||||||
|
dryRun?: boolean;
|
||||||
|
actorId: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function commitSetupCsvImport(input: CommitSetupCsvImportInput): Promise<ImportReport> {
|
||||||
|
const preview = await previewSetupCsvImport(input.files);
|
||||||
|
const plan = buildCsvCommitPlan(preview);
|
||||||
|
|
||||||
|
if (!input.previewHash) {
|
||||||
|
return buildImportReport({
|
||||||
|
previewHash: preview.previewHash,
|
||||||
|
committedHash: plan.committedHash,
|
||||||
|
dryRun: Boolean(input.dryRun),
|
||||||
|
files: [
|
||||||
|
{
|
||||||
|
fileName: '__request__',
|
||||||
|
template: 'setup-csv-commit',
|
||||||
|
status: 'FAIL',
|
||||||
|
imported: 0,
|
||||||
|
updated: 0,
|
||||||
|
skipped: 0,
|
||||||
|
errors: [
|
||||||
|
createImportError({
|
||||||
|
file: '__request__',
|
||||||
|
row: 1,
|
||||||
|
column: 'previewHash',
|
||||||
|
code: 'PREVIEW_HASH_REQUIRED',
|
||||||
|
message: 'previewHash is required before committing CSV import.',
|
||||||
|
suggestedFix: 'Call /api/setup/import/preview and submit the returned previewHash.'
|
||||||
|
})
|
||||||
|
],
|
||||||
|
warnings: [],
|
||||||
|
rows: []
|
||||||
|
}
|
||||||
|
]
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (preview.previewHash !== input.previewHash) {
|
||||||
|
return buildImportReport({
|
||||||
|
previewHash: preview.previewHash,
|
||||||
|
committedHash: plan.committedHash,
|
||||||
|
dryRun: Boolean(input.dryRun),
|
||||||
|
files: [
|
||||||
|
{
|
||||||
|
fileName: '__request__',
|
||||||
|
template: 'setup-csv-commit',
|
||||||
|
status: 'FAIL',
|
||||||
|
imported: 0,
|
||||||
|
updated: 0,
|
||||||
|
skipped: 0,
|
||||||
|
errors: [
|
||||||
|
createImportError({
|
||||||
|
file: '__request__',
|
||||||
|
row: 1,
|
||||||
|
column: 'previewHash',
|
||||||
|
code: 'PREVIEW_HASH_MISMATCH',
|
||||||
|
message: 'Submitted previewHash does not match the uploaded CSV files.',
|
||||||
|
suggestedFix: 'Re-run preview for the exact files being committed and retry with the new previewHash.'
|
||||||
|
})
|
||||||
|
],
|
||||||
|
warnings: [],
|
||||||
|
rows: []
|
||||||
|
}
|
||||||
|
]
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (preview.summary.error > 0) {
|
||||||
|
return buildImportReport({
|
||||||
|
previewHash: preview.previewHash,
|
||||||
|
committedHash: plan.committedHash,
|
||||||
|
dryRun: Boolean(input.dryRun),
|
||||||
|
files: preview.files.map((file) => ({
|
||||||
|
fileName: file.fileName,
|
||||||
|
template: file.template,
|
||||||
|
status: 'FAIL',
|
||||||
|
imported: 0,
|
||||||
|
updated: 0,
|
||||||
|
skipped: 0,
|
||||||
|
errors: file.errors,
|
||||||
|
warnings: file.warnings,
|
||||||
|
rows: file.rows.map((row) => ({
|
||||||
|
row: row.row,
|
||||||
|
action: row.action === 'error' ? 'error' : 'skip',
|
||||||
|
businessKey: row.key,
|
||||||
|
message: row.action === 'error' ? 'Preview row has blocking errors.' : 'Commit blocked because preview contains errors.',
|
||||||
|
warnings: row.warnings,
|
||||||
|
errors: row.errors
|
||||||
|
}))
|
||||||
|
}))
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
let files;
|
||||||
|
try {
|
||||||
|
files = await executeCsvCommitPlan(plan, {
|
||||||
|
actorId: input.actorId,
|
||||||
|
dryRun: Boolean(input.dryRun)
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof CsvCommitExecutionError) {
|
||||||
|
return buildImportReport({
|
||||||
|
previewHash: preview.previewHash,
|
||||||
|
committedHash: plan.committedHash,
|
||||||
|
dryRun: Boolean(input.dryRun),
|
||||||
|
files: error.files
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
return buildImportReport({
|
||||||
|
previewHash: preview.previewHash,
|
||||||
|
committedHash: plan.committedHash,
|
||||||
|
dryRun: Boolean(input.dryRun),
|
||||||
|
files
|
||||||
|
});
|
||||||
|
}
|
||||||
65
src/features/setup/server/csv/commit-builder.ts
Normal file
65
src/features/setup/server/csv/commit-builder.ts
Normal file
@@ -0,0 +1,65 @@
|
|||||||
|
import { createHash } from 'node:crypto';
|
||||||
|
import type { CsvPreviewFileResult, CsvPreviewResult, CsvPreviewRow } from './types';
|
||||||
|
|
||||||
|
export interface CsvCommitPlanRow {
|
||||||
|
fileName: string;
|
||||||
|
template: string;
|
||||||
|
importOrder: number;
|
||||||
|
row: CsvPreviewRow;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CsvCommitPlanFile {
|
||||||
|
fileName: string;
|
||||||
|
template: string;
|
||||||
|
importOrder: number;
|
||||||
|
rows: CsvCommitPlanRow[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CsvCommitPlan {
|
||||||
|
preview: CsvPreviewResult;
|
||||||
|
files: CsvCommitPlanFile[];
|
||||||
|
rows: CsvCommitPlanRow[];
|
||||||
|
committedHash: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
function stableRowPayload(file: CsvPreviewFileResult, row: CsvPreviewRow) {
|
||||||
|
return {
|
||||||
|
fileName: file.fileName,
|
||||||
|
importOrder: file.importOrder,
|
||||||
|
row: row.row,
|
||||||
|
key: row.key,
|
||||||
|
action: row.action,
|
||||||
|
values: row.values
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildCsvCommitPlan(preview: CsvPreviewResult): CsvCommitPlan {
|
||||||
|
const files = preview.files
|
||||||
|
.toSorted((left, right) => left.importOrder - right.importOrder || left.fileName.localeCompare(right.fileName))
|
||||||
|
.map((file) => ({
|
||||||
|
fileName: file.fileName,
|
||||||
|
template: file.template,
|
||||||
|
importOrder: file.importOrder,
|
||||||
|
rows: file.rows
|
||||||
|
.filter((row) => row.action !== 'error')
|
||||||
|
.toSorted((left, right) => left.row - right.row)
|
||||||
|
.map((row) => ({
|
||||||
|
fileName: file.fileName,
|
||||||
|
template: file.template,
|
||||||
|
importOrder: file.importOrder,
|
||||||
|
row
|
||||||
|
}))
|
||||||
|
}));
|
||||||
|
|
||||||
|
const rows = files.flatMap((file) => file.rows);
|
||||||
|
const committedHash = createHash('sha256')
|
||||||
|
.update(
|
||||||
|
JSON.stringify({
|
||||||
|
previewHash: preview.previewHash,
|
||||||
|
rows: preview.files.flatMap((file) => file.rows.map((row) => stableRowPayload(file, row)))
|
||||||
|
})
|
||||||
|
)
|
||||||
|
.digest('hex');
|
||||||
|
|
||||||
|
return { preview, files, rows, committedHash };
|
||||||
|
}
|
||||||
121
src/features/setup/server/csv/import-report.ts
Normal file
121
src/features/setup/server/csv/import-report.ts
Normal file
@@ -0,0 +1,121 @@
|
|||||||
|
import type { CsvPreviewError, CsvPreviewWarning } from './types';
|
||||||
|
|
||||||
|
export type ImportStatus = 'PASS' | 'WARNING' | 'FAIL';
|
||||||
|
export type ImportAction = 'create' | 'update' | 'skip' | 'error';
|
||||||
|
|
||||||
|
export interface ImportRowReport {
|
||||||
|
row: number;
|
||||||
|
action: ImportAction;
|
||||||
|
businessKey: string | null;
|
||||||
|
entityId?: string;
|
||||||
|
message: string;
|
||||||
|
warnings: CsvPreviewWarning[];
|
||||||
|
errors: CsvPreviewError[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ImportFileReport {
|
||||||
|
fileName: string;
|
||||||
|
template: string;
|
||||||
|
status: ImportStatus;
|
||||||
|
imported: number;
|
||||||
|
updated: number;
|
||||||
|
skipped: number;
|
||||||
|
errors: CsvPreviewError[];
|
||||||
|
warnings: CsvPreviewWarning[];
|
||||||
|
rows: ImportRowReport[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ImportReport {
|
||||||
|
status: ImportStatus;
|
||||||
|
generatedAt: string;
|
||||||
|
previewHash: string;
|
||||||
|
committedHash: string;
|
||||||
|
dryRun: boolean;
|
||||||
|
summary: {
|
||||||
|
imported: number;
|
||||||
|
updated: number;
|
||||||
|
skipped: number;
|
||||||
|
errors: number;
|
||||||
|
warnings: number;
|
||||||
|
};
|
||||||
|
files: ImportFileReport[];
|
||||||
|
errors: CsvPreviewError[];
|
||||||
|
warnings: CsvPreviewWarning[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createImportError(input: {
|
||||||
|
file: string;
|
||||||
|
row: number;
|
||||||
|
column: string;
|
||||||
|
code: string;
|
||||||
|
message: string;
|
||||||
|
suggestedFix: string;
|
||||||
|
}): CsvPreviewError {
|
||||||
|
return input;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createImportWarning(input: {
|
||||||
|
file: string;
|
||||||
|
row: number;
|
||||||
|
column: string;
|
||||||
|
code: string;
|
||||||
|
message: string;
|
||||||
|
suggestedFix: string;
|
||||||
|
}): CsvPreviewWarning {
|
||||||
|
return input;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildImportReport(input: {
|
||||||
|
previewHash: string;
|
||||||
|
committedHash: string;
|
||||||
|
dryRun: boolean;
|
||||||
|
files: ImportFileReport[];
|
||||||
|
}): ImportReport {
|
||||||
|
const errors = input.files.flatMap((file) => file.errors);
|
||||||
|
const warnings = input.files.flatMap((file) => file.warnings);
|
||||||
|
const summary = input.files.reduce(
|
||||||
|
(accumulator, file) => ({
|
||||||
|
imported: accumulator.imported + file.imported,
|
||||||
|
updated: accumulator.updated + file.updated,
|
||||||
|
skipped: accumulator.skipped + file.skipped,
|
||||||
|
errors: accumulator.errors + file.errors.length,
|
||||||
|
warnings: accumulator.warnings + file.warnings.length
|
||||||
|
}),
|
||||||
|
{ imported: 0, updated: 0, skipped: 0, errors: 0, warnings: 0 }
|
||||||
|
);
|
||||||
|
|
||||||
|
return {
|
||||||
|
status: errors.length > 0 ? 'FAIL' : warnings.length > 0 ? 'WARNING' : 'PASS',
|
||||||
|
generatedAt: new Date().toISOString(),
|
||||||
|
previewHash: input.previewHash,
|
||||||
|
committedHash: input.committedHash,
|
||||||
|
dryRun: input.dryRun,
|
||||||
|
summary,
|
||||||
|
files: input.files,
|
||||||
|
errors,
|
||||||
|
warnings
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function summarizeFileRows(input: {
|
||||||
|
fileName: string;
|
||||||
|
template: string;
|
||||||
|
rows: ImportRowReport[];
|
||||||
|
warnings?: CsvPreviewWarning[];
|
||||||
|
}): ImportFileReport {
|
||||||
|
const errors = input.rows.flatMap((row) => row.errors);
|
||||||
|
const rowWarnings = input.rows.flatMap((row) => row.warnings);
|
||||||
|
const warnings = [...rowWarnings, ...(input.warnings ?? [])];
|
||||||
|
|
||||||
|
return {
|
||||||
|
fileName: input.fileName,
|
||||||
|
template: input.template,
|
||||||
|
status: errors.length > 0 ? 'FAIL' : warnings.length > 0 ? 'WARNING' : 'PASS',
|
||||||
|
imported: input.rows.filter((row) => row.action === 'create').length,
|
||||||
|
updated: input.rows.filter((row) => row.action === 'update').length,
|
||||||
|
skipped: input.rows.filter((row) => row.action === 'skip').length,
|
||||||
|
errors,
|
||||||
|
warnings,
|
||||||
|
rows: input.rows
|
||||||
|
};
|
||||||
|
}
|
||||||
1244
src/features/setup/server/csv/upsert-executor.ts
Normal file
1244
src/features/setup/server/csv/upsert-executor.ts
Normal file
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user