task-d.7.4
This commit is contained in:
@@ -0,0 +1,47 @@
|
||||
# Task D.7.4 CSV Preview Validation Engine
|
||||
|
||||
Date: 2026-07-02
|
||||
Status: implemented
|
||||
|
||||
## Scope Delivered
|
||||
|
||||
- Added reusable preview-only CSV engine:
|
||||
- `src/features/setup/server/csv-preview.service.ts`
|
||||
- `src/features/setup/server/csv/parser.ts`
|
||||
- `src/features/setup/server/csv/template-registry.ts`
|
||||
- `src/features/setup/server/csv/header-validator.ts`
|
||||
- `src/features/setup/server/csv/value-validator.ts`
|
||||
- `src/features/setup/server/csv/dependency-resolver.ts`
|
||||
- `src/features/setup/server/csv/validator.ts`
|
||||
- `src/features/setup/server/csv/preview-builder.ts`
|
||||
- `src/features/setup/server/csv/types.ts`
|
||||
- Added protected preview route:
|
||||
- `POST /api/setup/import/preview`
|
||||
|
||||
## Behavior
|
||||
|
||||
- Accepts one or more CSV files through `multipart/form-data`.
|
||||
- Detects templates by filename from `setup/templates/templates.json`.
|
||||
- Treats `templates.json` plus checked-in CSV headers as runtime template truth.
|
||||
- Rejects unknown and unsupported templates with structured row/file errors.
|
||||
- Validates exact header names, order, duplicate columns, missing columns, and extra columns.
|
||||
- Parses UTF-8 comma-delimited CSV with quoted cell support.
|
||||
- Reports invalid encoding, empty rows, duplicate rows, inconsistent column counts, required fields, emails, booleans, integers, decimals, dates, enums, JSON metadata, format rules, and amount/line-total warnings.
|
||||
- Resolves dependencies from both uploaded batch files and existing database rows without writing.
|
||||
- Detects duplicate business keys inside each uploaded file.
|
||||
- Classifies rows as `create`, `update`, or `error`.
|
||||
- Returns structured file/row report with `PASS`, `WARNING`, `FAIL` status and `previewHash` for D.7.5.
|
||||
|
||||
## Safety Notes
|
||||
|
||||
- No database insert/update/delete logic added.
|
||||
- No seed execution added.
|
||||
- No reset/reseed behavior added.
|
||||
- No document number generation, approval triggering, PDF generation, or storage writes added.
|
||||
- Relationship checks use read-only Drizzle selects.
|
||||
- `quotation-topic-items.csv` resolves `topic_key` from the uploaded batch because `topic_key` is an import-only alias and is not persisted in `crm_quotation_topics`.
|
||||
|
||||
## Verification
|
||||
|
||||
- `npm run typecheck`: passed.
|
||||
- `npx oxlint src/features/setup/server/csv src/features/setup/server/csv-preview.service.ts src/app/api/setup/import/preview`: passed.
|
||||
437
plans/task-d.7.4.md
Normal file
437
plans/task-d.7.4.md
Normal file
@@ -0,0 +1,437 @@
|
||||
# Task D.7.4 – CSV Preview Validation Engine
|
||||
|
||||
## Objective
|
||||
|
||||
Implement a reusable, non-destructive CSV Preview Validation Engine for ALLA OS.
|
||||
|
||||
This engine must validate uploaded CSV files against the official template pack and return a complete preview report without modifying any database records.
|
||||
|
||||
The preview engine becomes the shared foundation for D.7.5 (CSV Commit Import Engine), Setup Wizard, and future bulk import features.
|
||||
|
||||
No database writes, seed execution, or reset operations are allowed in this task.
|
||||
|
||||
---
|
||||
|
||||
## Review Before Implementation
|
||||
|
||||
Review:
|
||||
|
||||
- AGENTS.md
|
||||
- plans/task-d.7.md
|
||||
- plans/task-d.7.1.md
|
||||
- plans/task-d.7.2.md
|
||||
- plans/task-d.7.3.md
|
||||
- docs/setup/seed-matrix.md
|
||||
- docs/setup/csv-template-spec.md
|
||||
- docs/setup/import-mapping.md
|
||||
- docs/setup/setup-wizard-blueprint.md
|
||||
- docs/setup/verification-matrix.md
|
||||
- setup/templates/*
|
||||
- setup/templates/templates.json
|
||||
|
||||
Treat templates.json as the runtime source of truth.
|
||||
|
||||
---
|
||||
|
||||
## Implementation Scope
|
||||
|
||||
Create:
|
||||
|
||||
src/features/setup/server/csv-preview.service.ts
|
||||
|
||||
Supporting modules
|
||||
|
||||
src/features/setup/server/csv/
|
||||
parser.ts
|
||||
validator.ts
|
||||
dependency-resolver.ts
|
||||
header-validator.ts
|
||||
value-validator.ts
|
||||
preview-builder.ts
|
||||
template-registry.ts
|
||||
|
||||
Create API
|
||||
|
||||
POST /api/setup/import/preview
|
||||
|
||||
---
|
||||
|
||||
## Input
|
||||
|
||||
Accept one or more CSV files.
|
||||
|
||||
Each file must match one registered template.
|
||||
|
||||
Files may be uploaded in any order.
|
||||
|
||||
Engine must automatically resolve dependency order.
|
||||
|
||||
---
|
||||
|
||||
## Validation Stages
|
||||
|
||||
Stage 1
|
||||
|
||||
Template Detection
|
||||
|
||||
- Match filename
|
||||
- Match template registry
|
||||
- Match supported template
|
||||
|
||||
Reject unknown templates.
|
||||
|
||||
---
|
||||
|
||||
Stage 2
|
||||
|
||||
Schema Validation
|
||||
|
||||
Validate
|
||||
|
||||
- header names
|
||||
- header order
|
||||
- duplicated columns
|
||||
- missing columns
|
||||
- extra columns
|
||||
- template version (if available)
|
||||
|
||||
---
|
||||
|
||||
Stage 3
|
||||
|
||||
CSV Validation
|
||||
|
||||
Validate
|
||||
|
||||
- encoding
|
||||
- delimiter
|
||||
- empty rows
|
||||
- duplicated rows
|
||||
- inconsistent column counts
|
||||
|
||||
---
|
||||
|
||||
Stage 4
|
||||
|
||||
Value Validation
|
||||
|
||||
Validate
|
||||
|
||||
Required fields
|
||||
|
||||
Email
|
||||
|
||||
Boolean
|
||||
|
||||
Integer
|
||||
|
||||
Decimal
|
||||
|
||||
Date (YYYY-MM-DD)
|
||||
|
||||
Enum
|
||||
|
||||
Length
|
||||
|
||||
Regex
|
||||
|
||||
Custom validators
|
||||
|
||||
---
|
||||
|
||||
Stage 5
|
||||
|
||||
Relationship Resolution
|
||||
|
||||
Resolve
|
||||
|
||||
organization
|
||||
|
||||
branch
|
||||
|
||||
product type
|
||||
|
||||
users
|
||||
|
||||
memberships
|
||||
|
||||
CRM role
|
||||
|
||||
customer
|
||||
|
||||
contact
|
||||
|
||||
lead
|
||||
|
||||
opportunity
|
||||
|
||||
quotation
|
||||
|
||||
master options
|
||||
|
||||
Return
|
||||
|
||||
Resolved
|
||||
|
||||
Missing
|
||||
|
||||
Ambiguous
|
||||
|
||||
---
|
||||
|
||||
Stage 6
|
||||
|
||||
Dependency Validation
|
||||
|
||||
Read
|
||||
|
||||
templates.json
|
||||
|
||||
Validate
|
||||
|
||||
dependsOn
|
||||
|
||||
Detect
|
||||
|
||||
Missing parent files
|
||||
|
||||
Circular dependency
|
||||
|
||||
Invalid import order
|
||||
|
||||
---
|
||||
|
||||
Stage 7
|
||||
|
||||
Duplicate Detection
|
||||
|
||||
Classify
|
||||
|
||||
New
|
||||
|
||||
Existing
|
||||
|
||||
Update
|
||||
|
||||
Conflict
|
||||
|
||||
Duplicate inside CSV
|
||||
|
||||
---
|
||||
|
||||
Stage 8
|
||||
|
||||
Preview Report
|
||||
|
||||
Return
|
||||
|
||||
Imported
|
||||
|
||||
Updated
|
||||
|
||||
Skipped
|
||||
|
||||
Warnings
|
||||
|
||||
Errors
|
||||
|
||||
Grouped by file.
|
||||
|
||||
---
|
||||
|
||||
## Report Contract
|
||||
|
||||
```ts
|
||||
type PreviewResult = {
|
||||
fileName: string;
|
||||
|
||||
template: string;
|
||||
|
||||
status:
|
||||
| "PASS"
|
||||
| "WARNING"
|
||||
| "FAIL";
|
||||
|
||||
summary: {
|
||||
|
||||
rows:number;
|
||||
|
||||
valid:number;
|
||||
|
||||
warning:number;
|
||||
|
||||
error:number;
|
||||
|
||||
};
|
||||
|
||||
rows:[];
|
||||
}
|
||||
```
|
||||
|
||||
Overall response
|
||||
|
||||
```ts
|
||||
{
|
||||
summary,
|
||||
|
||||
files:[],
|
||||
|
||||
generatedAt,
|
||||
|
||||
previewHash
|
||||
}
|
||||
```
|
||||
|
||||
previewHash will be required by D.7.5.
|
||||
|
||||
---
|
||||
|
||||
## Rules
|
||||
|
||||
Do not
|
||||
|
||||
Insert
|
||||
|
||||
Update
|
||||
|
||||
Delete
|
||||
|
||||
Commit
|
||||
|
||||
Seed
|
||||
|
||||
Generate IDs
|
||||
|
||||
Generate document numbers
|
||||
|
||||
Trigger approval
|
||||
|
||||
Generate PDFs
|
||||
|
||||
Only preview.
|
||||
|
||||
---
|
||||
|
||||
## Error Contract
|
||||
|
||||
Every error
|
||||
|
||||
Must contain
|
||||
|
||||
File
|
||||
|
||||
Row
|
||||
|
||||
Column
|
||||
|
||||
Code
|
||||
|
||||
Message
|
||||
|
||||
Suggested Fix
|
||||
|
||||
Example
|
||||
|
||||
```json
|
||||
{
|
||||
"file":"customers.csv",
|
||||
|
||||
"row":12,
|
||||
|
||||
"column":"customer_type",
|
||||
|
||||
"code":"UNKNOWN_OPTION",
|
||||
|
||||
"message":"Customer type not found.",
|
||||
|
||||
"suggestedFix":"Import master-options.csv first."
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Reuse Rules
|
||||
|
||||
Reuse existing
|
||||
|
||||
Master Option services
|
||||
|
||||
Organization services
|
||||
|
||||
CRM services
|
||||
|
||||
Validation helpers
|
||||
|
||||
Document sequence helpers
|
||||
|
||||
Do not duplicate logic.
|
||||
|
||||
---
|
||||
|
||||
## Deliverables
|
||||
|
||||
src/features/setup/server/csv-preview.service.ts
|
||||
|
||||
src/features/setup/server/csv/
|
||||
|
||||
POST /api/setup/import/preview
|
||||
|
||||
Shared validation utilities
|
||||
|
||||
Structured preview response
|
||||
|
||||
previewHash generator
|
||||
|
||||
---
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
✓ Preview never writes database.
|
||||
|
||||
✓ Multiple CSVs supported.
|
||||
|
||||
✓ Dependency order resolved automatically.
|
||||
|
||||
✓ Unknown templates rejected.
|
||||
|
||||
✓ Header validation implemented.
|
||||
|
||||
✓ Data validation implemented.
|
||||
|
||||
✓ Relationship resolution implemented.
|
||||
|
||||
✓ Duplicate detection implemented.
|
||||
|
||||
✓ Structured report returned.
|
||||
|
||||
✓ previewHash generated.
|
||||
|
||||
✓ Uses templates.json as runtime registry.
|
||||
|
||||
✓ Typecheck passes.
|
||||
|
||||
---
|
||||
|
||||
## Future
|
||||
|
||||
D.7.5
|
||||
|
||||
Commit Import Engine
|
||||
|
||||
Uses
|
||||
|
||||
previewHash
|
||||
|
||||
↓
|
||||
|
||||
Transaction
|
||||
|
||||
↓
|
||||
|
||||
Upsert
|
||||
|
||||
↓
|
||||
|
||||
Import Report
|
||||
|
||||
No validation logic should be duplicated between D.7.4 and D.7.5.
|
||||
39
src/app/api/setup/import/preview/route.ts
Normal file
39
src/app/api/setup/import/preview/route.ts
Normal file
@@ -0,0 +1,39 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { previewSetupCsvImport } from '@/features/setup/server/csv-preview.service';
|
||||
import { AuthError, requireSystemRole } from '@/lib/auth/session';
|
||||
|
||||
function isFile(value: FormDataEntryValue): value is File {
|
||||
return typeof value === 'object' && 'arrayBuffer' in value && 'name' in value;
|
||||
}
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
await requireSystemRole('super_admin');
|
||||
|
||||
const formData = await request.formData();
|
||||
const uploadedFiles = Array.from(formData.values()).filter(isFile);
|
||||
|
||||
if (uploadedFiles.length === 0) {
|
||||
return NextResponse.json(
|
||||
{ message: 'Upload at least one setup CSV file using multipart/form-data.' },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
const files = await Promise.all(
|
||||
uploadedFiles.map(async (file) => ({
|
||||
fileName: file.name,
|
||||
bytes: new Uint8Array(await file.arrayBuffer())
|
||||
}))
|
||||
);
|
||||
const preview = await previewSetupCsvImport(files);
|
||||
|
||||
return NextResponse.json(preview);
|
||||
} catch (error) {
|
||||
if (error instanceof AuthError) {
|
||||
return NextResponse.json({ message: error.message }, { status: error.status });
|
||||
}
|
||||
|
||||
return NextResponse.json({ message: 'Unable to preview setup CSV import' }, { status: 500 });
|
||||
}
|
||||
}
|
||||
121
src/features/setup/server/csv-preview.service.ts
Normal file
121
src/features/setup/server/csv-preview.service.ts
Normal file
@@ -0,0 +1,121 @@
|
||||
import 'server-only';
|
||||
|
||||
import path from 'node:path';
|
||||
import { buildCsvPreviewBatchIndex } from './csv/dependency-resolver';
|
||||
import { parseCsvFile } from './csv/parser';
|
||||
import { buildFileResult, buildPreviewResult } from './csv/preview-builder';
|
||||
import {
|
||||
findTemplateByFileName,
|
||||
getCsvTemplateRegistry,
|
||||
isUnsupportedTemplate
|
||||
} from './csv/template-registry';
|
||||
import type {
|
||||
CsvParsedFile,
|
||||
CsvPreviewError,
|
||||
CsvPreviewFileInput,
|
||||
CsvPreviewResult
|
||||
} from './csv/types';
|
||||
import { validateCsvPreviewFile } from './csv/validator';
|
||||
|
||||
function templateDetectionError(fileName: string, code: string, message: string, suggestedFix: string): CsvParsedFile {
|
||||
const file = path.basename(fileName);
|
||||
const error: CsvPreviewError = {
|
||||
file,
|
||||
row: 1,
|
||||
column: '__file__',
|
||||
code,
|
||||
message,
|
||||
suggestedFix
|
||||
};
|
||||
|
||||
return {
|
||||
fileName: file,
|
||||
template: {
|
||||
file,
|
||||
template: 'unknown',
|
||||
seedType: 'unknown',
|
||||
importOrder: Number.MAX_SAFE_INTEGER,
|
||||
supported: false,
|
||||
requiredColumns: [],
|
||||
description: 'Unknown template',
|
||||
expectedHeaders: []
|
||||
},
|
||||
headers: [],
|
||||
rows: [],
|
||||
errors: [error],
|
||||
warnings: []
|
||||
};
|
||||
}
|
||||
|
||||
export async function previewSetupCsvImport(files: CsvPreviewFileInput[]): Promise<CsvPreviewResult> {
|
||||
const registry = await getCsvTemplateRegistry();
|
||||
const parsedFiles: CsvParsedFile[] = [];
|
||||
|
||||
for (const input of files) {
|
||||
const fileName = path.basename(input.fileName);
|
||||
const template = findTemplateByFileName(registry, fileName);
|
||||
|
||||
if (!template) {
|
||||
parsedFiles.push(
|
||||
templateDetectionError(
|
||||
fileName,
|
||||
isUnsupportedTemplate(registry, fileName) ? 'UNSUPPORTED_TEMPLATE' : 'UNKNOWN_TEMPLATE',
|
||||
isUnsupportedTemplate(registry, fileName)
|
||||
? 'CSV template is known but not supported by the setup import engine.'
|
||||
: 'CSV file does not match any registered setup template.',
|
||||
'Use one of the supported files from setup/templates/templates.json.'
|
||||
)
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!template.supported) {
|
||||
parsedFiles.push(
|
||||
templateDetectionError(
|
||||
fileName,
|
||||
'UNSUPPORTED_TEMPLATE',
|
||||
'CSV template is registered but not supported.',
|
||||
'Remove this file from the preview upload.'
|
||||
)
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
parsedFiles.push(parseCsvFile({ fileName, bytes: input.bytes, template }));
|
||||
}
|
||||
|
||||
const uploadCounts = new Map<string, number>();
|
||||
parsedFiles.forEach((file) => {
|
||||
uploadCounts.set(file.template.file, (uploadCounts.get(file.template.file) ?? 0) + 1);
|
||||
});
|
||||
parsedFiles.forEach((file) => {
|
||||
if (uploadCounts.get(file.template.file)! > 1 && file.template.supported) {
|
||||
file.errors.push({
|
||||
file: file.fileName,
|
||||
row: 1,
|
||||
column: '__file__',
|
||||
code: 'DUPLICATE_FILE',
|
||||
message: `Template ${file.template.file} was uploaded more than once.`,
|
||||
suggestedFix: 'Upload only one file for each setup template.'
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
const sortedFiles = parsedFiles.toSorted((left, right) => left.template.importOrder - right.template.importOrder);
|
||||
const batch = buildCsvPreviewBatchIndex(sortedFiles.filter((file) => file.template.supported));
|
||||
const fileResults = [];
|
||||
|
||||
for (const file of sortedFiles) {
|
||||
const rows = await validateCsvPreviewFile(file, batch);
|
||||
fileResults.push(
|
||||
buildFileResult({
|
||||
fileName: file.fileName,
|
||||
template: file.template.template,
|
||||
importOrder: file.template.importOrder,
|
||||
rows
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
return buildPreviewResult(fileResults);
|
||||
}
|
||||
1242
src/features/setup/server/csv/dependency-resolver.ts
Normal file
1242
src/features/setup/server/csv/dependency-resolver.ts
Normal file
File diff suppressed because it is too large
Load Diff
75
src/features/setup/server/csv/header-validator.ts
Normal file
75
src/features/setup/server/csv/header-validator.ts
Normal file
@@ -0,0 +1,75 @@
|
||||
import type { CsvParsedFile, CsvPreviewError } from './types';
|
||||
|
||||
function issue(
|
||||
file: string,
|
||||
column: string,
|
||||
code: string,
|
||||
message: string,
|
||||
suggestedFix: string
|
||||
): CsvPreviewError {
|
||||
return { file, row: 1, column, code, message, suggestedFix };
|
||||
}
|
||||
|
||||
export function validateCsvHeaders(file: CsvParsedFile): CsvPreviewError[] {
|
||||
const errors: CsvPreviewError[] = [];
|
||||
const seen = new Map<string, number>();
|
||||
|
||||
file.headers.forEach((header, index) => {
|
||||
const previousIndex = seen.get(header);
|
||||
if (previousIndex !== undefined) {
|
||||
errors.push(
|
||||
issue(
|
||||
file.fileName,
|
||||
header,
|
||||
'DUPLICATE_COLUMN',
|
||||
`Column "${header}" appears more than once.`,
|
||||
`Remove the duplicate column at position ${index + 1}; keep the official template order.`
|
||||
)
|
||||
);
|
||||
}
|
||||
seen.set(header, index);
|
||||
});
|
||||
|
||||
file.template.expectedHeaders.forEach((expectedHeader, index) => {
|
||||
if (!file.headers.includes(expectedHeader)) {
|
||||
errors.push(
|
||||
issue(
|
||||
file.fileName,
|
||||
expectedHeader,
|
||||
'MISSING_COLUMN',
|
||||
`Required template column "${expectedHeader}" is missing.`,
|
||||
'Download the latest setup CSV template and keep all official columns.'
|
||||
)
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (file.headers[index] !== expectedHeader) {
|
||||
errors.push(
|
||||
issue(
|
||||
file.fileName,
|
||||
expectedHeader,
|
||||
'HEADER_ORDER_MISMATCH',
|
||||
`Column "${expectedHeader}" is not in the expected position ${index + 1}.`,
|
||||
'Reorder columns to match the official template exactly.'
|
||||
)
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
file.headers.forEach((header) => {
|
||||
if (!file.template.expectedHeaders.includes(header)) {
|
||||
errors.push(
|
||||
issue(
|
||||
file.fileName,
|
||||
header,
|
||||
'EXTRA_COLUMN',
|
||||
`Column "${header}" is not part of the official template.`,
|
||||
'Remove extra columns before previewing the import.'
|
||||
)
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
return errors;
|
||||
}
|
||||
212
src/features/setup/server/csv/parser.ts
Normal file
212
src/features/setup/server/csv/parser.ts
Normal file
@@ -0,0 +1,212 @@
|
||||
import type {
|
||||
CsvParsedFile,
|
||||
CsvParsedRow,
|
||||
CsvPreviewError,
|
||||
CsvPreviewWarning,
|
||||
CsvTemplateDefinition
|
||||
} from './types';
|
||||
|
||||
function buildIssue(
|
||||
file: string,
|
||||
row: number,
|
||||
column: string,
|
||||
code: string,
|
||||
message: string,
|
||||
suggestedFix: string
|
||||
): CsvPreviewError {
|
||||
return { file, row, column, code, message, suggestedFix };
|
||||
}
|
||||
|
||||
function decodeUtf8(fileName: string, bytes: Uint8Array): {
|
||||
text: string;
|
||||
errors: CsvPreviewError[];
|
||||
warnings: CsvPreviewWarning[];
|
||||
} {
|
||||
const errors: CsvPreviewError[] = [];
|
||||
const warnings: CsvPreviewWarning[] = [];
|
||||
|
||||
if (bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) {
|
||||
warnings.push(
|
||||
buildIssue(
|
||||
fileName,
|
||||
1,
|
||||
'__file__',
|
||||
'UTF8_BOM',
|
||||
'CSV file contains a UTF-8 BOM.',
|
||||
'Export the CSV as UTF-8 without BOM when possible.'
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
return {
|
||||
text: new TextDecoder('utf-8', { fatal: true }).decode(bytes).replace(/^\uFEFF/, ''),
|
||||
errors,
|
||||
warnings
|
||||
};
|
||||
} catch {
|
||||
errors.push(
|
||||
buildIssue(
|
||||
fileName,
|
||||
1,
|
||||
'__file__',
|
||||
'INVALID_ENCODING',
|
||||
'CSV file is not valid UTF-8.',
|
||||
'Save the file as UTF-8 CSV and upload it again.'
|
||||
)
|
||||
);
|
||||
return { text: '', errors, warnings };
|
||||
}
|
||||
}
|
||||
|
||||
function parseCsvText(text: string): string[][] {
|
||||
const rows: string[][] = [];
|
||||
let currentRow: string[] = [];
|
||||
let currentCell = '';
|
||||
let inQuotes = false;
|
||||
|
||||
for (let index = 0; index < text.length; index += 1) {
|
||||
const char = text[index];
|
||||
const next = text[index + 1];
|
||||
|
||||
if (char === '"') {
|
||||
if (inQuotes && next === '"') {
|
||||
currentCell += '"';
|
||||
index += 1;
|
||||
} else {
|
||||
inQuotes = !inQuotes;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (char === ',' && !inQuotes) {
|
||||
currentRow.push(currentCell);
|
||||
currentCell = '';
|
||||
continue;
|
||||
}
|
||||
|
||||
if ((char === '\n' || char === '\r') && !inQuotes) {
|
||||
if (char === '\r' && next === '\n') index += 1;
|
||||
currentRow.push(currentCell);
|
||||
rows.push(currentRow);
|
||||
currentRow = [];
|
||||
currentCell = '';
|
||||
continue;
|
||||
}
|
||||
|
||||
currentCell += char;
|
||||
}
|
||||
|
||||
if (currentCell.length > 0 || currentRow.length > 0) {
|
||||
currentRow.push(currentCell);
|
||||
rows.push(currentRow);
|
||||
}
|
||||
|
||||
return rows;
|
||||
}
|
||||
|
||||
export function parseCsvFile(input: {
|
||||
fileName: string;
|
||||
bytes: Uint8Array;
|
||||
template: CsvTemplateDefinition;
|
||||
}): CsvParsedFile {
|
||||
const decoded = decodeUtf8(input.fileName, input.bytes);
|
||||
const errors = [...decoded.errors];
|
||||
const warnings = [...decoded.warnings];
|
||||
|
||||
if (decoded.errors.length > 0) {
|
||||
return {
|
||||
fileName: input.fileName,
|
||||
template: input.template,
|
||||
headers: [],
|
||||
rows: [],
|
||||
errors,
|
||||
warnings
|
||||
};
|
||||
}
|
||||
|
||||
const parsedRows = parseCsvText(decoded.text);
|
||||
const [headerRow, ...dataRows] = parsedRows;
|
||||
|
||||
if (!headerRow) {
|
||||
errors.push(
|
||||
buildIssue(
|
||||
input.fileName,
|
||||
1,
|
||||
'__header__',
|
||||
'EMPTY_FILE',
|
||||
'CSV file is empty.',
|
||||
'Upload a CSV file with the official template header row.'
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
const headers = (headerRow ?? []).map((header) => header.trim());
|
||||
const rows: CsvParsedRow[] = [];
|
||||
const rowFingerprints = new Map<string, number>();
|
||||
|
||||
dataRows.forEach((raw, index) => {
|
||||
const rowNumber = index + 2;
|
||||
const isEmpty = raw.every((cell) => cell.trim() === '');
|
||||
|
||||
if (isEmpty) {
|
||||
warnings.push(
|
||||
buildIssue(
|
||||
input.fileName,
|
||||
rowNumber,
|
||||
'__row__',
|
||||
'EMPTY_ROW',
|
||||
'CSV row is empty and will be skipped.',
|
||||
'Remove empty rows from the file.'
|
||||
)
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (raw.length !== headers.length) {
|
||||
errors.push(
|
||||
buildIssue(
|
||||
input.fileName,
|
||||
rowNumber,
|
||||
'__row__',
|
||||
'INCONSISTENT_COLUMN_COUNT',
|
||||
`Row has ${raw.length} columns but header has ${headers.length}.`,
|
||||
'Fix quoting or column separators so every row matches the template header.'
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
const fingerprint = raw.map((cell) => cell.trim()).join('\u001f');
|
||||
const firstRow = rowFingerprints.get(fingerprint);
|
||||
if (firstRow) {
|
||||
warnings.push(
|
||||
buildIssue(
|
||||
input.fileName,
|
||||
rowNumber,
|
||||
'__row__',
|
||||
'DUPLICATE_ROW',
|
||||
`Row duplicates row ${firstRow}.`,
|
||||
'Remove duplicate rows or confirm that the later row is intentional.'
|
||||
)
|
||||
);
|
||||
} else {
|
||||
rowFingerprints.set(fingerprint, rowNumber);
|
||||
}
|
||||
|
||||
const values = headers.reduce<Record<string, string>>((accumulator, header, cellIndex) => {
|
||||
accumulator[header] = (raw[cellIndex] ?? '').trim();
|
||||
return accumulator;
|
||||
}, {});
|
||||
|
||||
rows.push({ rowNumber, values, raw });
|
||||
});
|
||||
|
||||
return {
|
||||
fileName: input.fileName,
|
||||
template: input.template,
|
||||
headers,
|
||||
rows,
|
||||
errors,
|
||||
warnings
|
||||
};
|
||||
}
|
||||
83
src/features/setup/server/csv/preview-builder.ts
Normal file
83
src/features/setup/server/csv/preview-builder.ts
Normal file
@@ -0,0 +1,83 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import type {
|
||||
CsvPreviewFileResult,
|
||||
CsvPreviewResult,
|
||||
CsvPreviewRow,
|
||||
CsvPreviewStatus
|
||||
} from './types';
|
||||
|
||||
export function resolveRowStatus(row: CsvPreviewRow): CsvPreviewStatus {
|
||||
if (row.errors.length > 0) return 'FAIL';
|
||||
if (row.warnings.length > 0) return 'WARNING';
|
||||
return 'PASS';
|
||||
}
|
||||
|
||||
export function buildFileResult(input: {
|
||||
fileName: string;
|
||||
template: string;
|
||||
importOrder: number;
|
||||
rows: CsvPreviewRow[];
|
||||
}): CsvPreviewFileResult {
|
||||
const errors = input.rows.flatMap((row) => row.errors);
|
||||
const warnings = input.rows.flatMap((row) => row.warnings);
|
||||
const status: CsvPreviewStatus = errors.length > 0 ? 'FAIL' : warnings.length > 0 ? 'WARNING' : 'PASS';
|
||||
|
||||
return {
|
||||
fileName: input.fileName,
|
||||
template: input.template,
|
||||
importOrder: input.importOrder,
|
||||
status,
|
||||
summary: {
|
||||
rows: input.rows.length,
|
||||
valid: input.rows.filter((row) => row.status === 'PASS').length,
|
||||
warning: input.rows.filter((row) => row.status === 'WARNING').length,
|
||||
error: input.rows.filter((row) => row.status === 'FAIL').length,
|
||||
create: input.rows.filter((row) => row.action === 'create').length,
|
||||
update: input.rows.filter((row) => row.action === 'update').length,
|
||||
skip: input.rows.filter((row) => row.action === 'skip').length
|
||||
},
|
||||
rows: input.rows,
|
||||
errors,
|
||||
warnings
|
||||
};
|
||||
}
|
||||
|
||||
export function buildPreviewResult(files: CsvPreviewFileResult[]): CsvPreviewResult {
|
||||
const summary = files.reduce(
|
||||
(accumulator, file) => ({
|
||||
files: accumulator.files + 1,
|
||||
rows: accumulator.rows + file.summary.rows,
|
||||
valid: accumulator.valid + file.summary.valid,
|
||||
warning: accumulator.warning + file.summary.warning,
|
||||
error: accumulator.error + file.summary.error,
|
||||
create: accumulator.create + file.summary.create,
|
||||
update: accumulator.update + file.summary.update,
|
||||
skip: accumulator.skip + file.summary.skip
|
||||
}),
|
||||
{ files: 0, rows: 0, valid: 0, warning: 0, error: 0, create: 0, update: 0, skip: 0 }
|
||||
);
|
||||
|
||||
const generatedAt = new Date().toISOString();
|
||||
const previewHash = createHash('sha256')
|
||||
.update(
|
||||
JSON.stringify({
|
||||
files: files.map((file) => ({
|
||||
fileName: file.fileName,
|
||||
template: file.template,
|
||||
importOrder: file.importOrder,
|
||||
rows: file.rows.map((row) => ({
|
||||
row: row.row,
|
||||
key: row.key,
|
||||
values: row.values,
|
||||
status: row.status,
|
||||
action: row.action,
|
||||
errors: row.errors.map((error) => error.code),
|
||||
warnings: row.warnings.map((warning) => warning.code)
|
||||
}))
|
||||
}))
|
||||
})
|
||||
)
|
||||
.digest('hex');
|
||||
|
||||
return { summary, files, generatedAt, previewHash };
|
||||
}
|
||||
66
src/features/setup/server/csv/template-registry.ts
Normal file
66
src/features/setup/server/csv/template-registry.ts
Normal file
@@ -0,0 +1,66 @@
|
||||
import 'server-only';
|
||||
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import type { CsvTemplateDefinition } from './types';
|
||||
|
||||
interface TemplateMetadataFile {
|
||||
version: string;
|
||||
generatedFrom: string;
|
||||
templates: Array<{
|
||||
file: string;
|
||||
seedType: string;
|
||||
importOrder: number;
|
||||
supported: boolean;
|
||||
requiredColumns: string[];
|
||||
description: string;
|
||||
}>;
|
||||
unsupportedTemplates: string[];
|
||||
}
|
||||
|
||||
export interface CsvTemplateRegistry {
|
||||
version: string;
|
||||
generatedFrom: string;
|
||||
templates: CsvTemplateDefinition[];
|
||||
unsupportedTemplates: string[];
|
||||
}
|
||||
|
||||
async function readTemplateHeader(file: string): Promise<string[]> {
|
||||
const templatePath = path.join(process.cwd(), 'setup', 'templates', file);
|
||||
const text = await readFile(templatePath, 'utf8');
|
||||
const [header = ''] = text.replace(/^\uFEFF/, '').split(/\r?\n/);
|
||||
return header.split(',').map((column) => column.trim());
|
||||
}
|
||||
|
||||
export async function getCsvTemplateRegistry(): Promise<CsvTemplateRegistry> {
|
||||
const metadataPath = path.join(process.cwd(), 'setup', 'templates', 'templates.json');
|
||||
const metadata = JSON.parse(await readFile(metadataPath, 'utf8')) as TemplateMetadataFile;
|
||||
|
||||
const templates = await Promise.all(
|
||||
metadata.templates.map(async (template) => ({
|
||||
...template,
|
||||
template: template.file.replace(/\.csv$/i, ''),
|
||||
expectedHeaders: await readTemplateHeader(template.file)
|
||||
}))
|
||||
);
|
||||
|
||||
return {
|
||||
version: metadata.version,
|
||||
generatedFrom: metadata.generatedFrom,
|
||||
templates,
|
||||
unsupportedTemplates: metadata.unsupportedTemplates
|
||||
};
|
||||
}
|
||||
|
||||
export function findTemplateByFileName(
|
||||
registry: CsvTemplateRegistry,
|
||||
fileName: string
|
||||
): CsvTemplateDefinition | null {
|
||||
const normalized = path.basename(fileName).toLowerCase();
|
||||
return registry.templates.find((template) => template.file.toLowerCase() === normalized) ?? null;
|
||||
}
|
||||
|
||||
export function isUnsupportedTemplate(registry: CsvTemplateRegistry, fileName: string): boolean {
|
||||
const normalized = path.basename(fileName).toLowerCase();
|
||||
return registry.unsupportedTemplates.some((template) => template.toLowerCase() === normalized);
|
||||
}
|
||||
116
src/features/setup/server/csv/types.ts
Normal file
116
src/features/setup/server/csv/types.ts
Normal file
@@ -0,0 +1,116 @@
|
||||
export type CsvPreviewStatus = 'PASS' | 'WARNING' | 'FAIL';
|
||||
|
||||
export type CsvPreviewAction = 'create' | 'update' | 'skip' | 'error';
|
||||
|
||||
export interface CsvPreviewError {
|
||||
file: string;
|
||||
row: number;
|
||||
column: string;
|
||||
code: string;
|
||||
message: string;
|
||||
suggestedFix: string;
|
||||
}
|
||||
|
||||
export interface CsvPreviewWarning extends CsvPreviewError {}
|
||||
|
||||
export interface CsvTemplateDefinition {
|
||||
file: string;
|
||||
template: string;
|
||||
seedType: string;
|
||||
importOrder: number;
|
||||
supported: boolean;
|
||||
requiredColumns: string[];
|
||||
description: string;
|
||||
expectedHeaders: string[];
|
||||
}
|
||||
|
||||
export interface CsvParsedRow {
|
||||
rowNumber: number;
|
||||
values: Record<string, string>;
|
||||
raw: string[];
|
||||
}
|
||||
|
||||
export interface CsvParsedFile {
|
||||
fileName: string;
|
||||
template: CsvTemplateDefinition;
|
||||
headers: string[];
|
||||
rows: CsvParsedRow[];
|
||||
errors: CsvPreviewError[];
|
||||
warnings: CsvPreviewWarning[];
|
||||
}
|
||||
|
||||
export interface CsvPreviewRow {
|
||||
row: number;
|
||||
status: CsvPreviewStatus;
|
||||
action: CsvPreviewAction;
|
||||
key: string | null;
|
||||
values: Record<string, string>;
|
||||
errors: CsvPreviewError[];
|
||||
warnings: CsvPreviewWarning[];
|
||||
}
|
||||
|
||||
export interface CsvPreviewFileResult {
|
||||
fileName: string;
|
||||
template: string;
|
||||
importOrder: number;
|
||||
status: CsvPreviewStatus;
|
||||
summary: {
|
||||
rows: number;
|
||||
valid: number;
|
||||
warning: number;
|
||||
error: number;
|
||||
create: number;
|
||||
update: number;
|
||||
skip: number;
|
||||
};
|
||||
rows: CsvPreviewRow[];
|
||||
errors: CsvPreviewError[];
|
||||
warnings: CsvPreviewWarning[];
|
||||
}
|
||||
|
||||
export interface CsvPreviewResult {
|
||||
summary: {
|
||||
files: number;
|
||||
rows: number;
|
||||
valid: number;
|
||||
warning: number;
|
||||
error: number;
|
||||
create: number;
|
||||
update: number;
|
||||
skip: number;
|
||||
};
|
||||
files: CsvPreviewFileResult[];
|
||||
generatedAt: string;
|
||||
previewHash: string;
|
||||
}
|
||||
|
||||
export interface CsvPreviewFileInput {
|
||||
fileName: string;
|
||||
bytes: Uint8Array;
|
||||
}
|
||||
|
||||
export interface CsvPreviewBatchIndex {
|
||||
organizations: Set<string>;
|
||||
users: Set<string>;
|
||||
memberships: Set<string>;
|
||||
roleAssignments: Set<string>;
|
||||
masterOptions: Set<string>;
|
||||
branches: Set<string>;
|
||||
productTypes: Set<string>;
|
||||
documentSequences: Set<string>;
|
||||
approvalWorkflows: Set<string>;
|
||||
approvalSteps: Set<string>;
|
||||
approvalMatrices: Set<string>;
|
||||
customers: Set<string>;
|
||||
contactsByEmail: Set<string>;
|
||||
contactsByName: Set<string>;
|
||||
contactShares: Set<string>;
|
||||
leads: Set<string>;
|
||||
opportunities: Set<string>;
|
||||
opportunityParties: Set<string>;
|
||||
quotations: Set<string>;
|
||||
quotationItems: Set<string>;
|
||||
quotationParties: Set<string>;
|
||||
quotationTopics: Set<string>;
|
||||
quotationTopicItems: Set<string>;
|
||||
}
|
||||
221
src/features/setup/server/csv/validator.ts
Normal file
221
src/features/setup/server/csv/validator.ts
Normal file
@@ -0,0 +1,221 @@
|
||||
import type {
|
||||
CsvParsedFile,
|
||||
CsvPreviewBatchIndex,
|
||||
CsvPreviewError,
|
||||
CsvPreviewRow,
|
||||
CsvPreviewWarning
|
||||
} from './types';
|
||||
import { validateCsvHeaders } from './header-validator';
|
||||
import { resolveCsvRowDependencies } from './dependency-resolver';
|
||||
import { resolveRowStatus } from './preview-builder';
|
||||
import { normalizeEmail, normalizeLower, normalizeOrganizationCode } from './value-validator';
|
||||
import { validateCsvRowValues } from './value-validator';
|
||||
|
||||
function key(...parts: Array<string | number | null | undefined>): string {
|
||||
return parts.map((part) => String(part ?? '').trim().toLowerCase()).join('|');
|
||||
}
|
||||
|
||||
function issue(
|
||||
file: string,
|
||||
row: number,
|
||||
column: string,
|
||||
code: string,
|
||||
message: string,
|
||||
suggestedFix: string
|
||||
): CsvPreviewError {
|
||||
return { file, row, column, code, message, suggestedFix };
|
||||
}
|
||||
|
||||
function organizationCode(values: Record<string, string>): string {
|
||||
return normalizeOrganizationCode(values.organization_code);
|
||||
}
|
||||
|
||||
function businessKey(file: CsvParsedFile, values: Record<string, string>): string {
|
||||
const organization = organizationCode(values);
|
||||
|
||||
switch (file.template.file) {
|
||||
case 'users.csv':
|
||||
return key(normalizeEmail(values.email));
|
||||
case 'organizations.csv':
|
||||
return key(organization);
|
||||
case 'memberships.csv':
|
||||
return key(organization, values.user_email);
|
||||
case 'crm-role-assignments.csv':
|
||||
return key(organization, values.user_email, values.crm_role_code);
|
||||
case 'master-options.csv':
|
||||
return key(organization, values.category, values.code);
|
||||
case 'branches.csv':
|
||||
return key(organization, values.branch_code);
|
||||
case 'product-types.csv':
|
||||
return key(organization, values.product_type_code);
|
||||
case 'document-sequences.csv':
|
||||
return key(
|
||||
organization,
|
||||
values.branch_code,
|
||||
values.product_type_code || 'all',
|
||||
values.document_type,
|
||||
values.period
|
||||
);
|
||||
case 'approval-workflows.csv':
|
||||
return key(organization, values.workflow_code);
|
||||
case 'approval-steps.csv':
|
||||
return key(organization, values.workflow_code, values.step_number);
|
||||
case 'approval-matrix.csv':
|
||||
return key(
|
||||
organization,
|
||||
values.entity_type,
|
||||
values.workflow_code,
|
||||
values.branch_code,
|
||||
values.product_type_code,
|
||||
values.min_amount,
|
||||
values.max_amount,
|
||||
values.priority
|
||||
);
|
||||
case 'customers.csv':
|
||||
return key(organization, values.customer_code);
|
||||
case 'contacts.csv':
|
||||
return values.email
|
||||
? key(organization, values.customer_code, values.email)
|
||||
: key(organization, values.customer_code, values.contact_name);
|
||||
case 'contact-shares.csv':
|
||||
return key(organization, values.customer_code, values.contact_email, values.shared_to_user_email);
|
||||
case 'leads.csv':
|
||||
return key(organization, values.lead_code);
|
||||
case 'opportunities.csv':
|
||||
return key(organization, values.opportunity_code);
|
||||
case 'opportunity-parties.csv':
|
||||
return key(organization, values.opportunity_code, values.customer_code, values.role_code);
|
||||
case 'quotations.csv':
|
||||
return key(organization, values.quotation_code);
|
||||
case 'quotation-items.csv':
|
||||
return key(organization, values.quotation_code, values.item_number);
|
||||
case 'quotation-parties.csv':
|
||||
return key(organization, values.quotation_code, values.customer_code, values.role_code);
|
||||
case 'quotation-topics.csv':
|
||||
return key(organization, values.quotation_code, values.topic_key);
|
||||
case 'quotation-topic-items.csv':
|
||||
return key(organization, values.quotation_code, values.topic_key, values.sort_order, values.content);
|
||||
default:
|
||||
return key(organization, JSON.stringify(values));
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeRowValues(values: Record<string, string>): Record<string, string> {
|
||||
const normalized = { ...values };
|
||||
|
||||
Object.keys(normalized).forEach((column) => {
|
||||
if (column.endsWith('_email') || column === 'email') normalized[column] = normalizeEmail(normalized[column]);
|
||||
if (
|
||||
column.endsWith('_code') ||
|
||||
[
|
||||
'slug',
|
||||
'category',
|
||||
'system_role',
|
||||
'membership_role',
|
||||
'business_role',
|
||||
'crm_role_code',
|
||||
'entity_type',
|
||||
'status',
|
||||
'priority',
|
||||
'quotation_type',
|
||||
'currency',
|
||||
'discount_type',
|
||||
'topic_type',
|
||||
'role_code'
|
||||
].includes(column)
|
||||
) {
|
||||
normalized[column] = normalizeLower(normalized[column]);
|
||||
}
|
||||
if (column === 'organization_code') normalized[column] = normalizeOrganizationCode(normalized[column]);
|
||||
});
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function mapFileLevelIssues(file: CsvParsedFile): CsvPreviewRow[] {
|
||||
const issues = [...file.errors, ...file.warnings];
|
||||
if (issues.length === 0) return [];
|
||||
|
||||
const errors = issues.filter((item) =>
|
||||
['INVALID_ENCODING', 'EMPTY_FILE', 'INCONSISTENT_COLUMN_COUNT'].includes(item.code)
|
||||
);
|
||||
const warnings = issues.filter((item) => !errors.includes(item));
|
||||
const row: CsvPreviewRow = {
|
||||
row: 1,
|
||||
status: errors.length > 0 ? 'FAIL' : 'WARNING',
|
||||
action: errors.length > 0 ? 'error' : 'skip',
|
||||
key: null,
|
||||
values: {},
|
||||
errors,
|
||||
warnings,
|
||||
};
|
||||
|
||||
return [row];
|
||||
}
|
||||
|
||||
export async function validateCsvPreviewFile(
|
||||
file: CsvParsedFile,
|
||||
batch: CsvPreviewBatchIndex
|
||||
): Promise<CsvPreviewRow[]> {
|
||||
const headerErrors = validateCsvHeaders(file);
|
||||
if (file.errors.length > 0 || headerErrors.length > 0) {
|
||||
const issueRows = mapFileLevelIssues(file);
|
||||
if (headerErrors.length === 0) return issueRows;
|
||||
return [
|
||||
...issueRows,
|
||||
{
|
||||
row: 1,
|
||||
status: 'FAIL',
|
||||
action: 'error',
|
||||
key: null,
|
||||
values: {},
|
||||
errors: headerErrors,
|
||||
warnings: []
|
||||
}
|
||||
];
|
||||
}
|
||||
|
||||
const seenBusinessKeys = new Map<string, number>();
|
||||
const previewRows: CsvPreviewRow[] = [];
|
||||
|
||||
for (const row of file.rows) {
|
||||
const values = normalizeRowValues(row.values);
|
||||
const rowForValidation = { ...row, values };
|
||||
const valueValidation = validateCsvRowValues(file, rowForValidation);
|
||||
const dependencyValidation = await resolveCsvRowDependencies(file, rowForValidation, batch);
|
||||
const errors: CsvPreviewError[] = [...valueValidation.errors, ...dependencyValidation.errors];
|
||||
const warnings: CsvPreviewWarning[] = [...valueValidation.warnings, ...dependencyValidation.warnings];
|
||||
const keyValue = businessKey(file, values);
|
||||
const duplicateFirstRow = seenBusinessKeys.get(keyValue);
|
||||
|
||||
if (duplicateFirstRow) {
|
||||
errors.push(
|
||||
issue(
|
||||
file.fileName,
|
||||
row.rowNumber,
|
||||
'__row__',
|
||||
'DUPLICATE_KEY',
|
||||
`Business key duplicates row ${duplicateFirstRow}.`,
|
||||
'Keep only one row for each import business key.'
|
||||
)
|
||||
);
|
||||
} else {
|
||||
seenBusinessKeys.set(keyValue, row.rowNumber);
|
||||
}
|
||||
|
||||
const action = errors.length > 0 ? 'error' : dependencyValidation.exists ? 'update' : 'create';
|
||||
const previewRow: CsvPreviewRow = {
|
||||
row: row.rowNumber,
|
||||
status: 'PASS',
|
||||
action,
|
||||
key: keyValue,
|
||||
values,
|
||||
errors,
|
||||
warnings
|
||||
};
|
||||
previewRow.status = resolveRowStatus(previewRow);
|
||||
previewRows.push(previewRow);
|
||||
}
|
||||
|
||||
return previewRows;
|
||||
}
|
||||
391
src/features/setup/server/csv/value-validator.ts
Normal file
391
src/features/setup/server/csv/value-validator.ts
Normal file
@@ -0,0 +1,391 @@
|
||||
import type { CsvParsedFile, CsvParsedRow, CsvPreviewError, CsvPreviewWarning } from './types';
|
||||
|
||||
type RowValues = Record<string, string>;
|
||||
|
||||
const EMAIL_COLUMNS = new Set([
|
||||
'email',
|
||||
'created_by_email',
|
||||
'updated_by_email',
|
||||
'user_email',
|
||||
'owner_user_email',
|
||||
'owner_marketing_user_email',
|
||||
'assigned_sales_owner_email',
|
||||
'assigned_to_user_email',
|
||||
'salesman_email',
|
||||
'shared_to_user_email',
|
||||
'shared_by_user_email',
|
||||
'contact_email'
|
||||
]);
|
||||
|
||||
const BOOLEAN_COLUMNS = new Set([
|
||||
'reset_password',
|
||||
'is_primary',
|
||||
'is_active',
|
||||
'is_system',
|
||||
'is_required',
|
||||
'is_default',
|
||||
'is_hot_project',
|
||||
'is_sent'
|
||||
]);
|
||||
|
||||
const INTEGER_COLUMNS = new Set([
|
||||
'sort_order',
|
||||
'current_number',
|
||||
'padding_length',
|
||||
'step_number',
|
||||
'sla_hours',
|
||||
'priority',
|
||||
'item_number',
|
||||
'revision',
|
||||
'chance_percent'
|
||||
]);
|
||||
|
||||
const DECIMAL_COLUMNS = new Set([
|
||||
'estimated_value',
|
||||
'min_amount',
|
||||
'max_amount',
|
||||
'subtotal',
|
||||
'total_amount',
|
||||
'exchange_rate',
|
||||
'discount',
|
||||
'tax_rate',
|
||||
'tax_amount',
|
||||
'quantity',
|
||||
'unit_price',
|
||||
'total_price'
|
||||
]);
|
||||
|
||||
const DATE_COLUMNS = new Set(['quotation_date', 'valid_until', 'expected_close_date']);
|
||||
|
||||
const ENUMS: Record<string, readonly string[]> = {
|
||||
system_role: ['super_admin', 'user'],
|
||||
membership_role: ['admin', 'user'],
|
||||
branch_scope_mode: ['all', 'selected', 'none', 'inherit'],
|
||||
product_type_scope_mode: ['all', 'selected', 'none', 'inherit'],
|
||||
entity_type: ['quotation'],
|
||||
approval_mode: ['sequential', 'parallel', 'any'],
|
||||
calendar_mode: ['calendar_days', 'business_days'],
|
||||
timeout_action: ['none', 'auto_approve', 'auto_reject', 'escalate'],
|
||||
reset_policy: ['period', 'never', 'manual'],
|
||||
discount_type: ['percent', 'amount', ''],
|
||||
outcome: ['open', 'won', 'lost', ''],
|
||||
is_active: ['true', 'false', ''],
|
||||
is_primary: ['true', 'false', ''],
|
||||
is_system: ['true', 'false', ''],
|
||||
is_required: ['true', 'false', ''],
|
||||
is_default: ['true', 'false', ''],
|
||||
reset_password: ['true', 'false', ''],
|
||||
is_hot_project: ['true', 'false', ''],
|
||||
is_sent: ['true', 'false', '']
|
||||
};
|
||||
|
||||
function buildIssue(
|
||||
file: string,
|
||||
row: number,
|
||||
column: string,
|
||||
code: string,
|
||||
message: string,
|
||||
suggestedFix: string
|
||||
): CsvPreviewError {
|
||||
return { file, row, column, code, message, suggestedFix };
|
||||
}
|
||||
|
||||
function isValidEmail(value: string): boolean {
|
||||
return /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(value);
|
||||
}
|
||||
|
||||
function isInteger(value: string): boolean {
|
||||
return /^-?\d+$/.test(value);
|
||||
}
|
||||
|
||||
function isDecimal(value: string): boolean {
|
||||
return /^-?\d+(\.\d+)?$/.test(value);
|
||||
}
|
||||
|
||||
function isDate(value: string): boolean {
|
||||
if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) return false;
|
||||
const date = new Date(`${value}T00:00:00.000Z`);
|
||||
return !Number.isNaN(date.getTime()) && date.toISOString().startsWith(value);
|
||||
}
|
||||
|
||||
function isLowerSnake(value: string): boolean {
|
||||
return /^[a-z0-9]+(?:_[a-z0-9]+)*$/.test(value);
|
||||
}
|
||||
|
||||
function validateCodeShape(file: CsvParsedFile, row: CsvParsedRow, errors: CsvPreviewError[]): void {
|
||||
const values = row.values;
|
||||
const lowerSnakeColumns = [
|
||||
'slug',
|
||||
'branch_code',
|
||||
'product_type_code',
|
||||
'workflow_code',
|
||||
'topic_key',
|
||||
'category',
|
||||
'code',
|
||||
'crm_role_code',
|
||||
'role_code'
|
||||
];
|
||||
|
||||
if (values.organization_code && !/^[A-Z0-9_-]+$/.test(values.organization_code)) {
|
||||
errors.push(
|
||||
buildIssue(
|
||||
file.fileName,
|
||||
row.rowNumber,
|
||||
'organization_code',
|
||||
'INVALID_FORMAT',
|
||||
'Organization code must be uppercase letters, numbers, underscore, or dash.',
|
||||
'Use values like ALLA or ALLA_DEMO.'
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
lowerSnakeColumns.forEach((column) => {
|
||||
const value = values[column];
|
||||
if (!value || column === 'code') return;
|
||||
if (!isLowerSnake(value)) {
|
||||
errors.push(
|
||||
buildIssue(
|
||||
file.fileName,
|
||||
row.rowNumber,
|
||||
column,
|
||||
'INVALID_FORMAT',
|
||||
`${column} should use lowercase snake_case.`,
|
||||
'Use lowercase letters, numbers, and underscores only.'
|
||||
)
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function validateJson(file: CsvParsedFile, row: CsvParsedRow, errors: CsvPreviewError[]): void {
|
||||
const value = row.values.metadata_json;
|
||||
if (!value) return;
|
||||
try {
|
||||
const parsed = JSON.parse(value) as unknown;
|
||||
if (parsed === null || Array.isArray(parsed) || typeof parsed !== 'object') {
|
||||
errors.push(
|
||||
buildIssue(
|
||||
file.fileName,
|
||||
row.rowNumber,
|
||||
'metadata_json',
|
||||
'INVALID_JSON_OBJECT',
|
||||
'metadata_json must be a JSON object.',
|
||||
'Use an object such as {} or {"prefix":"HO"}.'
|
||||
)
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
errors.push(
|
||||
buildIssue(
|
||||
file.fileName,
|
||||
row.rowNumber,
|
||||
'metadata_json',
|
||||
'INVALID_JSON',
|
||||
'metadata_json is not valid JSON.',
|
||||
'Fix JSON syntax or leave the column empty.'
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function validateNumericalRelationships(
|
||||
file: CsvParsedFile,
|
||||
row: CsvParsedRow,
|
||||
errors: CsvPreviewError[],
|
||||
warnings: CsvPreviewWarning[]
|
||||
): void {
|
||||
const minAmount = row.values.min_amount;
|
||||
const maxAmount = row.values.max_amount;
|
||||
if (minAmount && maxAmount && Number(minAmount) > Number(maxAmount)) {
|
||||
errors.push(
|
||||
buildIssue(
|
||||
file.fileName,
|
||||
row.rowNumber,
|
||||
'max_amount',
|
||||
'INVALID_AMOUNT_RANGE',
|
||||
'max_amount must be greater than or equal to min_amount.',
|
||||
'Adjust min_amount or max_amount.'
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
const chancePercent = row.values.chance_percent;
|
||||
if (chancePercent && (Number(chancePercent) < 0 || Number(chancePercent) > 100)) {
|
||||
errors.push(
|
||||
buildIssue(
|
||||
file.fileName,
|
||||
row.rowNumber,
|
||||
'chance_percent',
|
||||
'INVALID_PERCENT',
|
||||
'chance_percent must be between 0 and 100.',
|
||||
'Use a whole number between 0 and 100.'
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
const quantity = row.values.quantity;
|
||||
const unitPrice = row.values.unit_price;
|
||||
const totalPrice = row.values.total_price;
|
||||
if (quantity && unitPrice && totalPrice) {
|
||||
const expected = Number(quantity) * Number(unitPrice);
|
||||
const actual = Number(totalPrice);
|
||||
if (Number.isFinite(expected) && Number.isFinite(actual) && Math.abs(expected - actual) > 0.01) {
|
||||
warnings.push(
|
||||
buildIssue(
|
||||
file.fileName,
|
||||
row.rowNumber,
|
||||
'total_price',
|
||||
'TOTAL_MISMATCH',
|
||||
'total_price does not match quantity multiplied by unit_price.',
|
||||
'Confirm the line total or adjust quantity/unit_price.'
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function validateCsvRowValues(file: CsvParsedFile, row: CsvParsedRow): {
|
||||
errors: CsvPreviewError[];
|
||||
warnings: CsvPreviewWarning[];
|
||||
} {
|
||||
const errors: CsvPreviewError[] = [];
|
||||
const warnings: CsvPreviewWarning[] = [];
|
||||
|
||||
file.template.requiredColumns.forEach((column) => {
|
||||
if (!row.values[column]) {
|
||||
errors.push(
|
||||
buildIssue(
|
||||
file.fileName,
|
||||
row.rowNumber,
|
||||
column,
|
||||
'REQUIRED_FIELD',
|
||||
`Column "${column}" is required.`,
|
||||
'Fill in the required value before previewing again.'
|
||||
)
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
Object.entries(row.values).forEach(([column, value]) => {
|
||||
if (!value) return;
|
||||
|
||||
if (EMAIL_COLUMNS.has(column) && !isValidEmail(value)) {
|
||||
errors.push(
|
||||
buildIssue(
|
||||
file.fileName,
|
||||
row.rowNumber,
|
||||
column,
|
||||
'INVALID_EMAIL',
|
||||
`"${value}" is not a valid email address.`,
|
||||
'Use a valid email address such as admin@example.com.'
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
if (BOOLEAN_COLUMNS.has(column) && !['true', 'false'].includes(value.toLowerCase())) {
|
||||
errors.push(
|
||||
buildIssue(
|
||||
file.fileName,
|
||||
row.rowNumber,
|
||||
column,
|
||||
'INVALID_BOOLEAN',
|
||||
`"${value}" is not a valid boolean.`,
|
||||
'Use true or false.'
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
if (INTEGER_COLUMNS.has(column) && !isInteger(value)) {
|
||||
errors.push(
|
||||
buildIssue(
|
||||
file.fileName,
|
||||
row.rowNumber,
|
||||
column,
|
||||
'INVALID_INTEGER',
|
||||
`"${value}" is not a valid integer.`,
|
||||
'Use a whole number.'
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
if (DECIMAL_COLUMNS.has(column) && !isDecimal(value)) {
|
||||
errors.push(
|
||||
buildIssue(
|
||||
file.fileName,
|
||||
row.rowNumber,
|
||||
column,
|
||||
'INVALID_DECIMAL',
|
||||
`"${value}" is not a valid decimal number.`,
|
||||
'Use a number such as 1000 or 1000.50.'
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
if (DATE_COLUMNS.has(column) && !isDate(value)) {
|
||||
errors.push(
|
||||
buildIssue(
|
||||
file.fileName,
|
||||
row.rowNumber,
|
||||
column,
|
||||
'INVALID_DATE',
|
||||
`"${value}" is not a valid YYYY-MM-DD date.`,
|
||||
'Use Gregorian date format YYYY-MM-DD.'
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
const allowedValues = ENUMS[column];
|
||||
if (allowedValues && !allowedValues.includes(value.toLowerCase())) {
|
||||
errors.push(
|
||||
buildIssue(
|
||||
file.fileName,
|
||||
row.rowNumber,
|
||||
column,
|
||||
'UNKNOWN_OPTION',
|
||||
`"${value}" is not an allowed value for ${column}.`,
|
||||
`Use one of: ${allowedValues.filter(Boolean).join(', ')}.`
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
if (value.length > 5000) {
|
||||
errors.push(
|
||||
buildIssue(
|
||||
file.fileName,
|
||||
row.rowNumber,
|
||||
column,
|
||||
'VALUE_TOO_LONG',
|
||||
`${column} exceeds the maximum preview length.`,
|
||||
'Shorten the value before importing.'
|
||||
)
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
validateCodeShape(file, row, errors);
|
||||
validateJson(file, row, errors);
|
||||
validateNumericalRelationships(file, row, errors, warnings);
|
||||
|
||||
return { errors, warnings };
|
||||
}
|
||||
|
||||
export function normalizeEmail(value: string): string {
|
||||
return value.trim().toLowerCase();
|
||||
}
|
||||
|
||||
export function normalizeOrganizationCode(value: string): string {
|
||||
return value.trim().toUpperCase();
|
||||
}
|
||||
|
||||
export function normalizeLower(value: string): string {
|
||||
return value.trim().toLowerCase();
|
||||
}
|
||||
|
||||
export function pipeList(value: string): string[] {
|
||||
return value
|
||||
.split('|')
|
||||
.map((part) => part.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
export type CsvRowValues = RowValues;
|
||||
@@ -1,9 +1,8 @@
|
||||
import { drizzle } from 'drizzle-orm/postgres-js';
|
||||
import postgres from 'postgres';
|
||||
import * as schema from '@/db/schema';
|
||||
import { drizzle } from "drizzle-orm/postgres-js";
|
||||
import postgres from "postgres";
|
||||
import * as schema from "@/db/schema";
|
||||
|
||||
const connectionString =
|
||||
process.env.DATABASE_URL ?? 'postgres://postgres:postgres@localhost:5432/training_system';
|
||||
const connectionString = process.env.DATABASE_URL ?? "";
|
||||
|
||||
const globalForDb = globalThis as typeof globalThis & {
|
||||
postgresClient?: ReturnType<typeof postgres>;
|
||||
@@ -13,12 +12,12 @@ const globalForDb = globalThis as typeof globalThis & {
|
||||
const client =
|
||||
globalForDb.postgresClient ??
|
||||
postgres(connectionString, {
|
||||
prepare: false
|
||||
prepare: false,
|
||||
});
|
||||
|
||||
export const db = globalForDb.drizzleDb ?? drizzle(client, { schema });
|
||||
|
||||
if (process.env.NODE_ENV !== 'production') {
|
||||
if (process.env.NODE_ENV !== "production") {
|
||||
globalForDb.postgresClient = client;
|
||||
globalForDb.drizzleDb = db;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user