diff --git a/docs/implementation/task-d74-csv-preview-validation-engine.md b/docs/implementation/task-d74-csv-preview-validation-engine.md new file mode 100644 index 0000000..f9975b6 --- /dev/null +++ b/docs/implementation/task-d74-csv-preview-validation-engine.md @@ -0,0 +1,47 @@ +# Task D.7.4 CSV Preview Validation Engine + +Date: 2026-07-02 +Status: implemented + +## Scope Delivered + +- Added reusable preview-only CSV engine: + - `src/features/setup/server/csv-preview.service.ts` + - `src/features/setup/server/csv/parser.ts` + - `src/features/setup/server/csv/template-registry.ts` + - `src/features/setup/server/csv/header-validator.ts` + - `src/features/setup/server/csv/value-validator.ts` + - `src/features/setup/server/csv/dependency-resolver.ts` + - `src/features/setup/server/csv/validator.ts` + - `src/features/setup/server/csv/preview-builder.ts` + - `src/features/setup/server/csv/types.ts` +- Added protected preview route: + - `POST /api/setup/import/preview` + +## Behavior + +- Accepts one or more CSV files through `multipart/form-data`. +- Detects templates by filename from `setup/templates/templates.json`. +- Treats `templates.json` plus checked-in CSV headers as runtime template truth. +- Rejects unknown and unsupported templates with structured row/file errors. +- Validates exact header names, order, duplicate columns, missing columns, and extra columns. +- Parses UTF-8 comma-delimited CSV with quoted cell support. +- Reports invalid encoding, empty rows, duplicate rows, inconsistent column counts, required fields, emails, booleans, integers, decimals, dates, enums, JSON metadata, format rules, and amount/line-total warnings. +- Resolves dependencies from both uploaded batch files and existing database rows without writing. +- Detects duplicate business keys inside each uploaded file. +- Classifies rows as `create`, `update`, or `error`. +- Returns structured file/row report with `PASS`, `WARNING`, `FAIL` status and `previewHash` for D.7.5. + +## Safety Notes + +- No database insert/update/delete logic added. +- No seed execution added. +- No reset/reseed behavior added. +- No document number generation, approval triggering, PDF generation, or storage writes added. +- Relationship checks use read-only Drizzle selects. +- `quotation-topic-items.csv` resolves `topic_key` from the uploaded batch because `topic_key` is an import-only alias and is not persisted in `crm_quotation_topics`. + +## Verification + +- `npm run typecheck`: passed. +- `npx oxlint src/features/setup/server/csv src/features/setup/server/csv-preview.service.ts src/app/api/setup/import/preview`: passed. diff --git a/plans/task-d.7.4.md b/plans/task-d.7.4.md new file mode 100644 index 0000000..6923a26 --- /dev/null +++ b/plans/task-d.7.4.md @@ -0,0 +1,437 @@ +# Task D.7.4 – CSV Preview Validation Engine + +## Objective + +Implement a reusable, non-destructive CSV Preview Validation Engine for ALLA OS. + +This engine must validate uploaded CSV files against the official template pack and return a complete preview report without modifying any database records. + +The preview engine becomes the shared foundation for D.7.5 (CSV Commit Import Engine), Setup Wizard, and future bulk import features. + +No database writes, seed execution, or reset operations are allowed in this task. + +--- + +## Review Before Implementation + +Review: + +- AGENTS.md +- plans/task-d.7.md +- plans/task-d.7.1.md +- plans/task-d.7.2.md +- plans/task-d.7.3.md +- docs/setup/seed-matrix.md +- docs/setup/csv-template-spec.md +- docs/setup/import-mapping.md +- docs/setup/setup-wizard-blueprint.md +- docs/setup/verification-matrix.md +- setup/templates/* +- setup/templates/templates.json + +Treat templates.json as the runtime source of truth. + +--- + +## Implementation Scope + +Create: + +src/features/setup/server/csv-preview.service.ts + +Supporting modules + +src/features/setup/server/csv/ + parser.ts + validator.ts + dependency-resolver.ts + header-validator.ts + value-validator.ts + preview-builder.ts + template-registry.ts + +Create API + +POST /api/setup/import/preview + +--- + +## Input + +Accept one or more CSV files. + +Each file must match one registered template. + +Files may be uploaded in any order. + +Engine must automatically resolve dependency order. + +--- + +## Validation Stages + +Stage 1 + +Template Detection + +- Match filename +- Match template registry +- Match supported template + +Reject unknown templates. + +--- + +Stage 2 + +Schema Validation + +Validate + +- header names +- header order +- duplicated columns +- missing columns +- extra columns +- template version (if available) + +--- + +Stage 3 + +CSV Validation + +Validate + +- encoding +- delimiter +- empty rows +- duplicated rows +- inconsistent column counts + +--- + +Stage 4 + +Value Validation + +Validate + +Required fields + +Email + +Boolean + +Integer + +Decimal + +Date (YYYY-MM-DD) + +Enum + +Length + +Regex + +Custom validators + +--- + +Stage 5 + +Relationship Resolution + +Resolve + +organization + +branch + +product type + +users + +memberships + +CRM role + +customer + +contact + +lead + +opportunity + +quotation + +master options + +Return + +Resolved + +Missing + +Ambiguous + +--- + +Stage 6 + +Dependency Validation + +Read + +templates.json + +Validate + +dependsOn + +Detect + +Missing parent files + +Circular dependency + +Invalid import order + +--- + +Stage 7 + +Duplicate Detection + +Classify + +New + +Existing + +Update + +Conflict + +Duplicate inside CSV + +--- + +Stage 8 + +Preview Report + +Return + +Imported + +Updated + +Skipped + +Warnings + +Errors + +Grouped by file. + +--- + +## Report Contract + +```ts +type PreviewResult = { + fileName: string; + + template: string; + + status: + | "PASS" + | "WARNING" + | "FAIL"; + + summary: { + + rows:number; + + valid:number; + + warning:number; + + error:number; + + }; + + rows:[]; +} +``` + +Overall response + +```ts +{ + summary, + + files:[], + + generatedAt, + + previewHash +} +``` + +previewHash will be required by D.7.5. + +--- + +## Rules + +Do not + +Insert + +Update + +Delete + +Commit + +Seed + +Generate IDs + +Generate document numbers + +Trigger approval + +Generate PDFs + +Only preview. + +--- + +## Error Contract + +Every error + +Must contain + +File + +Row + +Column + +Code + +Message + +Suggested Fix + +Example + +```json +{ + "file":"customers.csv", + + "row":12, + + "column":"customer_type", + + "code":"UNKNOWN_OPTION", + + "message":"Customer type not found.", + + "suggestedFix":"Import master-options.csv first." +} +``` + +--- + +## Reuse Rules + +Reuse existing + +Master Option services + +Organization services + +CRM services + +Validation helpers + +Document sequence helpers + +Do not duplicate logic. + +--- + +## Deliverables + +src/features/setup/server/csv-preview.service.ts + +src/features/setup/server/csv/ + +POST /api/setup/import/preview + +Shared validation utilities + +Structured preview response + +previewHash generator + +--- + +## Acceptance Criteria + +✓ Preview never writes database. + +✓ Multiple CSVs supported. + +✓ Dependency order resolved automatically. + +✓ Unknown templates rejected. + +✓ Header validation implemented. + +✓ Data validation implemented. + +✓ Relationship resolution implemented. + +✓ Duplicate detection implemented. + +✓ Structured report returned. + +✓ previewHash generated. + +✓ Uses templates.json as runtime registry. + +✓ Typecheck passes. + +--- + +## Future + +D.7.5 + +Commit Import Engine + +Uses + +previewHash + +↓ + +Transaction + +↓ + +Upsert + +↓ + +Import Report + +No validation logic should be duplicated between D.7.4 and D.7.5. diff --git a/src/app/api/setup/import/preview/route.ts b/src/app/api/setup/import/preview/route.ts new file mode 100644 index 0000000..da0b368 --- /dev/null +++ b/src/app/api/setup/import/preview/route.ts @@ -0,0 +1,39 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { previewSetupCsvImport } from '@/features/setup/server/csv-preview.service'; +import { AuthError, requireSystemRole } from '@/lib/auth/session'; + +function isFile(value: FormDataEntryValue): value is File { + return typeof value === 'object' && 'arrayBuffer' in value && 'name' in value; +} + +export async function POST(request: NextRequest) { + try { + await requireSystemRole('super_admin'); + + const formData = await request.formData(); + const uploadedFiles = Array.from(formData.values()).filter(isFile); + + if (uploadedFiles.length === 0) { + return NextResponse.json( + { message: 'Upload at least one setup CSV file using multipart/form-data.' }, + { status: 400 } + ); + } + + const files = await Promise.all( + uploadedFiles.map(async (file) => ({ + fileName: file.name, + bytes: new Uint8Array(await file.arrayBuffer()) + })) + ); + const preview = await previewSetupCsvImport(files); + + return NextResponse.json(preview); + } catch (error) { + if (error instanceof AuthError) { + return NextResponse.json({ message: error.message }, { status: error.status }); + } + + return NextResponse.json({ message: 'Unable to preview setup CSV import' }, { status: 500 }); + } +} diff --git a/src/features/setup/server/csv-preview.service.ts b/src/features/setup/server/csv-preview.service.ts new file mode 100644 index 0000000..977a501 --- /dev/null +++ b/src/features/setup/server/csv-preview.service.ts @@ -0,0 +1,121 @@ +import 'server-only'; + +import path from 'node:path'; +import { buildCsvPreviewBatchIndex } from './csv/dependency-resolver'; +import { parseCsvFile } from './csv/parser'; +import { buildFileResult, buildPreviewResult } from './csv/preview-builder'; +import { + findTemplateByFileName, + getCsvTemplateRegistry, + isUnsupportedTemplate +} from './csv/template-registry'; +import type { + CsvParsedFile, + CsvPreviewError, + CsvPreviewFileInput, + CsvPreviewResult +} from './csv/types'; +import { validateCsvPreviewFile } from './csv/validator'; + +function templateDetectionError(fileName: string, code: string, message: string, suggestedFix: string): CsvParsedFile { + const file = path.basename(fileName); + const error: CsvPreviewError = { + file, + row: 1, + column: '__file__', + code, + message, + suggestedFix + }; + + return { + fileName: file, + template: { + file, + template: 'unknown', + seedType: 'unknown', + importOrder: Number.MAX_SAFE_INTEGER, + supported: false, + requiredColumns: [], + description: 'Unknown template', + expectedHeaders: [] + }, + headers: [], + rows: [], + errors: [error], + warnings: [] + }; +} + +export async function previewSetupCsvImport(files: CsvPreviewFileInput[]): Promise { + const registry = await getCsvTemplateRegistry(); + const parsedFiles: CsvParsedFile[] = []; + + for (const input of files) { + const fileName = path.basename(input.fileName); + const template = findTemplateByFileName(registry, fileName); + + if (!template) { + parsedFiles.push( + templateDetectionError( + fileName, + isUnsupportedTemplate(registry, fileName) ? 'UNSUPPORTED_TEMPLATE' : 'UNKNOWN_TEMPLATE', + isUnsupportedTemplate(registry, fileName) + ? 'CSV template is known but not supported by the setup import engine.' + : 'CSV file does not match any registered setup template.', + 'Use one of the supported files from setup/templates/templates.json.' + ) + ); + continue; + } + + if (!template.supported) { + parsedFiles.push( + templateDetectionError( + fileName, + 'UNSUPPORTED_TEMPLATE', + 'CSV template is registered but not supported.', + 'Remove this file from the preview upload.' + ) + ); + continue; + } + + parsedFiles.push(parseCsvFile({ fileName, bytes: input.bytes, template })); + } + + const uploadCounts = new Map(); + parsedFiles.forEach((file) => { + uploadCounts.set(file.template.file, (uploadCounts.get(file.template.file) ?? 0) + 1); + }); + parsedFiles.forEach((file) => { + if (uploadCounts.get(file.template.file)! > 1 && file.template.supported) { + file.errors.push({ + file: file.fileName, + row: 1, + column: '__file__', + code: 'DUPLICATE_FILE', + message: `Template ${file.template.file} was uploaded more than once.`, + suggestedFix: 'Upload only one file for each setup template.' + }); + } + }); + + const sortedFiles = parsedFiles.toSorted((left, right) => left.template.importOrder - right.template.importOrder); + const batch = buildCsvPreviewBatchIndex(sortedFiles.filter((file) => file.template.supported)); + const fileResults = []; + + for (const file of sortedFiles) { + const rows = await validateCsvPreviewFile(file, batch); + fileResults.push( + buildFileResult({ + fileName: file.fileName, + template: file.template.template, + importOrder: file.template.importOrder, + rows + }) + ); + } + + return buildPreviewResult(fileResults); +} diff --git a/src/features/setup/server/csv/dependency-resolver.ts b/src/features/setup/server/csv/dependency-resolver.ts new file mode 100644 index 0000000..6e05ed8 --- /dev/null +++ b/src/features/setup/server/csv/dependency-resolver.ts @@ -0,0 +1,1242 @@ +import 'server-only'; + +import { and, eq, isNull, sql } from 'drizzle-orm'; +import { + crmApprovalMatrices, + crmApprovalSteps, + crmApprovalWorkflows, + crmContactShares, + crmCustomerContacts, + crmCustomers, + crmLeads, + crmOpportunities, + crmOpportunityCustomers, + crmQuotationCustomers, + crmQuotationItems, + crmQuotationTopics, + crmQuotations, + crmRoleProfiles, + crmUserRoleAssignments, + documentSequences, + memberships, + msOptions, + organizations, + users +} from '@/db/schema'; +import { db } from '@/lib/db'; +import { normalizeEmail, normalizeLower, normalizeOrganizationCode, pipeList } from './value-validator'; +import type { + CsvParsedFile, + CsvParsedRow, + CsvPreviewBatchIndex, + CsvPreviewError, + CsvPreviewWarning +} from './types'; + +type ExistingState = { + exists: boolean; +}; + +type ResolverResult = { + errors: CsvPreviewError[]; + warnings: CsvPreviewWarning[]; + exists: boolean; +}; + +type OrganizationReference = { + id: string | null; + key: string; + exists: boolean; +}; + +function key(...parts: Array): string { + return parts.map((part) => String(part ?? '').trim().toLowerCase()).join('|'); +} + +function issue( + file: string, + row: number, + column: string, + code: string, + message: string, + suggestedFix: string +): CsvPreviewError { + return { file, row, column, code, message, suggestedFix }; +} + +async function first(promise: Promise): Promise { + const [row] = await promise; + return row ?? null; +} + +async function existingUser(email: string): Promise { + if (!email) return { exists: false }; + const row = await first( + db + .select({ id: users.id }) + .from(users) + .where(sql`lower(${users.email}) = ${normalizeEmail(email)}`) + .limit(1) + ); + return { exists: Boolean(row) }; +} + +async function resolveOrganization( + organizationCode: string, + batch: CsvPreviewBatchIndex +): Promise { + const normalizedCode = normalizeOrganizationCode(organizationCode); + const slug = normalizedCode.toLowerCase(); + const existsInBatch = batch.organizations.has(normalizedCode); + const row = await first( + db + .select({ id: organizations.id }) + .from(organizations) + .where(eq(organizations.slug, slug)) + .limit(1) + ); + + return { + id: row?.id ?? null, + key: normalizedCode, + exists: Boolean(row) || existsInBatch + }; +} + +async function existingMembership(organizationId: string | null, userEmail: string): Promise { + if (!organizationId || !userEmail) return false; + const user = await first( + db + .select({ id: users.id }) + .from(users) + .where(sql`lower(${users.email}) = ${normalizeEmail(userEmail)}`) + .limit(1) + ); + if (!user) return false; + const membership = await first( + db + .select({ id: memberships.id }) + .from(memberships) + .where(and(eq(memberships.organizationId, organizationId), eq(memberships.userId, user.id))) + .limit(1) + ); + return Boolean(membership); +} + +async function existingOption( + organizationId: string | null, + category: string, + code: string +): Promise { + if (!organizationId || !category || !code) return { exists: false }; + const row = await first( + db + .select({ id: msOptions.id }) + .from(msOptions) + .where( + and( + eq(msOptions.organizationId, organizationId), + eq(msOptions.category, category), + eq(msOptions.code, code), + isNull(msOptions.deletedAt) + ) + ) + .limit(1) + ); + return { exists: Boolean(row) }; +} + +async function existingRoleProfile( + organizationId: string | null, + roleCode: string +): Promise { + if (!organizationId || !roleCode) return { exists: false }; + const row = await first( + db + .select({ id: crmRoleProfiles.id }) + .from(crmRoleProfiles) + .where( + and( + eq(crmRoleProfiles.organizationId, organizationId), + eq(crmRoleProfiles.code, roleCode), + isNull(crmRoleProfiles.deletedAt) + ) + ) + .limit(1) + ); + return { exists: Boolean(row) }; +} + +async function existingApprovalWorkflow( + organizationId: string | null, + workflowCode: string +): Promise { + if (!organizationId || !workflowCode) return { exists: false }; + const row = await first( + db + .select({ id: crmApprovalWorkflows.id }) + .from(crmApprovalWorkflows) + .where( + and( + eq(crmApprovalWorkflows.organizationId, organizationId), + eq(crmApprovalWorkflows.code, workflowCode), + isNull(crmApprovalWorkflows.deletedAt) + ) + ) + .limit(1) + ); + return { exists: Boolean(row) }; +} + +async function existingCustomer(organizationId: string | null, customerCode: string): Promise { + if (!organizationId || !customerCode) return { exists: false }; + const row = await first( + db + .select({ id: crmCustomers.id }) + .from(crmCustomers) + .where( + and( + eq(crmCustomers.organizationId, organizationId), + eq(crmCustomers.code, customerCode), + isNull(crmCustomers.deletedAt) + ) + ) + .limit(1) + ); + return { exists: Boolean(row) }; +} + +async function existingContact( + organizationId: string | null, + customerCode: string, + contactEmail: string +): Promise { + if (!organizationId || !customerCode || !contactEmail) return { exists: false }; + const customer = await first( + db + .select({ id: crmCustomers.id }) + .from(crmCustomers) + .where( + and( + eq(crmCustomers.organizationId, organizationId), + eq(crmCustomers.code, customerCode), + isNull(crmCustomers.deletedAt) + ) + ) + .limit(1) + ); + if (!customer) return { exists: false }; + const row = await first( + db + .select({ id: crmCustomerContacts.id }) + .from(crmCustomerContacts) + .where( + and( + eq(crmCustomerContacts.organizationId, organizationId), + eq(crmCustomerContacts.customerId, customer.id), + sql`lower(${crmCustomerContacts.email}) = ${normalizeEmail(contactEmail)}`, + isNull(crmCustomerContacts.deletedAt) + ) + ) + .limit(1) + ); + return { exists: Boolean(row) }; +} + +async function existingCodeRecord( + organizationId: string | null, + table: 'lead' | 'opportunity' | 'quotation', + codeValue: string +): Promise { + if (!organizationId || !codeValue) return { exists: false }; + + if (table === 'lead') { + const row = await first( + db + .select({ id: crmLeads.id }) + .from(crmLeads) + .where( + and( + eq(crmLeads.organizationId, organizationId), + eq(crmLeads.code, codeValue), + isNull(crmLeads.deletedAt) + ) + ) + .limit(1) + ); + return { exists: Boolean(row) }; + } + + if (table === 'opportunity') { + const row = await first( + db + .select({ id: crmOpportunities.id }) + .from(crmOpportunities) + .where( + and( + eq(crmOpportunities.organizationId, organizationId), + eq(crmOpportunities.code, codeValue), + isNull(crmOpportunities.deletedAt) + ) + ) + .limit(1) + ); + return { exists: Boolean(row) }; + } + + const row = await first( + db + .select({ id: crmQuotations.id }) + .from(crmQuotations) + .where( + and( + eq(crmQuotations.organizationId, organizationId), + eq(crmQuotations.code, codeValue), + isNull(crmQuotations.deletedAt) + ) + ) + .limit(1) + ); + return { exists: Boolean(row) }; +} + +function batchHasOption( + batch: CsvPreviewBatchIndex, + organizationCode: string, + category: string, + codeValue: string +): boolean { + const normalizedCategory = normalizeLower(category); + const normalizedCode = normalizeLower(codeValue); + const organizationKey = normalizeOrganizationCode(organizationCode); + if (normalizedCategory === 'crm_branch') return batch.branches.has(key(organizationKey, normalizedCode)); + if (normalizedCategory === 'crm_product_type') return batch.productTypes.has(key(organizationKey, normalizedCode)); + return batch.masterOptions.has(key(organizationKey, normalizedCategory, normalizedCode)); +} + +async function requireOrganization( + file: CsvParsedFile, + row: CsvParsedRow, + batch: CsvPreviewBatchIndex, + errors: CsvPreviewError[] +): Promise { + const organizationCode = row.values.organization_code; + const organization = await resolveOrganization(organizationCode, batch); + if (!organization.exists) { + errors.push( + issue( + file.fileName, + row.rowNumber, + 'organization_code', + 'UNRESOLVED_REFERENCE', + 'Organization was not found in uploaded files or database.', + 'Upload organizations.csv first or use an existing organization code/slug.' + ) + ); + } + return organization; +} + +async function requireUser( + file: CsvParsedFile, + row: CsvParsedRow, + column: string, + batch: CsvPreviewBatchIndex, + errors: CsvPreviewError[] +): Promise { + const email = row.values[column]; + if (!email) return; + if (batch.users.has(normalizeEmail(email)) || (await existingUser(email)).exists) return; + errors.push( + issue( + file.fileName, + row.rowNumber, + column, + 'UNRESOLVED_REFERENCE', + 'User email was not found in uploaded files or database.', + 'Upload users.csv first or use an existing user email.' + ) + ); +} + +async function requireOption(input: { + file: CsvParsedFile; + row: CsvParsedRow; + column: string; + category: string; + organization: OrganizationReference; + batch: CsvPreviewBatchIndex; + errors: CsvPreviewError[]; +}): Promise { + const codeValue = input.row.values[input.column]; + if (!codeValue) return; + const normalized = normalizeLower(codeValue); + const inBatch = batchHasOption(input.batch, input.organization.key, input.category, normalized); + const inDb = await existingOption(input.organization.id, input.category, normalized); + if (inBatch || inDb.exists) return; + input.errors.push( + issue( + input.file.fileName, + input.row.rowNumber, + input.column, + 'UNKNOWN_OPTION', + `${input.category} option "${codeValue}" was not found.`, + 'Upload the related master option file first or use an existing option code.' + ) + ); +} + +async function requireWorkflow( + file: CsvParsedFile, + row: CsvParsedRow, + organization: OrganizationReference, + batch: CsvPreviewBatchIndex, + errors: CsvPreviewError[] +): Promise { + const workflowCode = row.values.workflow_code; + if (!workflowCode) return; + if ( + batch.approvalWorkflows.has(key(organization.key, workflowCode)) || + (await existingApprovalWorkflow(organization.id, workflowCode)).exists + ) { + return; + } + errors.push( + issue( + file.fileName, + row.rowNumber, + 'workflow_code', + 'UNRESOLVED_REFERENCE', + 'Approval workflow was not found.', + 'Upload approval-workflows.csv before approval steps or matrix rows.' + ) + ); +} + +async function requireCustomer( + file: CsvParsedFile, + row: CsvParsedRow, + organization: OrganizationReference, + batch: CsvPreviewBatchIndex, + errors: CsvPreviewError[] +): Promise { + const customerCode = row.values.customer_code; + if (!customerCode) return; + if ( + batch.customers.has(key(organization.key, customerCode)) || + (await existingCustomer(organization.id, customerCode)).exists + ) { + return; + } + errors.push( + issue( + file.fileName, + row.rowNumber, + 'customer_code', + 'UNRESOLVED_REFERENCE', + 'Customer was not found.', + 'Upload customers.csv first or use an existing customer code.' + ) + ); +} + +async function requireContact( + file: CsvParsedFile, + row: CsvParsedRow, + organization: OrganizationReference, + batch: CsvPreviewBatchIndex, + errors: CsvPreviewError[] +): Promise { + const customerCode = row.values.customer_code; + const contactEmail = row.values.contact_email; + if (!customerCode || !contactEmail) return; + if ( + batch.contactsByEmail.has(key(organization.key, customerCode, contactEmail)) || + (await existingContact(organization.id, customerCode, contactEmail)).exists + ) { + return; + } + errors.push( + issue( + file.fileName, + row.rowNumber, + 'contact_email', + 'UNRESOLVED_REFERENCE', + 'Contact email was not found for the resolved customer.', + 'Upload contacts.csv first and ensure the contact belongs to the customer.' + ) + ); +} + +async function resolveExistingForTemplate( + file: CsvParsedFile, + row: CsvParsedRow, + organization: OrganizationReference +): Promise { + const values = row.values; + + switch (file.template.file) { + case 'users.csv': + return (await existingUser(values.email)).exists; + case 'organizations.csv': + return (await resolveOrganization(values.organization_code, emptyBatchIndex())).id !== null; + case 'memberships.csv': + return existingMembership(organization.id, values.user_email); + case 'master-options.csv': + return (await existingOption(organization.id, values.category, values.code)).exists; + case 'branches.csv': + return (await existingOption(organization.id, 'crm_branch', values.branch_code)).exists; + case 'product-types.csv': + return (await existingOption(organization.id, 'crm_product_type', values.product_type_code)).exists; + case 'approval-workflows.csv': + return (await existingApprovalWorkflow(organization.id, values.workflow_code)).exists; + case 'customers.csv': + return (await existingCustomer(organization.id, values.customer_code)).exists; + case 'leads.csv': + return (await existingCodeRecord(organization.id, 'lead', values.lead_code)).exists; + case 'opportunities.csv': + return (await existingCodeRecord(organization.id, 'opportunity', values.opportunity_code)).exists; + case 'quotations.csv': + return (await existingCodeRecord(organization.id, 'quotation', values.quotation_code)).exists; + case 'document-sequences.csv': + return existingDocumentSequence(organization.id, values); + case 'crm-role-assignments.csv': + return existingRoleAssignment(organization.id, values); + case 'approval-steps.csv': + return existingApprovalStep(organization.id, values); + case 'approval-matrix.csv': + return existingApprovalMatrix(organization.id, values); + case 'contacts.csv': + return existingContactByRow(organization.id, values); + case 'contact-shares.csv': + return existingContactShare(organization.id, values); + case 'opportunity-parties.csv': + return existingOpportunityParty(organization.id, values); + case 'quotation-items.csv': + return existingQuotationItem(organization.id, values); + case 'quotation-parties.csv': + return existingQuotationParty(organization.id, values); + case 'quotation-topics.csv': + return existingQuotationTopic(organization.id, values); + case 'quotation-topic-items.csv': + return existingQuotationTopicItem(organization.id, values); + default: + return false; + } +} + +async function existingDocumentSequence( + organizationId: string | null, + values: Record +): Promise { + if (!organizationId) return false; + const branchId = values.branch_code ? normalizeLower(values.branch_code) : ''; + const productType = values.product_type_code ? normalizeLower(values.product_type_code) : 'all'; + const row = await first( + db + .select({ id: documentSequences.id }) + .from(documentSequences) + .where( + and( + eq(documentSequences.organizationId, organizationId), + eq(documentSequences.branchId, branchId), + eq(documentSequences.productType, productType), + eq(documentSequences.documentType, values.document_type), + eq(documentSequences.period, values.period) + ) + ) + .limit(1) + ); + return Boolean(row); +} + +async function existingRoleAssignment( + organizationId: string | null, + values: Record +): Promise { + if (!organizationId) return false; + const user = await first( + db.select({ id: users.id }).from(users).where(sql`lower(${users.email}) = ${normalizeEmail(values.user_email)}`).limit(1) + ); + const role = await first( + db + .select({ id: crmRoleProfiles.id }) + .from(crmRoleProfiles) + .where(and(eq(crmRoleProfiles.organizationId, organizationId), eq(crmRoleProfiles.code, values.crm_role_code))) + .limit(1) + ); + if (!user || !role) return false; + const row = await first( + db + .select({ id: crmUserRoleAssignments.id }) + .from(crmUserRoleAssignments) + .where( + and( + eq(crmUserRoleAssignments.organizationId, organizationId), + eq(crmUserRoleAssignments.userId, user.id), + eq(crmUserRoleAssignments.roleProfileId, role.id), + isNull(crmUserRoleAssignments.deletedAt) + ) + ) + .limit(1) + ); + return Boolean(row); +} + +async function workflowIdByCode(organizationId: string | null, workflowCode: string): Promise { + if (!organizationId || !workflowCode) return null; + const workflow = await first( + db + .select({ id: crmApprovalWorkflows.id }) + .from(crmApprovalWorkflows) + .where(and(eq(crmApprovalWorkflows.organizationId, organizationId), eq(crmApprovalWorkflows.code, workflowCode))) + .limit(1) + ); + return workflow?.id ?? null; +} + +async function quotationIdByCode(organizationId: string | null, quotationCode: string): Promise { + if (!organizationId || !quotationCode) return null; + const quotation = await first( + db + .select({ id: crmQuotations.id }) + .from(crmQuotations) + .where(and(eq(crmQuotations.organizationId, organizationId), eq(crmQuotations.code, quotationCode))) + .limit(1) + ); + return quotation?.id ?? null; +} + +async function opportunityIdByCode(organizationId: string | null, opportunityCode: string): Promise { + if (!organizationId || !opportunityCode) return null; + const opportunity = await first( + db + .select({ id: crmOpportunities.id }) + .from(crmOpportunities) + .where(and(eq(crmOpportunities.organizationId, organizationId), eq(crmOpportunities.code, opportunityCode))) + .limit(1) + ); + return opportunity?.id ?? null; +} + +async function customerIdByCode(organizationId: string | null, customerCode: string): Promise { + if (!organizationId || !customerCode) return null; + const customer = await first( + db + .select({ id: crmCustomers.id }) + .from(crmCustomers) + .where(and(eq(crmCustomers.organizationId, organizationId), eq(crmCustomers.code, customerCode))) + .limit(1) + ); + return customer?.id ?? null; +} + +async function existingApprovalStep( + organizationId: string | null, + values: Record +): Promise { + const workflowId = await workflowIdByCode(organizationId, values.workflow_code); + if (!workflowId || !values.step_number) return false; + const row = await first( + db + .select({ id: crmApprovalSteps.id }) + .from(crmApprovalSteps) + .where( + and( + eq(crmApprovalSteps.workflowId, workflowId), + eq(crmApprovalSteps.stepNumber, Number(values.step_number)), + isNull(crmApprovalSteps.deletedAt) + ) + ) + .limit(1) + ); + return Boolean(row); +} + +async function existingApprovalMatrix( + organizationId: string | null, + values: Record +): Promise { + const workflowId = await workflowIdByCode(organizationId, values.workflow_code); + if (!organizationId || !workflowId) return false; + const row = await first( + db + .select({ id: crmApprovalMatrices.id }) + .from(crmApprovalMatrices) + .where( + and( + eq(crmApprovalMatrices.organizationId, organizationId), + eq(crmApprovalMatrices.workflowId, workflowId), + eq(crmApprovalMatrices.entityType, values.entity_type), + eq(crmApprovalMatrices.priority, Number(values.priority)), + isNull(crmApprovalMatrices.deletedAt) + ) + ) + .limit(1) + ); + return Boolean(row); +} + +async function existingContactByRow( + organizationId: string | null, + values: Record +): Promise { + if (values.email) return (await existingContact(organizationId, values.customer_code, values.email)).exists; + const customerId = await customerIdByCode(organizationId, values.customer_code); + if (!organizationId || !customerId) return false; + const row = await first( + db + .select({ id: crmCustomerContacts.id }) + .from(crmCustomerContacts) + .where( + and( + eq(crmCustomerContacts.organizationId, organizationId), + eq(crmCustomerContacts.customerId, customerId), + eq(crmCustomerContacts.name, values.contact_name), + isNull(crmCustomerContacts.deletedAt) + ) + ) + .limit(1) + ); + return Boolean(row); +} + +async function existingContactShare( + organizationId: string | null, + values: Record +): Promise { + const customerId = await customerIdByCode(organizationId, values.customer_code); + if (!organizationId || !customerId || !values.contact_email || !values.shared_to_user_email) return false; + const contact = await first( + db + .select({ id: crmCustomerContacts.id }) + .from(crmCustomerContacts) + .where( + and( + eq(crmCustomerContacts.organizationId, organizationId), + eq(crmCustomerContacts.customerId, customerId), + sql`lower(${crmCustomerContacts.email}) = ${normalizeEmail(values.contact_email)}`, + isNull(crmCustomerContacts.deletedAt) + ) + ) + .limit(1) + ); + const sharedTo = await first( + db + .select({ id: users.id }) + .from(users) + .where(sql`lower(${users.email}) = ${normalizeEmail(values.shared_to_user_email)}`) + .limit(1) + ); + if (!contact || !sharedTo) return false; + const row = await first( + db + .select({ id: crmContactShares.id }) + .from(crmContactShares) + .where( + and( + eq(crmContactShares.organizationId, organizationId), + eq(crmContactShares.contactId, contact.id), + eq(crmContactShares.sharedToUserId, sharedTo.id), + isNull(crmContactShares.deletedAt) + ) + ) + .limit(1) + ); + return Boolean(row); +} + +async function existingOpportunityParty( + organizationId: string | null, + values: Record +): Promise { + const opportunityId = await opportunityIdByCode(organizationId, values.opportunity_code); + const customerId = await customerIdByCode(organizationId, values.customer_code); + if (!opportunityId || !customerId) return false; + const row = await first( + db + .select({ id: crmOpportunityCustomers.id }) + .from(crmOpportunityCustomers) + .where( + and( + eq(crmOpportunityCustomers.opportunityId, opportunityId), + eq(crmOpportunityCustomers.customerId, customerId), + eq(crmOpportunityCustomers.role, values.role_code), + isNull(crmOpportunityCustomers.deletedAt) + ) + ) + .limit(1) + ); + return Boolean(row); +} + +async function existingQuotationItem( + organizationId: string | null, + values: Record +): Promise { + const quotationId = await quotationIdByCode(organizationId, values.quotation_code); + if (!quotationId || !values.item_number) return false; + const row = await first( + db + .select({ id: crmQuotationItems.id }) + .from(crmQuotationItems) + .where( + and( + eq(crmQuotationItems.quotationId, quotationId), + eq(crmQuotationItems.itemNumber, Number(values.item_number)), + isNull(crmQuotationItems.deletedAt) + ) + ) + .limit(1) + ); + return Boolean(row); +} + +async function existingQuotationParty( + organizationId: string | null, + values: Record +): Promise { + const quotationId = await quotationIdByCode(organizationId, values.quotation_code); + const customerId = await customerIdByCode(organizationId, values.customer_code); + if (!quotationId || !customerId) return false; + const row = await first( + db + .select({ id: crmQuotationCustomers.id }) + .from(crmQuotationCustomers) + .where( + and( + eq(crmQuotationCustomers.quotationId, quotationId), + eq(crmQuotationCustomers.customerId, customerId), + eq(crmQuotationCustomers.role, values.role_code), + isNull(crmQuotationCustomers.deletedAt) + ) + ) + .limit(1) + ); + return Boolean(row); +} + +async function existingQuotationTopic( + organizationId: string | null, + values: Record +): Promise { + const quotationId = await quotationIdByCode(organizationId, values.quotation_code); + if (!organizationId || !quotationId || !values.topic_type || !values.title) return false; + const row = await first( + db + .select({ id: crmQuotationTopics.id }) + .from(crmQuotationTopics) + .where( + and( + eq(crmQuotationTopics.organizationId, organizationId), + eq(crmQuotationTopics.quotationId, quotationId), + eq(crmQuotationTopics.topicType, values.topic_type), + eq(crmQuotationTopics.title, values.title), + isNull(crmQuotationTopics.deletedAt) + ) + ) + .limit(1) + ); + return Boolean(row); +} + +async function existingQuotationTopicItem( + organizationId: string | null, + values: Record +): Promise { + void organizationId; + void values; + return false; +} + +function emptyBatchIndex(): CsvPreviewBatchIndex { + return { + organizations: new Set(), + users: new Set(), + memberships: new Set(), + roleAssignments: new Set(), + masterOptions: new Set(), + branches: new Set(), + productTypes: new Set(), + documentSequences: new Set(), + approvalWorkflows: new Set(), + approvalSteps: new Set(), + approvalMatrices: new Set(), + customers: new Set(), + contactsByEmail: new Set(), + contactsByName: new Set(), + contactShares: new Set(), + leads: new Set(), + opportunities: new Set(), + opportunityParties: new Set(), + quotations: new Set(), + quotationItems: new Set(), + quotationParties: new Set(), + quotationTopics: new Set(), + quotationTopicItems: new Set() + }; +} + +export function buildCsvPreviewBatchIndex(files: CsvParsedFile[]): CsvPreviewBatchIndex { + const index = emptyBatchIndex(); + + files.forEach((file) => { + file.rows.forEach((row) => { + const values = row.values; + const organizationCode = normalizeOrganizationCode(values.organization_code); + + switch (file.template.file) { + case 'organizations.csv': + index.organizations.add(normalizeOrganizationCode(values.organization_code)); + break; + case 'users.csv': + index.users.add(normalizeEmail(values.email)); + break; + case 'memberships.csv': + index.memberships.add(key(organizationCode, values.user_email)); + break; + case 'crm-role-assignments.csv': + index.roleAssignments.add(key(organizationCode, values.user_email, values.crm_role_code)); + break; + case 'master-options.csv': + index.masterOptions.add(key(organizationCode, values.category, values.code)); + break; + case 'branches.csv': + index.branches.add(key(organizationCode, values.branch_code)); + break; + case 'product-types.csv': + index.productTypes.add(key(organizationCode, values.product_type_code)); + break; + case 'document-sequences.csv': + index.documentSequences.add( + key( + organizationCode, + values.branch_code, + values.product_type_code || 'all', + values.document_type, + values.period + ) + ); + break; + case 'approval-workflows.csv': + index.approvalWorkflows.add(key(organizationCode, values.workflow_code)); + break; + case 'approval-steps.csv': + index.approvalSteps.add(key(organizationCode, values.workflow_code, values.step_number)); + break; + case 'approval-matrix.csv': + index.approvalMatrices.add(key(organizationCode, values.entity_type, values.workflow_code, values.priority)); + break; + case 'customers.csv': + index.customers.add(key(organizationCode, values.customer_code)); + break; + case 'contacts.csv': + if (values.email) index.contactsByEmail.add(key(organizationCode, values.customer_code, values.email)); + index.contactsByName.add(key(organizationCode, values.customer_code, values.contact_name)); + break; + case 'contact-shares.csv': + index.contactShares.add(key(organizationCode, values.customer_code, values.contact_email, values.shared_to_user_email)); + break; + case 'leads.csv': + index.leads.add(key(organizationCode, values.lead_code)); + break; + case 'opportunities.csv': + index.opportunities.add(key(organizationCode, values.opportunity_code)); + break; + case 'opportunity-parties.csv': + index.opportunityParties.add(key(organizationCode, values.opportunity_code, values.customer_code, values.role_code)); + break; + case 'quotations.csv': + index.quotations.add(key(organizationCode, values.quotation_code)); + break; + case 'quotation-items.csv': + index.quotationItems.add(key(organizationCode, values.quotation_code, values.item_number)); + break; + case 'quotation-parties.csv': + index.quotationParties.add(key(organizationCode, values.quotation_code, values.customer_code, values.role_code)); + break; + case 'quotation-topics.csv': + index.quotationTopics.add(key(organizationCode, values.quotation_code, values.topic_key)); + break; + case 'quotation-topic-items.csv': + index.quotationTopicItems.add( + key(organizationCode, values.quotation_code, values.topic_key, values.sort_order, values.content) + ); + break; + default: + break; + } + }); + }); + + return index; +} + +async function validateCommonReferences( + file: CsvParsedFile, + row: CsvParsedRow, + batch: CsvPreviewBatchIndex, + errors: CsvPreviewError[] +): Promise { + const organization = row.values.organization_code + ? await requireOrganization(file, row, batch, errors) + : { id: null, key: '', exists: false }; + + const userColumns = [ + 'created_by_email', + 'updated_by_email', + 'user_email', + 'owner_user_email', + 'owner_marketing_user_email', + 'assigned_sales_owner_email', + 'assigned_to_user_email', + 'salesman_email', + 'shared_to_user_email', + 'shared_by_user_email' + ]; + for (const column of userColumns) { + await requireUser(file, row, column, batch, errors); + } + + if (row.values.branch_code) { + await requireOption({ file, row, column: 'branch_code', category: 'crm_branch', organization, batch, errors }); + } + if (row.values.product_type_code) { + await requireOption({ + file, + row, + column: 'product_type_code', + category: 'crm_product_type', + organization, + batch, + errors + }); + } + + return organization; +} + +export async function resolveCsvRowDependencies( + file: CsvParsedFile, + row: CsvParsedRow, + batch: CsvPreviewBatchIndex +): Promise { + const errors: CsvPreviewError[] = []; + const warnings: CsvPreviewWarning[] = []; + const organization = await validateCommonReferences(file, row, batch, errors); + + switch (file.template.file) { + case 'crm-role-assignments.csv': + if ( + row.values.crm_role_code && + !(await existingRoleProfile(organization.id, row.values.crm_role_code)).exists + ) { + errors.push( + issue( + file.fileName, + row.rowNumber, + 'crm_role_code', + 'UNRESOLVED_REFERENCE', + 'CRM role profile was not found.', + 'Run foundation role setup before assigning CRM roles.' + ) + ); + } + for (const branchCode of pipeList(row.values.branch_codes ?? '')) { + await requireOption({ + file, + row: { ...row, values: { ...row.values, branch_code: branchCode } }, + column: 'branch_code', + category: 'crm_branch', + organization, + batch, + errors + }); + } + for (const productTypeCode of pipeList(row.values.product_type_codes ?? '')) { + await requireOption({ + file, + row: { ...row, values: { ...row.values, product_type_code: productTypeCode } }, + column: 'product_type_code', + category: 'crm_product_type', + organization, + batch, + errors + }); + } + break; + case 'master-options.csv': + if (row.values.parent_category && row.values.parent_code) { + await requireOption({ + file, + row: { ...row, values: { ...row.values, parent_code: row.values.parent_code } }, + column: 'parent_code', + category: row.values.parent_category, + organization, + batch, + errors + }); + } + break; + case 'approval-steps.csv': + case 'approval-matrix.csv': + await requireWorkflow(file, row, organization, batch, errors); + if ( + file.template.file === 'approval-steps.csv' && + row.values.role_code && + !(await existingRoleProfile(organization.id, row.values.role_code)).exists + ) { + warnings.push( + issue( + file.fileName, + row.rowNumber, + 'role_code', + 'UNRESOLVED_ROLE_PROFILE', + 'Approval step role was not found in CRM role profiles.', + 'Confirm the role is an approved workflow role or seed the CRM role profile first.' + ) + ); + } + if (row.values.currency) { + await requireOption({ file, row, column: 'currency', category: 'crm_currency', organization, batch, errors }); + } + break; + case 'customers.csv': + for (const [column, category] of [ + ['customer_type', 'crm_customer_type'], + ['customer_status', 'crm_customer_status'], + ['lead_channel', 'crm_lead_channel'], + ['customer_group', 'crm_customer_group'], + ['customer_sub_group', 'crm_customer_sub_group'] + ] as const) { + await requireOption({ file, row, column, category, organization, batch, errors }); + } + break; + case 'contacts.csv': + case 'contact-shares.csv': + await requireCustomer(file, row, organization, batch, errors); + if (file.template.file === 'contact-shares.csv') { + await requireContact(file, row, organization, batch, errors); + if (row.values.shared_to_user_email === row.values.shared_by_user_email) { + warnings.push( + issue( + file.fileName, + row.rowNumber, + 'shared_to_user_email', + 'SELF_SHARE', + 'Contact is shared to the same user who shared it.', + 'Confirm whether this share row is needed.' + ) + ); + } + } + break; + case 'leads.csv': + await requireCustomer(file, row, organization, batch, errors); + await requireContact(file, row, organization, batch, errors); + for (const [column, category] of [ + ['status', 'crm_lead_status'], + ['lead_channel', 'crm_lead_channel'], + ['priority', 'crm_priority'], + ['awareness_code', 'crm_lead_awareness'], + ['followup_status', 'crm_lead_followup_status'], + ['lost_reason', 'crm_lead_lost_reason'] + ] as const) { + await requireOption({ file, row, column, category, organization, batch, errors }); + } + break; + case 'opportunities.csv': + await requireCustomer(file, row, organization, batch, errors); + await requireContact(file, row, organization, batch, errors); + if ( + row.values.lead_code && + !batch.leads.has(key(organization.key, row.values.lead_code)) && + !(await existingCodeRecord(organization.id, 'lead', row.values.lead_code)).exists + ) { + errors.push(issue(file.fileName, row.rowNumber, 'lead_code', 'UNRESOLVED_REFERENCE', 'Lead was not found.', 'Upload leads.csv first or use an existing lead code.')); + } + for (const [column, category] of [ + ['status', 'crm_opportunity_status'], + ['priority', 'crm_priority'], + ['lead_channel', 'crm_lead_channel'] + ] as const) { + await requireOption({ file, row, column, category, organization, batch, errors }); + } + break; + case 'opportunity-parties.csv': + await requireCustomer(file, row, organization, batch, errors); + if ( + !batch.opportunities.has(key(organization.key, row.values.opportunity_code)) && + !(await existingCodeRecord(organization.id, 'opportunity', row.values.opportunity_code)).exists + ) { + errors.push(issue(file.fileName, row.rowNumber, 'opportunity_code', 'UNRESOLVED_REFERENCE', 'Opportunity was not found.', 'Upload opportunities.csv first or use an existing opportunity code.')); + } + await requireOption({ file, row, column: 'role_code', category: 'crm_project_party_role', organization, batch, errors }); + break; + case 'quotations.csv': + await requireCustomer(file, row, organization, batch, errors); + await requireContact(file, row, organization, batch, errors); + if ( + row.values.opportunity_code && + !batch.opportunities.has(key(organization.key, row.values.opportunity_code)) && + !(await existingCodeRecord(organization.id, 'opportunity', row.values.opportunity_code)).exists + ) { + errors.push(issue(file.fileName, row.rowNumber, 'opportunity_code', 'UNRESOLVED_REFERENCE', 'Opportunity was not found.', 'Upload opportunities.csv first or use an existing opportunity code.')); + } + for (const [column, category] of [ + ['quotation_type', 'crm_quotation_type'], + ['status', 'crm_quotation_status'], + ['currency', 'crm_currency'], + ['discount_type', 'crm_discount_type'], + ['sent_via', 'crm_sent_via'] + ] as const) { + await requireOption({ file, row, column, category, organization, batch, errors }); + } + break; + case 'quotation-items.csv': + if ( + !batch.quotations.has(key(organization.key, row.values.quotation_code)) && + !(await existingCodeRecord(organization.id, 'quotation', row.values.quotation_code)).exists + ) { + errors.push(issue(file.fileName, row.rowNumber, 'quotation_code', 'UNRESOLVED_REFERENCE', 'Quotation was not found.', 'Upload quotations.csv first or use an existing quotation code.')); + } + await requireOption({ file, row, column: 'unit', category: 'crm_unit', organization, batch, errors }); + await requireOption({ file, row, column: 'discount_type', category: 'crm_discount_type', organization, batch, errors }); + break; + case 'quotation-parties.csv': + await requireCustomer(file, row, organization, batch, errors); + if ( + !batch.quotations.has(key(organization.key, row.values.quotation_code)) && + !(await existingCodeRecord(organization.id, 'quotation', row.values.quotation_code)).exists + ) { + errors.push(issue(file.fileName, row.rowNumber, 'quotation_code', 'UNRESOLVED_REFERENCE', 'Quotation was not found.', 'Upload quotations.csv first or use an existing quotation code.')); + } + await requireOption({ file, row, column: 'role_code', category: 'crm_project_party_role', organization, batch, errors }); + break; + case 'quotation-topics.csv': + if ( + !batch.quotations.has(key(organization.key, row.values.quotation_code)) && + !(await existingCodeRecord(organization.id, 'quotation', row.values.quotation_code)).exists + ) { + errors.push(issue(file.fileName, row.rowNumber, 'quotation_code', 'UNRESOLVED_REFERENCE', 'Quotation was not found.', 'Upload quotations.csv first or use an existing quotation code.')); + } + await requireOption({ file, row, column: 'topic_type', category: 'crm_quotation_topic_type', organization, batch, errors }); + break; + case 'quotation-topic-items.csv': + if (!batch.quotationTopics.has(key(organization.key, row.values.quotation_code, row.values.topic_key))) { + const existingTopic = await existingQuotationTopic(organization.id, row.values); + if (!existingTopic) { + errors.push(issue(file.fileName, row.rowNumber, 'topic_key', 'UNRESOLVED_REFERENCE', 'Quotation topic was not found.', 'Upload quotation-topics.csv first or use an existing topic key.')); + } + } + break; + default: + break; + } + + return { + errors, + warnings, + exists: await resolveExistingForTemplate(file, row, organization) + }; +} diff --git a/src/features/setup/server/csv/header-validator.ts b/src/features/setup/server/csv/header-validator.ts new file mode 100644 index 0000000..a934e09 --- /dev/null +++ b/src/features/setup/server/csv/header-validator.ts @@ -0,0 +1,75 @@ +import type { CsvParsedFile, CsvPreviewError } from './types'; + +function issue( + file: string, + column: string, + code: string, + message: string, + suggestedFix: string +): CsvPreviewError { + return { file, row: 1, column, code, message, suggestedFix }; +} + +export function validateCsvHeaders(file: CsvParsedFile): CsvPreviewError[] { + const errors: CsvPreviewError[] = []; + const seen = new Map(); + + file.headers.forEach((header, index) => { + const previousIndex = seen.get(header); + if (previousIndex !== undefined) { + errors.push( + issue( + file.fileName, + header, + 'DUPLICATE_COLUMN', + `Column "${header}" appears more than once.`, + `Remove the duplicate column at position ${index + 1}; keep the official template order.` + ) + ); + } + seen.set(header, index); + }); + + file.template.expectedHeaders.forEach((expectedHeader, index) => { + if (!file.headers.includes(expectedHeader)) { + errors.push( + issue( + file.fileName, + expectedHeader, + 'MISSING_COLUMN', + `Required template column "${expectedHeader}" is missing.`, + 'Download the latest setup CSV template and keep all official columns.' + ) + ); + return; + } + + if (file.headers[index] !== expectedHeader) { + errors.push( + issue( + file.fileName, + expectedHeader, + 'HEADER_ORDER_MISMATCH', + `Column "${expectedHeader}" is not in the expected position ${index + 1}.`, + 'Reorder columns to match the official template exactly.' + ) + ); + } + }); + + file.headers.forEach((header) => { + if (!file.template.expectedHeaders.includes(header)) { + errors.push( + issue( + file.fileName, + header, + 'EXTRA_COLUMN', + `Column "${header}" is not part of the official template.`, + 'Remove extra columns before previewing the import.' + ) + ); + } + }); + + return errors; +} diff --git a/src/features/setup/server/csv/parser.ts b/src/features/setup/server/csv/parser.ts new file mode 100644 index 0000000..fc84e6e --- /dev/null +++ b/src/features/setup/server/csv/parser.ts @@ -0,0 +1,212 @@ +import type { + CsvParsedFile, + CsvParsedRow, + CsvPreviewError, + CsvPreviewWarning, + CsvTemplateDefinition +} from './types'; + +function buildIssue( + file: string, + row: number, + column: string, + code: string, + message: string, + suggestedFix: string +): CsvPreviewError { + return { file, row, column, code, message, suggestedFix }; +} + +function decodeUtf8(fileName: string, bytes: Uint8Array): { + text: string; + errors: CsvPreviewError[]; + warnings: CsvPreviewWarning[]; +} { + const errors: CsvPreviewError[] = []; + const warnings: CsvPreviewWarning[] = []; + + if (bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) { + warnings.push( + buildIssue( + fileName, + 1, + '__file__', + 'UTF8_BOM', + 'CSV file contains a UTF-8 BOM.', + 'Export the CSV as UTF-8 without BOM when possible.' + ) + ); + } + + try { + return { + text: new TextDecoder('utf-8', { fatal: true }).decode(bytes).replace(/^\uFEFF/, ''), + errors, + warnings + }; + } catch { + errors.push( + buildIssue( + fileName, + 1, + '__file__', + 'INVALID_ENCODING', + 'CSV file is not valid UTF-8.', + 'Save the file as UTF-8 CSV and upload it again.' + ) + ); + return { text: '', errors, warnings }; + } +} + +function parseCsvText(text: string): string[][] { + const rows: string[][] = []; + let currentRow: string[] = []; + let currentCell = ''; + let inQuotes = false; + + for (let index = 0; index < text.length; index += 1) { + const char = text[index]; + const next = text[index + 1]; + + if (char === '"') { + if (inQuotes && next === '"') { + currentCell += '"'; + index += 1; + } else { + inQuotes = !inQuotes; + } + continue; + } + + if (char === ',' && !inQuotes) { + currentRow.push(currentCell); + currentCell = ''; + continue; + } + + if ((char === '\n' || char === '\r') && !inQuotes) { + if (char === '\r' && next === '\n') index += 1; + currentRow.push(currentCell); + rows.push(currentRow); + currentRow = []; + currentCell = ''; + continue; + } + + currentCell += char; + } + + if (currentCell.length > 0 || currentRow.length > 0) { + currentRow.push(currentCell); + rows.push(currentRow); + } + + return rows; +} + +export function parseCsvFile(input: { + fileName: string; + bytes: Uint8Array; + template: CsvTemplateDefinition; +}): CsvParsedFile { + const decoded = decodeUtf8(input.fileName, input.bytes); + const errors = [...decoded.errors]; + const warnings = [...decoded.warnings]; + + if (decoded.errors.length > 0) { + return { + fileName: input.fileName, + template: input.template, + headers: [], + rows: [], + errors, + warnings + }; + } + + const parsedRows = parseCsvText(decoded.text); + const [headerRow, ...dataRows] = parsedRows; + + if (!headerRow) { + errors.push( + buildIssue( + input.fileName, + 1, + '__header__', + 'EMPTY_FILE', + 'CSV file is empty.', + 'Upload a CSV file with the official template header row.' + ) + ); + } + + const headers = (headerRow ?? []).map((header) => header.trim()); + const rows: CsvParsedRow[] = []; + const rowFingerprints = new Map(); + + dataRows.forEach((raw, index) => { + const rowNumber = index + 2; + const isEmpty = raw.every((cell) => cell.trim() === ''); + + if (isEmpty) { + warnings.push( + buildIssue( + input.fileName, + rowNumber, + '__row__', + 'EMPTY_ROW', + 'CSV row is empty and will be skipped.', + 'Remove empty rows from the file.' + ) + ); + return; + } + + if (raw.length !== headers.length) { + errors.push( + buildIssue( + input.fileName, + rowNumber, + '__row__', + 'INCONSISTENT_COLUMN_COUNT', + `Row has ${raw.length} columns but header has ${headers.length}.`, + 'Fix quoting or column separators so every row matches the template header.' + ) + ); + } + + const fingerprint = raw.map((cell) => cell.trim()).join('\u001f'); + const firstRow = rowFingerprints.get(fingerprint); + if (firstRow) { + warnings.push( + buildIssue( + input.fileName, + rowNumber, + '__row__', + 'DUPLICATE_ROW', + `Row duplicates row ${firstRow}.`, + 'Remove duplicate rows or confirm that the later row is intentional.' + ) + ); + } else { + rowFingerprints.set(fingerprint, rowNumber); + } + + const values = headers.reduce>((accumulator, header, cellIndex) => { + accumulator[header] = (raw[cellIndex] ?? '').trim(); + return accumulator; + }, {}); + + rows.push({ rowNumber, values, raw }); + }); + + return { + fileName: input.fileName, + template: input.template, + headers, + rows, + errors, + warnings + }; +} diff --git a/src/features/setup/server/csv/preview-builder.ts b/src/features/setup/server/csv/preview-builder.ts new file mode 100644 index 0000000..c9cc662 --- /dev/null +++ b/src/features/setup/server/csv/preview-builder.ts @@ -0,0 +1,83 @@ +import { createHash } from 'node:crypto'; +import type { + CsvPreviewFileResult, + CsvPreviewResult, + CsvPreviewRow, + CsvPreviewStatus +} from './types'; + +export function resolveRowStatus(row: CsvPreviewRow): CsvPreviewStatus { + if (row.errors.length > 0) return 'FAIL'; + if (row.warnings.length > 0) return 'WARNING'; + return 'PASS'; +} + +export function buildFileResult(input: { + fileName: string; + template: string; + importOrder: number; + rows: CsvPreviewRow[]; +}): CsvPreviewFileResult { + const errors = input.rows.flatMap((row) => row.errors); + const warnings = input.rows.flatMap((row) => row.warnings); + const status: CsvPreviewStatus = errors.length > 0 ? 'FAIL' : warnings.length > 0 ? 'WARNING' : 'PASS'; + + return { + fileName: input.fileName, + template: input.template, + importOrder: input.importOrder, + status, + summary: { + rows: input.rows.length, + valid: input.rows.filter((row) => row.status === 'PASS').length, + warning: input.rows.filter((row) => row.status === 'WARNING').length, + error: input.rows.filter((row) => row.status === 'FAIL').length, + create: input.rows.filter((row) => row.action === 'create').length, + update: input.rows.filter((row) => row.action === 'update').length, + skip: input.rows.filter((row) => row.action === 'skip').length + }, + rows: input.rows, + errors, + warnings + }; +} + +export function buildPreviewResult(files: CsvPreviewFileResult[]): CsvPreviewResult { + const summary = files.reduce( + (accumulator, file) => ({ + files: accumulator.files + 1, + rows: accumulator.rows + file.summary.rows, + valid: accumulator.valid + file.summary.valid, + warning: accumulator.warning + file.summary.warning, + error: accumulator.error + file.summary.error, + create: accumulator.create + file.summary.create, + update: accumulator.update + file.summary.update, + skip: accumulator.skip + file.summary.skip + }), + { files: 0, rows: 0, valid: 0, warning: 0, error: 0, create: 0, update: 0, skip: 0 } + ); + + const generatedAt = new Date().toISOString(); + const previewHash = createHash('sha256') + .update( + JSON.stringify({ + files: files.map((file) => ({ + fileName: file.fileName, + template: file.template, + importOrder: file.importOrder, + rows: file.rows.map((row) => ({ + row: row.row, + key: row.key, + values: row.values, + status: row.status, + action: row.action, + errors: row.errors.map((error) => error.code), + warnings: row.warnings.map((warning) => warning.code) + })) + })) + }) + ) + .digest('hex'); + + return { summary, files, generatedAt, previewHash }; +} diff --git a/src/features/setup/server/csv/template-registry.ts b/src/features/setup/server/csv/template-registry.ts new file mode 100644 index 0000000..d9236d0 --- /dev/null +++ b/src/features/setup/server/csv/template-registry.ts @@ -0,0 +1,66 @@ +import 'server-only'; + +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; +import type { CsvTemplateDefinition } from './types'; + +interface TemplateMetadataFile { + version: string; + generatedFrom: string; + templates: Array<{ + file: string; + seedType: string; + importOrder: number; + supported: boolean; + requiredColumns: string[]; + description: string; + }>; + unsupportedTemplates: string[]; +} + +export interface CsvTemplateRegistry { + version: string; + generatedFrom: string; + templates: CsvTemplateDefinition[]; + unsupportedTemplates: string[]; +} + +async function readTemplateHeader(file: string): Promise { + const templatePath = path.join(process.cwd(), 'setup', 'templates', file); + const text = await readFile(templatePath, 'utf8'); + const [header = ''] = text.replace(/^\uFEFF/, '').split(/\r?\n/); + return header.split(',').map((column) => column.trim()); +} + +export async function getCsvTemplateRegistry(): Promise { + const metadataPath = path.join(process.cwd(), 'setup', 'templates', 'templates.json'); + const metadata = JSON.parse(await readFile(metadataPath, 'utf8')) as TemplateMetadataFile; + + const templates = await Promise.all( + metadata.templates.map(async (template) => ({ + ...template, + template: template.file.replace(/\.csv$/i, ''), + expectedHeaders: await readTemplateHeader(template.file) + })) + ); + + return { + version: metadata.version, + generatedFrom: metadata.generatedFrom, + templates, + unsupportedTemplates: metadata.unsupportedTemplates + }; +} + +export function findTemplateByFileName( + registry: CsvTemplateRegistry, + fileName: string +): CsvTemplateDefinition | null { + const normalized = path.basename(fileName).toLowerCase(); + return registry.templates.find((template) => template.file.toLowerCase() === normalized) ?? null; +} + +export function isUnsupportedTemplate(registry: CsvTemplateRegistry, fileName: string): boolean { + const normalized = path.basename(fileName).toLowerCase(); + return registry.unsupportedTemplates.some((template) => template.toLowerCase() === normalized); +} diff --git a/src/features/setup/server/csv/types.ts b/src/features/setup/server/csv/types.ts new file mode 100644 index 0000000..2bd9e78 --- /dev/null +++ b/src/features/setup/server/csv/types.ts @@ -0,0 +1,116 @@ +export type CsvPreviewStatus = 'PASS' | 'WARNING' | 'FAIL'; + +export type CsvPreviewAction = 'create' | 'update' | 'skip' | 'error'; + +export interface CsvPreviewError { + file: string; + row: number; + column: string; + code: string; + message: string; + suggestedFix: string; +} + +export interface CsvPreviewWarning extends CsvPreviewError {} + +export interface CsvTemplateDefinition { + file: string; + template: string; + seedType: string; + importOrder: number; + supported: boolean; + requiredColumns: string[]; + description: string; + expectedHeaders: string[]; +} + +export interface CsvParsedRow { + rowNumber: number; + values: Record; + raw: string[]; +} + +export interface CsvParsedFile { + fileName: string; + template: CsvTemplateDefinition; + headers: string[]; + rows: CsvParsedRow[]; + errors: CsvPreviewError[]; + warnings: CsvPreviewWarning[]; +} + +export interface CsvPreviewRow { + row: number; + status: CsvPreviewStatus; + action: CsvPreviewAction; + key: string | null; + values: Record; + errors: CsvPreviewError[]; + warnings: CsvPreviewWarning[]; +} + +export interface CsvPreviewFileResult { + fileName: string; + template: string; + importOrder: number; + status: CsvPreviewStatus; + summary: { + rows: number; + valid: number; + warning: number; + error: number; + create: number; + update: number; + skip: number; + }; + rows: CsvPreviewRow[]; + errors: CsvPreviewError[]; + warnings: CsvPreviewWarning[]; +} + +export interface CsvPreviewResult { + summary: { + files: number; + rows: number; + valid: number; + warning: number; + error: number; + create: number; + update: number; + skip: number; + }; + files: CsvPreviewFileResult[]; + generatedAt: string; + previewHash: string; +} + +export interface CsvPreviewFileInput { + fileName: string; + bytes: Uint8Array; +} + +export interface CsvPreviewBatchIndex { + organizations: Set; + users: Set; + memberships: Set; + roleAssignments: Set; + masterOptions: Set; + branches: Set; + productTypes: Set; + documentSequences: Set; + approvalWorkflows: Set; + approvalSteps: Set; + approvalMatrices: Set; + customers: Set; + contactsByEmail: Set; + contactsByName: Set; + contactShares: Set; + leads: Set; + opportunities: Set; + opportunityParties: Set; + quotations: Set; + quotationItems: Set; + quotationParties: Set; + quotationTopics: Set; + quotationTopicItems: Set; +} diff --git a/src/features/setup/server/csv/validator.ts b/src/features/setup/server/csv/validator.ts new file mode 100644 index 0000000..fc1450a --- /dev/null +++ b/src/features/setup/server/csv/validator.ts @@ -0,0 +1,221 @@ +import type { + CsvParsedFile, + CsvPreviewBatchIndex, + CsvPreviewError, + CsvPreviewRow, + CsvPreviewWarning +} from './types'; +import { validateCsvHeaders } from './header-validator'; +import { resolveCsvRowDependencies } from './dependency-resolver'; +import { resolveRowStatus } from './preview-builder'; +import { normalizeEmail, normalizeLower, normalizeOrganizationCode } from './value-validator'; +import { validateCsvRowValues } from './value-validator'; + +function key(...parts: Array): string { + return parts.map((part) => String(part ?? '').trim().toLowerCase()).join('|'); +} + +function issue( + file: string, + row: number, + column: string, + code: string, + message: string, + suggestedFix: string +): CsvPreviewError { + return { file, row, column, code, message, suggestedFix }; +} + +function organizationCode(values: Record): string { + return normalizeOrganizationCode(values.organization_code); +} + +function businessKey(file: CsvParsedFile, values: Record): string { + const organization = organizationCode(values); + + switch (file.template.file) { + case 'users.csv': + return key(normalizeEmail(values.email)); + case 'organizations.csv': + return key(organization); + case 'memberships.csv': + return key(organization, values.user_email); + case 'crm-role-assignments.csv': + return key(organization, values.user_email, values.crm_role_code); + case 'master-options.csv': + return key(organization, values.category, values.code); + case 'branches.csv': + return key(organization, values.branch_code); + case 'product-types.csv': + return key(organization, values.product_type_code); + case 'document-sequences.csv': + return key( + organization, + values.branch_code, + values.product_type_code || 'all', + values.document_type, + values.period + ); + case 'approval-workflows.csv': + return key(organization, values.workflow_code); + case 'approval-steps.csv': + return key(organization, values.workflow_code, values.step_number); + case 'approval-matrix.csv': + return key( + organization, + values.entity_type, + values.workflow_code, + values.branch_code, + values.product_type_code, + values.min_amount, + values.max_amount, + values.priority + ); + case 'customers.csv': + return key(organization, values.customer_code); + case 'contacts.csv': + return values.email + ? key(organization, values.customer_code, values.email) + : key(organization, values.customer_code, values.contact_name); + case 'contact-shares.csv': + return key(organization, values.customer_code, values.contact_email, values.shared_to_user_email); + case 'leads.csv': + return key(organization, values.lead_code); + case 'opportunities.csv': + return key(organization, values.opportunity_code); + case 'opportunity-parties.csv': + return key(organization, values.opportunity_code, values.customer_code, values.role_code); + case 'quotations.csv': + return key(organization, values.quotation_code); + case 'quotation-items.csv': + return key(organization, values.quotation_code, values.item_number); + case 'quotation-parties.csv': + return key(organization, values.quotation_code, values.customer_code, values.role_code); + case 'quotation-topics.csv': + return key(organization, values.quotation_code, values.topic_key); + case 'quotation-topic-items.csv': + return key(organization, values.quotation_code, values.topic_key, values.sort_order, values.content); + default: + return key(organization, JSON.stringify(values)); + } +} + +function normalizeRowValues(values: Record): Record { + const normalized = { ...values }; + + Object.keys(normalized).forEach((column) => { + if (column.endsWith('_email') || column === 'email') normalized[column] = normalizeEmail(normalized[column]); + if ( + column.endsWith('_code') || + [ + 'slug', + 'category', + 'system_role', + 'membership_role', + 'business_role', + 'crm_role_code', + 'entity_type', + 'status', + 'priority', + 'quotation_type', + 'currency', + 'discount_type', + 'topic_type', + 'role_code' + ].includes(column) + ) { + normalized[column] = normalizeLower(normalized[column]); + } + if (column === 'organization_code') normalized[column] = normalizeOrganizationCode(normalized[column]); + }); + + return normalized; +} + +function mapFileLevelIssues(file: CsvParsedFile): CsvPreviewRow[] { + const issues = [...file.errors, ...file.warnings]; + if (issues.length === 0) return []; + + const errors = issues.filter((item) => + ['INVALID_ENCODING', 'EMPTY_FILE', 'INCONSISTENT_COLUMN_COUNT'].includes(item.code) + ); + const warnings = issues.filter((item) => !errors.includes(item)); + const row: CsvPreviewRow = { + row: 1, + status: errors.length > 0 ? 'FAIL' : 'WARNING', + action: errors.length > 0 ? 'error' : 'skip', + key: null, + values: {}, + errors, + warnings, + }; + + return [row]; +} + +export async function validateCsvPreviewFile( + file: CsvParsedFile, + batch: CsvPreviewBatchIndex +): Promise { + const headerErrors = validateCsvHeaders(file); + if (file.errors.length > 0 || headerErrors.length > 0) { + const issueRows = mapFileLevelIssues(file); + if (headerErrors.length === 0) return issueRows; + return [ + ...issueRows, + { + row: 1, + status: 'FAIL', + action: 'error', + key: null, + values: {}, + errors: headerErrors, + warnings: [] + } + ]; + } + + const seenBusinessKeys = new Map(); + const previewRows: CsvPreviewRow[] = []; + + for (const row of file.rows) { + const values = normalizeRowValues(row.values); + const rowForValidation = { ...row, values }; + const valueValidation = validateCsvRowValues(file, rowForValidation); + const dependencyValidation = await resolveCsvRowDependencies(file, rowForValidation, batch); + const errors: CsvPreviewError[] = [...valueValidation.errors, ...dependencyValidation.errors]; + const warnings: CsvPreviewWarning[] = [...valueValidation.warnings, ...dependencyValidation.warnings]; + const keyValue = businessKey(file, values); + const duplicateFirstRow = seenBusinessKeys.get(keyValue); + + if (duplicateFirstRow) { + errors.push( + issue( + file.fileName, + row.rowNumber, + '__row__', + 'DUPLICATE_KEY', + `Business key duplicates row ${duplicateFirstRow}.`, + 'Keep only one row for each import business key.' + ) + ); + } else { + seenBusinessKeys.set(keyValue, row.rowNumber); + } + + const action = errors.length > 0 ? 'error' : dependencyValidation.exists ? 'update' : 'create'; + const previewRow: CsvPreviewRow = { + row: row.rowNumber, + status: 'PASS', + action, + key: keyValue, + values, + errors, + warnings + }; + previewRow.status = resolveRowStatus(previewRow); + previewRows.push(previewRow); + } + + return previewRows; +} diff --git a/src/features/setup/server/csv/value-validator.ts b/src/features/setup/server/csv/value-validator.ts new file mode 100644 index 0000000..2bac4dd --- /dev/null +++ b/src/features/setup/server/csv/value-validator.ts @@ -0,0 +1,391 @@ +import type { CsvParsedFile, CsvParsedRow, CsvPreviewError, CsvPreviewWarning } from './types'; + +type RowValues = Record; + +const EMAIL_COLUMNS = new Set([ + 'email', + 'created_by_email', + 'updated_by_email', + 'user_email', + 'owner_user_email', + 'owner_marketing_user_email', + 'assigned_sales_owner_email', + 'assigned_to_user_email', + 'salesman_email', + 'shared_to_user_email', + 'shared_by_user_email', + 'contact_email' +]); + +const BOOLEAN_COLUMNS = new Set([ + 'reset_password', + 'is_primary', + 'is_active', + 'is_system', + 'is_required', + 'is_default', + 'is_hot_project', + 'is_sent' +]); + +const INTEGER_COLUMNS = new Set([ + 'sort_order', + 'current_number', + 'padding_length', + 'step_number', + 'sla_hours', + 'priority', + 'item_number', + 'revision', + 'chance_percent' +]); + +const DECIMAL_COLUMNS = new Set([ + 'estimated_value', + 'min_amount', + 'max_amount', + 'subtotal', + 'total_amount', + 'exchange_rate', + 'discount', + 'tax_rate', + 'tax_amount', + 'quantity', + 'unit_price', + 'total_price' +]); + +const DATE_COLUMNS = new Set(['quotation_date', 'valid_until', 'expected_close_date']); + +const ENUMS: Record = { + system_role: ['super_admin', 'user'], + membership_role: ['admin', 'user'], + branch_scope_mode: ['all', 'selected', 'none', 'inherit'], + product_type_scope_mode: ['all', 'selected', 'none', 'inherit'], + entity_type: ['quotation'], + approval_mode: ['sequential', 'parallel', 'any'], + calendar_mode: ['calendar_days', 'business_days'], + timeout_action: ['none', 'auto_approve', 'auto_reject', 'escalate'], + reset_policy: ['period', 'never', 'manual'], + discount_type: ['percent', 'amount', ''], + outcome: ['open', 'won', 'lost', ''], + is_active: ['true', 'false', ''], + is_primary: ['true', 'false', ''], + is_system: ['true', 'false', ''], + is_required: ['true', 'false', ''], + is_default: ['true', 'false', ''], + reset_password: ['true', 'false', ''], + is_hot_project: ['true', 'false', ''], + is_sent: ['true', 'false', ''] +}; + +function buildIssue( + file: string, + row: number, + column: string, + code: string, + message: string, + suggestedFix: string +): CsvPreviewError { + return { file, row, column, code, message, suggestedFix }; +} + +function isValidEmail(value: string): boolean { + return /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(value); +} + +function isInteger(value: string): boolean { + return /^-?\d+$/.test(value); +} + +function isDecimal(value: string): boolean { + return /^-?\d+(\.\d+)?$/.test(value); +} + +function isDate(value: string): boolean { + if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) return false; + const date = new Date(`${value}T00:00:00.000Z`); + return !Number.isNaN(date.getTime()) && date.toISOString().startsWith(value); +} + +function isLowerSnake(value: string): boolean { + return /^[a-z0-9]+(?:_[a-z0-9]+)*$/.test(value); +} + +function validateCodeShape(file: CsvParsedFile, row: CsvParsedRow, errors: CsvPreviewError[]): void { + const values = row.values; + const lowerSnakeColumns = [ + 'slug', + 'branch_code', + 'product_type_code', + 'workflow_code', + 'topic_key', + 'category', + 'code', + 'crm_role_code', + 'role_code' + ]; + + if (values.organization_code && !/^[A-Z0-9_-]+$/.test(values.organization_code)) { + errors.push( + buildIssue( + file.fileName, + row.rowNumber, + 'organization_code', + 'INVALID_FORMAT', + 'Organization code must be uppercase letters, numbers, underscore, or dash.', + 'Use values like ALLA or ALLA_DEMO.' + ) + ); + } + + lowerSnakeColumns.forEach((column) => { + const value = values[column]; + if (!value || column === 'code') return; + if (!isLowerSnake(value)) { + errors.push( + buildIssue( + file.fileName, + row.rowNumber, + column, + 'INVALID_FORMAT', + `${column} should use lowercase snake_case.`, + 'Use lowercase letters, numbers, and underscores only.' + ) + ); + } + }); +} + +function validateJson(file: CsvParsedFile, row: CsvParsedRow, errors: CsvPreviewError[]): void { + const value = row.values.metadata_json; + if (!value) return; + try { + const parsed = JSON.parse(value) as unknown; + if (parsed === null || Array.isArray(parsed) || typeof parsed !== 'object') { + errors.push( + buildIssue( + file.fileName, + row.rowNumber, + 'metadata_json', + 'INVALID_JSON_OBJECT', + 'metadata_json must be a JSON object.', + 'Use an object such as {} or {"prefix":"HO"}.' + ) + ); + } + } catch { + errors.push( + buildIssue( + file.fileName, + row.rowNumber, + 'metadata_json', + 'INVALID_JSON', + 'metadata_json is not valid JSON.', + 'Fix JSON syntax or leave the column empty.' + ) + ); + } +} + +function validateNumericalRelationships( + file: CsvParsedFile, + row: CsvParsedRow, + errors: CsvPreviewError[], + warnings: CsvPreviewWarning[] +): void { + const minAmount = row.values.min_amount; + const maxAmount = row.values.max_amount; + if (minAmount && maxAmount && Number(minAmount) > Number(maxAmount)) { + errors.push( + buildIssue( + file.fileName, + row.rowNumber, + 'max_amount', + 'INVALID_AMOUNT_RANGE', + 'max_amount must be greater than or equal to min_amount.', + 'Adjust min_amount or max_amount.' + ) + ); + } + + const chancePercent = row.values.chance_percent; + if (chancePercent && (Number(chancePercent) < 0 || Number(chancePercent) > 100)) { + errors.push( + buildIssue( + file.fileName, + row.rowNumber, + 'chance_percent', + 'INVALID_PERCENT', + 'chance_percent must be between 0 and 100.', + 'Use a whole number between 0 and 100.' + ) + ); + } + + const quantity = row.values.quantity; + const unitPrice = row.values.unit_price; + const totalPrice = row.values.total_price; + if (quantity && unitPrice && totalPrice) { + const expected = Number(quantity) * Number(unitPrice); + const actual = Number(totalPrice); + if (Number.isFinite(expected) && Number.isFinite(actual) && Math.abs(expected - actual) > 0.01) { + warnings.push( + buildIssue( + file.fileName, + row.rowNumber, + 'total_price', + 'TOTAL_MISMATCH', + 'total_price does not match quantity multiplied by unit_price.', + 'Confirm the line total or adjust quantity/unit_price.' + ) + ); + } + } +} + +export function validateCsvRowValues(file: CsvParsedFile, row: CsvParsedRow): { + errors: CsvPreviewError[]; + warnings: CsvPreviewWarning[]; +} { + const errors: CsvPreviewError[] = []; + const warnings: CsvPreviewWarning[] = []; + + file.template.requiredColumns.forEach((column) => { + if (!row.values[column]) { + errors.push( + buildIssue( + file.fileName, + row.rowNumber, + column, + 'REQUIRED_FIELD', + `Column "${column}" is required.`, + 'Fill in the required value before previewing again.' + ) + ); + } + }); + + Object.entries(row.values).forEach(([column, value]) => { + if (!value) return; + + if (EMAIL_COLUMNS.has(column) && !isValidEmail(value)) { + errors.push( + buildIssue( + file.fileName, + row.rowNumber, + column, + 'INVALID_EMAIL', + `"${value}" is not a valid email address.`, + 'Use a valid email address such as admin@example.com.' + ) + ); + } + + if (BOOLEAN_COLUMNS.has(column) && !['true', 'false'].includes(value.toLowerCase())) { + errors.push( + buildIssue( + file.fileName, + row.rowNumber, + column, + 'INVALID_BOOLEAN', + `"${value}" is not a valid boolean.`, + 'Use true or false.' + ) + ); + } + + if (INTEGER_COLUMNS.has(column) && !isInteger(value)) { + errors.push( + buildIssue( + file.fileName, + row.rowNumber, + column, + 'INVALID_INTEGER', + `"${value}" is not a valid integer.`, + 'Use a whole number.' + ) + ); + } + + if (DECIMAL_COLUMNS.has(column) && !isDecimal(value)) { + errors.push( + buildIssue( + file.fileName, + row.rowNumber, + column, + 'INVALID_DECIMAL', + `"${value}" is not a valid decimal number.`, + 'Use a number such as 1000 or 1000.50.' + ) + ); + } + + if (DATE_COLUMNS.has(column) && !isDate(value)) { + errors.push( + buildIssue( + file.fileName, + row.rowNumber, + column, + 'INVALID_DATE', + `"${value}" is not a valid YYYY-MM-DD date.`, + 'Use Gregorian date format YYYY-MM-DD.' + ) + ); + } + + const allowedValues = ENUMS[column]; + if (allowedValues && !allowedValues.includes(value.toLowerCase())) { + errors.push( + buildIssue( + file.fileName, + row.rowNumber, + column, + 'UNKNOWN_OPTION', + `"${value}" is not an allowed value for ${column}.`, + `Use one of: ${allowedValues.filter(Boolean).join(', ')}.` + ) + ); + } + + if (value.length > 5000) { + errors.push( + buildIssue( + file.fileName, + row.rowNumber, + column, + 'VALUE_TOO_LONG', + `${column} exceeds the maximum preview length.`, + 'Shorten the value before importing.' + ) + ); + } + }); + + validateCodeShape(file, row, errors); + validateJson(file, row, errors); + validateNumericalRelationships(file, row, errors, warnings); + + return { errors, warnings }; +} + +export function normalizeEmail(value: string): string { + return value.trim().toLowerCase(); +} + +export function normalizeOrganizationCode(value: string): string { + return value.trim().toUpperCase(); +} + +export function normalizeLower(value: string): string { + return value.trim().toLowerCase(); +} + +export function pipeList(value: string): string[] { + return value + .split('|') + .map((part) => part.trim()) + .filter(Boolean); +} + +export type CsvRowValues = RowValues; diff --git a/src/lib/db.ts b/src/lib/db.ts index a10c54e..53c6bfc 100644 --- a/src/lib/db.ts +++ b/src/lib/db.ts @@ -1,9 +1,8 @@ -import { drizzle } from 'drizzle-orm/postgres-js'; -import postgres from 'postgres'; -import * as schema from '@/db/schema'; +import { drizzle } from "drizzle-orm/postgres-js"; +import postgres from "postgres"; +import * as schema from "@/db/schema"; -const connectionString = - process.env.DATABASE_URL ?? 'postgres://postgres:postgres@localhost:5432/training_system'; +const connectionString = process.env.DATABASE_URL ?? ""; const globalForDb = globalThis as typeof globalThis & { postgresClient?: ReturnType; @@ -13,12 +12,12 @@ const globalForDb = globalThis as typeof globalThis & { const client = globalForDb.postgresClient ?? postgres(connectionString, { - prepare: false + prepare: false, }); export const db = globalForDb.drizzleDb ?? drizzle(client, { schema }); -if (process.env.NODE_ENV !== 'production') { +if (process.env.NODE_ENV !== "production") { globalForDb.postgresClient = client; globalForDb.drizzleDb = db; }