diff --git a/docs/spec/revit-csv-phase-14-audit-and-plan.md b/docs/spec/revit-csv-phase-14-audit-and-plan.md index 8ce675b..5db8f6a 100644 --- a/docs/spec/revit-csv-phase-14-audit-and-plan.md +++ b/docs/spec/revit-csv-phase-14-audit-and-plan.md @@ -4,8 +4,8 @@ Phase 14.0 ist abgeschlossen und freigegeben. Dieses Dokument beschreibt den gegen den aktuellen Code geprüften Zielzuschnitt. Phase 14.1 befindet sich in -Umsetzung; ihre reinen Transportverträge und Konfigurationsvalidatoren sind -vorhanden. +Umsetzung; ihre reinen Transportverträge, Konfigurationsvalidatoren und die +zustandsfreie CSV-Transportgrundlage sind vorhanden. Fachliche Quelle bleibt [Revit-CSV-Integration – Anforderungen und Implementierungsanweisung](revit-csv-integration-requirements.md). @@ -241,11 +241,11 @@ keine Projektrevision und laufen über einen eigenen Audit-Store. ### 14.1 – CSV-Konfiguration und reine Vorschau 1. Transport-Domainverträge, Dialekt- und Mappingvalidatoren sowie kleine - synthetische Fixtures ergänzen. **Erledigt:** Verträge und Validatoren; - Parser-Fixtures folgen gemeinsam mit dem Parser. + synthetische Fixtures ergänzen. **Erledigt.** 2. Zustandsfreien Parser und Serializer mit Round-trip-Tests für UTF-8-BOM, CRLF, Semikolon, vollständige Quotierung, Kopfzeile in Zeile 2 sowie - Titel-, Leer-, Objekt- und Passthrough-Zeilen implementieren. + Titel-, Leer-, Objekt- und Passthrough-Zeilen implementieren. **Erledigt für + synthetische Testdaten; die Referenzdatei steht noch aus.** 3. Konfiguration über Migration, Snapshot v3, Transfer und den Command `external-csv-configuration.update` persistieren. 4. Vorschau-Endpunkt und deutschen Projekt-Wizard ergänzen. Der Server speichert diff --git a/src/external-model/csv/external-csv-transport.ts b/src/external-model/csv/external-csv-transport.ts new file mode 100644 index 0000000..d8131a0 --- /dev/null +++ b/src/external-model/csv/external-csv-transport.ts @@ -0,0 +1,350 @@ +import { assertExternalCsvConfiguration } from "./external-csv-configuration.js"; +import type { + ExternalCsvCell, + ExternalCsvConfiguration, + ExternalCsvDocument, + ExternalCsvLineEnding, + ExternalCsvRow, +} from "./external-csv-contracts.js"; + +export type ExternalCsvParseErrorCode = + | "invalid-encoding" + | "invalid-csv" + | "header-not-found" + | "ambiguous-header" + | "invalid-ifc-guid" + | "duplicate-ifc-guid"; + +export class ExternalCsvParseError extends Error { + constructor( + public readonly code: ExternalCsvParseErrorCode, + message: string + ) { + super(message); + this.name = "ExternalCsvParseError"; + } +} + +export function parseExternalCsv( + input: Uint8Array, + configuration: ExternalCsvConfiguration +): ExternalCsvDocument { + assertExternalCsvConfiguration(configuration); + const hasBom = hasUtf8Bom(input); + const contentBytes = hasBom ? input.subarray(3) : input; + let text: string; + try { + text = new TextDecoder("utf-8", { fatal: true }).decode(contentBytes); + } catch { + throw new ExternalCsvParseError( + "invalid-encoding", + "CSV is not valid UTF-8." + ); + } + + const parsed = parseRows(text, configuration.delimiter); + const headerMatches = findHeaderRows(parsed.rows, configuration); + if (headerMatches.length === 0) { + throw new ExternalCsvParseError( + "header-not-found", + "CSV does not contain exactly the configured required columns in one header row." + ); + } + if (headerMatches.length > 1) { + throw new ExternalCsvParseError( + "ambiguous-header", + "CSV contains more than one possible header row." + ); + } + + const headerRowIndex = headerMatches[0]; + const configuredColumnNames = [ + ...Object.values(configuration.columns), + ...configuration.additionalSourceMappings.map((mapping) => mapping.sourceColumn), + ]; + const headerValues = parsed.rows[headerRowIndex].map((cell) => cell.value); + if ( + configuredColumnNames.some( + (columnName) => headerValues.filter((value) => value === columnName).length !== 1 + ) + ) { + throw new ExternalCsvParseError( + "ambiguous-header", + "CSV header contains a configured column more than once." + ); + } + const headerLookup = createHeaderLookup(parsed.rows[headerRowIndex]); + const ifcGuidColumnIndex = headerLookup.get(configuration.columns.ifcGuid)!; + const objectBearingColumnIndexes = [ + configuration.columns.roomNumber, + configuration.columns.roomName, + configuration.columns.familyAndType, + configuration.columns.selectionMarker, + configuration.columns.power, + configuration.columns.quantity, + ...configuration.additionalSourceMappings.map((mapping) => mapping.sourceColumn), + ].map((column) => headerLookup.get(column)!); + const seenIfcGuids = new Set(); + + const rows: ExternalCsvRow[] = parsed.rows.map((cells, index) => { + if (index < headerRowIndex) { + return { index, cells, classification: "metadata" }; + } + if (index === headerRowIndex) { + return { index, cells, classification: "header" }; + } + + const rawIfcGuid = cells[ifcGuidColumnIndex]?.value ?? ""; + const ifcGuid = rawIfcGuid.trim(); + if (ifcGuid) { + if (ifcGuid !== rawIfcGuid || !isIfcGuid(ifcGuid)) { + throw new ExternalCsvParseError( + "invalid-ifc-guid", + `CSV row ${index + 1} contains an invalid IfcGUID.` + ); + } + if (seenIfcGuids.has(ifcGuid)) { + throw new ExternalCsvParseError( + "duplicate-ifc-guid", + `CSV contains the IfcGUID more than once: ${ifcGuid}` + ); + } + seenIfcGuids.add(ifcGuid); + return { index, cells, classification: "object" }; + } + + const looksLikeObject = objectBearingColumnIndexes.some( + (columnIndex) => (cells[columnIndex]?.value.trim() ?? "") !== "" + ); + return { + index, + cells, + classification: looksLikeObject ? "suspect-object" : "passthrough", + }; + }); + + return { + dialect: { + encoding: "utf-8", + hasBom, + delimiter: configuration.delimiter, + lineEnding: resolveLineEnding(parsed.lineEndings), + quoteCharacter: '"', + quoteAllFields: + rows.flatMap((row) => row.cells).length > 0 && + rows.flatMap((row) => row.cells).every((cell) => cell.wasQuoted), + hasTrailingLineEnding: parsed.hasTrailingLineEnding, + }, + headerRowIndex, + rows, + }; +} + +export function serializeExternalCsv(document: ExternalCsvDocument): Uint8Array { + assertSerializableDocument(document); + const serializedRows = document.rows.map((row) => + row.cells + .map((cell) => serializeCell(cell, document.dialect.delimiter)) + .join(document.dialect.delimiter) + ); + let text = serializedRows.join(document.dialect.lineEnding); + if (document.dialect.hasTrailingLineEnding && document.rows.length > 0) { + text += document.dialect.lineEnding; + } + const encoded = new TextEncoder().encode(text); + if (!document.dialect.hasBom) { + return encoded; + } + const result = new Uint8Array(encoded.length + 3); + result.set([0xef, 0xbb, 0xbf]); + result.set(encoded, 3); + return result; +} + +function parseRows(text: string, delimiter: string) { + const rows: ExternalCsvCell[][] = []; + const lineEndings: ExternalCsvLineEnding[] = []; + let cells: ExternalCsvCell[] = []; + let value = ""; + let wasQuoted = false; + let inQuotes = false; + let afterClosingQuote = false; + let fieldStarted = false; + let hasTrailingLineEnding = false; + + const finishCell = () => { + cells.push({ value, wasQuoted }); + value = ""; + wasQuoted = false; + inQuotes = false; + afterClosingQuote = false; + fieldStarted = false; + }; + const finishRow = (lineEnding: ExternalCsvLineEnding) => { + finishCell(); + rows.push(cells); + cells = []; + lineEndings.push(lineEnding); + hasTrailingLineEnding = true; + }; + + for (let index = 0; index < text.length; index += 1) { + const character = text[index]; + if (inQuotes) { + if (character === '"') { + if (text[index + 1] === '"') { + value += '"'; + index += 1; + } else { + inQuotes = false; + afterClosingQuote = true; + } + } else { + value += character; + } + hasTrailingLineEnding = false; + continue; + } + + if (afterClosingQuote) { + if (character !== delimiter && character !== "\r" && character !== "\n") { + throw new ExternalCsvParseError( + "invalid-csv", + "CSV contains characters after a closing quote." + ); + } + } + + if (character === delimiter) { + finishCell(); + hasTrailingLineEnding = false; + continue; + } + if (character === "\r" || character === "\n") { + const lineEnding: ExternalCsvLineEnding = + character === "\r" && text[index + 1] === "\n" + ? "\r\n" + : character; + if (lineEnding === "\r\n") { + index += 1; + } + finishRow(lineEnding); + continue; + } + if (character === '"') { + if (fieldStarted || value.length > 0 || afterClosingQuote) { + throw new ExternalCsvParseError( + "invalid-csv", + "CSV contains a quote inside an unquoted field." + ); + } + wasQuoted = true; + inQuotes = true; + fieldStarted = true; + hasTrailingLineEnding = false; + continue; + } + if (afterClosingQuote) { + throw new ExternalCsvParseError( + "invalid-csv", + "CSV contains invalid data after a quoted field." + ); + } + value += character; + fieldStarted = true; + hasTrailingLineEnding = false; + } + + if (inQuotes) { + throw new ExternalCsvParseError( + "invalid-csv", + "CSV contains an unterminated quoted field." + ); + } + if (!hasTrailingLineEnding || cells.length > 0 || fieldStarted || value.length > 0) { + finishCell(); + rows.push(cells); + } + + return { rows, lineEndings, hasTrailingLineEnding }; +} + +function findHeaderRows( + rows: ExternalCsvCell[][], + configuration: ExternalCsvConfiguration +) { + const requiredColumns = new Set([ + ...Object.values(configuration.columns), + ...configuration.additionalSourceMappings.map((mapping) => mapping.sourceColumn), + ]); + const matches: number[] = []; + rows.forEach((row, index) => { + const values = new Set(row.map((cell) => cell.value)); + if ([...requiredColumns].every((column) => values.has(column))) { + matches.push(index); + } + }); + return matches; +} + +function createHeaderLookup(cells: ExternalCsvCell[]) { + return new Map(cells.map((cell, index) => [cell.value, index])); +} + +function isIfcGuid(value: string) { + return /^[0-9A-Za-z_$]{22}$/.test(value); +} + +function resolveLineEnding(lineEndings: ExternalCsvLineEnding[]): ExternalCsvLineEnding { + if (lineEndings.length === 0) { + return "\r\n"; + } + const unique = new Set(lineEndings); + if (unique.size !== 1) { + throw new ExternalCsvParseError( + "invalid-csv", + "CSV contains mixed line endings that cannot be preserved reliably." + ); + } + return lineEndings[0]; +} + +function serializeCell(cell: ExternalCsvCell, delimiter: string) { + const mustQuote = + cell.wasQuoted || + cell.value.includes(delimiter) || + cell.value.includes('"') || + cell.value.includes("\r") || + cell.value.includes("\n"); + if (!mustQuote) { + return cell.value; + } + return `"${cell.value.replaceAll('"', '""')}"`; +} + +function assertSerializableDocument(document: ExternalCsvDocument) { + if (document.dialect.encoding !== "utf-8") { + throw new Error("Only UTF-8 external CSV documents can be serialized."); + } + if ( + document.headerRowIndex < 0 || + document.headerRowIndex >= document.rows.length || + document.rows[document.headerRowIndex]?.classification !== "header" + ) { + throw new Error("External CSV document has an invalid header row."); + } + document.rows.forEach((row, index) => { + if (row.index !== index) { + throw new Error("External CSV row indexes must be contiguous and stable."); + } + for (const cell of row.cells) { + if (typeof cell.value !== "string" || typeof cell.wasQuoted !== "boolean") { + throw new Error("External CSV document contains an invalid cell."); + } + } + }); +} + +function hasUtf8Bom(input: Uint8Array) { + return input.length >= 3 && input[0] === 0xef && input[1] === 0xbb && input[2] === 0xbf; +} diff --git a/tests/external-csv-transport.test.ts b/tests/external-csv-transport.test.ts new file mode 100644 index 0000000..8d69474 --- /dev/null +++ b/tests/external-csv-transport.test.ts @@ -0,0 +1,124 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { + ExternalCsvParseError, + parseExternalCsv, + serializeExternalCsv, +} from "../src/external-model/csv/external-csv-transport.js"; +import { + externalCsvTestConfiguration, + firstIfcGuid, + quotedRevitCsv, + utf8Bytes, +} from "./fixtures/revit-csv-fixtures.js"; + +describe("external CSV transport", () => { + it("detects the observed Revit dialect and classifies every row", () => { + const document = parseExternalCsv( + utf8Bytes(quotedRevitCsv, true), + externalCsvTestConfiguration + ); + + assert.deepEqual(document.dialect, { + encoding: "utf-8", + hasBom: true, + delimiter: ";", + lineEnding: "\r\n", + quoteCharacter: '"', + quoteAllFields: true, + hasTrailingLineEnding: true, + }); + assert.equal(document.headerRowIndex, 1); + assert.deepEqual( + document.rows.map((row) => row.classification), + [ + "metadata", + "header", + "passthrough", + "object", + "passthrough", + "suspect-object", + "object", + ] + ); + assert.equal(document.rows[6].cells[2].value, 'Büro "Nord"'); + }); + + it("round-trips BOM, CRLF, quotation, order and cell values byte for byte", () => { + const source = utf8Bytes(quotedRevitCsv, true); + const document = parseExternalCsv(source, externalCsvTestConfiguration); + + assert.deepEqual(serializeExternalCsv(document), source); + }); + + it("preserves embedded delimiters and line endings inside quoted cells", () => { + const csv = quotedRevitCsv.replace( + '"Steckdose: Doppelsteckdose"', + '"Steckdose; Doppel\r\nsteckdose"' + ); + const source = utf8Bytes(csv, true); + const document = parseExternalCsv(source, externalCsvTestConfiguration); + + assert.equal( + document.rows[3].cells[3].value, + "Steckdose; Doppel\r\nsteckdose" + ); + assert.deepEqual(serializeExternalCsv(document), source); + }); + + it("blocks missing and ambiguous configured headers", () => { + assertParseError( + utf8Bytes(quotedRevitCsv.replace("IfcGUID", "Andere ID")), + "header-not-found" + ); + const duplicatedHeader = quotedRevitCsv.replace( + '"";"";"";"";"";"";"";""', + quotedRevitCsv.split("\r\n")[1] + ); + assertParseError(utf8Bytes(duplicatedHeader), "ambiguous-header"); + + const duplicateConfiguredColumn = quotedRevitCsv.replace( + '"Stromkreis";"Raumnummer"', + '"Stromkreis";"Raumnummer";"Raumnummer"' + ); + assertParseError( + utf8Bytes(duplicateConfiguredColumn), + "ambiguous-header" + ); + }); + + it("blocks invalid and duplicate non-empty IfcGUIDs", () => { + assertParseError( + utf8Bytes(quotedRevitCsv.replace(firstIfcGuid, "not-an-ifc-guid")), + "invalid-ifc-guid" + ); + assertParseError( + utf8Bytes(quotedRevitCsv.replace(firstIfcGuid, ` ${firstIfcGuid}`)), + "invalid-ifc-guid" + ); + const duplicate = quotedRevitCsv.replace( + "1AbCdEfGhIjKlMnOpQrStu", + firstIfcGuid + ); + assertParseError(utf8Bytes(duplicate), "duplicate-ifc-guid"); + }); + + it("blocks invalid UTF-8, unterminated quotes and mixed row endings", () => { + assertParseError(new Uint8Array([0xff, 0xfe, 0xfd]), "invalid-encoding"); + assertParseError( + utf8Bytes(quotedRevitCsv.replace(/"\r\n$/, "\r\n")), + "invalid-csv" + ); + assertParseError( + utf8Bytes(quotedRevitCsv.replace("\r\n", "\n")), + "invalid-csv" + ); + }); +}); + +function assertParseError(input: Uint8Array, code: ExternalCsvParseError["code"]) { + assert.throws( + () => parseExternalCsv(input, externalCsvTestConfiguration), + (error: unknown) => error instanceof ExternalCsvParseError && error.code === code + ); +} diff --git a/tests/fixtures/revit-csv-fixtures.ts b/tests/fixtures/revit-csv-fixtures.ts new file mode 100644 index 0000000..3c0f975 --- /dev/null +++ b/tests/fixtures/revit-csv-fixtures.ts @@ -0,0 +1,42 @@ +import { + createDefaultExternalCsvConfiguration, + type ExternalCsvColumnMapping, +} from "../../src/external-model/csv/external-csv-contracts.js"; + +export const externalCsvTestColumns: ExternalCsvColumnMapping = { + ifcGuid: "IfcGUID", + roomNumber: "Raumnummer", + roomName: "Raumname", + familyAndType: "Familie und Typ", + selectionMarker: "CAx_Auswahlkenner", + circuitIdentifier: "Stromkreis", + power: "Elektrische Leistung", + quantity: "Anzahl", +}; + +export const externalCsvTestConfiguration = + createDefaultExternalCsvConfiguration(externalCsvTestColumns); + +export const firstIfcGuid = "0AbCdEfGhIjKlMnOpQrStu"; +export const secondIfcGuid = "1AbCdEfGhIjKlMnOpQrStu"; + +export const quotedRevitCsv = [ + '"Projekt-Export";"";"";"";"";"";"";""', + '"Stromkreis";"Raumnummer";"Raumname";"Familie und Typ";"CAx_Auswahlkenner";"Elektrische Leistung";"Anzahl";"IfcGUID"', + '"";"";"";"";"";"";"";""', + `"UV_AV_01-2F1";"01/101";"Technik";"Steckdose: Doppelsteckdose";"Arbeitsplatz";"120,5";"2";"${firstIfcGuid}"`, + '"UV_AV_01-2F1: 2";"";"";"";"";"";"";""', + '"";"01/102";"Büro";"Steckdose: Standard";"Arbeitsplatz";"100";"1";""', + `"";"01/103";"Büro \"\"Nord\"\"";"Steckdose: Standard";"Arbeitsplatz";"100";"1";"${secondIfcGuid}"`, +].join("\r\n") + "\r\n"; + +export function utf8Bytes(value: string, withBom = false) { + const encoded = new TextEncoder().encode(value); + if (!withBom) { + return encoded; + } + const result = new Uint8Array(encoded.length + 3); + result.set([0xef, 0xbb, 0xbf]); + result.set(encoded, 3); + return result; +}