Add Revit CSV transport

docs/spec/revit-csv-phase-14-audit-and-plan.md

@@ -4,8 +4,8 @@
 
 Phase 14.0 ist abgeschlossen und freigegeben. Dieses Dokument beschreibt den
 gegen den aktuellen Code geprüften Zielzuschnitt. Phase 14.1 befindet sich in
-Umsetzung; ihre reinen Transportverträge und Konfigurationsvalidatoren sind
+Umsetzung; ihre reinen Transportverträge, Konfigurationsvalidatoren und die
-vorhanden.
+zustandsfreie CSV-Transportgrundlage sind vorhanden.
 
 Fachliche Quelle bleibt
 [Revit-CSV-Integration – Anforderungen und Implementierungsanweisung](revit-csv-integration-requirements.md).
@@ -241,11 +241,11 @@ keine Projektrevision und laufen über einen eigenen Audit-Store.
 ### 14.1 – CSV-Konfiguration und reine Vorschau
 
 1. Transport-Domainverträge, Dialekt- und Mappingvalidatoren sowie kleine
-   synthetische Fixtures ergänzen. **Erledigt:** Verträge und Validatoren;
+   synthetische Fixtures ergänzen. **Erledigt.**
-   Parser-Fixtures folgen gemeinsam mit dem Parser.
 2. Zustandsfreien Parser und Serializer mit Round-trip-Tests für UTF-8-BOM,
    CRLF, Semikolon, vollständige Quotierung, Kopfzeile in Zeile 2 sowie
-   Titel-, Leer-, Objekt- und Passthrough-Zeilen implementieren.
+   Titel-, Leer-, Objekt- und Passthrough-Zeilen implementieren. **Erledigt für
+   synthetische Testdaten; die Referenzdatei steht noch aus.**
 3. Konfiguration über Migration, Snapshot v3, Transfer und den Command
    `external-csv-configuration.update` persistieren.
 4. Vorschau-Endpunkt und deutschen Projekt-Wizard ergänzen. Der Server speichert

src/external-model/csv/external-csv-transport.ts

@@ -0,0 +1,350 @@
+import { assertExternalCsvConfiguration } from "./external-csv-configuration.js";
+import type {
+  ExternalCsvCell,
+  ExternalCsvConfiguration,
+  ExternalCsvDocument,
+  ExternalCsvLineEnding,
+  ExternalCsvRow,
+} from "./external-csv-contracts.js";
+
+export type ExternalCsvParseErrorCode =
+  | "invalid-encoding"
+  | "invalid-csv"
+  | "header-not-found"
+  | "ambiguous-header"
+  | "invalid-ifc-guid"
+  | "duplicate-ifc-guid";
+
+export class ExternalCsvParseError extends Error {
+  constructor(
+    public readonly code: ExternalCsvParseErrorCode,
+    message: string
+  ) {
+    super(message);
+    this.name = "ExternalCsvParseError";
+  }
+}
+
+export function parseExternalCsv(
+  input: Uint8Array,
+  configuration: ExternalCsvConfiguration
+): ExternalCsvDocument {
+  assertExternalCsvConfiguration(configuration);
+  const hasBom = hasUtf8Bom(input);
+  const contentBytes = hasBom ? input.subarray(3) : input;
+  let text: string;
+  try {
+    text = new TextDecoder("utf-8", { fatal: true }).decode(contentBytes);
+  } catch {
+    throw new ExternalCsvParseError(
+      "invalid-encoding",
+      "CSV is not valid UTF-8."
+    );
+  }
+
+  const parsed = parseRows(text, configuration.delimiter);
+  const headerMatches = findHeaderRows(parsed.rows, configuration);
+  if (headerMatches.length === 0) {
+    throw new ExternalCsvParseError(
+      "header-not-found",
+      "CSV does not contain exactly the configured required columns in one header row."
+    );
+  }
+  if (headerMatches.length > 1) {
+    throw new ExternalCsvParseError(
+      "ambiguous-header",
+      "CSV contains more than one possible header row."
+    );
+  }
+
+  const headerRowIndex = headerMatches[0];
+  const configuredColumnNames = [
+    ...Object.values(configuration.columns),
+    ...configuration.additionalSourceMappings.map((mapping) => mapping.sourceColumn),
+  ];
+  const headerValues = parsed.rows[headerRowIndex].map((cell) => cell.value);
+  if (
+    configuredColumnNames.some(
+      (columnName) => headerValues.filter((value) => value === columnName).length !== 1
+    )
+  ) {
+    throw new ExternalCsvParseError(
+      "ambiguous-header",
+      "CSV header contains a configured column more than once."
+    );
+  }
+  const headerLookup = createHeaderLookup(parsed.rows[headerRowIndex]);
+  const ifcGuidColumnIndex = headerLookup.get(configuration.columns.ifcGuid)!;
+  const objectBearingColumnIndexes = [
+    configuration.columns.roomNumber,
+    configuration.columns.roomName,
+    configuration.columns.familyAndType,
+    configuration.columns.selectionMarker,
+    configuration.columns.power,
+    configuration.columns.quantity,
+    ...configuration.additionalSourceMappings.map((mapping) => mapping.sourceColumn),
+  ].map((column) => headerLookup.get(column)!);
+  const seenIfcGuids = new Set<string>();
+
+  const rows: ExternalCsvRow[] = parsed.rows.map((cells, index) => {
+    if (index < headerRowIndex) {
+      return { index, cells, classification: "metadata" };
+    }
+    if (index === headerRowIndex) {
+      return { index, cells, classification: "header" };
+    }
+
+    const rawIfcGuid = cells[ifcGuidColumnIndex]?.value ?? "";
+    const ifcGuid = rawIfcGuid.trim();
+    if (ifcGuid) {
+      if (ifcGuid !== rawIfcGuid || !isIfcGuid(ifcGuid)) {
+        throw new ExternalCsvParseError(
+          "invalid-ifc-guid",
+          `CSV row ${index + 1} contains an invalid IfcGUID.`
+        );
+      }
+      if (seenIfcGuids.has(ifcGuid)) {
+        throw new ExternalCsvParseError(
+          "duplicate-ifc-guid",
+          `CSV contains the IfcGUID more than once: ${ifcGuid}`
+        );
+      }
+      seenIfcGuids.add(ifcGuid);
+      return { index, cells, classification: "object" };
+    }
+
+    const looksLikeObject = objectBearingColumnIndexes.some(
+      (columnIndex) => (cells[columnIndex]?.value.trim() ?? "") !== ""
+    );
+    return {
+      index,
+      cells,
+      classification: looksLikeObject ? "suspect-object" : "passthrough",
+    };
+  });
+
+  return {
+    dialect: {
+      encoding: "utf-8",
+      hasBom,
+      delimiter: configuration.delimiter,
+      lineEnding: resolveLineEnding(parsed.lineEndings),
+      quoteCharacter: '"',
+      quoteAllFields:
+        rows.flatMap((row) => row.cells).length > 0 &&
+        rows.flatMap((row) => row.cells).every((cell) => cell.wasQuoted),
+      hasTrailingLineEnding: parsed.hasTrailingLineEnding,
+    },
+    headerRowIndex,
+    rows,
+  };
+}
+
+export function serializeExternalCsv(document: ExternalCsvDocument): Uint8Array {
+  assertSerializableDocument(document);
+  const serializedRows = document.rows.map((row) =>
+    row.cells
+      .map((cell) => serializeCell(cell, document.dialect.delimiter))
+      .join(document.dialect.delimiter)
+  );
+  let text = serializedRows.join(document.dialect.lineEnding);
+  if (document.dialect.hasTrailingLineEnding && document.rows.length > 0) {
+    text += document.dialect.lineEnding;
+  }
+  const encoded = new TextEncoder().encode(text);
+  if (!document.dialect.hasBom) {
+    return encoded;
+  }
+  const result = new Uint8Array(encoded.length + 3);
+  result.set([0xef, 0xbb, 0xbf]);
+  result.set(encoded, 3);
+  return result;
+}
+
+function parseRows(text: string, delimiter: string) {
+  const rows: ExternalCsvCell[][] = [];
+  const lineEndings: ExternalCsvLineEnding[] = [];
+  let cells: ExternalCsvCell[] = [];
+  let value = "";
+  let wasQuoted = false;
+  let inQuotes = false;
+  let afterClosingQuote = false;
+  let fieldStarted = false;
+  let hasTrailingLineEnding = false;
+
+  const finishCell = () => {
+    cells.push({ value, wasQuoted });
+    value = "";
+    wasQuoted = false;
+    inQuotes = false;
+    afterClosingQuote = false;
+    fieldStarted = false;
+  };
+  const finishRow = (lineEnding: ExternalCsvLineEnding) => {
+    finishCell();
+    rows.push(cells);
+    cells = [];
+    lineEndings.push(lineEnding);
+    hasTrailingLineEnding = true;
+  };
+
+  for (let index = 0; index < text.length; index += 1) {
+    const character = text[index];
+    if (inQuotes) {
+      if (character === '"') {
+        if (text[index + 1] === '"') {
+          value += '"';
+          index += 1;
+        } else {
+          inQuotes = false;
+          afterClosingQuote = true;
+        }
+      } else {
+        value += character;
+      }
+      hasTrailingLineEnding = false;
+      continue;
+    }
+
+    if (afterClosingQuote) {
+      if (character !== delimiter && character !== "\r" && character !== "\n") {
+        throw new ExternalCsvParseError(
+          "invalid-csv",
+          "CSV contains characters after a closing quote."
+        );
+      }
+    }
+
+    if (character === delimiter) {
+      finishCell();
+      hasTrailingLineEnding = false;
+      continue;
+    }
+    if (character === "\r" || character === "\n") {
+      const lineEnding: ExternalCsvLineEnding =
+        character === "\r" && text[index + 1] === "\n"
+          ? "\r\n"
+          : character;
+      if (lineEnding === "\r\n") {
+        index += 1;
+      }
+      finishRow(lineEnding);
+      continue;
+    }
+    if (character === '"') {
+      if (fieldStarted || value.length > 0 || afterClosingQuote) {
+        throw new ExternalCsvParseError(
+          "invalid-csv",
+          "CSV contains a quote inside an unquoted field."
+        );
+      }
+      wasQuoted = true;
+      inQuotes = true;
+      fieldStarted = true;
+      hasTrailingLineEnding = false;
+      continue;
+    }
+    if (afterClosingQuote) {
+      throw new ExternalCsvParseError(
+        "invalid-csv",
+        "CSV contains invalid data after a quoted field."
+      );
+    }
+    value += character;
+    fieldStarted = true;
+    hasTrailingLineEnding = false;
+  }
+
+  if (inQuotes) {
+    throw new ExternalCsvParseError(
+      "invalid-csv",
+      "CSV contains an unterminated quoted field."
+    );
+  }
+  if (!hasTrailingLineEnding || cells.length > 0 || fieldStarted || value.length > 0) {
+    finishCell();
+    rows.push(cells);
+  }
+
+  return { rows, lineEndings, hasTrailingLineEnding };
+}
+
+function findHeaderRows(
+  rows: ExternalCsvCell[][],
+  configuration: ExternalCsvConfiguration
+) {
+  const requiredColumns = new Set([
+    ...Object.values(configuration.columns),
+    ...configuration.additionalSourceMappings.map((mapping) => mapping.sourceColumn),
+  ]);
+  const matches: number[] = [];
+  rows.forEach((row, index) => {
+    const values = new Set(row.map((cell) => cell.value));
+    if ([...requiredColumns].every((column) => values.has(column))) {
+      matches.push(index);
+    }
+  });
+  return matches;
+}
+
+function createHeaderLookup(cells: ExternalCsvCell[]) {
+  return new Map(cells.map((cell, index) => [cell.value, index]));
+}
+
+function isIfcGuid(value: string) {
+  return /^[0-9A-Za-z_$]{22}$/.test(value);
+}
+
+function resolveLineEnding(lineEndings: ExternalCsvLineEnding[]): ExternalCsvLineEnding {
+  if (lineEndings.length === 0) {
+    return "\r\n";
+  }
+  const unique = new Set(lineEndings);
+  if (unique.size !== 1) {
+    throw new ExternalCsvParseError(
+      "invalid-csv",
+      "CSV contains mixed line endings that cannot be preserved reliably."
+    );
+  }
+  return lineEndings[0];
+}
+
+function serializeCell(cell: ExternalCsvCell, delimiter: string) {
+  const mustQuote =
+    cell.wasQuoted ||
+    cell.value.includes(delimiter) ||
+    cell.value.includes('"') ||
+    cell.value.includes("\r") ||
+    cell.value.includes("\n");
+  if (!mustQuote) {
+    return cell.value;
+  }
+  return `"${cell.value.replaceAll('"', '""')}"`;
+}
+
+function assertSerializableDocument(document: ExternalCsvDocument) {
+  if (document.dialect.encoding !== "utf-8") {
+    throw new Error("Only UTF-8 external CSV documents can be serialized.");
+  }
+  if (
+    document.headerRowIndex < 0 ||
+    document.headerRowIndex >= document.rows.length ||
+    document.rows[document.headerRowIndex]?.classification !== "header"
+  ) {
+    throw new Error("External CSV document has an invalid header row.");
+  }
+  document.rows.forEach((row, index) => {
+    if (row.index !== index) {
+      throw new Error("External CSV row indexes must be contiguous and stable.");
+    }
+    for (const cell of row.cells) {
+      if (typeof cell.value !== "string" || typeof cell.wasQuoted !== "boolean") {
+        throw new Error("External CSV document contains an invalid cell.");
+      }
+    }
+  });
+}
+
+function hasUtf8Bom(input: Uint8Array) {
+  return input.length >= 3 && input[0] === 0xef && input[1] === 0xbb && input[2] === 0xbf;
+}

tests/external-csv-transport.test.ts

@@ -0,0 +1,124 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import {
+  ExternalCsvParseError,
+  parseExternalCsv,
+  serializeExternalCsv,
+} from "../src/external-model/csv/external-csv-transport.js";
+import {
+  externalCsvTestConfiguration,
+  firstIfcGuid,
+  quotedRevitCsv,
+  utf8Bytes,
+} from "./fixtures/revit-csv-fixtures.js";
+
+describe("external CSV transport", () => {
+  it("detects the observed Revit dialect and classifies every row", () => {
+    const document = parseExternalCsv(
+      utf8Bytes(quotedRevitCsv, true),
+      externalCsvTestConfiguration
+    );
+
+    assert.deepEqual(document.dialect, {
+      encoding: "utf-8",
+      hasBom: true,
+      delimiter: ";",
+      lineEnding: "\r\n",
+      quoteCharacter: '"',
+      quoteAllFields: true,
+      hasTrailingLineEnding: true,
+    });
+    assert.equal(document.headerRowIndex, 1);
+    assert.deepEqual(
+      document.rows.map((row) => row.classification),
+      [
+        "metadata",
+        "header",
+        "passthrough",
+        "object",
+        "passthrough",
+        "suspect-object",
+        "object",
+      ]
+    );
+    assert.equal(document.rows[6].cells[2].value, 'Büro "Nord"');
+  });
+
+  it("round-trips BOM, CRLF, quotation, order and cell values byte for byte", () => {
+    const source = utf8Bytes(quotedRevitCsv, true);
+    const document = parseExternalCsv(source, externalCsvTestConfiguration);
+
+    assert.deepEqual(serializeExternalCsv(document), source);
+  });
+
+  it("preserves embedded delimiters and line endings inside quoted cells", () => {
+    const csv = quotedRevitCsv.replace(
+      '"Steckdose: Doppelsteckdose"',
+      '"Steckdose; Doppel\r\nsteckdose"'
+    );
+    const source = utf8Bytes(csv, true);
+    const document = parseExternalCsv(source, externalCsvTestConfiguration);
+
+    assert.equal(
+      document.rows[3].cells[3].value,
+      "Steckdose; Doppel\r\nsteckdose"
+    );
+    assert.deepEqual(serializeExternalCsv(document), source);
+  });
+
+  it("blocks missing and ambiguous configured headers", () => {
+    assertParseError(
+      utf8Bytes(quotedRevitCsv.replace("IfcGUID", "Andere ID")),
+      "header-not-found"
+    );
+    const duplicatedHeader = quotedRevitCsv.replace(
+      '"";"";"";"";"";"";"";""',
+      quotedRevitCsv.split("\r\n")[1]
+    );
+    assertParseError(utf8Bytes(duplicatedHeader), "ambiguous-header");
+
+    const duplicateConfiguredColumn = quotedRevitCsv.replace(
+      '"Stromkreis";"Raumnummer"',
+      '"Stromkreis";"Raumnummer";"Raumnummer"'
+    );
+    assertParseError(
+      utf8Bytes(duplicateConfiguredColumn),
+      "ambiguous-header"
+    );
+  });
+
+  it("blocks invalid and duplicate non-empty IfcGUIDs", () => {
+    assertParseError(
+      utf8Bytes(quotedRevitCsv.replace(firstIfcGuid, "not-an-ifc-guid")),
+      "invalid-ifc-guid"
+    );
+    assertParseError(
+      utf8Bytes(quotedRevitCsv.replace(firstIfcGuid, ` ${firstIfcGuid}`)),
+      "invalid-ifc-guid"
+    );
+    const duplicate = quotedRevitCsv.replace(
+      "1AbCdEfGhIjKlMnOpQrStu",
+      firstIfcGuid
+    );
+    assertParseError(utf8Bytes(duplicate), "duplicate-ifc-guid");
+  });
+
+  it("blocks invalid UTF-8, unterminated quotes and mixed row endings", () => {
+    assertParseError(new Uint8Array([0xff, 0xfe, 0xfd]), "invalid-encoding");
+    assertParseError(
+      utf8Bytes(quotedRevitCsv.replace(/"\r\n$/, "\r\n")),
+      "invalid-csv"
+    );
+    assertParseError(
+      utf8Bytes(quotedRevitCsv.replace("\r\n", "\n")),
+      "invalid-csv"
+    );
+  });
+});
+
+function assertParseError(input: Uint8Array, code: ExternalCsvParseError["code"]) {
+  assert.throws(
+    () => parseExternalCsv(input, externalCsvTestConfiguration),
+    (error: unknown) => error instanceof ExternalCsvParseError && error.code === code
+  );
+}

tests/fixtures/revit-csv-fixtures.ts

@@ -0,0 +1,42 @@
+import {
+  createDefaultExternalCsvConfiguration,
+  type ExternalCsvColumnMapping,
+} from "../../src/external-model/csv/external-csv-contracts.js";
+
+export const externalCsvTestColumns: ExternalCsvColumnMapping = {
+  ifcGuid: "IfcGUID",
+  roomNumber: "Raumnummer",
+  roomName: "Raumname",
+  familyAndType: "Familie und Typ",
+  selectionMarker: "CAx_Auswahlkenner",
+  circuitIdentifier: "Stromkreis",
+  power: "Elektrische Leistung",
+  quantity: "Anzahl",
+};
+
+export const externalCsvTestConfiguration =
+  createDefaultExternalCsvConfiguration(externalCsvTestColumns);
+
+export const firstIfcGuid = "0AbCdEfGhIjKlMnOpQrStu";
+export const secondIfcGuid = "1AbCdEfGhIjKlMnOpQrStu";
+
+export const quotedRevitCsv = [
+  '"Projekt-Export";"";"";"";"";"";"";""',
+  '"Stromkreis";"Raumnummer";"Raumname";"Familie und Typ";"CAx_Auswahlkenner";"Elektrische Leistung";"Anzahl";"IfcGUID"',
+  '"";"";"";"";"";"";"";""',
+  `"UV_AV_01-2F1";"01/101";"Technik";"Steckdose: Doppelsteckdose";"Arbeitsplatz";"120,5";"2";"${firstIfcGuid}"`,
+  '"UV_AV_01-2F1: 2";"";"";"";"";"";"";""',
+  '"";"01/102";"Büro";"Steckdose: Standard";"Arbeitsplatz";"100";"1";""',
+  `"";"01/103";"Büro \"\"Nord\"\"";"Steckdose: Standard";"Arbeitsplatz";"100";"1";"${secondIfcGuid}"`,
+].join("\r\n") + "\r\n";
+
+export function utf8Bytes(value: string, withBom = false) {
+  const encoded = new TextEncoder().encode(value);
+  if (!withBom) {
+    return encoded;
+  }
+  const result = new Uint8Array(encoded.length + 3);
+  result.set([0xef, 0xbb, 0xbf]);
+  result.set(encoded, 3);
+  return result;
+}