fix: export sql + import metadata lib (#902)

src/dialogs/create-diagram-dialog/create-diagram-dialog.tsx

@@ -3,7 +3,7 @@ import { Dialog, DialogContent } from '@/components/dialog/dialog';
 import { DatabaseType } from '@/lib/domain/database-type';
 import { useStorage } from '@/hooks/use-storage';
 import type { Diagram } from '@/lib/domain/diagram';
-import { loadFromDatabaseMetadata } from '@/lib/domain/diagram';
+import { loadFromDatabaseMetadata } from '@/lib/data/import-metadata/import';
 import { useNavigate } from 'react-router-dom';
 import { useConfig } from '@/hooks/use-config';
 import type { DatabaseMetadata } from '@/lib/data/import-metadata/metadata-types/database-metadata';

src/dialogs/export-sql-dialog/export-sql-dialog.tsx

@@ -17,7 +17,7 @@ import { useDialog } from '@/hooks/use-dialog';
 import {
     exportBaseSQL,
     exportSQL,
-} from '@/lib/data/export-metadata/export-sql-script';
+} from '@/lib/data/sql-export/export-sql-script';
 import { databaseTypeToLabelMap } from '@/lib/databases';
 import { DatabaseType } from '@/lib/domain/database-type';
 import { Annoyed, Sparkles } from 'lucide-react';

src/dialogs/import-database-dialog/import-database-dialog.tsx

@@ -7,7 +7,7 @@ import type { DatabaseEdition } from '@/lib/domain/database-edition';
 import type { DatabaseMetadata } from '@/lib/data/import-metadata/metadata-types/database-metadata';
 import { loadDatabaseMetadata } from '@/lib/data/import-metadata/metadata-types/database-metadata';
 import type { Diagram } from '@/lib/domain/diagram';
-import { loadFromDatabaseMetadata } from '@/lib/domain/diagram';
+import { loadFromDatabaseMetadata } from '@/lib/data/import-metadata/import';
 import { useChartDB } from '@/hooks/use-chartdb';
 import { useRedoUndoStack } from '@/hooks/use-redo-undo-stack';
 import { Trans, useTranslation } from 'react-i18next';

src/lib/data/import-metadata/import/custom-types.ts

@@ -0,0 +1,21 @@
+import type { DBCustomType, DBCustomTypeKind } from '@/lib/domain';
+import { schemaNameToDomainSchemaName } from '@/lib/domain';
+import type { DBCustomTypeInfo } from '../metadata-types/custom-type-info';
+import { generateId } from '@/lib/utils';
+
+export const createCustomTypesFromMetadata = ({
+    customTypes,
+}: {
+    customTypes: DBCustomTypeInfo[];
+}): DBCustomType[] => {
+    return customTypes.map((customType) => {
+        return {
+            id: generateId(),
+            schema: schemaNameToDomainSchemaName(customType.schema),
+            name: customType.type,
+            kind: customType.kind as DBCustomTypeKind,
+            values: customType.values,
+            fields: customType.fields,
+        };
+    });
+};

src/lib/data/import-metadata/import/dependencies.ts

@@ -0,0 +1,351 @@
+import { generateId } from '@/lib/utils';
+import type { AST } from 'node-sql-parser';
+import type { DBDependency, DBTable } from '@/lib/domain';
+import { DatabaseType, schemaNameToDomainSchemaName } from '@/lib/domain';
+import type { ViewInfo } from '../metadata-types/view-info';
+import { decodeViewDefinition } from './tables';
+
+const astDatabaseTypes: Record<DatabaseType, string> = {
+    [DatabaseType.POSTGRESQL]: 'postgresql',
+    [DatabaseType.MYSQL]: 'postgresql',
+    [DatabaseType.MARIADB]: 'postgresql',
+    [DatabaseType.GENERIC]: 'postgresql',
+    [DatabaseType.SQLITE]: 'postgresql',
+    [DatabaseType.SQL_SERVER]: 'postgresql',
+    [DatabaseType.CLICKHOUSE]: 'postgresql',
+    [DatabaseType.COCKROACHDB]: 'postgresql',
+    [DatabaseType.ORACLE]: 'postgresql',
+};
+
+export const createDependenciesFromMetadata = async ({
+    views,
+    tables,
+    databaseType,
+}: {
+    views: ViewInfo[];
+    tables: DBTable[];
+    databaseType: DatabaseType;
+}): Promise<DBDependency[]> => {
+    if (!views || views.length === 0) {
+        return [];
+    }
+
+    const { Parser } = await import('node-sql-parser');
+    const parser = new Parser();
+
+    const dependencies = views
+        .flatMap((view) => {
+            const viewSchema = schemaNameToDomainSchemaName(view.schema);
+            const viewTable = tables.find(
+                (table) =>
+                    table.name === view.view_name && viewSchema === table.schema
+            );
+
+            if (!viewTable) {
+                console.warn(
+                    `Source table for view ${view.view_name} not found (schema: ${viewSchema})`
+                );
+                return []; // Skip this view and proceed to the next
+            }
+
+            if (view.view_definition) {
+                try {
+                    const decodedViewDefinition = decodeViewDefinition(
+                        databaseType,
+                        view.view_definition
+                    );
+
+                    let modifiedViewDefinition = '';
+                    if (
+                        databaseType === DatabaseType.MYSQL ||
+                        databaseType === DatabaseType.MARIADB
+                    ) {
+                        modifiedViewDefinition = preprocessViewDefinitionMySQL(
+                            decodedViewDefinition
+                        );
+                    } else if (databaseType === DatabaseType.SQL_SERVER) {
+                        modifiedViewDefinition =
+                            preprocessViewDefinitionSQLServer(
+                                decodedViewDefinition
+                            );
+                    } else {
+                        modifiedViewDefinition = preprocessViewDefinition(
+                            decodedViewDefinition
+                        );
+                    }
+
+                    // Parse using the appropriate dialect
+                    const ast = parser.astify(modifiedViewDefinition, {
+                        database: astDatabaseTypes[databaseType],
+                        type: 'select', // Parsing a SELECT statement
+                    });
+
+                    let relatedTables = extractTablesFromAST(ast);
+
+                    // Filter out duplicate tables without schema
+                    relatedTables = filterDuplicateTables(relatedTables);
+
+                    return relatedTables.map((relTable) => {
+                        const relSchema = relTable.schema || view.schema; // Use view's schema if relSchema is undefined
+                        const relTableName = relTable.tableName;
+
+                        const table = tables.find(
+                            (table) =>
+                                table.name === relTableName &&
+                                (table.schema || '') === relSchema
+                        );
+
+                        if (table) {
+                            const dependency: DBDependency = {
+                                id: generateId(),
+                                schema: view.schema,
+                                tableId: table.id, // related table
+                                dependentSchema: table.schema,
+                                dependentTableId: viewTable.id, // dependent view
+                                createdAt: Date.now(),
+                            };
+
+                            return dependency;
+                        } else {
+                            console.warn(
+                                `Dependent table ${relSchema}.${relTableName} not found for view ${view.schema}.${view.view_name}`
+                            );
+                            return null;
+                        }
+                    });
+                } catch (error) {
+                    console.error(
+                        `Error parsing view ${view.schema}.${view.view_name}:`,
+                        error
+                    );
+                    return [];
+                }
+            } else {
+                console.warn(
+                    `View definition missing for ${view.schema}.${view.view_name}`
+                );
+                return [];
+            }
+        })
+        .filter((dependency) => dependency !== null);
+
+    return dependencies;
+};
+
+// Add this new function to filter out duplicate tables
+function filterDuplicateTables(
+    tables: { schema?: string; tableName: string }[]
+): { schema?: string; tableName: string }[] {
+    const tableMap = new Map<string, { schema?: string; tableName: string }>();
+
+    for (const table of tables) {
+        const key = table.tableName;
+        const existingTable = tableMap.get(key);
+
+        if (!existingTable || (table.schema && !existingTable.schema)) {
+            tableMap.set(key, table);
+        }
+    }
+
+    return Array.from(tableMap.values());
+}
+
+// Preprocess the view_definition to remove schema from CREATE VIEW
+function preprocessViewDefinition(viewDefinition: string): string {
+    if (!viewDefinition) {
+        return '';
+    }
+
+    // Remove leading and trailing whitespace
+    viewDefinition = viewDefinition.replace(/\s+/g, ' ').trim();
+
+    // Replace escaped double quotes with regular ones
+    viewDefinition = viewDefinition.replace(/\\"/g, '"');
+
+    // Replace 'CREATE MATERIALIZED VIEW' with 'CREATE VIEW'
+    viewDefinition = viewDefinition.replace(
+        /CREATE\s+MATERIALIZED\s+VIEW/i,
+        'CREATE VIEW'
+    );
+
+    // Regular expression to match 'CREATE VIEW [schema.]view_name [ (column definitions) ] AS'
+    // This regex captures the view name and skips any content between the view name and 'AS'
+    const regex =
+        /CREATE\s+VIEW\s+(?:(?:`[^`]+`|"[^"]+"|\w+)\.)?(?:`([^`]+)`|"([^"]+)"|(\w+))[\s\S]*?\bAS\b\s+/i;
+
+    const match = viewDefinition.match(regex);
+    let modifiedDefinition: string;
+
+    if (match) {
+        const viewName = match[1] || match[2] || match[3];
+        // Extract the SQL after the 'AS' keyword
+        const restOfDefinition = viewDefinition.substring(
+            match.index! + match[0].length
+        );
+
+        // Replace double-quoted identifiers with unquoted ones
+        let modifiedSQL = restOfDefinition.replace(/"(\w+)"/g, '$1');
+
+        // Replace '::' type casts with 'CAST' expressions
+        modifiedSQL = modifiedSQL.replace(
+            /\(([^()]+)\)::(\w+)/g,
+            'CAST($1 AS $2)'
+        );
+
+        // Remove ClickHouse-specific syntax that may still be present
+        // For example, remove SETTINGS clauses inside the SELECT statement
+        modifiedSQL = modifiedSQL.replace(/\bSETTINGS\b[\s\S]*$/i, '');
+
+        modifiedDefinition = `CREATE VIEW ${viewName} AS ${modifiedSQL}`;
+    } else {
+        console.warn('Could not preprocess view definition:', viewDefinition);
+        modifiedDefinition = viewDefinition;
+    }
+
+    return modifiedDefinition;
+}
+
+// Preprocess the view_definition for SQL Server
+function preprocessViewDefinitionSQLServer(viewDefinition: string): string {
+    if (!viewDefinition) {
+        return '';
+    }
+
+    // Remove BOM if present
+    viewDefinition = viewDefinition.replace(/^\uFEFF/, '');
+
+    // Normalize whitespace
+    viewDefinition = viewDefinition.replace(/\s+/g, ' ').trim();
+
+    // Remove square brackets and replace with double quotes
+    viewDefinition = viewDefinition.replace(/\[([^\]]+)\]/g, '"$1"');
+
+    // Remove database names from fully qualified identifiers
+    viewDefinition = viewDefinition.replace(
+        /"([a-zA-Z0-9_]+)"\."([a-zA-Z0-9_]+)"\."([a-zA-Z0-9_]+)"/g,
+        '"$2"."$3"'
+    );
+
+    // Replace SQL Server functions with PostgreSQL equivalents
+    viewDefinition = viewDefinition.replace(/\bGETDATE\(\)/gi, 'NOW()');
+    viewDefinition = viewDefinition.replace(/\bISNULL\(/gi, 'COALESCE(');
+
+    // Replace 'TOP N' with 'LIMIT N' at the end of the query
+    const topMatch = viewDefinition.match(/SELECT\s+TOP\s+(\d+)/i);
+    if (topMatch) {
+        const topN = topMatch[1];
+        viewDefinition = viewDefinition.replace(
+            /SELECT\s+TOP\s+\d+/i,
+            'SELECT'
+        );
+        viewDefinition = viewDefinition.replace(/;+\s*$/, ''); // Remove semicolons at the end
+        viewDefinition += ` LIMIT ${topN}`;
+    }
+
+    viewDefinition = viewDefinition.replace(/\n/g, ''); // Remove newlines
+
+    // Adjust CREATE VIEW syntax
+    const regex =
+        /CREATE\s+VIEW\s+(?:"?([^".\s]+)"?\.)?"?([^".\s]+)"?\s+AS\s+/i;
+    const match = viewDefinition.match(regex);
+    let modifiedDefinition: string;
+
+    if (match) {
+        const viewName = match[2];
+        const modifiedSQL = viewDefinition.substring(
+            match.index! + match[0].length
+        );
+
+        // Remove semicolons at the end
+        const finalSQL = modifiedSQL.replace(/;+\s*$/, '');
+
+        modifiedDefinition = `CREATE VIEW "${viewName}" AS ${finalSQL}`;
+    } else {
+        console.warn('Could not preprocess view definition:', viewDefinition);
+        modifiedDefinition = viewDefinition;
+    }
+
+    return modifiedDefinition;
+}
+
+// Preprocess the view_definition to remove schema from CREATE VIEW
+function preprocessViewDefinitionMySQL(viewDefinition: string): string {
+    if (!viewDefinition) {
+        return '';
+    }
+
+    // Remove any trailing semicolons
+    viewDefinition = viewDefinition.replace(/;\s*$/, '');
+
+    // Remove backticks from identifiers
+    viewDefinition = viewDefinition.replace(/`/g, '');
+
+    // Remove unnecessary parentheses around joins and ON clauses
+    viewDefinition = removeRedundantParentheses(viewDefinition);
+
+    return viewDefinition;
+}
+
+function removeRedundantParentheses(sql: string): string {
+    // Regular expressions to match unnecessary parentheses
+    const patterns = [
+        /\(\s*(JOIN\s+[^()]+?)\s*\)/gi,
+        /\(\s*(ON\s+[^()]+?)\s*\)/gi,
+        // Additional patterns if necessary
+    ];
+
+    let prevSql;
+    do {
+        prevSql = sql;
+        patterns.forEach((pattern) => {
+            sql = sql.replace(pattern, '$1');
+        });
+    } while (sql !== prevSql);
+
+    return sql;
+}
+
+function extractTablesFromAST(
+    ast: AST | AST[]
+): { schema?: string; tableName: string }[] {
+    const tablesMap = new Map<string, { schema: string; tableName: string }>();
+    const visitedNodes = new Set();
+
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    function traverse(node: any) {
+        if (!node || visitedNodes.has(node)) return;
+        visitedNodes.add(node);
+
+        if (Array.isArray(node)) {
+            node.forEach(traverse);
+        } else if (typeof node === 'object') {
+            // Check if node represents a table
+            if (
+                Object.hasOwnProperty.call(node, 'table') &&
+                typeof node.table === 'string'
+            ) {
+                let schema = node.db || node.schema;
+                const tableName = node.table;
+                if (tableName) {
+                    // Assign default schema if undefined
+                    schema = schemaNameToDomainSchemaName(schema) || '';
+                    const key = `${schema}.${tableName}`;
+                    if (!tablesMap.has(key)) {
+                        tablesMap.set(key, { schema, tableName });
+                    }
+                }
+            }
+
+            // Recursively traverse all properties
+            for (const key in node) {
+                if (Object.hasOwnProperty.call(node, key)) {
+                    traverse(node[key]);
+                }
+            }
+        }
+    }
+
+    traverse(ast);
+
+    return Array.from(tablesMap.values());
+}

src/lib/data/import-metadata/import/fields.ts

@@ -0,0 +1,64 @@
+import type { DBField } from '@/lib/domain';
+import type { ColumnInfo } from '../metadata-types/column-info';
+import type { AggregatedIndexInfo } from '../metadata-types/index-info';
+import type { PrimaryKeyInfo } from '../metadata-types/primary-key-info';
+import type { TableInfo } from '../metadata-types/table-info';
+import { generateId } from '@/lib/utils';
+
+export const createFieldsFromMetadata = ({
+    tableColumns,
+    tablePrimaryKeys,
+    aggregatedIndexes,
+}: {
+    tableColumns: ColumnInfo[];
+    tableSchema?: string;
+    tableInfo: TableInfo;
+    tablePrimaryKeys: PrimaryKeyInfo[];
+    aggregatedIndexes: AggregatedIndexInfo[];
+}) => {
+    const uniqueColumns = tableColumns.reduce((acc, col) => {
+        if (!acc.has(col.name)) {
+            acc.set(col.name, col);
+        }
+        return acc;
+    }, new Map<string, ColumnInfo>());
+
+    const sortedColumns = Array.from(uniqueColumns.values()).sort(
+        (a, b) => a.ordinal_position - b.ordinal_position
+    );
+
+    const tablePrimaryKeysColumns = tablePrimaryKeys.map((pk) =>
+        pk.column.trim()
+    );
+
+    return sortedColumns.map(
+        (col: ColumnInfo): DBField => ({
+            id: generateId(),
+            name: col.name,
+            type: {
+                id: col.type.split(' ').join('_').toLowerCase(),
+                name: col.type.toLowerCase(),
+            },
+            primaryKey: tablePrimaryKeysColumns.includes(col.name),
+            unique: Object.values(aggregatedIndexes).some(
+                (idx) =>
+                    idx.unique &&
+                    idx.columns.length === 1 &&
+                    idx.columns[0].name === col.name
+            ),
+            nullable: Boolean(col.nullable),
+            ...(col.character_maximum_length &&
+            col.character_maximum_length !== 'null'
+                ? { characterMaximumLength: col.character_maximum_length }
+                : {}),
+            ...(col.precision?.precision
+                ? { precision: col.precision.precision }
+                : {}),
+            ...(col.precision?.scale ? { scale: col.precision.scale } : {}),
+            ...(col.default ? { default: col.default } : {}),
+            ...(col.collation ? { collation: col.collation } : {}),
+            createdAt: Date.now(),
+            comments: col.comment ? col.comment : undefined,
+        })
+    );
+};

src/lib/data/import-metadata/import/index.ts

@@ -0,0 +1,82 @@
+import type { DatabaseEdition, Diagram } from '@/lib/domain';
+import { adjustTablePositions, DatabaseType } from '@/lib/domain';
+import { generateDiagramId } from '@/lib/utils';
+import type { DatabaseMetadata } from '../metadata-types/database-metadata';
+import { createCustomTypesFromMetadata } from './custom-types';
+import { createRelationshipsFromMetadata } from './relationships';
+import { createTablesFromMetadata } from './tables';
+import { createDependenciesFromMetadata } from './dependencies';
+
+export const loadFromDatabaseMetadata = async ({
+    databaseType,
+    databaseMetadata,
+    diagramNumber,
+    databaseEdition,
+}: {
+    databaseType: DatabaseType;
+    databaseMetadata: DatabaseMetadata;
+    diagramNumber?: number;
+    databaseEdition?: DatabaseEdition;
+}): Promise<Diagram> => {
+    const {
+        fk_info: foreignKeys,
+        views: views,
+        custom_types: customTypes,
+    } = databaseMetadata;
+
+    const tables = createTablesFromMetadata({
+        databaseMetadata,
+        databaseType,
+    });
+
+    const relationships = createRelationshipsFromMetadata({
+        foreignKeys,
+        tables,
+    });
+
+    const dependencies = await createDependenciesFromMetadata({
+        views,
+        tables,
+        databaseType,
+    });
+
+    const dbCustomTypes = customTypes
+        ? createCustomTypesFromMetadata({
+              customTypes,
+          })
+        : [];
+
+    const adjustedTables = adjustTablePositions({
+        tables,
+        relationships,
+        mode: 'perSchema',
+    });
+
+    const sortedTables = adjustedTables.sort((a, b) => {
+        if (a.isView === b.isView) {
+            // Both are either tables or views, so sort alphabetically by name
+            return a.name.localeCompare(b.name);
+        }
+        // If one is a view and the other is not, put tables first
+        return a.isView ? 1 : -1;
+    });
+
+    const diagram: Diagram = {
+        id: generateDiagramId(),
+        name: databaseMetadata.database_name
+            ? `${databaseMetadata.database_name}-db`
+            : diagramNumber
+              ? `Diagram ${diagramNumber}`
+              : 'New Diagram',
+        databaseType: databaseType ?? DatabaseType.GENERIC,
+        databaseEdition,
+        tables: sortedTables,
+        relationships,
+        dependencies,
+        customTypes: dbCustomTypes,
+        createdAt: new Date(),
+        updatedAt: new Date(),
+    };
+
+    return diagram;
+};

src/lib/data/import-metadata/import/indexes.ts

@@ -0,0 +1,24 @@
+import type { DBField, DBIndex, IndexType } from '@/lib/domain';
+import type { AggregatedIndexInfo } from '../metadata-types/index-info';
+import { generateId } from '@/lib/utils';
+
+export const createIndexesFromMetadata = ({
+    aggregatedIndexes,
+    fields,
+}: {
+    aggregatedIndexes: AggregatedIndexInfo[];
+    fields: DBField[];
+}): DBIndex[] =>
+    aggregatedIndexes.map(
+        (idx): DBIndex => ({
+            id: generateId(),
+            name: idx.name,
+            unique: Boolean(idx.unique),
+            fieldIds: idx.columns
+                .sort((a, b) => a.position - b.position)
+                .map((c) => fields.find((f) => f.name === c.name)?.id)
+                .filter((id): id is string => id !== undefined),
+            createdAt: Date.now(),
+            type: idx.index_type?.toLowerCase() as IndexType,
+        })
+    );

src/lib/data/import-metadata/import/relationships.ts

@@ -0,0 +1,85 @@
+import type {
+    Cardinality,
+    DBField,
+    DBRelationship,
+    DBTable,
+} from '@/lib/domain';
+import { schemaNameToDomainSchemaName } from '@/lib/domain';
+import type { ForeignKeyInfo } from '../metadata-types/foreign-key-info';
+import { generateId } from '@/lib/utils';
+
+const determineCardinality = (
+    field: DBField,
+    isTablePKComplex: boolean
+): Cardinality => {
+    return field.unique || (field.primaryKey && !isTablePKComplex)
+        ? 'one'
+        : 'many';
+};
+
+export const createRelationshipsFromMetadata = ({
+    foreignKeys,
+    tables,
+}: {
+    foreignKeys: ForeignKeyInfo[];
+    tables: DBTable[];
+}): DBRelationship[] => {
+    return foreignKeys
+        .map((fk: ForeignKeyInfo): DBRelationship | null => {
+            const schema = schemaNameToDomainSchemaName(fk.schema);
+            const sourceTable = tables.find(
+                (table) => table.name === fk.table && table.schema === schema
+            );
+
+            const targetSchema = schemaNameToDomainSchemaName(
+                fk.reference_schema
+            );
+
+            const targetTable = tables.find(
+                (table) =>
+                    table.name === fk.reference_table &&
+                    table.schema === targetSchema
+            );
+            const sourceField = sourceTable?.fields.find(
+                (field) => field.name === fk.column
+            );
+            const targetField = targetTable?.fields.find(
+                (field) => field.name === fk.reference_column
+            );
+
+            const isSourceTablePKComplex =
+                (sourceTable?.fields.filter((field) => field.primaryKey) ?? [])
+                    .length > 1;
+            const isTargetTablePKComplex =
+                (targetTable?.fields.filter((field) => field.primaryKey) ?? [])
+                    .length > 1;
+
+            if (sourceTable && targetTable && sourceField && targetField) {
+                const sourceCardinality = determineCardinality(
+                    sourceField,
+                    isSourceTablePKComplex
+                );
+                const targetCardinality = determineCardinality(
+                    targetField,
+                    isTargetTablePKComplex
+                );
+
+                return {
+                    id: generateId(),
+                    name: fk.foreign_key_name,
+                    sourceSchema: schema,
+                    targetSchema: targetSchema,
+                    sourceTableId: sourceTable.id,
+                    targetTableId: targetTable.id,
+                    sourceFieldId: sourceField.id,
+                    targetFieldId: targetField.id,
+                    sourceCardinality,
+                    targetCardinality,
+                    createdAt: Date.now(),
+                };
+            }
+
+            return null;
+        })
+        .filter((rel) => rel !== null) as DBRelationship[];
+};

src/lib/data/import-metadata/import/tables.ts

@@ -0,0 +1,228 @@
+import type { DBIndex, DBTable } from '@/lib/domain';
+import {
+    DatabaseType,
+    generateTableKey,
+    schemaNameToDomainSchemaName,
+} from '@/lib/domain';
+import type { DatabaseMetadata } from '../metadata-types/database-metadata';
+import type { TableInfo } from '../metadata-types/table-info';
+import { createAggregatedIndexes } from '../metadata-types/index-info';
+import {
+    decodeBase64ToUtf16LE,
+    decodeBase64ToUtf8,
+    generateId,
+} from '@/lib/utils';
+import {
+    defaultTableColor,
+    materializedViewColor,
+    viewColor,
+} from '@/lib/colors';
+import { createFieldsFromMetadata } from './fields';
+import { createIndexesFromMetadata } from './indexes';
+
+export const decodeViewDefinition = (
+    databaseType: DatabaseType,
+    viewDefinition?: string
+): string => {
+    if (!viewDefinition) {
+        return '';
+    }
+
+    let decodedViewDefinition: string;
+    if (databaseType === DatabaseType.SQL_SERVER) {
+        decodedViewDefinition = decodeBase64ToUtf16LE(viewDefinition);
+    } else {
+        decodedViewDefinition = decodeBase64ToUtf8(viewDefinition);
+    }
+
+    return decodedViewDefinition;
+};
+
+export const createTablesFromMetadata = ({
+    databaseMetadata,
+    databaseType,
+}: {
+    databaseMetadata: DatabaseMetadata;
+    databaseType: DatabaseType;
+}): DBTable[] => {
+    const {
+        tables: tableInfos,
+        pk_info: primaryKeys,
+        columns,
+        indexes,
+        views: views,
+    } = databaseMetadata;
+
+    // Pre-compute view names for faster lookup if there are views
+    const viewNamesSet = new Set<string>();
+    const materializedViewNamesSet = new Set<string>();
+
+    if (views && views.length > 0) {
+        views.forEach((view) => {
+            const key = generateTableKey({
+                schemaName: view.schema,
+                tableName: view.view_name,
+            });
+            viewNamesSet.add(key);
+
+            if (
+                view.view_definition &&
+                decodeViewDefinition(databaseType, view.view_definition)
+                    .toLowerCase()
+                    .includes('materialized')
+            ) {
+                materializedViewNamesSet.add(key);
+            }
+        });
+    }
+
+    // Pre-compute lookup maps for better performance
+    const columnsByTable = new Map<string, (typeof columns)[0][]>();
+    const indexesByTable = new Map<string, (typeof indexes)[0][]>();
+    const primaryKeysByTable = new Map<string, (typeof primaryKeys)[0][]>();
+
+    // Group columns by table
+    columns.forEach((col) => {
+        const key = generateTableKey({
+            schemaName: col.schema,
+            tableName: col.table,
+        });
+        if (!columnsByTable.has(key)) {
+            columnsByTable.set(key, []);
+        }
+        columnsByTable.get(key)!.push(col);
+    });
+
+    // Group indexes by table
+    indexes.forEach((idx) => {
+        const key = generateTableKey({
+            schemaName: idx.schema,
+            tableName: idx.table,
+        });
+        if (!indexesByTable.has(key)) {
+            indexesByTable.set(key, []);
+        }
+        indexesByTable.get(key)!.push(idx);
+    });
+
+    // Group primary keys by table
+    primaryKeys.forEach((pk) => {
+        const key = generateTableKey({
+            schemaName: pk.schema,
+            tableName: pk.table,
+        });
+        if (!primaryKeysByTable.has(key)) {
+            primaryKeysByTable.set(key, []);
+        }
+        primaryKeysByTable.get(key)!.push(pk);
+    });
+
+    const result = tableInfos.map((tableInfo: TableInfo) => {
+        const tableSchema = schemaNameToDomainSchemaName(tableInfo.schema);
+        const tableKey = generateTableKey({
+            schemaName: tableInfo.schema,
+            tableName: tableInfo.table,
+        });
+
+        // Use pre-computed lookups instead of filtering entire arrays
+        const tableIndexes = indexesByTable.get(tableKey) || [];
+        const tablePrimaryKeys = primaryKeysByTable.get(tableKey) || [];
+        const tableColumns = columnsByTable.get(tableKey) || [];
+
+        // Aggregate indexes with multiple columns
+        const aggregatedIndexes = createAggregatedIndexes({
+            tableInfo,
+            tableSchema,
+            tableIndexes,
+        });
+
+        const fields = createFieldsFromMetadata({
+            aggregatedIndexes,
+            tableColumns,
+            tablePrimaryKeys,
+            tableInfo,
+            tableSchema,
+        });
+
+        // Check for composite primary key and find matching index name
+        const primaryKeyFields = fields.filter((f) => f.primaryKey);
+        let pkMatchingIndexName: string | undefined;
+        let pkIndex: DBIndex | undefined;
+
+        if (primaryKeyFields.length >= 1) {
+            // We have a composite primary key, look for an index that matches all PK columns
+            const pkFieldNames = primaryKeyFields.map((f) => f.name).sort();
+
+            // Find an index that matches the primary key columns exactly
+            const matchingIndex = aggregatedIndexes.find((index) => {
+                const indexColumnNames = index.columns
+                    .map((c) => c.name)
+                    .sort();
+                return (
+                    indexColumnNames.length === pkFieldNames.length &&
+                    indexColumnNames.every((col, i) => col === pkFieldNames[i])
+                );
+            });
+
+            if (matchingIndex) {
+                pkMatchingIndexName = matchingIndex.name;
+                // Create a special PK index
+                pkIndex = {
+                    id: generateId(),
+                    name: matchingIndex.name,
+                    unique: true,
+                    fieldIds: primaryKeyFields.map((f) => f.id),
+                    createdAt: Date.now(),
+                    isPrimaryKey: true,
+                };
+            }
+        }
+
+        // Filter out the index that matches the composite PK (to avoid duplication)
+        const filteredAggregatedIndexes = pkMatchingIndexName
+            ? aggregatedIndexes.filter(
+                  (idx) => idx.name !== pkMatchingIndexName
+              )
+            : aggregatedIndexes;
+
+        const dbIndexes = createIndexesFromMetadata({
+            aggregatedIndexes: filteredAggregatedIndexes,
+            fields,
+        });
+
+        // Add the PK index if it exists
+        if (pkIndex) {
+            dbIndexes.push(pkIndex);
+        }
+
+        // Determine if the current table is a view by checking against pre-computed sets
+        const viewKey = generateTableKey({
+            schemaName: tableSchema,
+            tableName: tableInfo.table,
+        });
+        const isView = viewNamesSet.has(viewKey);
+        const isMaterializedView = materializedViewNamesSet.has(viewKey);
+
+        // Initial random positions; these will be adjusted later
+        return {
+            id: generateId(),
+            name: tableInfo.table,
+            schema: tableSchema,
+            x: Math.random() * 1000, // Placeholder X
+            y: Math.random() * 800, // Placeholder Y
+            fields,
+            indexes: dbIndexes,
+            color: isMaterializedView
+                ? materializedViewColor
+                : isView
+                  ? viewColor
+                  : defaultTableColor,
+            isView: isView,
+            isMaterializedView: isMaterializedView,
+            createdAt: Date.now(),
+            comments: tableInfo.comment ? tableInfo.comment : undefined,
+        };
+    });
+
+    return result;
+};

src/lib/dbml/dbml-export/dbml-export.ts

@@ -1,5 +1,5 @@
 import { importer } from '@dbml/core';
-import { exportBaseSQL } from '@/lib/data/export-metadata/export-sql-script';
+import { exportBaseSQL } from '@/lib/data/sql-export/export-sql-script';
 import type { Diagram } from '@/lib/domain/diagram';
 import { DatabaseType } from '@/lib/domain/database-type';
 import type { DBTable } from '@/lib/domain/db-table';

src/lib/dbml/dbml-import/__tests__/composite-pk-name.test.ts

@@ -1,8 +1,8 @@
 import { describe, it, expect } from 'vitest';
 import { importDBMLToDiagram } from '../dbml-import';
-import { exportPostgreSQL } from '@/lib/data/export-metadata/export-per-type/postgresql';
-import { exportMySQL } from '@/lib/data/export-metadata/export-per-type/mysql';
-import { exportMSSQL } from '@/lib/data/export-metadata/export-per-type/mssql';
+import { exportPostgreSQL } from '@/lib/data/sql-export/export-per-type/postgresql';
+import { exportMySQL } from '@/lib/data/sql-export/export-per-type/mysql';
+import { exportMSSQL } from '@/lib/data/sql-export/export-per-type/mssql';
 import { DatabaseType } from '@/lib/domain/database-type';
 
 describe('Composite Primary Key with Name', () => {

src/lib/domain/__tests__/composite-pk-metadata-import.test.ts

@@ -1,5 +1,5 @@
 import { describe, it, expect } from 'vitest';
-import { createTablesFromMetadata } from '../db-table';
+import { createTablesFromMetadata } from '@/lib/data/import-metadata/import/tables';
 import { DatabaseType } from '../database-type';
 import type { DatabaseMetadata } from '@/lib/data/import-metadata/metadata-types/database-metadata';
 

src/lib/domain/db-custom-type.ts

@@ -1,7 +1,4 @@
 import { z } from 'zod';
-import type { DBCustomTypeInfo } from '@/lib/data/import-metadata/metadata-types/custom-type-info';
-import { generateId } from '../utils';
-import { schemaNameToDomainSchemaName } from './db-schema';
 
 export enum DBCustomTypeKind {
     enum = 'enum',
@@ -38,23 +35,6 @@ export const dbCustomTypeSchema: z.ZodType<DBCustomType> = z.object({
     order: z.number().or(z.null()).optional(),
 });
 
-export const createCustomTypesFromMetadata = ({
-    customTypes,
-}: {
-    customTypes: DBCustomTypeInfo[];
-}): DBCustomType[] => {
-    return customTypes.map((customType) => {
-        return {
-            id: generateId(),
-            schema: schemaNameToDomainSchemaName(customType.schema),
-            name: customType.type,
-            kind: customType.kind as DBCustomTypeKind,
-            values: customType.values,
-            fields: customType.fields,
-        };
-    });
-};
-
 export const customTypeKindToLabel: Record<DBCustomTypeKind, string> = {
     enum: 'Enum',
     composite: 'Composite',

src/lib/domain/db-dependency.ts

@@ -1,10 +1,4 @@
 import { z } from 'zod';
-import type { ViewInfo } from '../data/import-metadata/metadata-types/view-info';
-import { DatabaseType } from './database-type';
-import { schemaNameToDomainSchemaName } from './db-schema';
-import { decodeViewDefinition, type DBTable } from './db-table';
-import { generateId } from '@/lib/utils';
-import type { AST } from 'node-sql-parser';
 
 export interface DBDependency {
     id: string;
@@ -23,348 +17,3 @@ export const dbDependencySchema: z.ZodType<DBDependency> = z.object({
     dependentTableId: z.string(),
     createdAt: z.number(),
 });
-
-const astDatabaseTypes: Record<DatabaseType, string> = {
-    [DatabaseType.POSTGRESQL]: 'postgresql',
-    [DatabaseType.MYSQL]: 'postgresql',
-    [DatabaseType.MARIADB]: 'postgresql',
-    [DatabaseType.GENERIC]: 'postgresql',
-    [DatabaseType.SQLITE]: 'postgresql',
-    [DatabaseType.SQL_SERVER]: 'postgresql',
-    [DatabaseType.CLICKHOUSE]: 'postgresql',
-    [DatabaseType.COCKROACHDB]: 'postgresql',
-    [DatabaseType.ORACLE]: 'postgresql',
-};
-
-export const createDependenciesFromMetadata = async ({
-    views,
-    tables,
-    databaseType,
-}: {
-    views: ViewInfo[];
-    tables: DBTable[];
-    databaseType: DatabaseType;
-}): Promise<DBDependency[]> => {
-    if (!views || views.length === 0) {
-        return [];
-    }
-
-    const { Parser } = await import('node-sql-parser');
-    const parser = new Parser();
-
-    const dependencies = views
-        .flatMap((view) => {
-            const viewSchema = schemaNameToDomainSchemaName(view.schema);
-            const viewTable = tables.find(
-                (table) =>
-                    table.name === view.view_name && viewSchema === table.schema
-            );
-
-            if (!viewTable) {
-                console.warn(
-                    `Source table for view ${view.view_name} not found (schema: ${viewSchema})`
-                );
-                return []; // Skip this view and proceed to the next
-            }
-
-            if (view.view_definition) {
-                try {
-                    const decodedViewDefinition = decodeViewDefinition(
-                        databaseType,
-                        view.view_definition
-                    );
-
-                    let modifiedViewDefinition = '';
-                    if (
-                        databaseType === DatabaseType.MYSQL ||
-                        databaseType === DatabaseType.MARIADB
-                    ) {
-                        modifiedViewDefinition = preprocessViewDefinitionMySQL(
-                            decodedViewDefinition
-                        );
-                    } else if (databaseType === DatabaseType.SQL_SERVER) {
-                        modifiedViewDefinition =
-                            preprocessViewDefinitionSQLServer(
-                                decodedViewDefinition
-                            );
-                    } else {
-                        modifiedViewDefinition = preprocessViewDefinition(
-                            decodedViewDefinition
-                        );
-                    }
-
-                    // Parse using the appropriate dialect
-                    const ast = parser.astify(modifiedViewDefinition, {
-                        database: astDatabaseTypes[databaseType],
-                        type: 'select', // Parsing a SELECT statement
-                    });
-
-                    let relatedTables = extractTablesFromAST(ast);
-
-                    // Filter out duplicate tables without schema
-                    relatedTables = filterDuplicateTables(relatedTables);
-
-                    return relatedTables.map((relTable) => {
-                        const relSchema = relTable.schema || view.schema; // Use view's schema if relSchema is undefined
-                        const relTableName = relTable.tableName;
-
-                        const table = tables.find(
-                            (table) =>
-                                table.name === relTableName &&
-                                (table.schema || '') === relSchema
-                        );
-
-                        if (table) {
-                            const dependency: DBDependency = {
-                                id: generateId(),
-                                schema: view.schema,
-                                tableId: table.id, // related table
-                                dependentSchema: table.schema,
-                                dependentTableId: viewTable.id, // dependent view
-                                createdAt: Date.now(),
-                            };
-
-                            return dependency;
-                        } else {
-                            console.warn(
-                                `Dependent table ${relSchema}.${relTableName} not found for view ${view.schema}.${view.view_name}`
-                            );
-                            return null;
-                        }
-                    });
-                } catch (error) {
-                    console.error(
-                        `Error parsing view ${view.schema}.${view.view_name}:`,
-                        error
-                    );
-                    return [];
-                }
-            } else {
-                console.warn(
-                    `View definition missing for ${view.schema}.${view.view_name}`
-                );
-                return [];
-            }
-        })
-        .filter((dependency) => dependency !== null);
-
-    return dependencies;
-};
-
-// Add this new function to filter out duplicate tables
-function filterDuplicateTables(
-    tables: { schema?: string; tableName: string }[]
-): { schema?: string; tableName: string }[] {
-    const tableMap = new Map<string, { schema?: string; tableName: string }>();
-
-    for (const table of tables) {
-        const key = table.tableName;
-        const existingTable = tableMap.get(key);
-
-        if (!existingTable || (table.schema && !existingTable.schema)) {
-            tableMap.set(key, table);
-        }
-    }
-
-    return Array.from(tableMap.values());
-}
-
-// Preprocess the view_definition to remove schema from CREATE VIEW
-function preprocessViewDefinition(viewDefinition: string): string {
-    if (!viewDefinition) {
-        return '';
-    }
-
-    // Remove leading and trailing whitespace
-    viewDefinition = viewDefinition.replace(/\s+/g, ' ').trim();
-
-    // Replace escaped double quotes with regular ones
-    viewDefinition = viewDefinition.replace(/\\"/g, '"');
-
-    // Replace 'CREATE MATERIALIZED VIEW' with 'CREATE VIEW'
-    viewDefinition = viewDefinition.replace(
-        /CREATE\s+MATERIALIZED\s+VIEW/i,
-        'CREATE VIEW'
-    );
-
-    // Regular expression to match 'CREATE VIEW [schema.]view_name [ (column definitions) ] AS'
-    // This regex captures the view name and skips any content between the view name and 'AS'
-    const regex =
-        /CREATE\s+VIEW\s+(?:(?:`[^`]+`|"[^"]+"|\w+)\.)?(?:`([^`]+)`|"([^"]+)"|(\w+))[\s\S]*?\bAS\b\s+/i;
-
-    const match = viewDefinition.match(regex);
-    let modifiedDefinition: string;
-
-    if (match) {
-        const viewName = match[1] || match[2] || match[3];
-        // Extract the SQL after the 'AS' keyword
-        const restOfDefinition = viewDefinition.substring(
-            match.index! + match[0].length
-        );
-
-        // Replace double-quoted identifiers with unquoted ones
-        let modifiedSQL = restOfDefinition.replace(/"(\w+)"/g, '$1');
-
-        // Replace '::' type casts with 'CAST' expressions
-        modifiedSQL = modifiedSQL.replace(
-            /\(([^()]+)\)::(\w+)/g,
-            'CAST($1 AS $2)'
-        );
-
-        // Remove ClickHouse-specific syntax that may still be present
-        // For example, remove SETTINGS clauses inside the SELECT statement
-        modifiedSQL = modifiedSQL.replace(/\bSETTINGS\b[\s\S]*$/i, '');
-
-        modifiedDefinition = `CREATE VIEW ${viewName} AS ${modifiedSQL}`;
-    } else {
-        console.warn('Could not preprocess view definition:', viewDefinition);
-        modifiedDefinition = viewDefinition;
-    }
-
-    return modifiedDefinition;
-}
-
-// Preprocess the view_definition for SQL Server
-function preprocessViewDefinitionSQLServer(viewDefinition: string): string {
-    if (!viewDefinition) {
-        return '';
-    }
-
-    // Remove BOM if present
-    viewDefinition = viewDefinition.replace(/^\uFEFF/, '');
-
-    // Normalize whitespace
-    viewDefinition = viewDefinition.replace(/\s+/g, ' ').trim();
-
-    // Remove square brackets and replace with double quotes
-    viewDefinition = viewDefinition.replace(/\[([^\]]+)\]/g, '"$1"');
-
-    // Remove database names from fully qualified identifiers
-    viewDefinition = viewDefinition.replace(
-        /"([a-zA-Z0-9_]+)"\."([a-zA-Z0-9_]+)"\."([a-zA-Z0-9_]+)"/g,
-        '"$2"."$3"'
-    );
-
-    // Replace SQL Server functions with PostgreSQL equivalents
-    viewDefinition = viewDefinition.replace(/\bGETDATE\(\)/gi, 'NOW()');
-    viewDefinition = viewDefinition.replace(/\bISNULL\(/gi, 'COALESCE(');
-
-    // Replace 'TOP N' with 'LIMIT N' at the end of the query
-    const topMatch = viewDefinition.match(/SELECT\s+TOP\s+(\d+)/i);
-    if (topMatch) {
-        const topN = topMatch[1];
-        viewDefinition = viewDefinition.replace(
-            /SELECT\s+TOP\s+\d+/i,
-            'SELECT'
-        );
-        viewDefinition = viewDefinition.replace(/;+\s*$/, ''); // Remove semicolons at the end
-        viewDefinition += ` LIMIT ${topN}`;
-    }
-
-    viewDefinition = viewDefinition.replace(/\n/g, ''); // Remove newlines
-
-    // Adjust CREATE VIEW syntax
-    const regex =
-        /CREATE\s+VIEW\s+(?:"?([^".\s]+)"?\.)?"?([^".\s]+)"?\s+AS\s+/i;
-    const match = viewDefinition.match(regex);
-    let modifiedDefinition: string;
-
-    if (match) {
-        const viewName = match[2];
-        const modifiedSQL = viewDefinition.substring(
-            match.index! + match[0].length
-        );
-
-        // Remove semicolons at the end
-        const finalSQL = modifiedSQL.replace(/;+\s*$/, '');
-
-        modifiedDefinition = `CREATE VIEW "${viewName}" AS ${finalSQL}`;
-    } else {
-        console.warn('Could not preprocess view definition:', viewDefinition);
-        modifiedDefinition = viewDefinition;
-    }
-
-    return modifiedDefinition;
-}
-
-// Preprocess the view_definition to remove schema from CREATE VIEW
-function preprocessViewDefinitionMySQL(viewDefinition: string): string {
-    if (!viewDefinition) {
-        return '';
-    }
-
-    // Remove any trailing semicolons
-    viewDefinition = viewDefinition.replace(/;\s*$/, '');
-
-    // Remove backticks from identifiers
-    viewDefinition = viewDefinition.replace(/`/g, '');
-
-    // Remove unnecessary parentheses around joins and ON clauses
-    viewDefinition = removeRedundantParentheses(viewDefinition);
-
-    return viewDefinition;
-}
-
-function removeRedundantParentheses(sql: string): string {
-    // Regular expressions to match unnecessary parentheses
-    const patterns = [
-        /\(\s*(JOIN\s+[^()]+?)\s*\)/gi,
-        /\(\s*(ON\s+[^()]+?)\s*\)/gi,
-        // Additional patterns if necessary
-    ];
-
-    let prevSql;
-    do {
-        prevSql = sql;
-        patterns.forEach((pattern) => {
-            sql = sql.replace(pattern, '$1');
-        });
-    } while (sql !== prevSql);
-
-    return sql;
-}
-
-function extractTablesFromAST(
-    ast: AST | AST[]
-): { schema?: string; tableName: string }[] {
-    const tablesMap = new Map<string, { schema: string; tableName: string }>();
-    const visitedNodes = new Set();
-
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    function traverse(node: any) {
-        if (!node || visitedNodes.has(node)) return;
-        visitedNodes.add(node);
-
-        if (Array.isArray(node)) {
-            node.forEach(traverse);
-        } else if (typeof node === 'object') {
-            // Check if node represents a table
-            if (
-                Object.hasOwnProperty.call(node, 'table') &&
-                typeof node.table === 'string'
-            ) {
-                let schema = node.db || node.schema;
-                const tableName = node.table;
-                if (tableName) {
-                    // Assign default schema if undefined
-                    schema = schemaNameToDomainSchemaName(schema) || '';
-                    const key = `${schema}.${tableName}`;
-                    if (!tablesMap.has(key)) {
-                        tablesMap.set(key, { schema, tableName });
-                    }
-                }
-            }
-
-            // Recursively traverse all properties
-            for (const key in node) {
-                if (Object.hasOwnProperty.call(node, key)) {
-                    traverse(node[key]);
-                }
-            }
-        }
-    }
-
-    traverse(ast);
-
-    return Array.from(tablesMap.values());
-}

src/lib/domain/db-field.ts

@@ -4,11 +4,6 @@ import {
     findDataTypeDataById,
     type DataType,
 } from '../data/data-types/data-types';
-import type { ColumnInfo } from '../data/import-metadata/metadata-types/column-info';
-import type { AggregatedIndexInfo } from '../data/import-metadata/metadata-types/index-info';
-import type { PrimaryKeyInfo } from '../data/import-metadata/metadata-types/primary-key-info';
-import type { TableInfo } from '../data/import-metadata/metadata-types/table-info';
-import { generateId } from '../utils';
 import type { DatabaseType } from './database-type';
 
 export interface DBField {
@@ -45,64 +40,6 @@ export const dbFieldSchema: z.ZodType<DBField> = z.object({
     comments: z.string().or(z.null()).optional(),
 });
 
-export const createFieldsFromMetadata = ({
-    tableColumns,
-    tablePrimaryKeys,
-    aggregatedIndexes,
-}: {
-    tableColumns: ColumnInfo[];
-    tableSchema?: string;
-    tableInfo: TableInfo;
-    tablePrimaryKeys: PrimaryKeyInfo[];
-    aggregatedIndexes: AggregatedIndexInfo[];
-}) => {
-    const uniqueColumns = tableColumns.reduce((acc, col) => {
-        if (!acc.has(col.name)) {
-            acc.set(col.name, col);
-        }
-        return acc;
-    }, new Map<string, ColumnInfo>());
-
-    const sortedColumns = Array.from(uniqueColumns.values()).sort(
-        (a, b) => a.ordinal_position - b.ordinal_position
-    );
-
-    const tablePrimaryKeysColumns = tablePrimaryKeys.map((pk) =>
-        pk.column.trim()
-    );
-
-    return sortedColumns.map(
-        (col: ColumnInfo): DBField => ({
-            id: generateId(),
-            name: col.name,
-            type: {
-                id: col.type.split(' ').join('_').toLowerCase(),
-                name: col.type.toLowerCase(),
-            },
-            primaryKey: tablePrimaryKeysColumns.includes(col.name),
-            unique: Object.values(aggregatedIndexes).some(
-                (idx) =>
-                    idx.unique &&
-                    idx.columns.length === 1 &&
-                    idx.columns[0].name === col.name
-            ),
-            nullable: Boolean(col.nullable),
-            ...(col.character_maximum_length &&
-            col.character_maximum_length !== 'null'
-                ? { characterMaximumLength: col.character_maximum_length }
-                : {}),
-            ...(col.precision?.precision
-                ? { precision: col.precision.precision }
-                : {}),
-            ...(col.precision?.scale ? { scale: col.precision.scale } : {}),
-            ...(col.default ? { default: col.default } : {}),
-            ...(col.collation ? { collation: col.collation } : {}),
-            createdAt: Date.now(),
-            comments: col.comment ? col.comment : undefined,
-        })
-    );
-};
-
 export const generateDBFieldSuffix = (
     field: DBField,
     {

src/lib/domain/db-index.ts

@@ -1,7 +1,5 @@
 import { z } from 'zod';
-import type { AggregatedIndexInfo } from '../data/import-metadata/metadata-types/index-info';
 import { generateId } from '../utils';
-import type { DBField } from './db-field';
 import { DatabaseType } from './database-type';
 import type { DBTable } from './db-table';
 
@@ -43,27 +41,6 @@ export const dbIndexSchema: z.ZodType<DBIndex> = z.object({
     isPrimaryKey: z.boolean().or(z.null()).optional(),
 });
 
-export const createIndexesFromMetadata = ({
-    aggregatedIndexes,
-    fields,
-}: {
-    aggregatedIndexes: AggregatedIndexInfo[];
-    fields: DBField[];
-}): DBIndex[] =>
-    aggregatedIndexes.map(
-        (idx): DBIndex => ({
-            id: generateId(),
-            name: idx.name,
-            unique: Boolean(idx.unique),
-            fieldIds: idx.columns
-                .sort((a, b) => a.position - b.position)
-                .map((c) => fields.find((f) => f.name === c.name)?.id)
-                .filter((id): id is string => id !== undefined),
-            createdAt: Date.now(),
-            type: idx.index_type?.toLowerCase() as IndexType,
-        })
-    );
-
 export const databaseIndexTypes: { [key in DatabaseType]?: IndexType[] } = {
     [DatabaseType.POSTGRESQL]: ['btree', 'hash'],
 };

src/lib/domain/db-relationship.ts

@@ -1,9 +1,4 @@
 import { z } from 'zod';
-import type { ForeignKeyInfo } from '../data/import-metadata/metadata-types/foreign-key-info';
-import type { DBField } from './db-field';
-import { schemaNameToDomainSchemaName } from './db-schema';
-import type { DBTable } from './db-table';
-import { generateId } from '@/lib/utils';
 
 export interface DBRelationship {
     id: string;
@@ -40,82 +35,6 @@ export type RelationshipType =
     | 'many_to_many';
 export type Cardinality = 'one' | 'many';
 
-const determineCardinality = (
-    field: DBField,
-    isTablePKComplex: boolean
-): Cardinality => {
-    return field.unique || (field.primaryKey && !isTablePKComplex)
-        ? 'one'
-        : 'many';
-};
-
-export const createRelationshipsFromMetadata = ({
-    foreignKeys,
-    tables,
-}: {
-    foreignKeys: ForeignKeyInfo[];
-    tables: DBTable[];
-}): DBRelationship[] => {
-    return foreignKeys
-        .map((fk: ForeignKeyInfo): DBRelationship | null => {
-            const schema = schemaNameToDomainSchemaName(fk.schema);
-            const sourceTable = tables.find(
-                (table) => table.name === fk.table && table.schema === schema
-            );
-
-            const targetSchema = schemaNameToDomainSchemaName(
-                fk.reference_schema
-            );
-
-            const targetTable = tables.find(
-                (table) =>
-                    table.name === fk.reference_table &&
-                    table.schema === targetSchema
-            );
-            const sourceField = sourceTable?.fields.find(
-                (field) => field.name === fk.column
-            );
-            const targetField = targetTable?.fields.find(
-                (field) => field.name === fk.reference_column
-            );
-
-            const isSourceTablePKComplex =
-                (sourceTable?.fields.filter((field) => field.primaryKey) ?? [])
-                    .length > 1;
-            const isTargetTablePKComplex =
-                (targetTable?.fields.filter((field) => field.primaryKey) ?? [])
-                    .length > 1;
-
-            if (sourceTable && targetTable && sourceField && targetField) {
-                const sourceCardinality = determineCardinality(
-                    sourceField,
-                    isSourceTablePKComplex
-                );
-                const targetCardinality = determineCardinality(
-                    targetField,
-                    isTargetTablePKComplex
-                );
-
-                return {
-                    id: generateId(),
-                    name: fk.foreign_key_name,
-                    sourceSchema: schema,
-                    targetSchema: targetSchema,
-                    sourceTableId: sourceTable.id,
-                    targetTableId: targetTable.id,
-                    sourceFieldId: sourceField.id,
-                    targetFieldId: targetField.id,
-                    sourceCardinality,
-                    targetCardinality,
-                    createdAt: Date.now(),
-                };
-            }
-
-            return null;
-        })
-        .filter((rel) => rel !== null) as DBRelationship[];
-};
-
 export const determineRelationshipType = ({
     sourceCardinality,
     targetCardinality,

src/lib/domain/db-table.ts

@@ -1,30 +1,8 @@
-import {
-    createIndexesFromMetadata,
-    dbIndexSchema,
-    type DBIndex,
-} from './db-index';
-import {
-    createFieldsFromMetadata,
-    dbFieldSchema,
-    type DBField,
-} from './db-field';
-import type { TableInfo } from '../data/import-metadata/metadata-types/table-info';
-import { createAggregatedIndexes } from '../data/import-metadata/metadata-types/index-info';
-import {
-    materializedViewColor,
-    viewColor,
-    defaultTableColor,
-} from '@/lib/colors';
+import { dbIndexSchema, type DBIndex } from './db-index';
+import { dbFieldSchema, type DBField } from './db-field';
 import type { DBRelationship } from './db-relationship';
-import {
-    decodeBase64ToUtf16LE,
-    decodeBase64ToUtf8,
-    deepCopy,
-    generateId,
-} from '../utils';
+import { deepCopy } from '../utils';
 import { schemaNameToDomainSchemaName } from './db-schema';
-import { DatabaseType } from './database-type';
-import type { DatabaseMetadata } from '../data/import-metadata/metadata-types/database-metadata';
 import { z } from 'zod';
 import type { Area } from './area';
 
@@ -79,213 +57,6 @@ export const generateTableKey = ({
     tableName: string;
 }) => `${schemaNameToDomainSchemaName(schemaName) ?? ''}.${tableName}`;
 
-export const decodeViewDefinition = (
-    databaseType: DatabaseType,
-    viewDefinition?: string
-): string => {
-    if (!viewDefinition) {
-        return '';
-    }
-
-    let decodedViewDefinition: string;
-    if (databaseType === DatabaseType.SQL_SERVER) {
-        decodedViewDefinition = decodeBase64ToUtf16LE(viewDefinition);
-    } else {
-        decodedViewDefinition = decodeBase64ToUtf8(viewDefinition);
-    }
-
-    return decodedViewDefinition;
-};
-
-export const createTablesFromMetadata = ({
-    databaseMetadata,
-    databaseType,
-}: {
-    databaseMetadata: DatabaseMetadata;
-    databaseType: DatabaseType;
-}): DBTable[] => {
-    const {
-        tables: tableInfos,
-        pk_info: primaryKeys,
-        columns,
-        indexes,
-        views: views,
-    } = databaseMetadata;
-
-    // Pre-compute view names for faster lookup if there are views
-    const viewNamesSet = new Set<string>();
-    const materializedViewNamesSet = new Set<string>();
-
-    if (views && views.length > 0) {
-        views.forEach((view) => {
-            const key = generateTableKey({
-                schemaName: view.schema,
-                tableName: view.view_name,
-            });
-            viewNamesSet.add(key);
-
-            if (
-                view.view_definition &&
-                decodeViewDefinition(databaseType, view.view_definition)
-                    .toLowerCase()
-                    .includes('materialized')
-            ) {
-                materializedViewNamesSet.add(key);
-            }
-        });
-    }
-
-    // Pre-compute lookup maps for better performance
-    const columnsByTable = new Map<string, (typeof columns)[0][]>();
-    const indexesByTable = new Map<string, (typeof indexes)[0][]>();
-    const primaryKeysByTable = new Map<string, (typeof primaryKeys)[0][]>();
-
-    // Group columns by table
-    columns.forEach((col) => {
-        const key = generateTableKey({
-            schemaName: col.schema,
-            tableName: col.table,
-        });
-        if (!columnsByTable.has(key)) {
-            columnsByTable.set(key, []);
-        }
-        columnsByTable.get(key)!.push(col);
-    });
-
-    // Group indexes by table
-    indexes.forEach((idx) => {
-        const key = generateTableKey({
-            schemaName: idx.schema,
-            tableName: idx.table,
-        });
-        if (!indexesByTable.has(key)) {
-            indexesByTable.set(key, []);
-        }
-        indexesByTable.get(key)!.push(idx);
-    });
-
-    // Group primary keys by table
-    primaryKeys.forEach((pk) => {
-        const key = generateTableKey({
-            schemaName: pk.schema,
-            tableName: pk.table,
-        });
-        if (!primaryKeysByTable.has(key)) {
-            primaryKeysByTable.set(key, []);
-        }
-        primaryKeysByTable.get(key)!.push(pk);
-    });
-
-    const result = tableInfos.map((tableInfo: TableInfo) => {
-        const tableSchema = schemaNameToDomainSchemaName(tableInfo.schema);
-        const tableKey = generateTableKey({
-            schemaName: tableInfo.schema,
-            tableName: tableInfo.table,
-        });
-
-        // Use pre-computed lookups instead of filtering entire arrays
-        const tableIndexes = indexesByTable.get(tableKey) || [];
-        const tablePrimaryKeys = primaryKeysByTable.get(tableKey) || [];
-        const tableColumns = columnsByTable.get(tableKey) || [];
-
-        // Aggregate indexes with multiple columns
-        const aggregatedIndexes = createAggregatedIndexes({
-            tableInfo,
-            tableSchema,
-            tableIndexes,
-        });
-
-        const fields = createFieldsFromMetadata({
-            aggregatedIndexes,
-            tableColumns,
-            tablePrimaryKeys,
-            tableInfo,
-            tableSchema,
-        });
-
-        // Check for composite primary key and find matching index name
-        const primaryKeyFields = fields.filter((f) => f.primaryKey);
-        let pkMatchingIndexName: string | undefined;
-        let pkIndex: DBIndex | undefined;
-
-        if (primaryKeyFields.length >= 1) {
-            // We have a composite primary key, look for an index that matches all PK columns
-            const pkFieldNames = primaryKeyFields.map((f) => f.name).sort();
-
-            // Find an index that matches the primary key columns exactly
-            const matchingIndex = aggregatedIndexes.find((index) => {
-                const indexColumnNames = index.columns
-                    .map((c) => c.name)
-                    .sort();
-                return (
-                    indexColumnNames.length === pkFieldNames.length &&
-                    indexColumnNames.every((col, i) => col === pkFieldNames[i])
-                );
-            });
-
-            if (matchingIndex) {
-                pkMatchingIndexName = matchingIndex.name;
-                // Create a special PK index
-                pkIndex = {
-                    id: generateId(),
-                    name: matchingIndex.name,
-                    unique: true,
-                    fieldIds: primaryKeyFields.map((f) => f.id),
-                    createdAt: Date.now(),
-                    isPrimaryKey: true,
-                };
-            }
-        }
-
-        // Filter out the index that matches the composite PK (to avoid duplication)
-        const filteredAggregatedIndexes = pkMatchingIndexName
-            ? aggregatedIndexes.filter(
-                  (idx) => idx.name !== pkMatchingIndexName
-              )
-            : aggregatedIndexes;
-
-        const dbIndexes = createIndexesFromMetadata({
-            aggregatedIndexes: filteredAggregatedIndexes,
-            fields,
-        });
-
-        // Add the PK index if it exists
-        if (pkIndex) {
-            dbIndexes.push(pkIndex);
-        }
-
-        // Determine if the current table is a view by checking against pre-computed sets
-        const viewKey = generateTableKey({
-            schemaName: tableSchema,
-            tableName: tableInfo.table,
-        });
-        const isView = viewNamesSet.has(viewKey);
-        const isMaterializedView = materializedViewNamesSet.has(viewKey);
-
-        // Initial random positions; these will be adjusted later
-        return {
-            id: generateId(),
-            name: tableInfo.table,
-            schema: tableSchema,
-            x: Math.random() * 1000, // Placeholder X
-            y: Math.random() * 800, // Placeholder Y
-            fields,
-            indexes: dbIndexes,
-            color: isMaterializedView
-                ? materializedViewColor
-                : isView
-                  ? viewColor
-                  : defaultTableColor,
-            isView: isView,
-            isMaterializedView: isMaterializedView,
-            createdAt: Date.now(),
-            comments: tableInfo.comment ? tableInfo.comment : undefined,
-        };
-    });
-
-    return result;
-};
-
 export const adjustTablePositions = ({
     relationships: inputRelationships,
     tables: inputTables,

src/lib/domain/diagram.ts

@@ -1,30 +1,15 @@
 import { z } from 'zod';
-import type { DatabaseMetadata } from '../data/import-metadata/metadata-types/database-metadata';
 import { DatabaseEdition } from './database-edition';
 import { DatabaseType } from './database-type';
 import type { DBDependency } from './db-dependency';
-import {
-    createDependenciesFromMetadata,
-    dbDependencySchema,
-} from './db-dependency';
+import { dbDependencySchema } from './db-dependency';
 import type { DBRelationship } from './db-relationship';
-import {
-    createRelationshipsFromMetadata,
-    dbRelationshipSchema,
-} from './db-relationship';
+import { dbRelationshipSchema } from './db-relationship';
 import type { DBTable } from './db-table';
-import {
-    adjustTablePositions,
-    createTablesFromMetadata,
-    dbTableSchema,
-} from './db-table';
-import { generateDiagramId } from '@/lib/utils';
+import { dbTableSchema } from './db-table';
 import { areaSchema, type Area } from './area';
 import type { DBCustomType } from './db-custom-type';
-import {
-    dbCustomTypeSchema,
-    createCustomTypesFromMetadata,
-} from './db-custom-type';
+import { dbCustomTypeSchema } from './db-custom-type';
 
 export interface Diagram {
     id: string;
@@ -53,77 +38,3 @@ export const diagramSchema: z.ZodType<Diagram> = z.object({
     createdAt: z.date(),
     updatedAt: z.date(),
 });
-
-export const loadFromDatabaseMetadata = async ({
-    databaseType,
-    databaseMetadata,
-    diagramNumber,
-    databaseEdition,
-}: {
-    databaseType: DatabaseType;
-    databaseMetadata: DatabaseMetadata;
-    diagramNumber?: number;
-    databaseEdition?: DatabaseEdition;
-}): Promise<Diagram> => {
-    const {
-        fk_info: foreignKeys,
-        views: views,
-        custom_types: customTypes,
-    } = databaseMetadata;
-
-    const tables = createTablesFromMetadata({
-        databaseMetadata,
-        databaseType,
-    });
-
-    const relationships = createRelationshipsFromMetadata({
-        foreignKeys,
-        tables,
-    });
-
-    const dependencies = await createDependenciesFromMetadata({
-        views,
-        tables,
-        databaseType,
-    });
-
-    const dbCustomTypes = customTypes
-        ? createCustomTypesFromMetadata({
-              customTypes,
-          })
-        : [];
-
-    const adjustedTables = adjustTablePositions({
-        tables,
-        relationships,
-        mode: 'perSchema',
-    });
-
-    const sortedTables = adjustedTables.sort((a, b) => {
-        if (a.isView === b.isView) {
-            // Both are either tables or views, so sort alphabetically by name
-            return a.name.localeCompare(b.name);
-        }
-        // If one is a view and the other is not, put tables first
-        return a.isView ? 1 : -1;
-    });
-
-    const diagram: Diagram = {
-        id: generateDiagramId(),
-        name: databaseMetadata.database_name
-            ? `${databaseMetadata.database_name}-db`
-            : diagramNumber
-              ? `Diagram ${diagramNumber}`
-              : 'New Diagram',
-        databaseType: databaseType ?? DatabaseType.GENERIC,
-        databaseEdition,
-        tables: sortedTables,
-        relationships,
-        dependencies,
-        customTypes: dbCustomTypes,
-        createdAt: new Date(),
-        updatedAt: new Date(),
-    };
-
-    return diagram;
-};