From ffddcdcc987bacb0e0d7e8dea27d08d3a8c5a8c8 Mon Sep 17 00:00:00 2001 From: Guy Ben-Aharon Date: Thu, 4 Sep 2025 12:10:56 +0300 Subject: [PATCH] fix: export sql + import metadata lib (#902) --- .../create-diagram-dialog.tsx | 2 +- .../export-sql-dialog/export-sql-dialog.tsx | 2 +- .../import-database-dialog.tsx | 2 +- .../import-metadata/import/custom-types.ts | 21 ++ .../import-metadata/import/dependencies.ts | 351 ++++++++++++++++++ src/lib/data/import-metadata/import/fields.ts | 64 ++++ src/lib/data/import-metadata/import/index.ts | 82 ++++ .../data/import-metadata/import/indexes.ts | 24 ++ .../import-metadata/import/relationships.ts | 85 +++++ src/lib/data/import-metadata/import/tables.ts | 228 ++++++++++++ .../__tests__/export-sql-dbml.test.ts | 0 .../export-sql-quoted-identifiers.test.ts | 0 .../export-per-type/common.ts | 0 .../export-per-type/mssql.ts | 0 .../export-per-type/mysql.ts | 0 .../export-per-type/postgresql.ts | 0 .../export-per-type/sqlite.ts | 0 .../export-sql-cache.ts | 0 .../export-sql-script.ts | 0 src/lib/dbml/dbml-export/dbml-export.ts | 2 +- .../__tests__/composite-pk-name.test.ts | 6 +- .../composite-pk-metadata-import.test.ts | 2 +- src/lib/domain/db-custom-type.ts | 20 - src/lib/domain/db-dependency.ts | 351 ------------------ src/lib/domain/db-field.ts | 63 ---- src/lib/domain/db-index.ts | 23 -- src/lib/domain/db-relationship.ts | 81 ---- src/lib/domain/db-table.ts | 235 +----------- src/lib/domain/diagram.ts | 97 +---- 29 files changed, 870 insertions(+), 871 deletions(-) create mode 100644 src/lib/data/import-metadata/import/custom-types.ts create mode 100644 src/lib/data/import-metadata/import/dependencies.ts create mode 100644 src/lib/data/import-metadata/import/fields.ts create mode 100644 src/lib/data/import-metadata/import/index.ts create mode 100644 src/lib/data/import-metadata/import/indexes.ts create mode 100644 src/lib/data/import-metadata/import/relationships.ts create mode 100644 src/lib/data/import-metadata/import/tables.ts rename src/lib/data/{export-metadata => sql-export}/__tests__/export-sql-dbml.test.ts (100%) rename src/lib/data/{export-metadata => sql-export}/__tests__/export-sql-quoted-identifiers.test.ts (100%) rename src/lib/data/{export-metadata => sql-export}/export-per-type/common.ts (100%) rename src/lib/data/{export-metadata => sql-export}/export-per-type/mssql.ts (100%) rename src/lib/data/{export-metadata => sql-export}/export-per-type/mysql.ts (100%) rename src/lib/data/{export-metadata => sql-export}/export-per-type/postgresql.ts (100%) rename src/lib/data/{export-metadata => sql-export}/export-per-type/sqlite.ts (100%) rename src/lib/data/{export-metadata => sql-export}/export-sql-cache.ts (100%) rename src/lib/data/{export-metadata => sql-export}/export-sql-script.ts (100%) diff --git a/src/dialogs/create-diagram-dialog/create-diagram-dialog.tsx b/src/dialogs/create-diagram-dialog/create-diagram-dialog.tsx index ab8af742..b08a8390 100644 --- a/src/dialogs/create-diagram-dialog/create-diagram-dialog.tsx +++ b/src/dialogs/create-diagram-dialog/create-diagram-dialog.tsx @@ -3,7 +3,7 @@ import { Dialog, DialogContent } from '@/components/dialog/dialog'; import { DatabaseType } from '@/lib/domain/database-type'; import { useStorage } from '@/hooks/use-storage'; import type { Diagram } from '@/lib/domain/diagram'; -import { loadFromDatabaseMetadata } from '@/lib/domain/diagram'; +import { loadFromDatabaseMetadata } from '@/lib/data/import-metadata/import'; import { useNavigate } from 'react-router-dom'; import { useConfig } from '@/hooks/use-config'; import type { DatabaseMetadata } from '@/lib/data/import-metadata/metadata-types/database-metadata'; diff --git a/src/dialogs/export-sql-dialog/export-sql-dialog.tsx b/src/dialogs/export-sql-dialog/export-sql-dialog.tsx index 949d3a53..df9d29cd 100644 --- a/src/dialogs/export-sql-dialog/export-sql-dialog.tsx +++ b/src/dialogs/export-sql-dialog/export-sql-dialog.tsx @@ -17,7 +17,7 @@ import { useDialog } from '@/hooks/use-dialog'; import { exportBaseSQL, exportSQL, -} from '@/lib/data/export-metadata/export-sql-script'; +} from '@/lib/data/sql-export/export-sql-script'; import { databaseTypeToLabelMap } from '@/lib/databases'; import { DatabaseType } from '@/lib/domain/database-type'; import { Annoyed, Sparkles } from 'lucide-react'; diff --git a/src/dialogs/import-database-dialog/import-database-dialog.tsx b/src/dialogs/import-database-dialog/import-database-dialog.tsx index ce29e75b..e7dd28af 100644 --- a/src/dialogs/import-database-dialog/import-database-dialog.tsx +++ b/src/dialogs/import-database-dialog/import-database-dialog.tsx @@ -7,7 +7,7 @@ import type { DatabaseEdition } from '@/lib/domain/database-edition'; import type { DatabaseMetadata } from '@/lib/data/import-metadata/metadata-types/database-metadata'; import { loadDatabaseMetadata } from '@/lib/data/import-metadata/metadata-types/database-metadata'; import type { Diagram } from '@/lib/domain/diagram'; -import { loadFromDatabaseMetadata } from '@/lib/domain/diagram'; +import { loadFromDatabaseMetadata } from '@/lib/data/import-metadata/import'; import { useChartDB } from '@/hooks/use-chartdb'; import { useRedoUndoStack } from '@/hooks/use-redo-undo-stack'; import { Trans, useTranslation } from 'react-i18next'; diff --git a/src/lib/data/import-metadata/import/custom-types.ts b/src/lib/data/import-metadata/import/custom-types.ts new file mode 100644 index 00000000..b3269f2a --- /dev/null +++ b/src/lib/data/import-metadata/import/custom-types.ts @@ -0,0 +1,21 @@ +import type { DBCustomType, DBCustomTypeKind } from '@/lib/domain'; +import { schemaNameToDomainSchemaName } from '@/lib/domain'; +import type { DBCustomTypeInfo } from '../metadata-types/custom-type-info'; +import { generateId } from '@/lib/utils'; + +export const createCustomTypesFromMetadata = ({ + customTypes, +}: { + customTypes: DBCustomTypeInfo[]; +}): DBCustomType[] => { + return customTypes.map((customType) => { + return { + id: generateId(), + schema: schemaNameToDomainSchemaName(customType.schema), + name: customType.type, + kind: customType.kind as DBCustomTypeKind, + values: customType.values, + fields: customType.fields, + }; + }); +}; diff --git a/src/lib/data/import-metadata/import/dependencies.ts b/src/lib/data/import-metadata/import/dependencies.ts new file mode 100644 index 00000000..1937c5ec --- /dev/null +++ b/src/lib/data/import-metadata/import/dependencies.ts @@ -0,0 +1,351 @@ +import { generateId } from '@/lib/utils'; +import type { AST } from 'node-sql-parser'; +import type { DBDependency, DBTable } from '@/lib/domain'; +import { DatabaseType, schemaNameToDomainSchemaName } from '@/lib/domain'; +import type { ViewInfo } from '../metadata-types/view-info'; +import { decodeViewDefinition } from './tables'; + +const astDatabaseTypes: Record = { + [DatabaseType.POSTGRESQL]: 'postgresql', + [DatabaseType.MYSQL]: 'postgresql', + [DatabaseType.MARIADB]: 'postgresql', + [DatabaseType.GENERIC]: 'postgresql', + [DatabaseType.SQLITE]: 'postgresql', + [DatabaseType.SQL_SERVER]: 'postgresql', + [DatabaseType.CLICKHOUSE]: 'postgresql', + [DatabaseType.COCKROACHDB]: 'postgresql', + [DatabaseType.ORACLE]: 'postgresql', +}; + +export const createDependenciesFromMetadata = async ({ + views, + tables, + databaseType, +}: { + views: ViewInfo[]; + tables: DBTable[]; + databaseType: DatabaseType; +}): Promise => { + if (!views || views.length === 0) { + return []; + } + + const { Parser } = await import('node-sql-parser'); + const parser = new Parser(); + + const dependencies = views + .flatMap((view) => { + const viewSchema = schemaNameToDomainSchemaName(view.schema); + const viewTable = tables.find( + (table) => + table.name === view.view_name && viewSchema === table.schema + ); + + if (!viewTable) { + console.warn( + `Source table for view ${view.view_name} not found (schema: ${viewSchema})` + ); + return []; // Skip this view and proceed to the next + } + + if (view.view_definition) { + try { + const decodedViewDefinition = decodeViewDefinition( + databaseType, + view.view_definition + ); + + let modifiedViewDefinition = ''; + if ( + databaseType === DatabaseType.MYSQL || + databaseType === DatabaseType.MARIADB + ) { + modifiedViewDefinition = preprocessViewDefinitionMySQL( + decodedViewDefinition + ); + } else if (databaseType === DatabaseType.SQL_SERVER) { + modifiedViewDefinition = + preprocessViewDefinitionSQLServer( + decodedViewDefinition + ); + } else { + modifiedViewDefinition = preprocessViewDefinition( + decodedViewDefinition + ); + } + + // Parse using the appropriate dialect + const ast = parser.astify(modifiedViewDefinition, { + database: astDatabaseTypes[databaseType], + type: 'select', // Parsing a SELECT statement + }); + + let relatedTables = extractTablesFromAST(ast); + + // Filter out duplicate tables without schema + relatedTables = filterDuplicateTables(relatedTables); + + return relatedTables.map((relTable) => { + const relSchema = relTable.schema || view.schema; // Use view's schema if relSchema is undefined + const relTableName = relTable.tableName; + + const table = tables.find( + (table) => + table.name === relTableName && + (table.schema || '') === relSchema + ); + + if (table) { + const dependency: DBDependency = { + id: generateId(), + schema: view.schema, + tableId: table.id, // related table + dependentSchema: table.schema, + dependentTableId: viewTable.id, // dependent view + createdAt: Date.now(), + }; + + return dependency; + } else { + console.warn( + `Dependent table ${relSchema}.${relTableName} not found for view ${view.schema}.${view.view_name}` + ); + return null; + } + }); + } catch (error) { + console.error( + `Error parsing view ${view.schema}.${view.view_name}:`, + error + ); + return []; + } + } else { + console.warn( + `View definition missing for ${view.schema}.${view.view_name}` + ); + return []; + } + }) + .filter((dependency) => dependency !== null); + + return dependencies; +}; + +// Add this new function to filter out duplicate tables +function filterDuplicateTables( + tables: { schema?: string; tableName: string }[] +): { schema?: string; tableName: string }[] { + const tableMap = new Map(); + + for (const table of tables) { + const key = table.tableName; + const existingTable = tableMap.get(key); + + if (!existingTable || (table.schema && !existingTable.schema)) { + tableMap.set(key, table); + } + } + + return Array.from(tableMap.values()); +} + +// Preprocess the view_definition to remove schema from CREATE VIEW +function preprocessViewDefinition(viewDefinition: string): string { + if (!viewDefinition) { + return ''; + } + + // Remove leading and trailing whitespace + viewDefinition = viewDefinition.replace(/\s+/g, ' ').trim(); + + // Replace escaped double quotes with regular ones + viewDefinition = viewDefinition.replace(/\\"/g, '"'); + + // Replace 'CREATE MATERIALIZED VIEW' with 'CREATE VIEW' + viewDefinition = viewDefinition.replace( + /CREATE\s+MATERIALIZED\s+VIEW/i, + 'CREATE VIEW' + ); + + // Regular expression to match 'CREATE VIEW [schema.]view_name [ (column definitions) ] AS' + // This regex captures the view name and skips any content between the view name and 'AS' + const regex = + /CREATE\s+VIEW\s+(?:(?:`[^`]+`|"[^"]+"|\w+)\.)?(?:`([^`]+)`|"([^"]+)"|(\w+))[\s\S]*?\bAS\b\s+/i; + + const match = viewDefinition.match(regex); + let modifiedDefinition: string; + + if (match) { + const viewName = match[1] || match[2] || match[3]; + // Extract the SQL after the 'AS' keyword + const restOfDefinition = viewDefinition.substring( + match.index! + match[0].length + ); + + // Replace double-quoted identifiers with unquoted ones + let modifiedSQL = restOfDefinition.replace(/"(\w+)"/g, '$1'); + + // Replace '::' type casts with 'CAST' expressions + modifiedSQL = modifiedSQL.replace( + /\(([^()]+)\)::(\w+)/g, + 'CAST($1 AS $2)' + ); + + // Remove ClickHouse-specific syntax that may still be present + // For example, remove SETTINGS clauses inside the SELECT statement + modifiedSQL = modifiedSQL.replace(/\bSETTINGS\b[\s\S]*$/i, ''); + + modifiedDefinition = `CREATE VIEW ${viewName} AS ${modifiedSQL}`; + } else { + console.warn('Could not preprocess view definition:', viewDefinition); + modifiedDefinition = viewDefinition; + } + + return modifiedDefinition; +} + +// Preprocess the view_definition for SQL Server +function preprocessViewDefinitionSQLServer(viewDefinition: string): string { + if (!viewDefinition) { + return ''; + } + + // Remove BOM if present + viewDefinition = viewDefinition.replace(/^\uFEFF/, ''); + + // Normalize whitespace + viewDefinition = viewDefinition.replace(/\s+/g, ' ').trim(); + + // Remove square brackets and replace with double quotes + viewDefinition = viewDefinition.replace(/\[([^\]]+)\]/g, '"$1"'); + + // Remove database names from fully qualified identifiers + viewDefinition = viewDefinition.replace( + /"([a-zA-Z0-9_]+)"\."([a-zA-Z0-9_]+)"\."([a-zA-Z0-9_]+)"/g, + '"$2"."$3"' + ); + + // Replace SQL Server functions with PostgreSQL equivalents + viewDefinition = viewDefinition.replace(/\bGETDATE\(\)/gi, 'NOW()'); + viewDefinition = viewDefinition.replace(/\bISNULL\(/gi, 'COALESCE('); + + // Replace 'TOP N' with 'LIMIT N' at the end of the query + const topMatch = viewDefinition.match(/SELECT\s+TOP\s+(\d+)/i); + if (topMatch) { + const topN = topMatch[1]; + viewDefinition = viewDefinition.replace( + /SELECT\s+TOP\s+\d+/i, + 'SELECT' + ); + viewDefinition = viewDefinition.replace(/;+\s*$/, ''); // Remove semicolons at the end + viewDefinition += ` LIMIT ${topN}`; + } + + viewDefinition = viewDefinition.replace(/\n/g, ''); // Remove newlines + + // Adjust CREATE VIEW syntax + const regex = + /CREATE\s+VIEW\s+(?:"?([^".\s]+)"?\.)?"?([^".\s]+)"?\s+AS\s+/i; + const match = viewDefinition.match(regex); + let modifiedDefinition: string; + + if (match) { + const viewName = match[2]; + const modifiedSQL = viewDefinition.substring( + match.index! + match[0].length + ); + + // Remove semicolons at the end + const finalSQL = modifiedSQL.replace(/;+\s*$/, ''); + + modifiedDefinition = `CREATE VIEW "${viewName}" AS ${finalSQL}`; + } else { + console.warn('Could not preprocess view definition:', viewDefinition); + modifiedDefinition = viewDefinition; + } + + return modifiedDefinition; +} + +// Preprocess the view_definition to remove schema from CREATE VIEW +function preprocessViewDefinitionMySQL(viewDefinition: string): string { + if (!viewDefinition) { + return ''; + } + + // Remove any trailing semicolons + viewDefinition = viewDefinition.replace(/;\s*$/, ''); + + // Remove backticks from identifiers + viewDefinition = viewDefinition.replace(/`/g, ''); + + // Remove unnecessary parentheses around joins and ON clauses + viewDefinition = removeRedundantParentheses(viewDefinition); + + return viewDefinition; +} + +function removeRedundantParentheses(sql: string): string { + // Regular expressions to match unnecessary parentheses + const patterns = [ + /\(\s*(JOIN\s+[^()]+?)\s*\)/gi, + /\(\s*(ON\s+[^()]+?)\s*\)/gi, + // Additional patterns if necessary + ]; + + let prevSql; + do { + prevSql = sql; + patterns.forEach((pattern) => { + sql = sql.replace(pattern, '$1'); + }); + } while (sql !== prevSql); + + return sql; +} + +function extractTablesFromAST( + ast: AST | AST[] +): { schema?: string; tableName: string }[] { + const tablesMap = new Map(); + const visitedNodes = new Set(); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + function traverse(node: any) { + if (!node || visitedNodes.has(node)) return; + visitedNodes.add(node); + + if (Array.isArray(node)) { + node.forEach(traverse); + } else if (typeof node === 'object') { + // Check if node represents a table + if ( + Object.hasOwnProperty.call(node, 'table') && + typeof node.table === 'string' + ) { + let schema = node.db || node.schema; + const tableName = node.table; + if (tableName) { + // Assign default schema if undefined + schema = schemaNameToDomainSchemaName(schema) || ''; + const key = `${schema}.${tableName}`; + if (!tablesMap.has(key)) { + tablesMap.set(key, { schema, tableName }); + } + } + } + + // Recursively traverse all properties + for (const key in node) { + if (Object.hasOwnProperty.call(node, key)) { + traverse(node[key]); + } + } + } + } + + traverse(ast); + + return Array.from(tablesMap.values()); +} diff --git a/src/lib/data/import-metadata/import/fields.ts b/src/lib/data/import-metadata/import/fields.ts new file mode 100644 index 00000000..01c901c8 --- /dev/null +++ b/src/lib/data/import-metadata/import/fields.ts @@ -0,0 +1,64 @@ +import type { DBField } from '@/lib/domain'; +import type { ColumnInfo } from '../metadata-types/column-info'; +import type { AggregatedIndexInfo } from '../metadata-types/index-info'; +import type { PrimaryKeyInfo } from '../metadata-types/primary-key-info'; +import type { TableInfo } from '../metadata-types/table-info'; +import { generateId } from '@/lib/utils'; + +export const createFieldsFromMetadata = ({ + tableColumns, + tablePrimaryKeys, + aggregatedIndexes, +}: { + tableColumns: ColumnInfo[]; + tableSchema?: string; + tableInfo: TableInfo; + tablePrimaryKeys: PrimaryKeyInfo[]; + aggregatedIndexes: AggregatedIndexInfo[]; +}) => { + const uniqueColumns = tableColumns.reduce((acc, col) => { + if (!acc.has(col.name)) { + acc.set(col.name, col); + } + return acc; + }, new Map()); + + const sortedColumns = Array.from(uniqueColumns.values()).sort( + (a, b) => a.ordinal_position - b.ordinal_position + ); + + const tablePrimaryKeysColumns = tablePrimaryKeys.map((pk) => + pk.column.trim() + ); + + return sortedColumns.map( + (col: ColumnInfo): DBField => ({ + id: generateId(), + name: col.name, + type: { + id: col.type.split(' ').join('_').toLowerCase(), + name: col.type.toLowerCase(), + }, + primaryKey: tablePrimaryKeysColumns.includes(col.name), + unique: Object.values(aggregatedIndexes).some( + (idx) => + idx.unique && + idx.columns.length === 1 && + idx.columns[0].name === col.name + ), + nullable: Boolean(col.nullable), + ...(col.character_maximum_length && + col.character_maximum_length !== 'null' + ? { characterMaximumLength: col.character_maximum_length } + : {}), + ...(col.precision?.precision + ? { precision: col.precision.precision } + : {}), + ...(col.precision?.scale ? { scale: col.precision.scale } : {}), + ...(col.default ? { default: col.default } : {}), + ...(col.collation ? { collation: col.collation } : {}), + createdAt: Date.now(), + comments: col.comment ? col.comment : undefined, + }) + ); +}; diff --git a/src/lib/data/import-metadata/import/index.ts b/src/lib/data/import-metadata/import/index.ts new file mode 100644 index 00000000..ecc907fb --- /dev/null +++ b/src/lib/data/import-metadata/import/index.ts @@ -0,0 +1,82 @@ +import type { DatabaseEdition, Diagram } from '@/lib/domain'; +import { adjustTablePositions, DatabaseType } from '@/lib/domain'; +import { generateDiagramId } from '@/lib/utils'; +import type { DatabaseMetadata } from '../metadata-types/database-metadata'; +import { createCustomTypesFromMetadata } from './custom-types'; +import { createRelationshipsFromMetadata } from './relationships'; +import { createTablesFromMetadata } from './tables'; +import { createDependenciesFromMetadata } from './dependencies'; + +export const loadFromDatabaseMetadata = async ({ + databaseType, + databaseMetadata, + diagramNumber, + databaseEdition, +}: { + databaseType: DatabaseType; + databaseMetadata: DatabaseMetadata; + diagramNumber?: number; + databaseEdition?: DatabaseEdition; +}): Promise => { + const { + fk_info: foreignKeys, + views: views, + custom_types: customTypes, + } = databaseMetadata; + + const tables = createTablesFromMetadata({ + databaseMetadata, + databaseType, + }); + + const relationships = createRelationshipsFromMetadata({ + foreignKeys, + tables, + }); + + const dependencies = await createDependenciesFromMetadata({ + views, + tables, + databaseType, + }); + + const dbCustomTypes = customTypes + ? createCustomTypesFromMetadata({ + customTypes, + }) + : []; + + const adjustedTables = adjustTablePositions({ + tables, + relationships, + mode: 'perSchema', + }); + + const sortedTables = adjustedTables.sort((a, b) => { + if (a.isView === b.isView) { + // Both are either tables or views, so sort alphabetically by name + return a.name.localeCompare(b.name); + } + // If one is a view and the other is not, put tables first + return a.isView ? 1 : -1; + }); + + const diagram: Diagram = { + id: generateDiagramId(), + name: databaseMetadata.database_name + ? `${databaseMetadata.database_name}-db` + : diagramNumber + ? `Diagram ${diagramNumber}` + : 'New Diagram', + databaseType: databaseType ?? DatabaseType.GENERIC, + databaseEdition, + tables: sortedTables, + relationships, + dependencies, + customTypes: dbCustomTypes, + createdAt: new Date(), + updatedAt: new Date(), + }; + + return diagram; +}; diff --git a/src/lib/data/import-metadata/import/indexes.ts b/src/lib/data/import-metadata/import/indexes.ts new file mode 100644 index 00000000..c020fd41 --- /dev/null +++ b/src/lib/data/import-metadata/import/indexes.ts @@ -0,0 +1,24 @@ +import type { DBField, DBIndex, IndexType } from '@/lib/domain'; +import type { AggregatedIndexInfo } from '../metadata-types/index-info'; +import { generateId } from '@/lib/utils'; + +export const createIndexesFromMetadata = ({ + aggregatedIndexes, + fields, +}: { + aggregatedIndexes: AggregatedIndexInfo[]; + fields: DBField[]; +}): DBIndex[] => + aggregatedIndexes.map( + (idx): DBIndex => ({ + id: generateId(), + name: idx.name, + unique: Boolean(idx.unique), + fieldIds: idx.columns + .sort((a, b) => a.position - b.position) + .map((c) => fields.find((f) => f.name === c.name)?.id) + .filter((id): id is string => id !== undefined), + createdAt: Date.now(), + type: idx.index_type?.toLowerCase() as IndexType, + }) + ); diff --git a/src/lib/data/import-metadata/import/relationships.ts b/src/lib/data/import-metadata/import/relationships.ts new file mode 100644 index 00000000..0b0fd7aa --- /dev/null +++ b/src/lib/data/import-metadata/import/relationships.ts @@ -0,0 +1,85 @@ +import type { + Cardinality, + DBField, + DBRelationship, + DBTable, +} from '@/lib/domain'; +import { schemaNameToDomainSchemaName } from '@/lib/domain'; +import type { ForeignKeyInfo } from '../metadata-types/foreign-key-info'; +import { generateId } from '@/lib/utils'; + +const determineCardinality = ( + field: DBField, + isTablePKComplex: boolean +): Cardinality => { + return field.unique || (field.primaryKey && !isTablePKComplex) + ? 'one' + : 'many'; +}; + +export const createRelationshipsFromMetadata = ({ + foreignKeys, + tables, +}: { + foreignKeys: ForeignKeyInfo[]; + tables: DBTable[]; +}): DBRelationship[] => { + return foreignKeys + .map((fk: ForeignKeyInfo): DBRelationship | null => { + const schema = schemaNameToDomainSchemaName(fk.schema); + const sourceTable = tables.find( + (table) => table.name === fk.table && table.schema === schema + ); + + const targetSchema = schemaNameToDomainSchemaName( + fk.reference_schema + ); + + const targetTable = tables.find( + (table) => + table.name === fk.reference_table && + table.schema === targetSchema + ); + const sourceField = sourceTable?.fields.find( + (field) => field.name === fk.column + ); + const targetField = targetTable?.fields.find( + (field) => field.name === fk.reference_column + ); + + const isSourceTablePKComplex = + (sourceTable?.fields.filter((field) => field.primaryKey) ?? []) + .length > 1; + const isTargetTablePKComplex = + (targetTable?.fields.filter((field) => field.primaryKey) ?? []) + .length > 1; + + if (sourceTable && targetTable && sourceField && targetField) { + const sourceCardinality = determineCardinality( + sourceField, + isSourceTablePKComplex + ); + const targetCardinality = determineCardinality( + targetField, + isTargetTablePKComplex + ); + + return { + id: generateId(), + name: fk.foreign_key_name, + sourceSchema: schema, + targetSchema: targetSchema, + sourceTableId: sourceTable.id, + targetTableId: targetTable.id, + sourceFieldId: sourceField.id, + targetFieldId: targetField.id, + sourceCardinality, + targetCardinality, + createdAt: Date.now(), + }; + } + + return null; + }) + .filter((rel) => rel !== null) as DBRelationship[]; +}; diff --git a/src/lib/data/import-metadata/import/tables.ts b/src/lib/data/import-metadata/import/tables.ts new file mode 100644 index 00000000..b45689b8 --- /dev/null +++ b/src/lib/data/import-metadata/import/tables.ts @@ -0,0 +1,228 @@ +import type { DBIndex, DBTable } from '@/lib/domain'; +import { + DatabaseType, + generateTableKey, + schemaNameToDomainSchemaName, +} from '@/lib/domain'; +import type { DatabaseMetadata } from '../metadata-types/database-metadata'; +import type { TableInfo } from '../metadata-types/table-info'; +import { createAggregatedIndexes } from '../metadata-types/index-info'; +import { + decodeBase64ToUtf16LE, + decodeBase64ToUtf8, + generateId, +} from '@/lib/utils'; +import { + defaultTableColor, + materializedViewColor, + viewColor, +} from '@/lib/colors'; +import { createFieldsFromMetadata } from './fields'; +import { createIndexesFromMetadata } from './indexes'; + +export const decodeViewDefinition = ( + databaseType: DatabaseType, + viewDefinition?: string +): string => { + if (!viewDefinition) { + return ''; + } + + let decodedViewDefinition: string; + if (databaseType === DatabaseType.SQL_SERVER) { + decodedViewDefinition = decodeBase64ToUtf16LE(viewDefinition); + } else { + decodedViewDefinition = decodeBase64ToUtf8(viewDefinition); + } + + return decodedViewDefinition; +}; + +export const createTablesFromMetadata = ({ + databaseMetadata, + databaseType, +}: { + databaseMetadata: DatabaseMetadata; + databaseType: DatabaseType; +}): DBTable[] => { + const { + tables: tableInfos, + pk_info: primaryKeys, + columns, + indexes, + views: views, + } = databaseMetadata; + + // Pre-compute view names for faster lookup if there are views + const viewNamesSet = new Set(); + const materializedViewNamesSet = new Set(); + + if (views && views.length > 0) { + views.forEach((view) => { + const key = generateTableKey({ + schemaName: view.schema, + tableName: view.view_name, + }); + viewNamesSet.add(key); + + if ( + view.view_definition && + decodeViewDefinition(databaseType, view.view_definition) + .toLowerCase() + .includes('materialized') + ) { + materializedViewNamesSet.add(key); + } + }); + } + + // Pre-compute lookup maps for better performance + const columnsByTable = new Map(); + const indexesByTable = new Map(); + const primaryKeysByTable = new Map(); + + // Group columns by table + columns.forEach((col) => { + const key = generateTableKey({ + schemaName: col.schema, + tableName: col.table, + }); + if (!columnsByTable.has(key)) { + columnsByTable.set(key, []); + } + columnsByTable.get(key)!.push(col); + }); + + // Group indexes by table + indexes.forEach((idx) => { + const key = generateTableKey({ + schemaName: idx.schema, + tableName: idx.table, + }); + if (!indexesByTable.has(key)) { + indexesByTable.set(key, []); + } + indexesByTable.get(key)!.push(idx); + }); + + // Group primary keys by table + primaryKeys.forEach((pk) => { + const key = generateTableKey({ + schemaName: pk.schema, + tableName: pk.table, + }); + if (!primaryKeysByTable.has(key)) { + primaryKeysByTable.set(key, []); + } + primaryKeysByTable.get(key)!.push(pk); + }); + + const result = tableInfos.map((tableInfo: TableInfo) => { + const tableSchema = schemaNameToDomainSchemaName(tableInfo.schema); + const tableKey = generateTableKey({ + schemaName: tableInfo.schema, + tableName: tableInfo.table, + }); + + // Use pre-computed lookups instead of filtering entire arrays + const tableIndexes = indexesByTable.get(tableKey) || []; + const tablePrimaryKeys = primaryKeysByTable.get(tableKey) || []; + const tableColumns = columnsByTable.get(tableKey) || []; + + // Aggregate indexes with multiple columns + const aggregatedIndexes = createAggregatedIndexes({ + tableInfo, + tableSchema, + tableIndexes, + }); + + const fields = createFieldsFromMetadata({ + aggregatedIndexes, + tableColumns, + tablePrimaryKeys, + tableInfo, + tableSchema, + }); + + // Check for composite primary key and find matching index name + const primaryKeyFields = fields.filter((f) => f.primaryKey); + let pkMatchingIndexName: string | undefined; + let pkIndex: DBIndex | undefined; + + if (primaryKeyFields.length >= 1) { + // We have a composite primary key, look for an index that matches all PK columns + const pkFieldNames = primaryKeyFields.map((f) => f.name).sort(); + + // Find an index that matches the primary key columns exactly + const matchingIndex = aggregatedIndexes.find((index) => { + const indexColumnNames = index.columns + .map((c) => c.name) + .sort(); + return ( + indexColumnNames.length === pkFieldNames.length && + indexColumnNames.every((col, i) => col === pkFieldNames[i]) + ); + }); + + if (matchingIndex) { + pkMatchingIndexName = matchingIndex.name; + // Create a special PK index + pkIndex = { + id: generateId(), + name: matchingIndex.name, + unique: true, + fieldIds: primaryKeyFields.map((f) => f.id), + createdAt: Date.now(), + isPrimaryKey: true, + }; + } + } + + // Filter out the index that matches the composite PK (to avoid duplication) + const filteredAggregatedIndexes = pkMatchingIndexName + ? aggregatedIndexes.filter( + (idx) => idx.name !== pkMatchingIndexName + ) + : aggregatedIndexes; + + const dbIndexes = createIndexesFromMetadata({ + aggregatedIndexes: filteredAggregatedIndexes, + fields, + }); + + // Add the PK index if it exists + if (pkIndex) { + dbIndexes.push(pkIndex); + } + + // Determine if the current table is a view by checking against pre-computed sets + const viewKey = generateTableKey({ + schemaName: tableSchema, + tableName: tableInfo.table, + }); + const isView = viewNamesSet.has(viewKey); + const isMaterializedView = materializedViewNamesSet.has(viewKey); + + // Initial random positions; these will be adjusted later + return { + id: generateId(), + name: tableInfo.table, + schema: tableSchema, + x: Math.random() * 1000, // Placeholder X + y: Math.random() * 800, // Placeholder Y + fields, + indexes: dbIndexes, + color: isMaterializedView + ? materializedViewColor + : isView + ? viewColor + : defaultTableColor, + isView: isView, + isMaterializedView: isMaterializedView, + createdAt: Date.now(), + comments: tableInfo.comment ? tableInfo.comment : undefined, + }; + }); + + return result; +}; diff --git a/src/lib/data/export-metadata/__tests__/export-sql-dbml.test.ts b/src/lib/data/sql-export/__tests__/export-sql-dbml.test.ts similarity index 100% rename from src/lib/data/export-metadata/__tests__/export-sql-dbml.test.ts rename to src/lib/data/sql-export/__tests__/export-sql-dbml.test.ts diff --git a/src/lib/data/export-metadata/__tests__/export-sql-quoted-identifiers.test.ts b/src/lib/data/sql-export/__tests__/export-sql-quoted-identifiers.test.ts similarity index 100% rename from src/lib/data/export-metadata/__tests__/export-sql-quoted-identifiers.test.ts rename to src/lib/data/sql-export/__tests__/export-sql-quoted-identifiers.test.ts diff --git a/src/lib/data/export-metadata/export-per-type/common.ts b/src/lib/data/sql-export/export-per-type/common.ts similarity index 100% rename from src/lib/data/export-metadata/export-per-type/common.ts rename to src/lib/data/sql-export/export-per-type/common.ts diff --git a/src/lib/data/export-metadata/export-per-type/mssql.ts b/src/lib/data/sql-export/export-per-type/mssql.ts similarity index 100% rename from src/lib/data/export-metadata/export-per-type/mssql.ts rename to src/lib/data/sql-export/export-per-type/mssql.ts diff --git a/src/lib/data/export-metadata/export-per-type/mysql.ts b/src/lib/data/sql-export/export-per-type/mysql.ts similarity index 100% rename from src/lib/data/export-metadata/export-per-type/mysql.ts rename to src/lib/data/sql-export/export-per-type/mysql.ts diff --git a/src/lib/data/export-metadata/export-per-type/postgresql.ts b/src/lib/data/sql-export/export-per-type/postgresql.ts similarity index 100% rename from src/lib/data/export-metadata/export-per-type/postgresql.ts rename to src/lib/data/sql-export/export-per-type/postgresql.ts diff --git a/src/lib/data/export-metadata/export-per-type/sqlite.ts b/src/lib/data/sql-export/export-per-type/sqlite.ts similarity index 100% rename from src/lib/data/export-metadata/export-per-type/sqlite.ts rename to src/lib/data/sql-export/export-per-type/sqlite.ts diff --git a/src/lib/data/export-metadata/export-sql-cache.ts b/src/lib/data/sql-export/export-sql-cache.ts similarity index 100% rename from src/lib/data/export-metadata/export-sql-cache.ts rename to src/lib/data/sql-export/export-sql-cache.ts diff --git a/src/lib/data/export-metadata/export-sql-script.ts b/src/lib/data/sql-export/export-sql-script.ts similarity index 100% rename from src/lib/data/export-metadata/export-sql-script.ts rename to src/lib/data/sql-export/export-sql-script.ts diff --git a/src/lib/dbml/dbml-export/dbml-export.ts b/src/lib/dbml/dbml-export/dbml-export.ts index 5030085d..b0049583 100644 --- a/src/lib/dbml/dbml-export/dbml-export.ts +++ b/src/lib/dbml/dbml-export/dbml-export.ts @@ -1,5 +1,5 @@ import { importer } from '@dbml/core'; -import { exportBaseSQL } from '@/lib/data/export-metadata/export-sql-script'; +import { exportBaseSQL } from '@/lib/data/sql-export/export-sql-script'; import type { Diagram } from '@/lib/domain/diagram'; import { DatabaseType } from '@/lib/domain/database-type'; import type { DBTable } from '@/lib/domain/db-table'; diff --git a/src/lib/dbml/dbml-import/__tests__/composite-pk-name.test.ts b/src/lib/dbml/dbml-import/__tests__/composite-pk-name.test.ts index a2fe3e5d..6cc40513 100644 --- a/src/lib/dbml/dbml-import/__tests__/composite-pk-name.test.ts +++ b/src/lib/dbml/dbml-import/__tests__/composite-pk-name.test.ts @@ -1,8 +1,8 @@ import { describe, it, expect } from 'vitest'; import { importDBMLToDiagram } from '../dbml-import'; -import { exportPostgreSQL } from '@/lib/data/export-metadata/export-per-type/postgresql'; -import { exportMySQL } from '@/lib/data/export-metadata/export-per-type/mysql'; -import { exportMSSQL } from '@/lib/data/export-metadata/export-per-type/mssql'; +import { exportPostgreSQL } from '@/lib/data/sql-export/export-per-type/postgresql'; +import { exportMySQL } from '@/lib/data/sql-export/export-per-type/mysql'; +import { exportMSSQL } from '@/lib/data/sql-export/export-per-type/mssql'; import { DatabaseType } from '@/lib/domain/database-type'; describe('Composite Primary Key with Name', () => { diff --git a/src/lib/domain/__tests__/composite-pk-metadata-import.test.ts b/src/lib/domain/__tests__/composite-pk-metadata-import.test.ts index 252ff579..22d1c76a 100644 --- a/src/lib/domain/__tests__/composite-pk-metadata-import.test.ts +++ b/src/lib/domain/__tests__/composite-pk-metadata-import.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from 'vitest'; -import { createTablesFromMetadata } from '../db-table'; +import { createTablesFromMetadata } from '@/lib/data/import-metadata/import/tables'; import { DatabaseType } from '../database-type'; import type { DatabaseMetadata } from '@/lib/data/import-metadata/metadata-types/database-metadata'; diff --git a/src/lib/domain/db-custom-type.ts b/src/lib/domain/db-custom-type.ts index 59a00679..2252d031 100644 --- a/src/lib/domain/db-custom-type.ts +++ b/src/lib/domain/db-custom-type.ts @@ -1,7 +1,4 @@ import { z } from 'zod'; -import type { DBCustomTypeInfo } from '@/lib/data/import-metadata/metadata-types/custom-type-info'; -import { generateId } from '../utils'; -import { schemaNameToDomainSchemaName } from './db-schema'; export enum DBCustomTypeKind { enum = 'enum', @@ -38,23 +35,6 @@ export const dbCustomTypeSchema: z.ZodType = z.object({ order: z.number().or(z.null()).optional(), }); -export const createCustomTypesFromMetadata = ({ - customTypes, -}: { - customTypes: DBCustomTypeInfo[]; -}): DBCustomType[] => { - return customTypes.map((customType) => { - return { - id: generateId(), - schema: schemaNameToDomainSchemaName(customType.schema), - name: customType.type, - kind: customType.kind as DBCustomTypeKind, - values: customType.values, - fields: customType.fields, - }; - }); -}; - export const customTypeKindToLabel: Record = { enum: 'Enum', composite: 'Composite', diff --git a/src/lib/domain/db-dependency.ts b/src/lib/domain/db-dependency.ts index 347194e2..4f9c4bba 100644 --- a/src/lib/domain/db-dependency.ts +++ b/src/lib/domain/db-dependency.ts @@ -1,10 +1,4 @@ import { z } from 'zod'; -import type { ViewInfo } from '../data/import-metadata/metadata-types/view-info'; -import { DatabaseType } from './database-type'; -import { schemaNameToDomainSchemaName } from './db-schema'; -import { decodeViewDefinition, type DBTable } from './db-table'; -import { generateId } from '@/lib/utils'; -import type { AST } from 'node-sql-parser'; export interface DBDependency { id: string; @@ -23,348 +17,3 @@ export const dbDependencySchema: z.ZodType = z.object({ dependentTableId: z.string(), createdAt: z.number(), }); - -const astDatabaseTypes: Record = { - [DatabaseType.POSTGRESQL]: 'postgresql', - [DatabaseType.MYSQL]: 'postgresql', - [DatabaseType.MARIADB]: 'postgresql', - [DatabaseType.GENERIC]: 'postgresql', - [DatabaseType.SQLITE]: 'postgresql', - [DatabaseType.SQL_SERVER]: 'postgresql', - [DatabaseType.CLICKHOUSE]: 'postgresql', - [DatabaseType.COCKROACHDB]: 'postgresql', - [DatabaseType.ORACLE]: 'postgresql', -}; - -export const createDependenciesFromMetadata = async ({ - views, - tables, - databaseType, -}: { - views: ViewInfo[]; - tables: DBTable[]; - databaseType: DatabaseType; -}): Promise => { - if (!views || views.length === 0) { - return []; - } - - const { Parser } = await import('node-sql-parser'); - const parser = new Parser(); - - const dependencies = views - .flatMap((view) => { - const viewSchema = schemaNameToDomainSchemaName(view.schema); - const viewTable = tables.find( - (table) => - table.name === view.view_name && viewSchema === table.schema - ); - - if (!viewTable) { - console.warn( - `Source table for view ${view.view_name} not found (schema: ${viewSchema})` - ); - return []; // Skip this view and proceed to the next - } - - if (view.view_definition) { - try { - const decodedViewDefinition = decodeViewDefinition( - databaseType, - view.view_definition - ); - - let modifiedViewDefinition = ''; - if ( - databaseType === DatabaseType.MYSQL || - databaseType === DatabaseType.MARIADB - ) { - modifiedViewDefinition = preprocessViewDefinitionMySQL( - decodedViewDefinition - ); - } else if (databaseType === DatabaseType.SQL_SERVER) { - modifiedViewDefinition = - preprocessViewDefinitionSQLServer( - decodedViewDefinition - ); - } else { - modifiedViewDefinition = preprocessViewDefinition( - decodedViewDefinition - ); - } - - // Parse using the appropriate dialect - const ast = parser.astify(modifiedViewDefinition, { - database: astDatabaseTypes[databaseType], - type: 'select', // Parsing a SELECT statement - }); - - let relatedTables = extractTablesFromAST(ast); - - // Filter out duplicate tables without schema - relatedTables = filterDuplicateTables(relatedTables); - - return relatedTables.map((relTable) => { - const relSchema = relTable.schema || view.schema; // Use view's schema if relSchema is undefined - const relTableName = relTable.tableName; - - const table = tables.find( - (table) => - table.name === relTableName && - (table.schema || '') === relSchema - ); - - if (table) { - const dependency: DBDependency = { - id: generateId(), - schema: view.schema, - tableId: table.id, // related table - dependentSchema: table.schema, - dependentTableId: viewTable.id, // dependent view - createdAt: Date.now(), - }; - - return dependency; - } else { - console.warn( - `Dependent table ${relSchema}.${relTableName} not found for view ${view.schema}.${view.view_name}` - ); - return null; - } - }); - } catch (error) { - console.error( - `Error parsing view ${view.schema}.${view.view_name}:`, - error - ); - return []; - } - } else { - console.warn( - `View definition missing for ${view.schema}.${view.view_name}` - ); - return []; - } - }) - .filter((dependency) => dependency !== null); - - return dependencies; -}; - -// Add this new function to filter out duplicate tables -function filterDuplicateTables( - tables: { schema?: string; tableName: string }[] -): { schema?: string; tableName: string }[] { - const tableMap = new Map(); - - for (const table of tables) { - const key = table.tableName; - const existingTable = tableMap.get(key); - - if (!existingTable || (table.schema && !existingTable.schema)) { - tableMap.set(key, table); - } - } - - return Array.from(tableMap.values()); -} - -// Preprocess the view_definition to remove schema from CREATE VIEW -function preprocessViewDefinition(viewDefinition: string): string { - if (!viewDefinition) { - return ''; - } - - // Remove leading and trailing whitespace - viewDefinition = viewDefinition.replace(/\s+/g, ' ').trim(); - - // Replace escaped double quotes with regular ones - viewDefinition = viewDefinition.replace(/\\"/g, '"'); - - // Replace 'CREATE MATERIALIZED VIEW' with 'CREATE VIEW' - viewDefinition = viewDefinition.replace( - /CREATE\s+MATERIALIZED\s+VIEW/i, - 'CREATE VIEW' - ); - - // Regular expression to match 'CREATE VIEW [schema.]view_name [ (column definitions) ] AS' - // This regex captures the view name and skips any content between the view name and 'AS' - const regex = - /CREATE\s+VIEW\s+(?:(?:`[^`]+`|"[^"]+"|\w+)\.)?(?:`([^`]+)`|"([^"]+)"|(\w+))[\s\S]*?\bAS\b\s+/i; - - const match = viewDefinition.match(regex); - let modifiedDefinition: string; - - if (match) { - const viewName = match[1] || match[2] || match[3]; - // Extract the SQL after the 'AS' keyword - const restOfDefinition = viewDefinition.substring( - match.index! + match[0].length - ); - - // Replace double-quoted identifiers with unquoted ones - let modifiedSQL = restOfDefinition.replace(/"(\w+)"/g, '$1'); - - // Replace '::' type casts with 'CAST' expressions - modifiedSQL = modifiedSQL.replace( - /\(([^()]+)\)::(\w+)/g, - 'CAST($1 AS $2)' - ); - - // Remove ClickHouse-specific syntax that may still be present - // For example, remove SETTINGS clauses inside the SELECT statement - modifiedSQL = modifiedSQL.replace(/\bSETTINGS\b[\s\S]*$/i, ''); - - modifiedDefinition = `CREATE VIEW ${viewName} AS ${modifiedSQL}`; - } else { - console.warn('Could not preprocess view definition:', viewDefinition); - modifiedDefinition = viewDefinition; - } - - return modifiedDefinition; -} - -// Preprocess the view_definition for SQL Server -function preprocessViewDefinitionSQLServer(viewDefinition: string): string { - if (!viewDefinition) { - return ''; - } - - // Remove BOM if present - viewDefinition = viewDefinition.replace(/^\uFEFF/, ''); - - // Normalize whitespace - viewDefinition = viewDefinition.replace(/\s+/g, ' ').trim(); - - // Remove square brackets and replace with double quotes - viewDefinition = viewDefinition.replace(/\[([^\]]+)\]/g, '"$1"'); - - // Remove database names from fully qualified identifiers - viewDefinition = viewDefinition.replace( - /"([a-zA-Z0-9_]+)"\."([a-zA-Z0-9_]+)"\."([a-zA-Z0-9_]+)"/g, - '"$2"."$3"' - ); - - // Replace SQL Server functions with PostgreSQL equivalents - viewDefinition = viewDefinition.replace(/\bGETDATE\(\)/gi, 'NOW()'); - viewDefinition = viewDefinition.replace(/\bISNULL\(/gi, 'COALESCE('); - - // Replace 'TOP N' with 'LIMIT N' at the end of the query - const topMatch = viewDefinition.match(/SELECT\s+TOP\s+(\d+)/i); - if (topMatch) { - const topN = topMatch[1]; - viewDefinition = viewDefinition.replace( - /SELECT\s+TOP\s+\d+/i, - 'SELECT' - ); - viewDefinition = viewDefinition.replace(/;+\s*$/, ''); // Remove semicolons at the end - viewDefinition += ` LIMIT ${topN}`; - } - - viewDefinition = viewDefinition.replace(/\n/g, ''); // Remove newlines - - // Adjust CREATE VIEW syntax - const regex = - /CREATE\s+VIEW\s+(?:"?([^".\s]+)"?\.)?"?([^".\s]+)"?\s+AS\s+/i; - const match = viewDefinition.match(regex); - let modifiedDefinition: string; - - if (match) { - const viewName = match[2]; - const modifiedSQL = viewDefinition.substring( - match.index! + match[0].length - ); - - // Remove semicolons at the end - const finalSQL = modifiedSQL.replace(/;+\s*$/, ''); - - modifiedDefinition = `CREATE VIEW "${viewName}" AS ${finalSQL}`; - } else { - console.warn('Could not preprocess view definition:', viewDefinition); - modifiedDefinition = viewDefinition; - } - - return modifiedDefinition; -} - -// Preprocess the view_definition to remove schema from CREATE VIEW -function preprocessViewDefinitionMySQL(viewDefinition: string): string { - if (!viewDefinition) { - return ''; - } - - // Remove any trailing semicolons - viewDefinition = viewDefinition.replace(/;\s*$/, ''); - - // Remove backticks from identifiers - viewDefinition = viewDefinition.replace(/`/g, ''); - - // Remove unnecessary parentheses around joins and ON clauses - viewDefinition = removeRedundantParentheses(viewDefinition); - - return viewDefinition; -} - -function removeRedundantParentheses(sql: string): string { - // Regular expressions to match unnecessary parentheses - const patterns = [ - /\(\s*(JOIN\s+[^()]+?)\s*\)/gi, - /\(\s*(ON\s+[^()]+?)\s*\)/gi, - // Additional patterns if necessary - ]; - - let prevSql; - do { - prevSql = sql; - patterns.forEach((pattern) => { - sql = sql.replace(pattern, '$1'); - }); - } while (sql !== prevSql); - - return sql; -} - -function extractTablesFromAST( - ast: AST | AST[] -): { schema?: string; tableName: string }[] { - const tablesMap = new Map(); - const visitedNodes = new Set(); - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - function traverse(node: any) { - if (!node || visitedNodes.has(node)) return; - visitedNodes.add(node); - - if (Array.isArray(node)) { - node.forEach(traverse); - } else if (typeof node === 'object') { - // Check if node represents a table - if ( - Object.hasOwnProperty.call(node, 'table') && - typeof node.table === 'string' - ) { - let schema = node.db || node.schema; - const tableName = node.table; - if (tableName) { - // Assign default schema if undefined - schema = schemaNameToDomainSchemaName(schema) || ''; - const key = `${schema}.${tableName}`; - if (!tablesMap.has(key)) { - tablesMap.set(key, { schema, tableName }); - } - } - } - - // Recursively traverse all properties - for (const key in node) { - if (Object.hasOwnProperty.call(node, key)) { - traverse(node[key]); - } - } - } - } - - traverse(ast); - - return Array.from(tablesMap.values()); -} diff --git a/src/lib/domain/db-field.ts b/src/lib/domain/db-field.ts index dfc1a032..042530e3 100644 --- a/src/lib/domain/db-field.ts +++ b/src/lib/domain/db-field.ts @@ -4,11 +4,6 @@ import { findDataTypeDataById, type DataType, } from '../data/data-types/data-types'; -import type { ColumnInfo } from '../data/import-metadata/metadata-types/column-info'; -import type { AggregatedIndexInfo } from '../data/import-metadata/metadata-types/index-info'; -import type { PrimaryKeyInfo } from '../data/import-metadata/metadata-types/primary-key-info'; -import type { TableInfo } from '../data/import-metadata/metadata-types/table-info'; -import { generateId } from '../utils'; import type { DatabaseType } from './database-type'; export interface DBField { @@ -45,64 +40,6 @@ export const dbFieldSchema: z.ZodType = z.object({ comments: z.string().or(z.null()).optional(), }); -export const createFieldsFromMetadata = ({ - tableColumns, - tablePrimaryKeys, - aggregatedIndexes, -}: { - tableColumns: ColumnInfo[]; - tableSchema?: string; - tableInfo: TableInfo; - tablePrimaryKeys: PrimaryKeyInfo[]; - aggregatedIndexes: AggregatedIndexInfo[]; -}) => { - const uniqueColumns = tableColumns.reduce((acc, col) => { - if (!acc.has(col.name)) { - acc.set(col.name, col); - } - return acc; - }, new Map()); - - const sortedColumns = Array.from(uniqueColumns.values()).sort( - (a, b) => a.ordinal_position - b.ordinal_position - ); - - const tablePrimaryKeysColumns = tablePrimaryKeys.map((pk) => - pk.column.trim() - ); - - return sortedColumns.map( - (col: ColumnInfo): DBField => ({ - id: generateId(), - name: col.name, - type: { - id: col.type.split(' ').join('_').toLowerCase(), - name: col.type.toLowerCase(), - }, - primaryKey: tablePrimaryKeysColumns.includes(col.name), - unique: Object.values(aggregatedIndexes).some( - (idx) => - idx.unique && - idx.columns.length === 1 && - idx.columns[0].name === col.name - ), - nullable: Boolean(col.nullable), - ...(col.character_maximum_length && - col.character_maximum_length !== 'null' - ? { characterMaximumLength: col.character_maximum_length } - : {}), - ...(col.precision?.precision - ? { precision: col.precision.precision } - : {}), - ...(col.precision?.scale ? { scale: col.precision.scale } : {}), - ...(col.default ? { default: col.default } : {}), - ...(col.collation ? { collation: col.collation } : {}), - createdAt: Date.now(), - comments: col.comment ? col.comment : undefined, - }) - ); -}; - export const generateDBFieldSuffix = ( field: DBField, { diff --git a/src/lib/domain/db-index.ts b/src/lib/domain/db-index.ts index b9a8c900..b9668daf 100644 --- a/src/lib/domain/db-index.ts +++ b/src/lib/domain/db-index.ts @@ -1,7 +1,5 @@ import { z } from 'zod'; -import type { AggregatedIndexInfo } from '../data/import-metadata/metadata-types/index-info'; import { generateId } from '../utils'; -import type { DBField } from './db-field'; import { DatabaseType } from './database-type'; import type { DBTable } from './db-table'; @@ -43,27 +41,6 @@ export const dbIndexSchema: z.ZodType = z.object({ isPrimaryKey: z.boolean().or(z.null()).optional(), }); -export const createIndexesFromMetadata = ({ - aggregatedIndexes, - fields, -}: { - aggregatedIndexes: AggregatedIndexInfo[]; - fields: DBField[]; -}): DBIndex[] => - aggregatedIndexes.map( - (idx): DBIndex => ({ - id: generateId(), - name: idx.name, - unique: Boolean(idx.unique), - fieldIds: idx.columns - .sort((a, b) => a.position - b.position) - .map((c) => fields.find((f) => f.name === c.name)?.id) - .filter((id): id is string => id !== undefined), - createdAt: Date.now(), - type: idx.index_type?.toLowerCase() as IndexType, - }) - ); - export const databaseIndexTypes: { [key in DatabaseType]?: IndexType[] } = { [DatabaseType.POSTGRESQL]: ['btree', 'hash'], }; diff --git a/src/lib/domain/db-relationship.ts b/src/lib/domain/db-relationship.ts index fd5debd2..c470a002 100644 --- a/src/lib/domain/db-relationship.ts +++ b/src/lib/domain/db-relationship.ts @@ -1,9 +1,4 @@ import { z } from 'zod'; -import type { ForeignKeyInfo } from '../data/import-metadata/metadata-types/foreign-key-info'; -import type { DBField } from './db-field'; -import { schemaNameToDomainSchemaName } from './db-schema'; -import type { DBTable } from './db-table'; -import { generateId } from '@/lib/utils'; export interface DBRelationship { id: string; @@ -40,82 +35,6 @@ export type RelationshipType = | 'many_to_many'; export type Cardinality = 'one' | 'many'; -const determineCardinality = ( - field: DBField, - isTablePKComplex: boolean -): Cardinality => { - return field.unique || (field.primaryKey && !isTablePKComplex) - ? 'one' - : 'many'; -}; - -export const createRelationshipsFromMetadata = ({ - foreignKeys, - tables, -}: { - foreignKeys: ForeignKeyInfo[]; - tables: DBTable[]; -}): DBRelationship[] => { - return foreignKeys - .map((fk: ForeignKeyInfo): DBRelationship | null => { - const schema = schemaNameToDomainSchemaName(fk.schema); - const sourceTable = tables.find( - (table) => table.name === fk.table && table.schema === schema - ); - - const targetSchema = schemaNameToDomainSchemaName( - fk.reference_schema - ); - - const targetTable = tables.find( - (table) => - table.name === fk.reference_table && - table.schema === targetSchema - ); - const sourceField = sourceTable?.fields.find( - (field) => field.name === fk.column - ); - const targetField = targetTable?.fields.find( - (field) => field.name === fk.reference_column - ); - - const isSourceTablePKComplex = - (sourceTable?.fields.filter((field) => field.primaryKey) ?? []) - .length > 1; - const isTargetTablePKComplex = - (targetTable?.fields.filter((field) => field.primaryKey) ?? []) - .length > 1; - - if (sourceTable && targetTable && sourceField && targetField) { - const sourceCardinality = determineCardinality( - sourceField, - isSourceTablePKComplex - ); - const targetCardinality = determineCardinality( - targetField, - isTargetTablePKComplex - ); - - return { - id: generateId(), - name: fk.foreign_key_name, - sourceSchema: schema, - targetSchema: targetSchema, - sourceTableId: sourceTable.id, - targetTableId: targetTable.id, - sourceFieldId: sourceField.id, - targetFieldId: targetField.id, - sourceCardinality, - targetCardinality, - createdAt: Date.now(), - }; - } - - return null; - }) - .filter((rel) => rel !== null) as DBRelationship[]; -}; - export const determineRelationshipType = ({ sourceCardinality, targetCardinality, diff --git a/src/lib/domain/db-table.ts b/src/lib/domain/db-table.ts index 4879f469..87f6e62f 100644 --- a/src/lib/domain/db-table.ts +++ b/src/lib/domain/db-table.ts @@ -1,30 +1,8 @@ -import { - createIndexesFromMetadata, - dbIndexSchema, - type DBIndex, -} from './db-index'; -import { - createFieldsFromMetadata, - dbFieldSchema, - type DBField, -} from './db-field'; -import type { TableInfo } from '../data/import-metadata/metadata-types/table-info'; -import { createAggregatedIndexes } from '../data/import-metadata/metadata-types/index-info'; -import { - materializedViewColor, - viewColor, - defaultTableColor, -} from '@/lib/colors'; +import { dbIndexSchema, type DBIndex } from './db-index'; +import { dbFieldSchema, type DBField } from './db-field'; import type { DBRelationship } from './db-relationship'; -import { - decodeBase64ToUtf16LE, - decodeBase64ToUtf8, - deepCopy, - generateId, -} from '../utils'; +import { deepCopy } from '../utils'; import { schemaNameToDomainSchemaName } from './db-schema'; -import { DatabaseType } from './database-type'; -import type { DatabaseMetadata } from '../data/import-metadata/metadata-types/database-metadata'; import { z } from 'zod'; import type { Area } from './area'; @@ -79,213 +57,6 @@ export const generateTableKey = ({ tableName: string; }) => `${schemaNameToDomainSchemaName(schemaName) ?? ''}.${tableName}`; -export const decodeViewDefinition = ( - databaseType: DatabaseType, - viewDefinition?: string -): string => { - if (!viewDefinition) { - return ''; - } - - let decodedViewDefinition: string; - if (databaseType === DatabaseType.SQL_SERVER) { - decodedViewDefinition = decodeBase64ToUtf16LE(viewDefinition); - } else { - decodedViewDefinition = decodeBase64ToUtf8(viewDefinition); - } - - return decodedViewDefinition; -}; - -export const createTablesFromMetadata = ({ - databaseMetadata, - databaseType, -}: { - databaseMetadata: DatabaseMetadata; - databaseType: DatabaseType; -}): DBTable[] => { - const { - tables: tableInfos, - pk_info: primaryKeys, - columns, - indexes, - views: views, - } = databaseMetadata; - - // Pre-compute view names for faster lookup if there are views - const viewNamesSet = new Set(); - const materializedViewNamesSet = new Set(); - - if (views && views.length > 0) { - views.forEach((view) => { - const key = generateTableKey({ - schemaName: view.schema, - tableName: view.view_name, - }); - viewNamesSet.add(key); - - if ( - view.view_definition && - decodeViewDefinition(databaseType, view.view_definition) - .toLowerCase() - .includes('materialized') - ) { - materializedViewNamesSet.add(key); - } - }); - } - - // Pre-compute lookup maps for better performance - const columnsByTable = new Map(); - const indexesByTable = new Map(); - const primaryKeysByTable = new Map(); - - // Group columns by table - columns.forEach((col) => { - const key = generateTableKey({ - schemaName: col.schema, - tableName: col.table, - }); - if (!columnsByTable.has(key)) { - columnsByTable.set(key, []); - } - columnsByTable.get(key)!.push(col); - }); - - // Group indexes by table - indexes.forEach((idx) => { - const key = generateTableKey({ - schemaName: idx.schema, - tableName: idx.table, - }); - if (!indexesByTable.has(key)) { - indexesByTable.set(key, []); - } - indexesByTable.get(key)!.push(idx); - }); - - // Group primary keys by table - primaryKeys.forEach((pk) => { - const key = generateTableKey({ - schemaName: pk.schema, - tableName: pk.table, - }); - if (!primaryKeysByTable.has(key)) { - primaryKeysByTable.set(key, []); - } - primaryKeysByTable.get(key)!.push(pk); - }); - - const result = tableInfos.map((tableInfo: TableInfo) => { - const tableSchema = schemaNameToDomainSchemaName(tableInfo.schema); - const tableKey = generateTableKey({ - schemaName: tableInfo.schema, - tableName: tableInfo.table, - }); - - // Use pre-computed lookups instead of filtering entire arrays - const tableIndexes = indexesByTable.get(tableKey) || []; - const tablePrimaryKeys = primaryKeysByTable.get(tableKey) || []; - const tableColumns = columnsByTable.get(tableKey) || []; - - // Aggregate indexes with multiple columns - const aggregatedIndexes = createAggregatedIndexes({ - tableInfo, - tableSchema, - tableIndexes, - }); - - const fields = createFieldsFromMetadata({ - aggregatedIndexes, - tableColumns, - tablePrimaryKeys, - tableInfo, - tableSchema, - }); - - // Check for composite primary key and find matching index name - const primaryKeyFields = fields.filter((f) => f.primaryKey); - let pkMatchingIndexName: string | undefined; - let pkIndex: DBIndex | undefined; - - if (primaryKeyFields.length >= 1) { - // We have a composite primary key, look for an index that matches all PK columns - const pkFieldNames = primaryKeyFields.map((f) => f.name).sort(); - - // Find an index that matches the primary key columns exactly - const matchingIndex = aggregatedIndexes.find((index) => { - const indexColumnNames = index.columns - .map((c) => c.name) - .sort(); - return ( - indexColumnNames.length === pkFieldNames.length && - indexColumnNames.every((col, i) => col === pkFieldNames[i]) - ); - }); - - if (matchingIndex) { - pkMatchingIndexName = matchingIndex.name; - // Create a special PK index - pkIndex = { - id: generateId(), - name: matchingIndex.name, - unique: true, - fieldIds: primaryKeyFields.map((f) => f.id), - createdAt: Date.now(), - isPrimaryKey: true, - }; - } - } - - // Filter out the index that matches the composite PK (to avoid duplication) - const filteredAggregatedIndexes = pkMatchingIndexName - ? aggregatedIndexes.filter( - (idx) => idx.name !== pkMatchingIndexName - ) - : aggregatedIndexes; - - const dbIndexes = createIndexesFromMetadata({ - aggregatedIndexes: filteredAggregatedIndexes, - fields, - }); - - // Add the PK index if it exists - if (pkIndex) { - dbIndexes.push(pkIndex); - } - - // Determine if the current table is a view by checking against pre-computed sets - const viewKey = generateTableKey({ - schemaName: tableSchema, - tableName: tableInfo.table, - }); - const isView = viewNamesSet.has(viewKey); - const isMaterializedView = materializedViewNamesSet.has(viewKey); - - // Initial random positions; these will be adjusted later - return { - id: generateId(), - name: tableInfo.table, - schema: tableSchema, - x: Math.random() * 1000, // Placeholder X - y: Math.random() * 800, // Placeholder Y - fields, - indexes: dbIndexes, - color: isMaterializedView - ? materializedViewColor - : isView - ? viewColor - : defaultTableColor, - isView: isView, - isMaterializedView: isMaterializedView, - createdAt: Date.now(), - comments: tableInfo.comment ? tableInfo.comment : undefined, - }; - }); - - return result; -}; - export const adjustTablePositions = ({ relationships: inputRelationships, tables: inputTables, diff --git a/src/lib/domain/diagram.ts b/src/lib/domain/diagram.ts index ed171e72..a0548e41 100644 --- a/src/lib/domain/diagram.ts +++ b/src/lib/domain/diagram.ts @@ -1,30 +1,15 @@ import { z } from 'zod'; -import type { DatabaseMetadata } from '../data/import-metadata/metadata-types/database-metadata'; import { DatabaseEdition } from './database-edition'; import { DatabaseType } from './database-type'; import type { DBDependency } from './db-dependency'; -import { - createDependenciesFromMetadata, - dbDependencySchema, -} from './db-dependency'; +import { dbDependencySchema } from './db-dependency'; import type { DBRelationship } from './db-relationship'; -import { - createRelationshipsFromMetadata, - dbRelationshipSchema, -} from './db-relationship'; +import { dbRelationshipSchema } from './db-relationship'; import type { DBTable } from './db-table'; -import { - adjustTablePositions, - createTablesFromMetadata, - dbTableSchema, -} from './db-table'; -import { generateDiagramId } from '@/lib/utils'; +import { dbTableSchema } from './db-table'; import { areaSchema, type Area } from './area'; import type { DBCustomType } from './db-custom-type'; -import { - dbCustomTypeSchema, - createCustomTypesFromMetadata, -} from './db-custom-type'; +import { dbCustomTypeSchema } from './db-custom-type'; export interface Diagram { id: string; @@ -53,77 +38,3 @@ export const diagramSchema: z.ZodType = z.object({ createdAt: z.date(), updatedAt: z.date(), }); - -export const loadFromDatabaseMetadata = async ({ - databaseType, - databaseMetadata, - diagramNumber, - databaseEdition, -}: { - databaseType: DatabaseType; - databaseMetadata: DatabaseMetadata; - diagramNumber?: number; - databaseEdition?: DatabaseEdition; -}): Promise => { - const { - fk_info: foreignKeys, - views: views, - custom_types: customTypes, - } = databaseMetadata; - - const tables = createTablesFromMetadata({ - databaseMetadata, - databaseType, - }); - - const relationships = createRelationshipsFromMetadata({ - foreignKeys, - tables, - }); - - const dependencies = await createDependenciesFromMetadata({ - views, - tables, - databaseType, - }); - - const dbCustomTypes = customTypes - ? createCustomTypesFromMetadata({ - customTypes, - }) - : []; - - const adjustedTables = adjustTablePositions({ - tables, - relationships, - mode: 'perSchema', - }); - - const sortedTables = adjustedTables.sort((a, b) => { - if (a.isView === b.isView) { - // Both are either tables or views, so sort alphabetically by name - return a.name.localeCompare(b.name); - } - // If one is a view and the other is not, put tables first - return a.isView ? 1 : -1; - }); - - const diagram: Diagram = { - id: generateDiagramId(), - name: databaseMetadata.database_name - ? `${databaseMetadata.database_name}-db` - : diagramNumber - ? `Diagram ${diagramNumber}` - : 'New Diagram', - databaseType: databaseType ?? DatabaseType.GENERIC, - databaseEdition, - tables: sortedTables, - relationships, - dependencies, - customTypes: dbCustomTypes, - createdAt: new Date(), - updatedAt: new Date(), - }; - - return diagram; -};