From 0d11b0c55a94a12a764785cfdcf2ba10437241d6 Mon Sep 17 00:00:00 2001 From: Jonathan Fishner Date: Wed, 28 May 2025 10:10:33 +0300 Subject: [PATCH] fix(import-database): remove the default fetch from import database (#718) * fix(import-database): remove the default fetch from import database * fix(import-default): keep the option to fetch extras like default and comments --- .../import-metadata/scripts/clickhouse-script.ts | 4 ++-- .../import-metadata/scripts/cockroachdb-script.ts | 2 +- src/lib/data/import-metadata/scripts/maria-script.ts | 6 +++++- src/lib/data/import-metadata/scripts/mysql-script.ts | 9 +++++++-- .../data/import-metadata/scripts/oracle-script.ts | 2 +- .../data/import-metadata/scripts/postgres-script.ts | 12 ++++++++++-- .../data/import-metadata/scripts/sqlite-script.ts | 9 +++++++-- .../data/import-metadata/scripts/sqlserver-script.ts | 11 +++++++---- 8 files changed, 40 insertions(+), 15 deletions(-) diff --git a/src/lib/data/import-metadata/scripts/clickhouse-script.ts b/src/lib/data/import-metadata/scripts/clickhouse-script.ts index 26478edd..b4742299 100644 --- a/src/lib/data/import-metadata/scripts/clickhouse-script.ts +++ b/src/lib/data/import-metadata/scripts/clickhouse-script.ts @@ -7,7 +7,7 @@ cols AS ( ',"ordinal_position":', toString(col_tuple.4), ',"type":"', col_tuple.5, '"', ',"nullable":"', if(col_tuple.6 = 'NULLABLE', 'true', 'false'), '"', - ',"default":"', if(col_tuple.7 = '', 'null', col_tuple.7), '"', + ',"default":"', col_tuple.7, '"', ',"comment":', if(col_tuple.8 = '', '""', toString(toJSONString(col_tuple.8))), '}'), groupArray(( col.database, @@ -15,7 +15,7 @@ cols AS ( col.name, col.position, col.type, - col.default_kind, + null as default_kind, col.default_expression, col.comment )) diff --git a/src/lib/data/import-metadata/scripts/cockroachdb-script.ts b/src/lib/data/import-metadata/scripts/cockroachdb-script.ts index 55135376..a220dc6b 100644 --- a/src/lib/data/import-metadata/scripts/cockroachdb-script.ts +++ b/src/lib/data/import-metadata/scripts/cockroachdb-script.ts @@ -124,7 +124,7 @@ cols AS ( ELSE 'null' END, ',"nullable":', CASE WHEN (cols.IS_NULLABLE = 'YES') THEN true ELSE false END::TEXT, - ',"default":"', COALESCE(replace(replace(cols.column_default::TEXT, '"', '\\"'), '\\x', '\\\\x'), ''), + ',"default":"', null, '","collation":"', COALESCE(cols.COLLATION_NAME::TEXT, ''), '","comment":"', COALESCE(replace(replace(dsc.description::TEXT, '"', '\\"'), '\\x', '\\\\x'), ''), '"}')), ',') AS cols_metadata diff --git a/src/lib/data/import-metadata/scripts/maria-script.ts b/src/lib/data/import-metadata/scripts/maria-script.ts index c928f8bf..58e04c6e 100644 --- a/src/lib/data/import-metadata/scripts/maria-script.ts +++ b/src/lib/data/import-metadata/scripts/maria-script.ts @@ -1,3 +1,7 @@ +const withExtras = false; +const withDefault = `IFNULL(REPLACE(REPLACE(cols.column_default, '\\\\', ''), '"', 'ֿֿֿ\\"'), '')`; +const withoutDefault = `""`; + export const mariaDBQuery = `WITH fk_info as ( (SELECT (@fk_info:=NULL), (SELECT (0) @@ -76,7 +80,7 @@ export const mariaDBQuery = `WITH fk_info as ( END, ',"ordinal_position":', cols.ordinal_position, ',"nullable":', IF(cols.is_nullable = 'YES', 'true', 'false'), - ',"default":"', IFNULL(REPLACE(REPLACE(cols.column_default, '\\\\', ''), '"', '\\"'), ''), + ',"default":"', ${withExtras ? withDefault : withoutDefault}, '","collation":"', IFNULL(cols.collation_name, ''), '"}' ))))) ), indexes as ( diff --git a/src/lib/data/import-metadata/scripts/mysql-script.ts b/src/lib/data/import-metadata/scripts/mysql-script.ts index 57d0b908..1967f26f 100644 --- a/src/lib/data/import-metadata/scripts/mysql-script.ts +++ b/src/lib/data/import-metadata/scripts/mysql-script.ts @@ -8,6 +8,11 @@ export const getMySQLQuery = ( const databaseEdition: DatabaseEdition | undefined = options.databaseEdition; + const withExtras = false; + + const withDefault = `IFNULL(REPLACE(REPLACE(cols.column_default, '\\\\', ''), '"', 'ֿֿֿ\\"'), '')`; + const withoutDefault = `""`; + const newMySQLQuery = `WITH fk_info as ( (SELECT (@fk_info:=NULL), (SELECT (0) @@ -86,7 +91,7 @@ export const getMySQLQuery = ( END, ',"ordinal_position":', cols.ordinal_position, ',"nullable":', IF(cols.is_nullable = 'YES', 'true', 'false'), - ',"default":"', IFNULL(REPLACE(REPLACE(cols.column_default, '\\\\', ''), '"', 'ֿֿֿ\\"'), ''), + ',"default":"', ${withExtras ? withDefault : withoutDefault}, '","collation":"', IFNULL(cols.collation_name, ''), '"}' ))))) ), indexes as ( @@ -211,7 +216,7 @@ export const getMySQLQuery = ( ',"scale":', IFNULL(cols.numeric_scale, 'null'), '}'), 'null'), ',"ordinal_position":', cols.ordinal_position, ',"nullable":', IF(cols.is_nullable = 'YES', 'true', 'false'), - ',"default":"', IFNULL(REPLACE(REPLACE(cols.column_default, '\\\\', ''), '"', '\\"'), ''), + ',"default":"', ${withExtras ? withDefault : withoutDefault}, '","collation":"', IFNULL(cols.collation_name, ''), '"}') ) FROM ( SELECT cols.table_schema, diff --git a/src/lib/data/import-metadata/scripts/oracle-script.ts b/src/lib/data/import-metadata/scripts/oracle-script.ts index 4cd557e1..2f3a7fc5 100644 --- a/src/lib/data/import-metadata/scripts/oracle-script.ts +++ b/src/lib/data/import-metadata/scripts/oracle-script.ts @@ -80,7 +80,7 @@ WITH fk_info AS ( KEY 'ordinal_position' VALUE column_id, KEY 'nullable' VALUE CASE nullable WHEN 'Y' THEN 'true' ELSE 'false' END FORMAT JSON, - KEY 'default' VALUE NULL, + KEY 'default' VALUE '""' FORMAT JSON, KEY 'collation' VALUE '""' FORMAT JSON RETURNING CLOB ) AS json_data diff --git a/src/lib/data/import-metadata/scripts/postgres-script.ts b/src/lib/data/import-metadata/scripts/postgres-script.ts index c57ceded..645a1a95 100644 --- a/src/lib/data/import-metadata/scripts/postgres-script.ts +++ b/src/lib/data/import-metadata/scripts/postgres-script.ts @@ -55,6 +55,14 @@ export const getPostgresQuery = ( AND views.schemaname !~ '^(timescaledb_|_timescaledb_)' `; + const withExtras = false; + + const withDefault = `COALESCE(replace(replace(cols.column_default, '"', '\\"'), '\\x', '\\\\x'), '')`; + const withoutDefault = `null`; + + const withComments = `COALESCE(replace(replace(dsc.description, '"', '\\"'), '\\x', '\\\\x'), '')`; + const withoutComments = `null`; + // Define the base query const query = `${`/* ${databaseEdition ? databaseEditionToLabelMap[databaseEdition] : 'PostgreSQL'} edition */`} WITH fk_info${databaseEdition ? '_' + databaseEdition : ''} AS ( @@ -175,9 +183,9 @@ cols AS ( ELSE 'null' END, ',"nullable":', CASE WHEN (cols.IS_NULLABLE = 'YES') THEN 'true' ELSE 'false' END, - ',"default":"', COALESCE(replace(replace(cols.column_default, '"', '\\"'), '\\x', '\\\\x'), ''), + ',"default":"', ${withExtras ? withDefault : withoutDefault}, '","collation":"', COALESCE(cols.COLLATION_NAME, ''), - '","comment":"', COALESCE(replace(replace(dsc.description, '"', '\\"'), '\\x', '\\\\x'), ''), + '","comment":"', ${withExtras ? withComments : withoutComments}, '"}')), ',') AS cols_metadata FROM information_schema.columns cols LEFT JOIN pg_catalog.pg_class c diff --git a/src/lib/data/import-metadata/scripts/sqlite-script.ts b/src/lib/data/import-metadata/scripts/sqlite-script.ts index 3936e7b0..3ece3a73 100644 --- a/src/lib/data/import-metadata/scripts/sqlite-script.ts +++ b/src/lib/data/import-metadata/scripts/sqlite-script.ts @@ -1,6 +1,11 @@ import { DatabaseEdition } from '@/lib/domain/database-edition'; import { DatabaseClient } from '@/lib/domain/database-clients'; +const withExtras = true; + +const withDefault = `COALESCE(REPLACE(p.dflt_value, '"', '\\"'), '')`; +const withoutDefault = `null`; + const sqliteQuery = `${`/* Standard SQLite */`} WITH fk_info AS ( SELECT @@ -114,7 +119,7 @@ WITH fk_info AS ( END ELSE null END, - 'default', COALESCE(REPLACE(p.dflt_value, '"', '\\"'), '') + 'default', ${withExtras ? withDefault : withoutDefault} ) ) AS cols_metadata FROM @@ -287,7 +292,7 @@ WITH fk_info AS ( END ELSE null END, - 'default', COALESCE(REPLACE(p.dflt_value, '"', '\\"'), '') + 'default', ${withExtras ? withDefault : withoutDefault} ) ) AS cols_metadata FROM diff --git a/src/lib/data/import-metadata/scripts/sqlserver-script.ts b/src/lib/data/import-metadata/scripts/sqlserver-script.ts index fe26e615..b6a6ee04 100644 --- a/src/lib/data/import-metadata/scripts/sqlserver-script.ts +++ b/src/lib/data/import-metadata/scripts/sqlserver-script.ts @@ -1,5 +1,10 @@ import { DatabaseEdition } from '@/lib/domain/database-edition'; +const withExtras = false; + +const withDefault = `'"' + STRING_ESCAPE(COALESCE(REPLACE(CAST(cols.COLUMN_DEFAULT AS NVARCHAR(MAX)), '"', '\\"'), ''), 'json') + '"'`; +const withoutDefault = `'""'`; + const sqlServerQuery = `${`/* SQL Server 2017 and above edition (14.0, 15.0, 16.0, 17.0)*/`} WITH fk_info AS ( SELECT @@ -81,8 +86,7 @@ cols AS ( ELSE 'null' END + ', "nullable": ' + CASE WHEN cols.IS_NULLABLE = 'YES' THEN 'true' ELSE 'false' END + - ', "default": ' + - '"' + STRING_ESCAPE(COALESCE(REPLACE(CAST(cols.COLUMN_DEFAULT AS NVARCHAR(MAX)), '"', '\\"'), ''), 'json') + '"' + + ', "default": ' + ${withExtras ? withDefault : withoutDefault} + ', "collation": ' + CASE WHEN cols.COLLATION_NAME IS NULL THEN 'null' ELSE '"' + STRING_ESCAPE(cols.COLLATION_NAME, 'json') + '"' @@ -275,8 +279,7 @@ cols AS ( ELSE 'null' END + ', "nullable": ' + CASE WHEN cols.IS_NULLABLE = 'YES' THEN 'true' ELSE 'false' END + - ', "default": ' + - '"' + STRING_ESCAPE(COALESCE(REPLACE(CAST(cols.COLUMN_DEFAULT AS NVARCHAR(MAX)), '"', '\\"'), ''), 'json') + '"' + + ', "default": ' + ${withExtras ? withDefault : withoutDefault} + ', "collation": ' + CASE WHEN cols.COLLATION_NAME IS NULL THEN 'null'