fix(sql-import): handle SQL Server DDL with multiple tables, inline foreign keys, and case-insensitive field matching (#897)

This commit is contained in:
Jonathan Fishner
2025-09-02 15:15:15 +03:00
committed by GitHub
parent e5e1d59327
commit 2a64deebb8
6 changed files with 514 additions and 4 deletions

View File

@@ -779,10 +779,10 @@ export function convertToChartDBDiagram(
}
const sourceField = sourceTable.fields.find(
(f) => f.name === rel.sourceColumn
(f) => f.name.toLowerCase() === rel.sourceColumn.toLowerCase()
);
const targetField = targetTable.fields.find(
(f) => f.name === rel.targetColumn
(f) => f.name.toLowerCase() === rel.targetColumn.toLowerCase()
);
if (!sourceField || !targetField) {

View File

@@ -0,0 +1,91 @@
import { describe, it, expect } from 'vitest';
import { fromSQLServer } from '../sqlserver';
describe('SQL Server Complex Fantasy Case', () => {
it('should parse complex SQL with SpellDefinition and SpellComponent tables', async () => {
// Complex SQL with same structure as user's case but fantasy-themed
const sql = `CREATE TABLE [DBO].[SpellDefinition](
[SPELLID] (VARCHAR)(32),
[HASVERBALCOMP] BOOLEAN,
[INCANTATION] [VARCHAR](128),
[INCANTATIONFIX] BOOLEAN,
[ITSCOMPONENTREL] [VARCHAR](32), FOREIGN KEY (itscomponentrel) REFERENCES SpellComponent(SPELLID),
[SHOWVISUALS] BOOLEAN, ) ON [PRIMARY]
CREATE TABLE [DBO].[SpellComponent](
[ALIAS] CHAR (50),
[SPELLID] (VARCHAR)(32),
[ISOPTIONAL] BOOLEAN,
[ITSPARENTCOMP] [VARCHAR](32), FOREIGN KEY (itsparentcomp) REFERENCES SpellComponent(SPELLID),
[ITSSCHOOLMETA] [VARCHAR](32), FOREIGN KEY (itsschoolmeta) REFERENCES MagicSchool(SCHOOLID),
[KEYATTR] CHAR (100), ) ON [PRIMARY]`;
console.log('Testing complex fantasy SQL...');
console.log(
'Number of CREATE TABLE statements:',
(sql.match(/CREATE\s+TABLE/gi) || []).length
);
const result = await fromSQLServer(sql);
console.log(
'Result tables:',
result.tables.map((t) => t.name)
);
console.log('Result relationships:', result.relationships.length);
// Debug: Show actual relationships
if (result.relationships.length === 0) {
console.log('WARNING: No relationships found!');
} else {
console.log('Relationships found:');
result.relationships.forEach((r) => {
console.log(
` ${r.sourceTable}.${r.sourceColumn} -> ${r.targetTable}.${r.targetColumn}`
);
});
}
// Should create TWO tables
expect(result.tables).toHaveLength(2);
// Check first table
const spellDef = result.tables.find(
(t) => t.name === 'SpellDefinition'
);
expect(spellDef).toBeDefined();
expect(spellDef?.schema).toBe('DBO');
expect(spellDef?.columns).toHaveLength(6);
// Check second table
const spellComp = result.tables.find(
(t) => t.name === 'SpellComponent'
);
expect(spellComp).toBeDefined();
expect(spellComp?.schema).toBe('DBO');
expect(spellComp?.columns).toHaveLength(6);
// Check foreign key relationships (should have at least 2)
expect(result.relationships.length).toBeGreaterThanOrEqual(2);
// Check FK from SpellDefinition to SpellComponent
const fkDefToComp = result.relationships.find(
(r) =>
r.sourceTable === 'SpellDefinition' &&
r.targetTable === 'SpellComponent' &&
r.sourceColumn === 'itscomponentrel'
);
expect(fkDefToComp).toBeDefined();
expect(fkDefToComp?.targetColumn).toBe('SPELLID');
// Check self-referential FK in SpellComponent
const selfRefFK = result.relationships.find(
(r) =>
r.sourceTable === 'SpellComponent' &&
r.targetTable === 'SpellComponent' &&
r.sourceColumn === 'itsparentcomp'
);
expect(selfRefFK).toBeDefined();
expect(selfRefFK?.targetColumn).toBe('SPELLID');
});
});

View File

@@ -0,0 +1,102 @@
import { describe, it, expect } from 'vitest';
import { sqlImportToDiagram } from '../../../index';
import { DatabaseType } from '@/lib/domain/database-type';
describe('SQL Server Full Import Flow', () => {
it('should create relationships when importing through the full flow', async () => {
const sql = `CREATE TABLE [DBO].[SpellDefinition](
[SPELLID] (VARCHAR)(32),
[HASVERBALCOMP] BOOLEAN,
[INCANTATION] [VARCHAR](128),
[INCANTATIONFIX] BOOLEAN,
[ITSCOMPONENTREL] [VARCHAR](32), FOREIGN KEY (itscomponentrel) REFERENCES SpellComponent(SPELLID),
[SHOWVISUALS] BOOLEAN, ) ON [PRIMARY]
CREATE TABLE [DBO].[SpellComponent](
[ALIAS] CHAR (50),
[SPELLID] (VARCHAR)(32),
[ISOPTIONAL] BOOLEAN,
[ITSPARENTCOMP] [VARCHAR](32), FOREIGN KEY (itsparentcomp) REFERENCES SpellComponent(SPELLID),
[ITSSCHOOLMETA] [VARCHAR](32), FOREIGN KEY (itsschoolmeta) REFERENCES MagicSchool(SCHOOLID),
[KEYATTR] CHAR (100), ) ON [PRIMARY]`;
// Test the full import flow as the application uses it
const diagram = await sqlImportToDiagram({
sqlContent: sql,
sourceDatabaseType: DatabaseType.SQL_SERVER,
targetDatabaseType: DatabaseType.SQL_SERVER,
});
// Verify tables
expect(diagram.tables).toHaveLength(2);
const tableNames = diagram.tables?.map((t) => t.name).sort();
expect(tableNames).toEqual(['SpellComponent', 'SpellDefinition']);
// Verify relationships are created in the diagram
expect(diagram.relationships).toBeDefined();
expect(diagram.relationships?.length).toBeGreaterThanOrEqual(2);
// Check specific relationships
const fk1 = diagram.relationships?.find(
(r) =>
r.sourceFieldId &&
r.targetFieldId && // Must have field IDs
diagram.tables?.some(
(t) =>
t.id === r.sourceTableId && t.name === 'SpellDefinition'
)
);
expect(fk1).toBeDefined();
const fk2 = diagram.relationships?.find(
(r) =>
r.sourceFieldId &&
r.targetFieldId && // Must have field IDs
diagram.tables?.some(
(t) =>
t.id === r.sourceTableId &&
t.name === 'SpellComponent' &&
t.id === r.targetTableId // self-reference
)
);
expect(fk2).toBeDefined();
console.log(
'Full flow test - Relationships created:',
diagram.relationships?.length
);
diagram.relationships?.forEach((r) => {
const sourceTable = diagram.tables?.find(
(t) => t.id === r.sourceTableId
);
const targetTable = diagram.tables?.find(
(t) => t.id === r.targetTableId
);
const sourceField = sourceTable?.fields.find(
(f) => f.id === r.sourceFieldId
);
const targetField = targetTable?.fields.find(
(f) => f.id === r.targetFieldId
);
console.log(
` ${sourceTable?.name}.${sourceField?.name} -> ${targetTable?.name}.${targetField?.name}`
);
});
});
it('should handle case-insensitive field matching', async () => {
const sql = `CREATE TABLE DragonLair (
[LAIRID] INT PRIMARY KEY,
[parentLairId] INT, FOREIGN KEY (PARENTLAIRID) REFERENCES DragonLair(lairid)
)`;
const diagram = await sqlImportToDiagram({
sqlContent: sql,
sourceDatabaseType: DatabaseType.SQL_SERVER,
targetDatabaseType: DatabaseType.SQL_SERVER,
});
// Should create the self-referential relationship despite case differences
expect(diagram.relationships?.length).toBe(1);
});
});

View File

@@ -0,0 +1,132 @@
import { describe, it, expect } from 'vitest';
import { fromSQLServer } from '../sqlserver';
describe('SQL Server Multiple Tables with Foreign Keys', () => {
it('should parse multiple tables with foreign keys in user format', async () => {
const sql = `
CREATE TABLE [DBO].[QuestReward](
[BOID] (VARCHAR)(32),
[HASEXTRACOL] BOOLEAN,
[REWARDCODE] [VARCHAR](128),
[REWARDFIX] BOOLEAN,
[ITSQUESTREL] [VARCHAR](32), FOREIGN KEY (itsquestrel) REFERENCES QuestRelation(BOID),
[SHOWDETAILS] BOOLEAN,
) ON [PRIMARY]
CREATE TABLE [DBO].[QuestRelation](
[ALIAS] CHAR (50),
[BOID] (VARCHAR)(32),
[ISOPTIONAL] BOOLEAN,
[ITSPARENTREL] [VARCHAR](32), FOREIGN KEY (itsparentrel) REFERENCES QuestRelation(BOID),
[ITSGUILDMETA] [VARCHAR](32), FOREIGN KEY (itsguildmeta) REFERENCES GuildMeta(BOID),
[KEYATTR] CHAR (100),
) ON [PRIMARY]
`;
const result = await fromSQLServer(sql);
// Should create both tables
expect(result.tables).toHaveLength(2);
// Check first table
const questReward = result.tables.find((t) => t.name === 'QuestReward');
expect(questReward).toBeDefined();
expect(questReward?.schema).toBe('DBO');
expect(questReward?.columns).toHaveLength(6);
// Check second table
const questRelation = result.tables.find(
(t) => t.name === 'QuestRelation'
);
expect(questRelation).toBeDefined();
expect(questRelation?.schema).toBe('DBO');
expect(questRelation?.columns).toHaveLength(6);
// Check foreign key relationships
expect(result.relationships).toHaveLength(2); // Should have 2 FKs (one self-referential in QuestRelation, one from QuestReward to QuestRelation)
// Check FK from QuestReward to QuestRelation
const fkToRelation = result.relationships.find(
(r) =>
r.sourceTable === 'QuestReward' &&
r.targetTable === 'QuestRelation'
);
expect(fkToRelation).toBeDefined();
expect(fkToRelation?.sourceColumn).toBe('itsquestrel');
expect(fkToRelation?.targetColumn).toBe('BOID');
// Check self-referential FK in QuestRelation
const selfRefFK = result.relationships.find(
(r) =>
r.sourceTable === 'QuestRelation' &&
r.targetTable === 'QuestRelation' &&
r.sourceColumn === 'itsparentrel'
);
expect(selfRefFK).toBeDefined();
expect(selfRefFK?.targetColumn).toBe('BOID');
});
it('should parse multiple tables with circular dependencies', async () => {
const sql = `
CREATE TABLE [DBO].[Dragon](
[DRAGONID] (VARCHAR)(32),
[NAME] [VARCHAR](100),
[ITSLAIRREL] [VARCHAR](32), FOREIGN KEY (itslairrel) REFERENCES DragonLair(LAIRID),
[POWER] INT,
) ON [PRIMARY]
CREATE TABLE [DBO].[DragonLair](
[LAIRID] (VARCHAR)(32),
[LOCATION] [VARCHAR](200),
[ITSGUARDIAN] [VARCHAR](32), FOREIGN KEY (itsguardian) REFERENCES Dragon(DRAGONID),
[TREASURES] INT,
) ON [PRIMARY]
`;
const result = await fromSQLServer(sql);
// Should create both tables despite circular dependency
expect(result.tables).toHaveLength(2);
const dragon = result.tables.find((t) => t.name === 'Dragon');
expect(dragon).toBeDefined();
const dragonLair = result.tables.find((t) => t.name === 'DragonLair');
expect(dragonLair).toBeDefined();
// Check foreign key relationships (may have one or both depending on parser behavior with circular deps)
expect(result.relationships.length).toBeGreaterThanOrEqual(1);
});
it('should handle exact user input format', async () => {
// Exact copy of the user's input with fantasy theme
const sql = `CREATE TABLE [DBO].[WizardDef](
[BOID] (VARCHAR)(32),
[HASEXTRACNTCOL] BOOLEAN,
[HISTORYCD] [VARCHAR](128),
[HISTORYCDFIX] BOOLEAN,
[ITSADWIZARDREL] [VARCHAR](32), FOREIGN KEY (itsadwizardrel) REFERENCES WizardRel(BOID),
[SHOWDETAILS] BOOLEAN, ) ON [PRIMARY]
CREATE TABLE [DBO].[WizardRel](
[ALIAS] CHAR (50),
[BOID] (VARCHAR)(32),
[ISOPTIONAL] BOOLEAN,
[ITSARWIZARDREL] [VARCHAR](32), FOREIGN KEY (itsarwizardrel) REFERENCES WizardRel(BOID),
[ITSARMETABO] [VARCHAR](32), FOREIGN KEY (itsarmetabo) REFERENCES MetaBO(BOID),
[KEYATTR] CHAR (100), ) ON [PRIMARY]`;
const result = await fromSQLServer(sql);
// This should create TWO tables, not just one
expect(result.tables).toHaveLength(2);
const wizardDef = result.tables.find((t) => t.name === 'WizardDef');
expect(wizardDef).toBeDefined();
expect(wizardDef?.columns).toHaveLength(6);
const wizardRel = result.tables.find((t) => t.name === 'WizardRel');
expect(wizardRel).toBeDefined();
expect(wizardRel?.columns).toHaveLength(6);
});
});

View File

@@ -0,0 +1,93 @@
import { describe, it, expect } from 'vitest';
import { fromSQLServer } from '../sqlserver';
describe('SQL Server FK Verification', () => {
it('should correctly parse FKs from complex fantasy SQL', async () => {
const sql = `CREATE TABLE [DBO].[SpellDefinition](
[SPELLID] (VARCHAR)(32),
[HASVERBALCOMP] BOOLEAN,
[INCANTATION] [VARCHAR](128),
[INCANTATIONFIX] BOOLEAN,
[ITSCOMPONENTREL] [VARCHAR](32), FOREIGN KEY (itscomponentrel) REFERENCES SpellComponent(SPELLID),
[SHOWVISUALS] BOOLEAN, ) ON [PRIMARY]
CREATE TABLE [DBO].[SpellComponent](
[ALIAS] CHAR (50),
[SPELLID] (VARCHAR)(32),
[ISOPTIONAL] BOOLEAN,
[ITSPARENTCOMP] [VARCHAR](32), FOREIGN KEY (itsparentcomp) REFERENCES SpellComponent(SPELLID),
[ITSSCHOOLMETA] [VARCHAR](32), FOREIGN KEY (itsschoolmeta) REFERENCES MagicSchool(SCHOOLID),
[KEYATTR] CHAR (100), ) ON [PRIMARY]`;
const result = await fromSQLServer(sql);
// Verify tables
expect(result.tables).toHaveLength(2);
expect(result.tables.map((t) => t.name).sort()).toEqual([
'SpellComponent',
'SpellDefinition',
]);
// Verify that FKs were found (even if MagicSchool doesn't exist)
// The parsing should find 3 FKs initially, but linkRelationships will filter out the one to MagicSchool
expect(result.relationships.length).toBeGreaterThanOrEqual(2);
// Verify specific FKs that should exist
const fk1 = result.relationships.find(
(r) =>
r.sourceTable === 'SpellDefinition' &&
r.sourceColumn.toLowerCase() === 'itscomponentrel' &&
r.targetTable === 'SpellComponent'
);
expect(fk1).toBeDefined();
expect(fk1?.targetColumn).toBe('SPELLID');
expect(fk1?.sourceTableId).toBeTruthy();
expect(fk1?.targetTableId).toBeTruthy();
const fk2 = result.relationships.find(
(r) =>
r.sourceTable === 'SpellComponent' &&
r.sourceColumn.toLowerCase() === 'itsparentcomp' &&
r.targetTable === 'SpellComponent'
);
expect(fk2).toBeDefined();
expect(fk2?.targetColumn).toBe('SPELLID');
expect(fk2?.sourceTableId).toBeTruthy();
expect(fk2?.targetTableId).toBeTruthy();
// Log for debugging
console.log('\n=== FK Verification Results ===');
console.log(
'Tables:',
result.tables.map((t) => `${t.schema}.${t.name}`)
);
console.log('Total FKs found:', result.relationships.length);
result.relationships.forEach((r, i) => {
console.log(
`FK ${i + 1}: ${r.sourceTable}.${r.sourceColumn} -> ${r.targetTable}.${r.targetColumn}`
);
console.log(` IDs: ${r.sourceTableId} -> ${r.targetTableId}`);
});
});
it('should parse inline FOREIGN KEY syntax correctly', async () => {
// Simplified test with just one FK to ensure parsing works
const sql = `CREATE TABLE [DBO].[WizardTower](
[TOWERID] INT,
[MASTERKEY] [VARCHAR](32), FOREIGN KEY (MASTERKEY) REFERENCES ArcaneGuild(GUILDID),
[NAME] VARCHAR(100)
) ON [PRIMARY]
CREATE TABLE [DBO].[ArcaneGuild](
[GUILDID] [VARCHAR](32),
[GUILDNAME] VARCHAR(100)
) ON [PRIMARY]`;
const result = await fromSQLServer(sql);
expect(result.tables).toHaveLength(2);
expect(result.relationships).toHaveLength(1);
expect(result.relationships[0].sourceColumn).toBe('MASTERKEY');
expect(result.relationships[0].targetColumn).toBe('GUILDID');
});
});

View File

@@ -342,6 +342,35 @@ function parseCreateTableManually(
// Process each part (column or constraint)
for (const part of parts) {
// Handle standalone FOREIGN KEY definitions (without CONSTRAINT keyword)
// Format: FOREIGN KEY (column) REFERENCES Table(column)
if (part.match(/^\s*FOREIGN\s+KEY/i)) {
const fkMatch = part.match(
/FOREIGN\s+KEY\s*\(([^)]+)\)\s+REFERENCES\s+(?:\[?(\w+)\]?\.)??\[?(\w+)\]?\s*\(([^)]+)\)/i
);
if (fkMatch) {
const [
,
sourceCol,
targetSchema = 'dbo',
targetTable,
targetCol,
] = fkMatch;
relationships.push({
name: `FK_${tableName}_${sourceCol.trim().replace(/\[|\]/g, '')}`,
sourceTable: tableName,
sourceSchema: schema,
sourceColumn: sourceCol.trim().replace(/\[|\]/g, ''),
targetTable: targetTable || targetSchema,
targetSchema: targetTable ? targetSchema : 'dbo',
targetColumn: targetCol.trim().replace(/\[|\]/g, ''),
sourceTableId: tableId,
targetTableId: '', // Will be filled later
});
}
continue;
}
// Handle constraint definitions
if (part.match(/^\s*CONSTRAINT/i)) {
// Parse constraints
@@ -435,6 +464,13 @@ function parseCreateTableManually(
columnMatch = part.match(/^\s*(\w+)\s+(\w+)\s+([\d,\s]+)\s+(.*)$/i);
}
// Handle unusual format: [COLUMN_NAME] (VARCHAR)(32)
if (!columnMatch) {
columnMatch = part.match(
/^\s*\[?(\w+)\]?\s+\((\w+)\)\s*\(([\d,\s]+|max)\)(.*)$/i
);
}
if (columnMatch) {
const [, colName, baseType, typeArgs, rest] = columnMatch;
@@ -446,7 +482,37 @@ function parseCreateTableManually(
const inlineFkMatch = rest.match(
/FOREIGN\s+KEY\s+REFERENCES\s+(?:\[?(\w+)\]?\.)??\[?(\w+)\]?\s*\(([^)]+)\)/i
);
if (inlineFkMatch) {
// Also check if there's a FOREIGN KEY after a comma with column name
// Format: , FOREIGN KEY (columnname) REFERENCES Table(column)
if (!inlineFkMatch && rest.includes('FOREIGN KEY')) {
const fkWithColumnMatch = rest.match(
/,\s*FOREIGN\s+KEY\s*\((\w+)\)\s+REFERENCES\s+(?:\[?(\w+)\]?\.)??\[?(\w+)\]?\s*\(([^)]+)\)/i
);
if (fkWithColumnMatch) {
const [, srcCol, targetSchema, targetTable, targetCol] =
fkWithColumnMatch;
// Only process if srcCol matches current colName (case-insensitive)
if (srcCol.toLowerCase() === colName.toLowerCase()) {
// Create FK relationship
relationships.push({
name: `FK_${tableName}_${colName}`,
sourceTable: tableName,
sourceSchema: schema,
sourceColumn: colName,
targetTable: targetTable || targetSchema,
targetSchema: targetTable
? targetSchema || 'dbo'
: 'dbo',
targetColumn: targetCol
.trim()
.replace(/\[|\]/g, ''),
sourceTableId: tableId,
targetTableId: '', // Will be filled later
});
}
}
} else if (inlineFkMatch) {
const [, targetSchema = 'dbo', targetTable, targetCol] =
inlineFkMatch;
relationships.push({
@@ -536,10 +602,36 @@ export async function fromSQLServer(
try {
// First, handle ALTER TABLE statements for foreign keys
// Split by GO or semicolon for SQL Server
const statements = sqlContent
let statements = sqlContent
.split(/(?:GO\s*$|;\s*$)/im)
.filter((stmt) => stmt.trim().length > 0);
// Additional splitting for CREATE TABLE statements that might not be separated by semicolons
// If we have a statement with multiple CREATE TABLE, split them
const expandedStatements: string[] = [];
for (const stmt of statements) {
// Check if this statement contains multiple CREATE TABLE statements
if ((stmt.match(/CREATE\s+TABLE/gi) || []).length > 1) {
// Split by ") ON [PRIMARY]" followed by CREATE TABLE
const parts = stmt.split(
/\)\s*ON\s*\[PRIMARY\]\s*(?=CREATE\s+TABLE)/gi
);
for (let i = 0; i < parts.length; i++) {
let part = parts[i].trim();
// Re-add ") ON [PRIMARY]" to all parts except the last (which should already have it)
if (i < parts.length - 1 && part.length > 0) {
part += ') ON [PRIMARY]';
}
if (part.trim().length > 0) {
expandedStatements.push(part);
}
}
} else {
expandedStatements.push(stmt);
}
}
statements = expandedStatements;
const alterTableStatements = statements.filter(
(stmt) =>
stmt.trim().toUpperCase().includes('ALTER TABLE') &&