feat(import): optimize large diagrams with filtering and smart reordering

src/dialogs/create-diagram-dialog/create-diagram-dialog.tsx

@@ -19,6 +19,7 @@ import { SelectTables } from '../common/select-tables/select-tables';
 import { useTranslation } from 'react-i18next';
 import type { BaseDialogProps } from '../common/base-dialog-props';
 import { sqlImportToDiagram } from '@/lib/data/sql-import';
+import { getInitialFilterForLargeDiagram } from '@/lib/export-import-utils';
 import type { SelectedTable } from '@/lib/data/import-metadata/filter-metadata';
 import { filterMetadataByTables } from '@/lib/data/import-metadata/filter-metadata';
 import { MAX_TABLES_WITHOUT_SHOWING_FILTER } from '../common/select-tables/constants';
@@ -43,7 +44,7 @@ export const CreateDiagramDialog: React.FC<CreateDiagramDialogProps> = ({
     const [step, setStep] = useState<CreateDiagramDialogStep>(
         CreateDiagramDialogStep.SELECT_DATABASE
     );
-    const { listDiagrams, addDiagram } = useStorage();
+    const { listDiagrams, addDiagram, updateDiagramFilter } = useStorage();
     const [diagramNumber, setDiagramNumber] = useState<number>(1);
     const navigate = useNavigate();
     const [parsedMetadata, setParsedMetadata] = useState<DatabaseMetadata>();
@@ -89,6 +90,12 @@ export const CreateDiagramDialog: React.FC<CreateDiagramDialogProps> = ({
                     sourceDatabaseType: databaseType,
                     targetDatabaseType: databaseType,
                 });
+
+                // Check if we need a filter for large SQL imports
+                const initialFilter = getInitialFilterForLargeDiagram(diagram);
+                if (initialFilter) {
+                    await updateDiagramFilter(diagram.id, initialFilter);
+                }
             } else {
                 let metadata: DatabaseMetadata | undefined = databaseMetadata;
 
@@ -103,7 +110,7 @@ export const CreateDiagramDialog: React.FC<CreateDiagramDialogProps> = ({
                     });
                 }
 
-                diagram = await loadFromDatabaseMetadata({
+                const result = await loadFromDatabaseMetadata({
                     databaseType,
                     databaseMetadata: metadata,
                     diagramNumber,
@@ -112,6 +119,12 @@ export const CreateDiagramDialog: React.FC<CreateDiagramDialogProps> = ({
                             ? undefined
                             : databaseEdition,
                 });
+                diagram = result.diagram;
+
+                // Apply filter if needed for large diagrams
+                if (result.initialFilter) {
+                    await updateDiagramFilter(diagram.id, result.initialFilter);
+                }
             }
 
             await addDiagram({ diagram });
@@ -126,6 +139,7 @@ export const CreateDiagramDialog: React.FC<CreateDiagramDialogProps> = ({
             importMethod,
             databaseType,
             addDiagram,
+            updateDiagramFilter,
             databaseEdition,
             closeCreateDiagramDialog,
             navigate,

src/dialogs/import-database-dialog/import-database-dialog.tsx

@@ -69,7 +69,7 @@ export const ImportDatabaseDialog: React.FC<ImportDatabaseDialogProps> = ({
             const databaseMetadata: DatabaseMetadata =
                 loadDatabaseMetadata(scriptResult);
 
-            diagram = await loadFromDatabaseMetadata({
+            const result = await loadFromDatabaseMetadata({
                 databaseType,
                 databaseMetadata,
                 databaseEdition:
@@ -77,6 +77,9 @@ export const ImportDatabaseDialog: React.FC<ImportDatabaseDialogProps> = ({
                         ? undefined
                         : databaseEdition,
             });
+            diagram = result.diagram;
+            // Note: For importing into existing diagram, we don't apply the filter
+            // as it would affect the existing tables too
         }
 
         const tableIdsToRemove = tables

src/dialogs/import-diagram-dialog/import-diagram-dialog.tsx

@@ -16,7 +16,10 @@ import { useTranslation } from 'react-i18next';
 import { FileUploader } from '@/components/file-uploader/file-uploader';
 import { useStorage } from '@/hooks/use-storage';
 import { useNavigate } from 'react-router-dom';
-import { diagramFromJSONInput } from '@/lib/export-import-utils';
+import {
+    diagramFromJSONInput,
+    getInitialFilterForLargeDiagram,
+} from '@/lib/export-import-utils';
 import { Alert, AlertDescription, AlertTitle } from '@/components/alert/alert';
 import { AlertCircle } from 'lucide-react';
 
@@ -27,7 +30,7 @@ export const ImportDiagramDialog: React.FC<ImportDiagramDialogProps> = ({
 }) => {
     const { t } = useTranslation();
     const [file, setFile] = useState<File | null>(null);
-    const { addDiagram } = useStorage();
+    const { addDiagram, updateDiagramFilter } = useStorage();
     const navigate = useNavigate();
     const [error, setError] = useState(false);
 
@@ -58,8 +61,16 @@ export const ImportDiagramDialog: React.FC<ImportDiagramDialogProps> = ({
             try {
                 const diagram = diagramFromJSONInput(json);
 
+                // Check if we need to apply a filter for large diagrams
+                const initialFilter = getInitialFilterForLargeDiagram(diagram);
+
                 await addDiagram({ diagram });
 
+                // Apply the filter if needed (to hide isolated tables)
+                if (initialFilter) {
+                    await updateDiagramFilter(diagram.id, initialFilter);
+                }
+
                 closeImportDiagramDialog();
                 closeCreateDiagramDialog();
 
@@ -74,6 +85,7 @@ export const ImportDiagramDialog: React.FC<ImportDiagramDialogProps> = ({
     }, [
         file,
         addDiagram,
+        updateDiagramFilter,
         navigate,
         closeImportDiagramDialog,
         closeCreateDiagramDialog,

src/lib/data/sql-import/index.ts

@@ -220,11 +220,45 @@ export async function sqlImportToDiagram({
         targetDatabaseType
     );
 
-    const adjustedTables = adjustTablePositions({
-        tables: diagram.tables ?? [],
-        relationships: diagram.relationships ?? [],
-        mode: 'perSchema',
-    });
+    // Apply the same logic as loadFromDatabaseMetadata for large diagrams
+    const LARGE_DIAGRAM_THRESHOLD = 200;
+    const tables = diagram.tables ?? [];
+    const relationships = diagram.relationships ?? [];
+    let adjustedTables = tables;
+
+    if (tables.length > LARGE_DIAGRAM_THRESHOLD) {
+        // Create a set of table IDs that have relationships
+        const tablesWithRelationships = new Set<string>();
+        relationships.forEach((rel) => {
+            tablesWithRelationships.add(rel.sourceTableId);
+            tablesWithRelationships.add(rel.targetTableId);
+        });
+
+        // Separate tables into connected and isolated
+        const connectedTables = tables.filter((table) =>
+            tablesWithRelationships.has(table.id)
+        );
+        const isolatedTables = tables.filter(
+            (table) => !tablesWithRelationships.has(table.id)
+        );
+
+        // Only reorder connected tables
+        const reorderedConnectedTables = adjustTablePositions({
+            tables: connectedTables,
+            relationships,
+            mode: 'perSchema',
+        });
+
+        // Combine reordered connected tables with isolated tables
+        adjustedTables = [...reorderedConnectedTables, ...isolatedTables];
+    } else {
+        // For smaller diagrams, reorder all tables as before
+        adjustedTables = adjustTablePositions({
+            tables,
+            relationships,
+            mode: 'perSchema',
+        });
+    }
 
     const sortedTables = adjustedTables.sort((a, b) => {
         if (a.isView === b.isView) {

src/lib/domain/diagram.ts

@@ -64,7 +64,7 @@ export const loadFromDatabaseMetadata = async ({
     databaseMetadata: DatabaseMetadata;
     diagramNumber?: number;
     databaseEdition?: DatabaseEdition;
-}): Promise<Diagram> => {
+}): Promise<{ diagram: Diagram; initialFilter?: { tableIds: string[] } }> => {
     const {
         fk_info: foreignKeys,
         views: views,
@@ -93,11 +93,51 @@ export const loadFromDatabaseMetadata = async ({
           })
         : [];
 
-    const adjustedTables = adjustTablePositions({
-        tables,
-        relationships,
-        mode: 'perSchema',
-    });
+    // For large diagrams, apply special handling
+    const LARGE_DIAGRAM_THRESHOLD = 200;
+    let adjustedTables = tables;
+    let initialFilter: { tableIds: string[] } | undefined;
+
+    if (tables.length > LARGE_DIAGRAM_THRESHOLD) {
+        // Create a set of table IDs that have relationships
+        const tablesWithRelationships = new Set<string>();
+        relationships.forEach((rel) => {
+            tablesWithRelationships.add(rel.sourceTableId);
+            tablesWithRelationships.add(rel.targetTableId);
+        });
+
+        // Separate tables into connected and isolated
+        const connectedTables = tables.filter((table) =>
+            tablesWithRelationships.has(table.id)
+        );
+        const isolatedTables = tables.filter(
+            (table) => !tablesWithRelationships.has(table.id)
+        );
+
+        // Only reorder connected tables
+        const reorderedConnectedTables = adjustTablePositions({
+            tables: connectedTables,
+            relationships,
+            mode: 'perSchema',
+        });
+
+        // Combine reordered connected tables with isolated tables
+        adjustedTables = [...reorderedConnectedTables, ...isolatedTables];
+
+        // Set up filter to hide isolated tables if there are any
+        if (isolatedTables.length > 0) {
+            initialFilter = {
+                tableIds: connectedTables.map((t) => t.id),
+            };
+        }
+    } else {
+        // For smaller diagrams, reorder all tables as before
+        adjustedTables = adjustTablePositions({
+            tables,
+            relationships,
+            mode: 'perSchema',
+        });
+    }
 
     const sortedTables = adjustedTables.sort((a, b) => {
         if (a.isView === b.isView) {
@@ -125,5 +165,5 @@ export const loadFromDatabaseMetadata = async ({
         updatedAt: new Date(),
     };
 
-    return diagram;
+    return { diagram, initialFilter };
 };

src/lib/export-import-utils.ts

@@ -1,6 +1,7 @@
 import { cloneDiagram } from './clone';
 import { diagramSchema, type Diagram } from './domain/diagram';
 import { generateDiagramId } from './utils';
+import { adjustTablePositions } from './domain/db-table';
 
 export const runningIdGenerator = (): (() => string) => {
     let id = 0;
@@ -36,5 +37,82 @@ export const diagramFromJSONInput = (json: string): Diagram => {
         updatedAt: new Date(),
     });
 
-    return cloneDiagramWithIds(diagram);
+    const clonedDiagram = cloneDiagramWithIds(diagram);
+
+    // Apply reordering for large diagrams AFTER identifying which tables have relationships
+    const LARGE_DIAGRAM_THRESHOLD = 200;
+
+    if (
+        clonedDiagram.tables &&
+        clonedDiagram.tables.length > LARGE_DIAGRAM_THRESHOLD &&
+        clonedDiagram.relationships
+    ) {
+        // Create a set of table IDs that have relationships
+        const tablesWithRelationships = new Set<string>();
+        clonedDiagram.relationships.forEach((rel) => {
+            tablesWithRelationships.add(rel.sourceTableId);
+            tablesWithRelationships.add(rel.targetTableId);
+        });
+
+        // Filter tables to only those with relationships for reordering
+        const tablesToReorder = clonedDiagram.tables.filter((table) =>
+            tablesWithRelationships.has(table.id)
+        );
+
+        // Apply reordering only to tables with relationships
+        const reorderedTables = adjustTablePositions({
+            tables: tablesToReorder,
+            relationships: clonedDiagram.relationships || [],
+            areas: clonedDiagram.areas || [],
+            mode: 'all',
+        });
+
+        // Update positions for reordered tables
+        clonedDiagram.tables = clonedDiagram.tables.map((table) => {
+            const reorderedTable = reorderedTables.find(
+                (t) => t.id === table.id
+            );
+            if (reorderedTable) {
+                return {
+                    ...table,
+                    x: reorderedTable.x,
+                    y: reorderedTable.y,
+                };
+            }
+            return table;
+        });
+    }
+
+    return clonedDiagram;
+};
+
+export const getInitialFilterForLargeDiagram = (
+    diagram: Diagram
+): { tableIds?: string[] } | null => {
+    const LARGE_DIAGRAM_THRESHOLD = 200;
+
+    if (
+        diagram.tables &&
+        diagram.tables.length > LARGE_DIAGRAM_THRESHOLD &&
+        diagram.relationships
+    ) {
+        // Create a set of table IDs that have relationships
+        const tablesWithRelationships = new Set<string>();
+        diagram.relationships.forEach((rel) => {
+            tablesWithRelationships.add(rel.sourceTableId);
+            tablesWithRelationships.add(rel.targetTableId);
+        });
+
+        // Return only tables with relationships to be shown (filter will hide the rest)
+        const tablesToShow = diagram.tables
+            .filter((table) => tablesWithRelationships.has(table.id))
+            .map((table) => table.id);
+
+        // If there are tables to filter out, return the filter
+        if (tablesToShow.length < diagram.tables.length) {
+            return { tableIds: tablesToShow };
+        }
+    }
+
+    return null;
 };