From dac7143a19a7d577997454b6426c87d5b7278d5d Mon Sep 17 00:00:00 2001 From: Greirson Lee-Thorp Date: Mon, 3 Feb 2025 15:55:40 -0800 Subject: [PATCH 1/6] feat: improve file and folder upload handling with unique naming - Add support for batch uploads with unique folder and file naming - Implement getUniqueFilePath and getUniqueFolderPath to prevent file/folder overwrites - Add batch ID tracking for folder uploads - Enhance client-side file handling to support batch uploads - Improve file path generation and logging for uploads --- public/index.html | 9 +++++- server.js | 80 ++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 83 insertions(+), 6 deletions(-) diff --git a/public/index.html b/public/index.html index 71757ef..ab092da 100644 --- a/public/index.html +++ b/public/index.html @@ -88,7 +88,10 @@ const response = await fetch('/upload/init', { method: 'POST', - headers: { 'Content-Type': 'application/json' }, + headers: { + 'Content-Type': 'application/json', + 'X-Batch-ID': this.file.batchId || Date.now().toString() + }, body: JSON.stringify({ filename: uploadPath, fileSize: this.file.size @@ -230,11 +233,13 @@ // Helper function to process directory entries async function getAllFileEntries(dataTransferItems) { let fileEntries = []; + const batchId = Date.now().toString(); async function traverseEntry(entry, path = '') { if (entry.isFile) { const file = await new Promise((resolve) => entry.file(resolve)); file.relativePath = path; + file.batchId = batchId; fileEntries.push(file); } else if (entry.isDirectory) { const reader = entry.createReader(); @@ -312,11 +317,13 @@ } function handleFolders(e) { + const batchId = Date.now().toString(); files = [...e.target.files]; files.forEach(file => { const pathParts = file.webkitRelativePath.split('/'); pathParts.pop(); // Remove filename file.relativePath = pathParts.length > 0 ? pathParts.join('/') + '/' : ''; + file.batchId = batchId; }); updateFileList(); } diff --git a/server.js b/server.js index 235ad12..483dd46 100644 --- a/server.js +++ b/server.js @@ -218,12 +218,45 @@ app.use('/upload', requirePin); // Store ongoing uploads const uploads = new Map(); +// Store folder name mappings for batch uploads with timestamps +const folderMappings = new Map(); +// Store batch IDs for folder uploads +const batchUploads = new Map(); + +// Add these helper functions before the routes +function getUniqueFilePath(filePath) { + const dir = path.dirname(filePath); + const ext = path.extname(filePath); + const baseName = path.basename(filePath, ext); + let counter = 1; + let newPath = filePath; + + while (fs.existsSync(newPath)) { + newPath = path.join(dir, `${baseName} (${counter})${ext}`); + counter++; + } + + return newPath; +} + +function getUniqueFolderPath(folderPath) { + let counter = 1; + let newPath = folderPath; + + while (fs.existsSync(newPath)) { + newPath = `${folderPath} (${counter})`; + counter++; + } + + return newPath; +} // Routes app.post('/upload/init', async (req, res) => { const { filename, fileSize } = req.body; + const batchId = req.headers['x-batch-id'] || Date.now().toString(); - const safeFilename = path.normalize(filename).replace(/^(\.\.(\/|\\|$))+/, '') + const safeFilename = path.normalize(filename).replace(/^(\.\.(\/|\\|$))+/, ''); // Check file size limit if (fileSize > maxFileSize) { @@ -236,20 +269,57 @@ app.post('/upload/init', async (req, res) => { } const uploadId = Date.now().toString(); - const filePath = path.join(uploadDir, safeFilename); + let filePath = path.join(uploadDir, safeFilename); try { + // Handle file/folder duplication + const pathParts = safeFilename.split('/'); + + if (pathParts.length > 1) { + // This is a file within a folder + const originalFolderName = pathParts[0]; + const folderPath = path.join(uploadDir, originalFolderName); + + // Check if we already have a mapping for this folder in this batch + let newFolderName = folderMappings.get(`${originalFolderName}-${batchId}`); + + if (!newFolderName) { + // Always check if the folder exists, even for new uploads + if (fs.existsSync(folderPath)) { + const uniqueFolderPath = getUniqueFolderPath(folderPath); + newFolderName = path.basename(uniqueFolderPath); + log.info(`Folder "${originalFolderName}" exists, using "${newFolderName}" instead`); + } else { + newFolderName = originalFolderName; + } + folderMappings.set(`${originalFolderName}-${batchId}`, newFolderName); + + // Clean up mapping after 5 minutes + setTimeout(() => { + folderMappings.delete(`${originalFolderName}-${batchId}`); + }, 5 * 60 * 1000); + } + + // Replace the original folder path with the mapped one and keep original file name + pathParts[0] = newFolderName; + filePath = path.join(uploadDir, ...pathParts); + } else { + // This is a single file + filePath = getUniqueFilePath(filePath); + } + + // Ensure the directory exists before creating the write stream await ensureDirectoryExists(filePath); uploads.set(uploadId, { - safeFilename, + safeFilename: path.relative(uploadDir, filePath), filePath, fileSize, bytesReceived: 0, - writeStream: fs.createWriteStream(filePath) + writeStream: fs.createWriteStream(filePath, { flags: 'wx' }) }); - log.info(`Initialized upload for ${safeFilename} (${fileSize} bytes)`); + log.info(`Initialized upload for ${path.relative(uploadDir, filePath)} (${fileSize} bytes)`); res.json({ uploadId }); } catch (err) { log.error(`Failed to initialize upload: ${err.message}`); From 140d58cdc39ce667d1d7a1c1b2f3a682b49c07f9 Mon Sep 17 00:00:00 2001 From: Greirson Lee-Thorp Date: Mon, 3 Feb 2025 16:56:30 -0800 Subject: [PATCH 2/6] feat: improve batch ID generation and validation for file uploads - Add batch ID validation function with specific format requirements - Generate more secure and unique batch IDs using timestamp and random string - Update client-side batch ID generation to create consistent, unique identifiers - Enhance upload initialization route to validate batch ID before processing - Modify FileUploader to use generated batch ID during uploads --- public/index.html | 18 +++++++++++++----- server.js | 16 ++++++++++++++-- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/public/index.html b/public/index.html index ab092da..0ad97b4 100644 --- a/public/index.html +++ b/public/index.html @@ -61,9 +61,15 @@ const MAX_RETRIES = 3; const RETRY_DELAY = 1000; + // Utility function to generate a unique batch ID + function generateBatchId() { + return `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; + } + class FileUploader { - constructor(file) { + constructor(file, batchId) { this.file = file; + this.batchId = batchId; this.uploadId = null; this.position = 0; this.progressElement = null; @@ -90,7 +96,7 @@ method: 'POST', headers: { 'Content-Type': 'application/json', - 'X-Batch-ID': this.file.batchId || Date.now().toString() + 'X-Batch-ID': this.batchId }, body: JSON.stringify({ filename: uploadPath, @@ -233,7 +239,7 @@ // Helper function to process directory entries async function getAllFileEntries(dataTransferItems) { let fileEntries = []; - const batchId = Date.now().toString(); + const batchId = generateBatchId(); async function traverseEntry(entry, path = '') { if (entry.isFile) { @@ -317,7 +323,7 @@ } function handleFolders(e) { - const batchId = Date.now().toString(); + const batchId = generateBatchId(); files = [...e.target.files]; files.forEach(file => { const pathParts = file.webkitRelativePath.split('/'); @@ -361,11 +367,13 @@ document.getElementById('uploadProgress').innerHTML = ''; const groupedItems = groupFilesByFolder(files); + const batchId = generateBatchId(); // Generate a single batch ID for all files + const results = await Promise.all( groupedItems.map(async item => { let success = true; for (const file of item.files) { - const uploader = new FileUploader(file); + const uploader = new FileUploader(file, batchId); if (!await uploader.start()) { success = false; } diff --git a/server.js b/server.js index 483dd46..62261d9 100644 --- a/server.js +++ b/server.js @@ -251,10 +251,22 @@ function getUniqueFolderPath(folderPath) { return newPath; } +// Validate batch ID format +function isValidBatchId(batchId) { + // Batch ID should be in format: timestamp-randomstring + return /^\d+-[a-z0-9]{9}$/.test(batchId); +} + // Routes app.post('/upload/init', async (req, res) => { const { filename, fileSize } = req.body; - const batchId = req.headers['x-batch-id'] || Date.now().toString(); + const batchId = req.headers['x-batch-id']; + + // Validate batch ID + if (!batchId || !isValidBatchId(batchId)) { + log.error('Invalid or missing batch ID'); + return res.status(400).json({ error: 'Invalid or missing batch ID' }); + } const safeFilename = path.normalize(filename).replace(/^(\.\.(\/|\\|$))+/, ''); @@ -268,7 +280,7 @@ app.post('/upload/init', async (req, res) => { }); } - const uploadId = Date.now().toString(); + const uploadId = crypto.randomBytes(16).toString('hex'); let filePath = path.join(uploadDir, safeFilename); try { From a488c753db46e98816c3f8b9c782728191c4cda2 Mon Sep 17 00:00:00 2001 From: Greirson Lee-Thorp Date: Mon, 3 Feb 2025 17:07:43 -0800 Subject: [PATCH 3/6] feat: improve file upload handling with atomic file and folder creation - Refactor getUniqueFilePath and getUniqueFolderPath to use async/await and atomic file operations - Enhance upload initialization to handle file and folder naming conflicts more robustly - Implement file handle management to prevent resource leaks - Add error handling for file and folder creation scenarios - Ensure parent directories are created recursively when needed --- server.js | 93 +++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 67 insertions(+), 26 deletions(-) diff --git a/server.js b/server.js index 62261d9..85c3cfe 100644 --- a/server.js +++ b/server.js @@ -224,31 +224,55 @@ const folderMappings = new Map(); const batchUploads = new Map(); // Add these helper functions before the routes -function getUniqueFilePath(filePath) { +async function getUniqueFilePath(filePath) { const dir = path.dirname(filePath); const ext = path.extname(filePath); const baseName = path.basename(filePath, ext); let counter = 1; - let newPath = filePath; + let finalPath = filePath; - while (fs.existsSync(newPath)) { - newPath = path.join(dir, `${baseName} (${counter})${ext}`); - counter++; + while (true) { + try { + // Try to create the file exclusively - will fail if file exists + const fileHandle = await fs.promises.open(finalPath, 'wx'); + // Return both the path and handle instead of closing it + return { path: finalPath, handle: fileHandle }; + } catch (err) { + if (err.code === 'EEXIST') { + // File exists, try next number + finalPath = path.join(dir, `${baseName} (${counter})${ext}`); + counter++; + } else { + throw err; // Other errors should be handled by caller + } + } } - - return newPath; } -function getUniqueFolderPath(folderPath) { +async function getUniqueFolderPath(folderPath) { let counter = 1; - let newPath = folderPath; + let finalPath = folderPath; - while (fs.existsSync(newPath)) { - newPath = `${folderPath} (${counter})`; - counter++; + while (true) { + try { + // Try to create the directory - mkdir with recursive:false is atomic + await fs.promises.mkdir(finalPath, { recursive: false }); + return finalPath; + } catch (err) { + if (err.code === 'EEXIST') { + // Folder exists, try next number + finalPath = `${folderPath} (${counter})`; + counter++; + } else if (err.code === 'ENOENT') { + // Parent directory doesn't exist, create it first + await fs.promises.mkdir(path.dirname(finalPath), { recursive: true }); + // Then try again with the same path + continue; + } else { + throw err; // Other errors should be handled by caller + } + } } - - return newPath; } // Validate batch ID format @@ -282,6 +306,7 @@ app.post('/upload/init', async (req, res) => { const uploadId = crypto.randomBytes(16).toString('hex'); let filePath = path.join(uploadDir, safeFilename); + let fileHandle; try { // Handle file/folder duplication @@ -296,14 +321,21 @@ app.post('/upload/init', async (req, res) => { let newFolderName = folderMappings.get(`${originalFolderName}-${batchId}`); if (!newFolderName) { - // Always check if the folder exists, even for new uploads - if (fs.existsSync(folderPath)) { - const uniqueFolderPath = getUniqueFolderPath(folderPath); - newFolderName = path.basename(uniqueFolderPath); - log.info(`Folder "${originalFolderName}" exists, using "${newFolderName}" instead`); - } else { + try { + // Try to create the folder atomically first + await fs.promises.mkdir(folderPath, { recursive: false }); newFolderName = originalFolderName; + } catch (err) { + if (err.code === 'EEXIST') { + // Folder exists, get a unique name + const uniqueFolderPath = await getUniqueFolderPath(folderPath); + newFolderName = path.basename(uniqueFolderPath); + log.info(`Folder "${originalFolderName}" exists, using "${newFolderName}" instead`); + } else { + throw err; + } } + folderMappings.set(`${originalFolderName}-${batchId}`, newFolderName); // Clean up mapping after 5 minutes @@ -315,25 +347,34 @@ app.post('/upload/init', async (req, res) => { // Replace the original folder path with the mapped one and keep original file name pathParts[0] = newFolderName; filePath = path.join(uploadDir, ...pathParts); - } else { - // This is a single file - filePath = getUniqueFilePath(filePath); + + // Ensure parent directories exist + await fs.promises.mkdir(path.dirname(filePath), { recursive: true }); } - // Ensure the directory exists before creating the write stream - await ensureDirectoryExists(filePath); + // For both single files and files in folders, get a unique path and file handle + const result = await getUniqueFilePath(filePath); + filePath = result.path; + fileHandle = result.handle; + // Create upload entry (using the file handle we already have) uploads.set(uploadId, { safeFilename: path.relative(uploadDir, filePath), filePath, fileSize, bytesReceived: 0, - writeStream: fs.createWriteStream(filePath, { flags: 'wx' }) + writeStream: fileHandle.createWriteStream() }); log.info(`Initialized upload for ${path.relative(uploadDir, filePath)} (${fileSize} bytes)`); res.json({ uploadId }); } catch (err) { + // Clean up file handle if something went wrong + if (fileHandle) { + await fileHandle.close().catch(() => {}); + // Try to remove the file if it was created + fs.unlink(filePath).catch(() => {}); + } log.error(`Failed to initialize upload: ${err.message}`); res.status(500).json({ error: 'Failed to initialize upload' }); } From dec54b78032a6c52d44b34d4b96e3efd2e8331b4 Mon Sep 17 00:00:00 2001 From: Greirson Lee-Thorp Date: Mon, 3 Feb 2025 17:10:43 -0800 Subject: [PATCH 4/6] feat: implement batch upload inactivity cleanup mechanism - Add batchActivity Map to track batch upload timestamps - Create interval-based cleanup for inactive batch uploads - Update upload chunk route to refresh batch activity timestamp - Remove manual timeout for folder mappings in favor of centralized cleanup - Improve resource management for long-running batch uploads --- server.js | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/server.js b/server.js index 85c3cfe..fae67e0 100644 --- a/server.js +++ b/server.js @@ -222,6 +222,25 @@ const uploads = new Map(); const folderMappings = new Map(); // Store batch IDs for folder uploads const batchUploads = new Map(); +// Store batch activity timestamps +const batchActivity = new Map(); + +// Add cleanup interval for inactive batches +setInterval(() => { + const now = Date.now(); + for (const [batchId, lastActivity] of batchActivity.entries()) { + if (now - lastActivity >= 5 * 60 * 1000) { // 5 minutes of inactivity + // Clean up all folder mappings for this batch + for (const key of folderMappings.keys()) { + if (key.endsWith(`-${batchId}`)) { + folderMappings.delete(key); + } + } + batchActivity.delete(batchId); + log.info(`Cleaned up folder mappings for inactive batch: ${batchId}`); + } + } +}, 60000); // Check every minute // Add these helper functions before the routes async function getUniqueFilePath(filePath) { @@ -338,10 +357,8 @@ app.post('/upload/init', async (req, res) => { folderMappings.set(`${originalFolderName}-${batchId}`, newFolderName); - // Clean up mapping after 5 minutes - setTimeout(() => { - folderMappings.delete(`${originalFolderName}-${batchId}`); - }, 5 * 60 * 1000); + // Update batch activity timestamp instead of setting a timeout + batchActivity.set(batchId, Date.now()); } // Replace the original folder path with the mapped one and keep original file name @@ -393,6 +410,13 @@ app.post('/upload/chunk/:uploadId', express.raw({ } try { + // Get the batch ID from the request headers + const batchId = req.headers['x-batch-id']; + if (batchId && isValidBatchId(batchId)) { + // Update batch activity timestamp + batchActivity.set(batchId, Date.now()); + } + upload.writeStream.write(Buffer.from(req.body)); upload.bytesReceived += chunkSize; From 79f6c5387163f62ae2296e5463509801b012ab24 Mon Sep 17 00:00:00 2001 From: Greirson Lee-Thorp Date: Mon, 3 Feb 2025 17:14:03 -0800 Subject: [PATCH 5/6] feat: improve single file upload batch ID generation - Add automatic batch ID generation for single file uploads - Generate unique batch ID using timestamp and random string - Enhance batch ID validation to handle single file and multi-file upload scenarios - Improve error handling for batch ID format validation --- server.js | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/server.js b/server.js index fae67e0..a7667c2 100644 --- a/server.js +++ b/server.js @@ -303,12 +303,16 @@ function isValidBatchId(batchId) { // Routes app.post('/upload/init', async (req, res) => { const { filename, fileSize } = req.body; - const batchId = req.headers['x-batch-id']; + let batchId = req.headers['x-batch-id']; - // Validate batch ID - if (!batchId || !isValidBatchId(batchId)) { - log.error('Invalid or missing batch ID'); - return res.status(400).json({ error: 'Invalid or missing batch ID' }); + // For single file uploads without a batch ID, generate one + if (!batchId) { + const timestamp = Date.now(); + const randomStr = crypto.randomBytes(4).toString('hex').substring(0, 9); + batchId = `${timestamp}-${randomStr}`; + } else if (!isValidBatchId(batchId)) { + log.error('Invalid batch ID format'); + return res.status(400).json({ error: 'Invalid batch ID format' }); } const safeFilename = path.normalize(filename).replace(/^(\.\.(\/|\\|$))+/, ''); From bcc555939b5623543cee2aa20bc33fb0a69c1a3a Mon Sep 17 00:00:00 2001 From: Greirson Lee-Thorp Date: Mon, 3 Feb 2025 17:37:33 -0800 Subject: [PATCH 6/6] feat: improve batch ID handling for file and folder uploads - Update server-side upload initialization to always refresh batch activity timestamp - Enhance client-side file grouping to consistently track batch IDs for files and folders - Modify drop and file selection handlers to generate batch IDs for all upload scenarios - Ensure batch ID is preserved and used consistently across file upload groups --- public/index.html | 28 +++++++++++++++++++++++----- server.js | 6 +++--- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/public/index.html b/public/index.html index 0ad97b4..bb74968 100644 --- a/public/index.html +++ b/public/index.html @@ -216,10 +216,16 @@ name: folderName, isFolder: true, totalSize: 0, - files: [] + files: [], + // Use the first file's batch ID or generate a new one + batchId: file.batchId }); } const group = groups.get(folderName); + // If group doesn't have a batch ID yet, use the file's batch ID + if (!group.batchId) { + group.batchId = file.batchId; + } group.files.push(file); group.totalSize += file.size; } else { @@ -228,7 +234,8 @@ name: file.name, isFolder: false, totalSize: file.size, - files: [file] + files: [file], + batchId: file.batchId }); } }); @@ -245,7 +252,7 @@ if (entry.isFile) { const file = await new Promise((resolve) => entry.file(resolve)); file.relativePath = path; - file.batchId = batchId; + file.batchId = batchId; // Use the same batch ID for all files in this drop fileEntries.push(file); } else if (entry.isDirectory) { const reader = entry.createReader(); @@ -304,20 +311,29 @@ function handleDrop(e) { const items = e.dataTransfer.items; if (items && items[0].webkitGetAsEntry) { + // Handle folder/file drop using DataTransferItemList getAllFileEntries(items).then(newFiles => { files = newFiles; updateFileList(); }); } else { + // Handle single file drop + const batchId = generateBatchId(); files = [...e.dataTransfer.files]; + files.forEach(file => { + file.relativePath = ''; // No relative path for dropped files + file.batchId = batchId; + }); updateFileList(); } } function handleFiles(e) { + const batchId = generateBatchId(); files = [...e.target.files]; files.forEach(file => { file.relativePath = ''; // No relative path for individual files + file.batchId = batchId; }); updateFileList(); } @@ -367,13 +383,15 @@ document.getElementById('uploadProgress').innerHTML = ''; const groupedItems = groupFilesByFolder(files); - const batchId = generateBatchId(); // Generate a single batch ID for all files const results = await Promise.all( groupedItems.map(async item => { let success = true; + // Use the group's batch ID for all files in the group + const groupBatchId = item.batchId || generateBatchId(); for (const file of item.files) { - const uploader = new FileUploader(file, batchId); + // Always use the group's batch ID + const uploader = new FileUploader(file, groupBatchId); if (!await uploader.start()) { success = false; } diff --git a/server.js b/server.js index a7667c2..88aed01 100644 --- a/server.js +++ b/server.js @@ -315,6 +315,9 @@ app.post('/upload/init', async (req, res) => { return res.status(400).json({ error: 'Invalid batch ID format' }); } + // Always update batch activity timestamp for any upload + batchActivity.set(batchId, Date.now()); + const safeFilename = path.normalize(filename).replace(/^(\.\.(\/|\\|$))+/, ''); // Check file size limit @@ -360,9 +363,6 @@ app.post('/upload/init', async (req, res) => { } folderMappings.set(`${originalFolderName}-${batchId}`, newFolderName); - - // Update batch activity timestamp instead of setting a timeout - batchActivity.set(batchId, Date.now()); } // Replace the original folder path with the mapped one and keep original file name