diff --git a/apps/docs/package.json b/apps/docs/package.json index f91268a..63a202f 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -1,6 +1,6 @@ { "name": "palmr-docs", - "version": "3.1.4-beta", + "version": "3.1.5-beta", "description": "Docs for Palmr", "private": true, "author": "Daniel Luiz Alves ", diff --git a/apps/server/package.json b/apps/server/package.json index 37ee0c3..06feda8 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -1,6 +1,6 @@ { "name": "palmr-api", - "version": "3.1.4-beta", + "version": "3.1.5-beta", "description": "API for Palmr", "private": true, "author": "Daniel Luiz Alves ", diff --git a/apps/server/src/modules/filesystem/controller.ts b/apps/server/src/modules/filesystem/controller.ts index b3d04ee..7947d21 100644 --- a/apps/server/src/modules/filesystem/controller.ts +++ b/apps/server/src/modules/filesystem/controller.ts @@ -8,9 +8,6 @@ import { ChunkManager, ChunkMetadata } from "./chunk-manager"; export class FilesystemController { private chunkManager = ChunkManager.getInstance(); - /** - * Safely encode filename for Content-Disposition header - */ private encodeFilenameForHeader(filename: string): string { if (!filename || filename.trim() === "") { return 'attachment; filename="download"'; @@ -103,9 +100,6 @@ export class FilesystemController { await provider.uploadFileFromStream(objectName, request.raw); } - /** - * Extract chunk metadata from request headers - */ private extractChunkMetadata(request: FastifyRequest): ChunkMetadata | null { const fileId = request.headers["x-file-id"] as string; const chunkIndex = request.headers["x-chunk-index"] as string; @@ -132,9 +126,6 @@ export class FilesystemController { return metadata; } - /** - * Handle chunked upload with streaming - */ private async handleChunkedUpload(request: FastifyRequest, metadata: ChunkMetadata, originalObjectName: string) { const stream = request.raw; @@ -145,9 +136,6 @@ export class FilesystemController { return await this.chunkManager.processChunk(metadata, stream, originalObjectName); } - /** - * Get upload progress for chunked uploads - */ async getUploadProgress(request: FastifyRequest, reply: FastifyReply) { try { const { fileId } = request.params as { fileId: string }; @@ -164,9 +152,6 @@ export class FilesystemController { } } - /** - * Cancel chunked upload - */ async cancelUpload(request: FastifyRequest, reply: FastifyReply) { try { const { fileId } = request.params as { fileId: string }; @@ -194,7 +179,6 @@ export class FilesystemController { const filePath = provider.getFilePath(tokenData.objectName); const stats = await fs.promises.stat(filePath); const fileSize = stats.size; - const isLargeFile = fileSize > 50 * 1024 * 1024; const fileName = tokenData.fileName || "download"; const range = request.headers.range; @@ -207,28 +191,15 @@ export class FilesystemController { const parts = range.replace(/bytes=/, "").split("-"); const start = parseInt(parts[0], 10); const end = parts[1] ? parseInt(parts[1], 10) : fileSize - 1; - const chunkSize = end - start + 1; reply.status(206); reply.header("Content-Range", `bytes ${start}-${end}/${fileSize}`); - reply.header("Content-Length", chunkSize); + reply.header("Content-Length", end - start + 1); - if (isLargeFile) { - await this.downloadLargeFileRange(reply, provider, tokenData.objectName, start, end); - } else { - const buffer = await provider.downloadFile(tokenData.objectName); - const chunk = buffer.slice(start, end + 1); - reply.send(chunk); - } + await this.downloadFileRange(reply, provider, tokenData.objectName, start, end); } else { reply.header("Content-Length", fileSize); - - if (isLargeFile) { - await this.downloadLargeFile(reply, provider, filePath); - } else { - const stream = provider.createDecryptedReadStream(tokenData.objectName); - reply.send(stream); - } + await this.downloadFileStream(reply, provider, tokenData.objectName); } provider.consumeDownloadToken(token); @@ -237,32 +208,75 @@ export class FilesystemController { } } - private async downloadLargeFile(reply: FastifyReply, provider: FilesystemStorageProvider, filePath: string) { - const readStream = fs.createReadStream(filePath); - const decryptStream = provider.createDecryptStream(); - + private async downloadFileStream(reply: FastifyReply, provider: FilesystemStorageProvider, objectName: string) { try { - await pipeline(readStream, decryptStream, reply.raw); + FilesystemStorageProvider.logMemoryUsage(`Download start: ${objectName}`); + + const downloadStream = provider.createDownloadStream(objectName); + + downloadStream.on("error", (error) => { + console.error("Download stream error:", error); + FilesystemStorageProvider.logMemoryUsage(`Download error: ${objectName}`); + if (!reply.sent) { + reply.status(500).send({ error: "Download failed" }); + } + }); + + reply.raw.on("close", () => { + if (downloadStream.readable && typeof (downloadStream as any).destroy === "function") { + (downloadStream as any).destroy(); + } + FilesystemStorageProvider.logMemoryUsage(`Download client disconnect: ${objectName}`); + }); + + await pipeline(downloadStream, reply.raw); + + FilesystemStorageProvider.logMemoryUsage(`Download complete: ${objectName}`); } catch (error) { - throw error; + console.error("Download error:", error); + FilesystemStorageProvider.logMemoryUsage(`Download failed: ${objectName}`); + if (!reply.sent) { + reply.status(500).send({ error: "Download failed" }); + } } } - private async downloadLargeFileRange( + private async downloadFileRange( reply: FastifyReply, provider: FilesystemStorageProvider, objectName: string, start: number, end: number ) { - const filePath = provider.getFilePath(objectName); - const readStream = fs.createReadStream(filePath, { start, end }); - const decryptStream = provider.createDecryptStream(); - try { - await pipeline(readStream, decryptStream, reply.raw); + FilesystemStorageProvider.logMemoryUsage(`Range download start: ${objectName} (${start}-${end})`); + + const rangeStream = await provider.createDownloadRangeStream(objectName, start, end); + + rangeStream.on("error", (error) => { + console.error("Range download stream error:", error); + FilesystemStorageProvider.logMemoryUsage(`Range download error: ${objectName} (${start}-${end})`); + if (!reply.sent) { + reply.status(500).send({ error: "Download failed" }); + } + }); + + reply.raw.on("close", () => { + if (rangeStream.readable && typeof (rangeStream as any).destroy === "function") { + (rangeStream as any).destroy(); + } + FilesystemStorageProvider.logMemoryUsage(`Range download client disconnect: ${objectName} (${start}-${end})`); + }); + + await pipeline(rangeStream, reply.raw); + + FilesystemStorageProvider.logMemoryUsage(`Range download complete: ${objectName} (${start}-${end})`); } catch (error) { - throw error; + console.error("Range download error:", error); + FilesystemStorageProvider.logMemoryUsage(`Range download failed: ${objectName} (${start}-${end})`); + if (!reply.sent) { + reply.status(500).send({ error: "Download failed" }); + } } } } diff --git a/apps/server/src/providers/filesystem-storage.provider.ts b/apps/server/src/providers/filesystem-storage.provider.ts index b744a8b..f17ce96 100644 --- a/apps/server/src/providers/filesystem-storage.provider.ts +++ b/apps/server/src/providers/filesystem-storage.provider.ts @@ -22,7 +22,7 @@ export class FilesystemStorageProvider implements StorageProvider { this.ensureUploadsDir(); setInterval(() => this.cleanExpiredTokens(), 5 * 60 * 1000); - setInterval(() => this.cleanupEmptyTempDirs(), 10 * 60 * 1000); // Every 10 minutes + setInterval(() => this.cleanupEmptyTempDirs(), 10 * 60 * 1000); } public static getInstance(): FilesystemStorageProvider { @@ -32,14 +32,6 @@ export class FilesystemStorageProvider implements StorageProvider { return FilesystemStorageProvider.instance; } - public createDecryptedReadStream(objectName: string): NodeJS.ReadableStream { - const filePath = this.getFilePath(objectName); - const fileStream = fsSync.createReadStream(filePath); - const decryptStream = this.createDecryptStream(); - - return fileStream.pipe(decryptStream); - } - private async ensureUploadsDir(): Promise { try { await fs.access(this.uploadsDir); @@ -261,6 +253,183 @@ export class FilesystemStorageProvider implements StorageProvider { return this.decryptFileLegacy(fileBuffer); } + createDownloadStream(objectName: string): NodeJS.ReadableStream { + const filePath = this.getFilePath(objectName); + const fileStream = fsSync.createReadStream(filePath); + + if (this.isEncryptionDisabled) { + fileStream.on("end", () => { + if (global.gc) { + global.gc(); + } + }); + + fileStream.on("close", () => { + if (global.gc) { + global.gc(); + } + }); + + return fileStream; + } + + const decryptStream = this.createDecryptStream(); + let isDestroyed = false; + + const cleanup = () => { + if (isDestroyed) return; + isDestroyed = true; + + try { + if (fileStream && !fileStream.destroyed) { + fileStream.destroy(); + } + if (decryptStream && !decryptStream.destroyed) { + decryptStream.destroy(); + } + } catch (error) { + console.warn("Error during download stream cleanup:", error); + } + + if (global.gc) { + global.gc(); + } + }; + + fileStream.on("error", cleanup); + decryptStream.on("error", cleanup); + decryptStream.on("end", cleanup); + decryptStream.on("close", cleanup); + + decryptStream.on("pipe", (src: any) => { + if (src && src.on) { + src.on("close", cleanup); + src.on("error", cleanup); + } + }); + + return fileStream.pipe(decryptStream); + } + + async createDownloadRangeStream(objectName: string, start: number, end: number): Promise { + if (!this.isEncryptionDisabled) { + return this.createRangeStreamFromDecrypted(objectName, start, end); + } + + const filePath = this.getFilePath(objectName); + return fsSync.createReadStream(filePath, { start, end }); + } + + private createRangeStreamFromDecrypted(objectName: string, start: number, end: number): NodeJS.ReadableStream { + const { Transform, PassThrough } = require("stream"); + const filePath = this.getFilePath(objectName); + const fileStream = fsSync.createReadStream(filePath); + const decryptStream = this.createDecryptStream(); + const rangeStream = new PassThrough(); + + let bytesRead = 0; + let rangeEnded = false; + let isDestroyed = false; + + const rangeTransform = new Transform({ + transform(chunk: Buffer, encoding: any, callback: any) { + if (rangeEnded || isDestroyed) { + callback(); + return; + } + + const chunkStart = bytesRead; + const chunkEnd = bytesRead + chunk.length - 1; + bytesRead += chunk.length; + + if (chunkEnd < start) { + callback(); + return; + } + + if (chunkStart > end) { + rangeEnded = true; + this.end(); + callback(); + return; + } + + let sliceStart = 0; + let sliceEnd = chunk.length; + + if (chunkStart < start) { + sliceStart = start - chunkStart; + } + + if (chunkEnd > end) { + sliceEnd = end - chunkStart + 1; + rangeEnded = true; + } + + const slicedChunk = chunk.slice(sliceStart, sliceEnd); + this.push(slicedChunk); + + if (rangeEnded) { + this.end(); + } + + callback(); + }, + + flush(callback: any) { + if (global.gc) { + global.gc(); + } + callback(); + }, + }); + + const cleanup = () => { + if (isDestroyed) return; + isDestroyed = true; + + try { + if (fileStream && !fileStream.destroyed) { + fileStream.destroy(); + } + if (decryptStream && !decryptStream.destroyed) { + decryptStream.destroy(); + } + if (rangeTransform && !rangeTransform.destroyed) { + rangeTransform.destroy(); + } + if (rangeStream && !rangeStream.destroyed) { + rangeStream.destroy(); + } + } catch (error) { + console.warn("Error during stream cleanup:", error); + } + + if (global.gc) { + global.gc(); + } + }; + + fileStream.on("error", cleanup); + decryptStream.on("error", cleanup); + rangeTransform.on("error", cleanup); + rangeStream.on("error", cleanup); + + rangeStream.on("close", cleanup); + rangeStream.on("end", cleanup); + + rangeStream.on("pipe", (src: any) => { + if (src && src.on) { + src.on("close", cleanup); + src.on("error", cleanup); + } + }); + + fileStream.pipe(decryptStream).pipe(rangeTransform).pipe(rangeStream); + + return rangeStream; + } + private decryptFileBuffer(encryptedBuffer: Buffer): Buffer { const key = this.createEncryptionKey(); const iv = encryptedBuffer.slice(0, 16); @@ -277,6 +446,59 @@ export class FilesystemStorageProvider implements StorageProvider { return Buffer.from(decrypted.toString(CryptoJS.enc.Utf8), "base64"); } + static logMemoryUsage(context: string = "Unknown"): void { + const memUsage = process.memoryUsage(); + const formatBytes = (bytes: number) => { + const mb = bytes / 1024 / 1024; + return `${mb.toFixed(2)} MB`; + }; + + const rssInMB = memUsage.rss / 1024 / 1024; + const heapUsedInMB = memUsage.heapUsed / 1024 / 1024; + + if (rssInMB > 1024 || heapUsedInMB > 512) { + console.warn(`[MEMORY WARNING] ${context} - High memory usage detected:`); + console.warn(` RSS: ${formatBytes(memUsage.rss)}`); + console.warn(` Heap Used: ${formatBytes(memUsage.heapUsed)}`); + console.warn(` Heap Total: ${formatBytes(memUsage.heapTotal)}`); + console.warn(` External: ${formatBytes(memUsage.external)}`); + + if (global.gc) { + console.warn(" Forcing garbage collection..."); + global.gc(); + + const afterGC = process.memoryUsage(); + console.warn(` After GC - RSS: ${formatBytes(afterGC.rss)}, Heap: ${formatBytes(afterGC.heapUsed)}`); + } + } else { + console.log( + `[MEMORY INFO] ${context} - RSS: ${formatBytes(memUsage.rss)}, Heap: ${formatBytes(memUsage.heapUsed)}` + ); + } + } + + static forceGarbageCollection(context: string = "Manual"): void { + if (global.gc) { + const beforeGC = process.memoryUsage(); + global.gc(); + const afterGC = process.memoryUsage(); + + const formatBytes = (bytes: number) => `${(bytes / 1024 / 1024).toFixed(2)} MB`; + + console.log(`[GC] ${context} - Before: RSS ${formatBytes(beforeGC.rss)}, Heap ${formatBytes(beforeGC.heapUsed)}`); + console.log(`[GC] ${context} - After: RSS ${formatBytes(afterGC.rss)}, Heap ${formatBytes(afterGC.heapUsed)}`); + + const rssSaved = beforeGC.rss - afterGC.rss; + const heapSaved = beforeGC.heapUsed - afterGC.heapUsed; + + if (rssSaved > 0 || heapSaved > 0) { + console.log(`[GC] ${context} - Freed: RSS ${formatBytes(rssSaved)}, Heap ${formatBytes(heapSaved)}`); + } + } else { + console.warn(`[GC] ${context} - Garbage collection not available. Start Node.js with --expose-gc flag.`); + } + } + async fileExists(objectName: string): Promise { const filePath = this.getFilePath(objectName); try { @@ -321,9 +543,6 @@ export class FilesystemStorageProvider implements StorageProvider { this.downloadTokens.delete(token); } - /** - * Clean up temporary file and its parent directory if empty - */ private async cleanupTempFile(tempPath: string): Promise { try { await fs.unlink(tempPath); @@ -346,9 +565,6 @@ export class FilesystemStorageProvider implements StorageProvider { } } - /** - * Clean up empty temporary directories periodically - */ private async cleanupEmptyTempDirs(): Promise { try { const tempUploadsDir = directoriesConfig.tempUploads; diff --git a/apps/web/package.json b/apps/web/package.json index f46e21e..4bb55d8 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -1,6 +1,6 @@ { "name": "palmr-web", - "version": "3.1.4-beta", + "version": "3.1.5-beta", "description": "Frontend for Palmr", "private": true, "author": "Daniel Luiz Alves ", diff --git a/package.json b/package.json index 035a7cd..4db7245 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "palmr-monorepo", - "version": "3.1.4-beta", + "version": "3.1.5-beta", "description": "Palmr monorepo with Husky configuration", "private": true, "packageManager": "pnpm@10.6.0",