diff --git a/src/converters/main.ts b/src/converters/main.ts index a9f2e42..5bdbc78 100644 --- a/src/converters/main.ts +++ b/src/converters/main.ts @@ -1,4 +1,6 @@ -import { normalizeFiletype } from "../helpers/normalizeFiletype"; +import db from "../db/db"; +import { MAX_CONVERT_PROCESS } from "../helpers/env"; +import { normalizeFiletype, normalizeOutputFiletype } from "../helpers/normalizeFiletype"; import { convert as convertassimp, properties as propertiesassimp } from "./assimp"; import { convert as convertCalibre, properties as propertiesCalibre } from "./calibre"; import { convert as convertDvisvgm, properties as propertiesDvisvgm } from "./dvisvgm"; @@ -105,6 +107,64 @@ const properties: Record< }, }; +function chunks(arr: T[], size: number): T[][] { + if(size <= 0){ + return [arr] + } + return Array.from({ length: Math.ceil(arr.length / size) }, (_: T, i: number) => + arr.slice(i * size, i * size + size) + ); +} + +export async function handleConvert( + fileNames: string[], + userUploadsDir: string, + userOutputDir: string, + convertTo: string, + converterName: string, + jobId: any +) { + + const query = db.query( + "INSERT INTO file_names (job_id, file_name, output_file_name, status) VALUES (?1, ?2, ?3, ?4)", + ); + + + for (const chunk of chunks(fileNames, MAX_CONVERT_PROCESS)) { + for(const fileName of chunk) { + const toProcess: Promise[] = []; + const filePath = `${userUploadsDir}${fileName}`; + const fileTypeOrig = fileName.split(".").pop() ?? ""; + const fileType = normalizeFiletype(fileTypeOrig); + const newFileExt = normalizeOutputFiletype(convertTo); + const newFileName = fileName.replace( + new RegExp(`${fileTypeOrig}(?!.*${fileTypeOrig})`), + newFileExt, + ); + const targetPath = `${userOutputDir}${newFileName}`; + toProcess.push( + new Promise((resolve, reject) => { + mainConverter( + filePath, + fileType, + convertTo, + targetPath, + {}, + converterName, + ).then(r => { + if (jobId.value) { + query.run(jobId.value, fileName, newFileName, r); + } + resolve(r); + }).catch(c => reject(c)); + }) + ); + + await Promise.all(toProcess); + } + } +} + export async function mainConverter( inputFilePath: string, fileTypeOriginal: string, diff --git a/src/helpers/env.ts b/src/helpers/env.ts index cbdf423..29820b7 100644 --- a/src/helpers/env.ts +++ b/src/helpers/env.ts @@ -15,3 +15,5 @@ export const HIDE_HISTORY = process.env.HIDE_HISTORY?.toLowerCase() === "true" | export const WEBROOT = process.env.WEBROOT ?? ""; export const LANGUAGE = process.env.LANGUAGE?.toLowerCase() || "en"; + +export const MAX_CONVERT_PROCESS = process.env.MAX_CONVERT_PROCESS && Number(process.env.MAX_CONVERT_PROCESS) > 0 ? Number(process.env.MAX_CONVERT_PROCESS) : 0 \ No newline at end of file diff --git a/src/pages/convert.tsx b/src/pages/convert.tsx index a95302c..3dbb892 100644 --- a/src/pages/convert.tsx +++ b/src/pages/convert.tsx @@ -2,11 +2,11 @@ import { mkdir } from "node:fs/promises"; import { Elysia, t } from "elysia"; import sanitize from "sanitize-filename"; import { outputDir, uploadsDir } from ".."; -import { mainConverter } from "../converters/main"; +import { handleConvert } from "../converters/main"; import db from "../db/db"; import { Jobs } from "../db/types"; import { WEBROOT } from "../helpers/env"; -import { normalizeFiletype, normalizeOutputFiletype } from "../helpers/normalizeFiletype"; +import { normalizeFiletype } from "../helpers/normalizeFiletype"; import { userService } from "./user"; export const convert = new Elysia().use(userService).post( @@ -61,36 +61,8 @@ export const convert = new Elysia().use(userService).post( jobId.value, ); - const query = db.query( - "INSERT INTO file_names (job_id, file_name, output_file_name, status) VALUES (?1, ?2, ?3, ?4)", - ); - // Start the conversion process in the background - Promise.all( - fileNames.map(async (fileName) => { - const filePath = `${userUploadsDir}${fileName}`; - const fileTypeOrig = fileName.split(".").pop() ?? ""; - const fileType = normalizeFiletype(fileTypeOrig); - const newFileExt = normalizeOutputFiletype(convertTo); - const newFileName = fileName.replace( - new RegExp(`${fileTypeOrig}(?!.*${fileTypeOrig})`), - newFileExt, - ); - const targetPath = `${userOutputDir}${newFileName}`; - - const result = await mainConverter( - filePath, - fileType, - convertTo, - targetPath, - {}, - converterName, - ); - if (jobId.value) { - query.run(jobId.value, fileName, newFileName, result); - } - }), - ) + handleConvert(fileNames, userUploadsDir, userOutputDir, convertTo, converterName, jobId) .then(() => { // All conversions are done, update the job status to 'completed' if (jobId.value) {