Fixed repo count issue

Refactored code to remove duplicate backend api endpoints for counting
Improved connection persistence issues
Improved database connection pooling issues
Fixed redis connection efficiency
Changed version to 1.3.0
Fixed GO binary detection based on package manager rather than OS
This commit is contained in:
Muhammad Ibrahim
2025-10-19 17:53:10 +01:00
parent 30c89de134
commit c4d0d8bee8
42 changed files with 376 additions and 310 deletions

View File

@@ -1,6 +1,6 @@
/**
* Database configuration for multiple instances
* Optimizes connection pooling to prevent "too many connections" errors
* Centralized Prisma Client Singleton
* Prevents multiple Prisma clients from creating connection leaks
*/
const { PrismaClient } = require("@prisma/client");
@@ -26,22 +26,43 @@ function getOptimizedDatabaseUrl() {
return url.toString();
}
// Create optimized Prisma client
function createPrismaClient() {
const optimizedUrl = getOptimizedDatabaseUrl();
// Singleton Prisma client instance
let prismaInstance = null;
return new PrismaClient({
datasources: {
db: {
url: optimizedUrl,
function getPrismaClient() {
if (!prismaInstance) {
const optimizedUrl = getOptimizedDatabaseUrl();
prismaInstance = new PrismaClient({
datasources: {
db: {
url: optimizedUrl,
},
},
},
log:
process.env.PRISMA_LOG_QUERIES === "true"
? ["query", "info", "warn", "error"]
: ["warn", "error"],
errorFormat: "pretty",
});
log:
process.env.PRISMA_LOG_QUERIES === "true"
? ["query", "info", "warn", "error"]
: ["warn", "error"],
errorFormat: "pretty",
});
// Handle graceful shutdown
process.on("beforeExit", async () => {
await prismaInstance.$disconnect();
});
process.on("SIGINT", async () => {
await prismaInstance.$disconnect();
process.exit(0);
});
process.on("SIGTERM", async () => {
await prismaInstance.$disconnect();
process.exit(0);
});
}
return prismaInstance;
}
// Connection health check
@@ -50,7 +71,7 @@ async function checkDatabaseConnection(prisma) {
await prisma.$queryRaw`SELECT 1`;
return true;
} catch (error) {
console.error("Database connection failed:", error.message);
console.error("Database connection check failed:", error.message);
return false;
}
}
@@ -121,9 +142,8 @@ async function disconnectPrisma(prisma, maxRetries = 3) {
}
module.exports = {
createPrismaClient,
getPrismaClient,
checkDatabaseConnection,
waitForDatabase,
disconnectPrisma,
getOptimizedDatabaseUrl,
};

View File

@@ -1,12 +1,12 @@
const jwt = require("jsonwebtoken");
const { PrismaClient } = require("@prisma/client");
const { getPrismaClient } = require("../config/prisma");
const {
validate_session,
update_session_activity,
is_tfa_bypassed,
} = require("../utils/session_manager");
const prisma = new PrismaClient();
const prisma = getPrismaClient();
// Middleware to verify JWT token with session validation
const authenticateToken = async (req, res, next) => {

View File

@@ -1,5 +1,5 @@
const { PrismaClient } = require("@prisma/client");
const prisma = new PrismaClient();
const { getPrismaClient } = require("../config/prisma");
const prisma = getPrismaClient();
// Permission middleware factory
const requirePermission = (permission) => {

View File

@@ -1,7 +1,7 @@
const express = require("express");
const bcrypt = require("bcryptjs");
const jwt = require("jsonwebtoken");
const { PrismaClient } = require("@prisma/client");
const { getPrismaClient } = require("../config/prisma");
const { body, validationResult } = require("express-validator");
const { authenticateToken, _requireAdmin } = require("../middleware/auth");
const {
@@ -20,7 +20,7 @@ const {
} = require("../utils/session_manager");
const router = express.Router();
const prisma = new PrismaClient();
const prisma = getPrismaClient();
/**
* Parse user agent string to extract browser and OS info

View File

@@ -1,5 +1,5 @@
const express = require("express");
const { PrismaClient } = require("@prisma/client");
const { getPrismaClient } = require("../config/prisma");
const crypto = require("node:crypto");
const bcrypt = require("bcryptjs");
const { body, validationResult } = require("express-validator");
@@ -8,7 +8,7 @@ const { requireManageSettings } = require("../middleware/permissions");
const { v4: uuidv4 } = require("uuid");
const router = express.Router();
const prisma = new PrismaClient();
const prisma = getPrismaClient();
// Generate auto-enrollment token credentials
const generate_auto_enrollment_token = () => {

View File

@@ -1,11 +1,11 @@
const express = require("express");
const { body, validationResult } = require("express-validator");
const { PrismaClient } = require("@prisma/client");
const { getPrismaClient } = require("../config/prisma");
const { authenticateToken } = require("../middleware/auth");
const { v4: uuidv4 } = require("uuid");
const router = express.Router();
const prisma = new PrismaClient();
const prisma = getPrismaClient();
// Helper function to get user permissions based on role
async function getUserPermissions(userRole) {

View File

@@ -1,5 +1,5 @@
const express = require("express");
const { PrismaClient } = require("@prisma/client");
const { getPrismaClient } = require("../config/prisma");
const moment = require("moment");
const { authenticateToken } = require("../middleware/auth");
const {
@@ -11,7 +11,7 @@ const {
const { queueManager } = require("../services/automation");
const router = express.Router();
const prisma = new PrismaClient();
const prisma = getPrismaClient();
// Get dashboard statistics
router.get(
@@ -61,9 +61,15 @@ router.get(
},
}),
// Total outdated packages across all hosts
prisma.host_packages.count({
where: { needs_update: true },
// Total unique packages that need updates
prisma.packages.count({
where: {
host_packages: {
some: {
needs_update: true,
},
},
},
}),
// Errored hosts (not updated within threshold based on update interval)
@@ -76,11 +82,15 @@ router.get(
},
}),
// Security updates count
prisma.host_packages.count({
// Security updates count (unique packages)
prisma.packages.count({
where: {
needs_update: true,
is_security_update: true,
host_packages: {
some: {
needs_update: true,
is_security_update: true,
},
},
},
}),

View File

@@ -1,9 +1,9 @@
const express = require("express");
const { authenticateToken } = require("../middleware/auth");
const { PrismaClient } = require("@prisma/client");
const { getPrismaClient } = require("../config/prisma");
const { v4: uuidv4 } = require("uuid");
const prisma = new PrismaClient();
const prisma = getPrismaClient();
const router = express.Router();
// Helper function to convert BigInt fields to strings for JSON serialization

View File

@@ -1,9 +1,9 @@
const express = require("express");
const { createPrismaClient } = require("../config/database");
const { getPrismaClient } = require("../config/prisma");
const bcrypt = require("bcryptjs");
const router = express.Router();
const prisma = createPrismaClient();
const prisma = getPrismaClient();
// Middleware to authenticate API key
const authenticateApiKey = async (req, res, next) => {
@@ -114,9 +114,15 @@ router.get("/stats", authenticateApiKey, async (_req, res) => {
where: { status: "active" },
});
// Get total outdated packages count
const totalOutdatedPackages = await prisma.host_packages.count({
where: { needs_update: true },
// Get total unique packages that need updates (consistent with dashboard)
const totalOutdatedPackages = await prisma.packages.count({
where: {
host_packages: {
some: {
needs_update: true,
},
},
},
});
// Get total repositories count
@@ -136,11 +142,15 @@ router.get("/stats", authenticateApiKey, async (_req, res) => {
},
});
// Get security updates count
const securityUpdates = await prisma.host_packages.count({
// Get security updates count (unique packages - consistent with dashboard)
const securityUpdates = await prisma.packages.count({
where: {
needs_update: true,
is_security_update: true,
host_packages: {
some: {
needs_update: true,
is_security_update: true,
},
},
},
});

View File

@@ -1,12 +1,12 @@
const express = require("express");
const { body, validationResult } = require("express-validator");
const { PrismaClient } = require("@prisma/client");
const { getPrismaClient } = require("../config/prisma");
const { randomUUID } = require("node:crypto");
const { authenticateToken } = require("../middleware/auth");
const { requireManageHosts } = require("../middleware/permissions");
const router = express.Router();
const prisma = new PrismaClient();
const prisma = getPrismaClient();
// Get all host groups
router.get("/", authenticateToken, async (_req, res) => {

View File

@@ -1,5 +1,5 @@
const express = require("express");
const { PrismaClient } = require("@prisma/client");
const { getPrismaClient } = require("../config/prisma");
const { body, validationResult } = require("express-validator");
const { v4: uuidv4 } = require("uuid");
const crypto = require("node:crypto");
@@ -12,7 +12,7 @@ const {
} = require("../middleware/permissions");
const router = express.Router();
const prisma = new PrismaClient();
const prisma = getPrismaClient();
// Secure endpoint to download the agent script/binary (requires API authentication)
router.get("/agent/download", async (req, res) => {
@@ -39,7 +39,10 @@ router.get("/agent/download", async (req, res) => {
// Check if this is a legacy agent (bash script) requesting update
// Legacy agents will have agent_version < 1.2.9 (excluding 1.2.9 itself)
// But allow forcing binary download for fresh installations
const forceBinary = req.query.force === "binary";
const isLegacyAgent =
!forceBinary &&
host.agent_version &&
((host.agent_version.startsWith("1.2.") &&
host.agent_version !== "1.2.9") ||

View File

@@ -1,8 +1,8 @@
const express = require("express");
const { PrismaClient } = require("@prisma/client");
const { getPrismaClient } = require("../config/prisma");
const router = express.Router();
const prisma = new PrismaClient();
const prisma = getPrismaClient();
// Get all packages with their update status
router.get("/", async (req, res) => {

View File

@@ -1,5 +1,5 @@
const express = require("express");
const { PrismaClient } = require("@prisma/client");
const { getPrismaClient } = require("../config/prisma");
const { authenticateToken } = require("../middleware/auth");
const {
requireManageSettings,
@@ -7,7 +7,7 @@ const {
} = require("../middleware/permissions");
const router = express.Router();
const prisma = new PrismaClient();
const prisma = getPrismaClient();
// Get all role permissions (allow users who can manage users to view roles)
router.get(

View File

@@ -1,6 +1,6 @@
const express = require("express");
const { body, validationResult } = require("express-validator");
const { PrismaClient } = require("@prisma/client");
const { getPrismaClient } = require("../config/prisma");
const { authenticateToken } = require("../middleware/auth");
const {
requireViewHosts,
@@ -8,7 +8,7 @@ const {
} = require("../middleware/permissions");
const router = express.Router();
const prisma = new PrismaClient();
const prisma = getPrismaClient();
// Get all repositories with host count
router.get("/", authenticateToken, requireViewHosts, async (_req, res) => {

View File

@@ -1,9 +1,9 @@
const express = require("express");
const router = express.Router();
const { createPrismaClient } = require("../config/database");
const { getPrismaClient } = require("../config/prisma");
const { authenticateToken } = require("../middleware/auth");
const prisma = createPrismaClient();
const prisma = getPrismaClient();
/**
* Global search endpoint

View File

@@ -1,12 +1,12 @@
const express = require("express");
const { body, validationResult } = require("express-validator");
const { PrismaClient } = require("@prisma/client");
const { getPrismaClient } = require("../config/prisma");
const { authenticateToken } = require("../middleware/auth");
const { requireManageSettings } = require("../middleware/permissions");
const { getSettings, updateSettings } = require("../services/settingsService");
const router = express.Router();
const prisma = new PrismaClient();
const prisma = getPrismaClient();
// WebSocket broadcaster for agent policy updates (no longer used - queue-based delivery preferred)
// const { broadcastSettingsUpdate } = require("../services/agentWs");

View File

@@ -1,12 +1,12 @@
const express = require("express");
const { PrismaClient } = require("@prisma/client");
const { getPrismaClient } = require("../config/prisma");
const speakeasy = require("speakeasy");
const QRCode = require("qrcode");
const { authenticateToken } = require("../middleware/auth");
const { body, validationResult } = require("express-validator");
const router = express.Router();
const prisma = new PrismaClient();
const prisma = getPrismaClient();
// Generate TFA secret and QR code
router.get("/setup", authenticateToken, async (req, res) => {

View File

@@ -1,9 +1,9 @@
const express = require("express");
const { authenticateToken } = require("../middleware/auth");
const { requireManageSettings } = require("../middleware/permissions");
const { PrismaClient } = require("@prisma/client");
const { getPrismaClient } = require("../config/prisma");
const prisma = new PrismaClient();
const prisma = getPrismaClient();
// Default GitHub repository URL
const DEFAULT_GITHUB_REPO = "https://github.com/PatchMon/PatchMon.git";
@@ -14,13 +14,13 @@ const router = express.Router();
function getCurrentVersion() {
try {
const packageJson = require("../../package.json");
return packageJson?.version || "1.2.9";
return packageJson?.version || "1.3.0";
} catch (packageError) {
console.warn(
"Could not read version from package.json, using fallback:",
packageError.message,
);
return "1.2.9";
return "1.3.0";
}
}

View File

@@ -53,8 +53,6 @@ router.get("/status/:apiId/stream", async (req, res) => {
// Validate session (same as regular auth middleware)
const validation = await validate_session(decoded.sessionId, token);
if (!validation.valid) {
console.error("[SSE] Session validation failed:", validation.reason);
console.error("[SSE] Invalid session for api_id:", apiId);
return res.status(401).json({ error: "Invalid or expired session" });
}
@@ -62,9 +60,7 @@ router.get("/status/:apiId/stream", async (req, res) => {
await update_session_activity(decoded.sessionId);
req.user = validation.user;
} catch (err) {
console.error("[SSE] JWT verification failed:", err.message);
console.error("[SSE] Invalid token for api_id:", apiId);
} catch (_err) {
return res.status(401).json({ error: "Invalid or expired token" });
}

View File

@@ -41,10 +41,10 @@ const helmet = require("helmet");
const rateLimit = require("express-rate-limit");
const cookieParser = require("cookie-parser");
const {
createPrismaClient,
getPrismaClient,
waitForDatabase,
disconnectPrisma,
} = require("./config/database");
} = require("./config/prisma");
const winston = require("winston");
// Import routes
@@ -75,7 +75,7 @@ const { BullMQAdapter } = require("@bull-board/api/bullMQAdapter");
const { ExpressAdapter } = require("@bull-board/express");
// Initialize Prisma client with optimized connection pooling for multiple instances
const prisma = createPrismaClient();
const prisma = getPrismaClient();
// Function to check and create default role permissions on startup
async function checkAndCreateRolePermissions() {

View File

@@ -52,7 +52,7 @@ class GitHubUpdateCheck {
}
// Read version from package.json
let currentVersion = "1.2.7"; // fallback
let currentVersion = "1.3.0"; // fallback
try {
const packageJson = require("../../../package.json");
if (packageJson?.version) {

View File

@@ -99,16 +99,27 @@ class QueueManager {
* Initialize all workers
*/
async initializeWorkers() {
// Optimized worker options to reduce Redis connections
const workerOptions = {
connection: redisConnection,
concurrency: 1, // Keep concurrency low to reduce connections
// Connection optimization
maxStalledCount: 1,
stalledInterval: 30000,
// Reduce connection churn
settings: {
stalledInterval: 30000,
maxStalledCount: 1,
},
};
// GitHub Update Check Worker
this.workers[QUEUE_NAMES.GITHUB_UPDATE_CHECK] = new Worker(
QUEUE_NAMES.GITHUB_UPDATE_CHECK,
this.automations[QUEUE_NAMES.GITHUB_UPDATE_CHECK].process.bind(
this.automations[QUEUE_NAMES.GITHUB_UPDATE_CHECK],
),
{
connection: redisConnection,
concurrency: 1,
},
workerOptions,
);
// Session Cleanup Worker
@@ -117,10 +128,7 @@ class QueueManager {
this.automations[QUEUE_NAMES.SESSION_CLEANUP].process.bind(
this.automations[QUEUE_NAMES.SESSION_CLEANUP],
),
{
connection: redisConnection,
concurrency: 1,
},
workerOptions,
);
// Orphaned Repo Cleanup Worker
@@ -129,10 +137,7 @@ class QueueManager {
this.automations[QUEUE_NAMES.ORPHANED_REPO_CLEANUP].process.bind(
this.automations[QUEUE_NAMES.ORPHANED_REPO_CLEANUP],
),
{
connection: redisConnection,
concurrency: 1,
},
workerOptions,
);
// Orphaned Package Cleanup Worker
@@ -141,167 +146,33 @@ class QueueManager {
this.automations[QUEUE_NAMES.ORPHANED_PACKAGE_CLEANUP].process.bind(
this.automations[QUEUE_NAMES.ORPHANED_PACKAGE_CLEANUP],
),
{
connection: redisConnection,
concurrency: 1,
},
workerOptions,
);
// Agent Commands Worker
this.workers[QUEUE_NAMES.AGENT_COMMANDS] = new Worker(
QUEUE_NAMES.AGENT_COMMANDS,
async (job) => {
const { api_id, type, update_interval } = job.data || {};
console.log("[agent-commands] processing job", job.id, api_id, type);
const { api_id, type } = job.data;
console.log(`Processing agent command: ${type} for ${api_id}`);
// Log job attempt to history - use job.id as the unique identifier
const attemptNumber = job.attemptsMade || 1;
const historyId = job.id; // Single row per job, updated with each attempt
try {
if (!api_id || !type) {
throw new Error("invalid job data");
}
// Find host by api_id
const host = await prisma.hosts.findUnique({
where: { api_id },
select: { id: true },
});
// Ensure agent is connected; if not, retry later
if (!agentWs.isConnected(api_id)) {
const error = new Error("agent not connected");
// Log failed attempt
await prisma.job_history.upsert({
where: { id: historyId },
create: {
id: historyId,
job_id: job.id,
queue_name: QUEUE_NAMES.AGENT_COMMANDS,
job_name: type,
host_id: host?.id,
api_id,
status: "failed",
attempt_number: attemptNumber,
error_message: error.message,
created_at: new Date(),
updated_at: new Date(),
},
update: {
status: "failed",
attempt_number: attemptNumber,
error_message: error.message,
updated_at: new Date(),
},
});
console.log(
"[agent-commands] agent not connected, will retry",
api_id,
);
throw error;
}
// Process the command
let result;
if (type === "settings_update") {
agentWs.pushSettingsUpdate(api_id, update_interval);
console.log(
"[agent-commands] delivered settings_update",
api_id,
update_interval,
);
result = { delivered: true, update_interval };
} else if (type === "report_now") {
agentWs.pushReportNow(api_id);
console.log("[agent-commands] delivered report_now", api_id);
result = { delivered: true };
} else {
throw new Error("unsupported agent command");
}
// Log successful completion
await prisma.job_history.upsert({
where: { id: historyId },
create: {
id: historyId,
job_id: job.id,
queue_name: QUEUE_NAMES.AGENT_COMMANDS,
job_name: type,
host_id: host?.id,
api_id,
status: "completed",
attempt_number: attemptNumber,
output: result,
created_at: new Date(),
updated_at: new Date(),
completed_at: new Date(),
},
update: {
status: "completed",
attempt_number: attemptNumber,
output: result,
error_message: null,
updated_at: new Date(),
completed_at: new Date(),
},
});
return result;
} catch (error) {
// Log error to history (if not already logged above)
if (error.message !== "agent not connected") {
const host = await prisma.hosts
.findUnique({
where: { api_id },
select: { id: true },
})
.catch(() => null);
await prisma.job_history
.upsert({
where: { id: historyId },
create: {
id: historyId,
job_id: job.id,
queue_name: QUEUE_NAMES.AGENT_COMMANDS,
job_name: type || "unknown",
host_id: host?.id,
api_id,
status: "failed",
attempt_number: attemptNumber,
error_message: error.message,
created_at: new Date(),
updated_at: new Date(),
},
update: {
status: "failed",
attempt_number: attemptNumber,
error_message: error.message,
updated_at: new Date(),
},
})
.catch((err) =>
console.error("[agent-commands] failed to log error:", err),
);
}
throw error;
// Send command via WebSocket based on type
if (type === "report_now") {
agentWs.pushReportNow(api_id);
} else if (type === "settings_update") {
// For settings update, we need additional data
const { update_interval } = job.data;
agentWs.pushSettingsUpdate(api_id, update_interval);
} else {
console.error(`Unknown agent command type: ${type}`);
}
},
{
connection: redisConnection,
concurrency: 10,
},
workerOptions,
);
// Add error handling for all workers
Object.values(this.workers).forEach((worker) => {
worker.on("error", (error) => {
console.error("Worker error:", error);
});
});
console.log("✅ All workers initialized");
console.log(
"✅ All workers initialized with optimized connection settings",
);
}
/**

View File

@@ -1,5 +1,5 @@
const { PrismaClient } = require("@prisma/client");
const { getPrismaClient } = require("../../../config/prisma");
const prisma = new PrismaClient();
const prisma = getPrismaClient();
module.exports = { prisma };

View File

@@ -1,17 +1,56 @@
const IORedis = require("ioredis");
// Redis connection configuration
// Redis connection configuration with connection pooling
const redisConnection = {
host: process.env.REDIS_HOST || "localhost",
port: parseInt(process.env.REDIS_PORT, 10) || 6379,
password: process.env.REDIS_PASSWORD || undefined,
username: process.env.REDIS_USER || undefined,
db: parseInt(process.env.REDIS_DB, 10) || 0,
// Connection pooling settings
lazyConnect: true,
keepAlive: 30000,
connectTimeout: 30000, // Increased from 10s to 30s
commandTimeout: 30000, // Increased from 5s to 30s
enableReadyCheck: false,
// Reduce connection churn
family: 4, // Force IPv4
// Retry settings
retryDelayOnClusterDown: 300,
retryDelayOnFailover: 100,
maxRetriesPerRequest: null, // BullMQ requires this to be null
// Connection pool settings
maxLoadingTimeout: 30000,
};
// Create Redis connection
const redis = new IORedis(redisConnection);
// Create Redis connection with singleton pattern
let redisInstance = null;
module.exports = { redis, redisConnection };
function getRedisConnection() {
if (!redisInstance) {
redisInstance = new IORedis(redisConnection);
// Handle graceful shutdown
process.on("beforeExit", async () => {
await redisInstance.quit();
});
process.on("SIGINT", async () => {
await redisInstance.quit();
process.exit(0);
});
process.on("SIGTERM", async () => {
await redisInstance.quit();
process.exit(0);
});
}
return redisInstance;
}
module.exports = {
redis: getRedisConnection(),
redisConnection,
getRedisConnection,
};

View File

@@ -33,7 +33,7 @@ async function checkPublicRepo(owner, repo) {
try {
const httpsRepoUrl = `https://api.github.com/repos/${owner}/${repo}/releases/latest`;
let currentVersion = "1.2.7"; // fallback
let currentVersion = "1.3.0"; // fallback
try {
const packageJson = require("../../../package.json");
if (packageJson?.version) {

View File

@@ -1,7 +1,7 @@
const { PrismaClient } = require("@prisma/client");
const { getPrismaClient } = require("../config/prisma");
const { v4: uuidv4 } = require("uuid");
const prisma = new PrismaClient();
const prisma = getPrismaClient();
// Cached settings instance
let cachedSettings = null;

View File

@@ -1,8 +1,8 @@
const jwt = require("jsonwebtoken");
const crypto = require("node:crypto");
const { PrismaClient } = require("@prisma/client");
const { getPrismaClient } = require("../config/prisma");
const prisma = new PrismaClient();
const prisma = getPrismaClient();
/**
* Session Manager - Handles secure session management with inactivity timeout