mirror of
https://github.com/9technologygroup/patchmon.net.git
synced 2025-11-10 08:55:44 +00:00
fixed the issue with graphs on dashboard not showing information correctly, so now statistics are queued into a new table specifically for this and addd this in automation queue
This commit is contained in:
@@ -0,0 +1,16 @@
|
||||
-- CreateTable
|
||||
CREATE TABLE "system_statistics" (
|
||||
"id" TEXT NOT NULL,
|
||||
"unique_packages_count" INTEGER NOT NULL,
|
||||
"unique_security_count" INTEGER NOT NULL,
|
||||
"total_packages" INTEGER NOT NULL,
|
||||
"total_hosts" INTEGER NOT NULL,
|
||||
"hosts_needing_updates" INTEGER NOT NULL,
|
||||
"timestamp" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
CONSTRAINT "system_statistics_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "system_statistics_timestamp_idx" ON "system_statistics"("timestamp");
|
||||
|
||||
@@ -212,6 +212,18 @@ model update_history {
|
||||
hosts hosts @relation(fields: [host_id], references: [id], onDelete: Cascade)
|
||||
}
|
||||
|
||||
model system_statistics {
|
||||
id String @id
|
||||
unique_packages_count Int
|
||||
unique_security_count Int
|
||||
total_packages Int
|
||||
total_hosts Int
|
||||
hosts_needing_updates Int
|
||||
timestamp DateTime @default(now())
|
||||
|
||||
@@index([timestamp])
|
||||
}
|
||||
|
||||
model users {
|
||||
id String @id
|
||||
username String @unique
|
||||
|
||||
@@ -242,6 +242,30 @@ router.post(
|
||||
},
|
||||
);
|
||||
|
||||
// Trigger manual system statistics collection
|
||||
router.post(
|
||||
"/trigger/system-statistics",
|
||||
authenticateToken,
|
||||
async (_req, res) => {
|
||||
try {
|
||||
const job = await queueManager.triggerSystemStatistics();
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
jobId: job.id,
|
||||
message: "System statistics collection triggered successfully",
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("Error triggering system statistics collection:", error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: "Failed to trigger system statistics collection",
|
||||
});
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
// Get queue health status
|
||||
router.get("/health", authenticateToken, async (_req, res) => {
|
||||
try {
|
||||
@@ -300,6 +324,7 @@ router.get("/overview", authenticateToken, async (_req, res) => {
|
||||
queueManager.getRecentJobs(QUEUE_NAMES.ORPHANED_PACKAGE_CLEANUP, 1),
|
||||
queueManager.getRecentJobs(QUEUE_NAMES.DOCKER_INVENTORY_CLEANUP, 1),
|
||||
queueManager.getRecentJobs(QUEUE_NAMES.AGENT_COMMANDS, 1),
|
||||
queueManager.getRecentJobs(QUEUE_NAMES.SYSTEM_STATISTICS, 1),
|
||||
]);
|
||||
|
||||
// Calculate overview metrics
|
||||
@@ -309,21 +334,24 @@ router.get("/overview", authenticateToken, async (_req, res) => {
|
||||
stats[QUEUE_NAMES.SESSION_CLEANUP].delayed +
|
||||
stats[QUEUE_NAMES.ORPHANED_REPO_CLEANUP].delayed +
|
||||
stats[QUEUE_NAMES.ORPHANED_PACKAGE_CLEANUP].delayed +
|
||||
stats[QUEUE_NAMES.DOCKER_INVENTORY_CLEANUP].delayed,
|
||||
stats[QUEUE_NAMES.DOCKER_INVENTORY_CLEANUP].delayed +
|
||||
stats[QUEUE_NAMES.SYSTEM_STATISTICS].delayed,
|
||||
|
||||
runningTasks:
|
||||
stats[QUEUE_NAMES.GITHUB_UPDATE_CHECK].active +
|
||||
stats[QUEUE_NAMES.SESSION_CLEANUP].active +
|
||||
stats[QUEUE_NAMES.ORPHANED_REPO_CLEANUP].active +
|
||||
stats[QUEUE_NAMES.ORPHANED_PACKAGE_CLEANUP].active +
|
||||
stats[QUEUE_NAMES.DOCKER_INVENTORY_CLEANUP].active,
|
||||
stats[QUEUE_NAMES.DOCKER_INVENTORY_CLEANUP].active +
|
||||
stats[QUEUE_NAMES.SYSTEM_STATISTICS].active,
|
||||
|
||||
failedTasks:
|
||||
stats[QUEUE_NAMES.GITHUB_UPDATE_CHECK].failed +
|
||||
stats[QUEUE_NAMES.SESSION_CLEANUP].failed +
|
||||
stats[QUEUE_NAMES.ORPHANED_REPO_CLEANUP].failed +
|
||||
stats[QUEUE_NAMES.ORPHANED_PACKAGE_CLEANUP].failed +
|
||||
stats[QUEUE_NAMES.DOCKER_INVENTORY_CLEANUP].failed,
|
||||
stats[QUEUE_NAMES.DOCKER_INVENTORY_CLEANUP].failed +
|
||||
stats[QUEUE_NAMES.SYSTEM_STATISTICS].failed,
|
||||
|
||||
totalAutomations: Object.values(stats).reduce((sum, queueStats) => {
|
||||
return (
|
||||
@@ -435,6 +463,22 @@ router.get("/overview", authenticateToken, async (_req, res) => {
|
||||
: "Never run",
|
||||
stats: stats[QUEUE_NAMES.AGENT_COMMANDS],
|
||||
},
|
||||
{
|
||||
name: "System Statistics Collection",
|
||||
queue: QUEUE_NAMES.SYSTEM_STATISTICS,
|
||||
description: "Collects aggregated system-wide package statistics",
|
||||
schedule: "Every 30 minutes",
|
||||
lastRun: recentJobs[6][0]?.finishedOn
|
||||
? new Date(recentJobs[6][0].finishedOn).toLocaleString()
|
||||
: "Never",
|
||||
lastRunTimestamp: recentJobs[6][0]?.finishedOn || 0,
|
||||
status: recentJobs[6][0]?.failedReason
|
||||
? "Failed"
|
||||
: recentJobs[6][0]
|
||||
? "Success"
|
||||
: "Never run",
|
||||
stats: stats[QUEUE_NAMES.SYSTEM_STATISTICS],
|
||||
},
|
||||
].sort((a, b) => {
|
||||
// Sort by last run timestamp (most recent first)
|
||||
// If both have never run (timestamp 0), maintain original order
|
||||
|
||||
@@ -564,22 +564,43 @@ router.get(
|
||||
const startDate = new Date();
|
||||
startDate.setDate(endDate.getDate() - daysInt);
|
||||
|
||||
// Build where clause
|
||||
const whereClause = {
|
||||
// Determine if we need aggregation based on host filter
|
||||
const needsAggregation =
|
||||
!hostId || hostId === "all" || hostId === "undefined";
|
||||
|
||||
let trendsData;
|
||||
|
||||
if (needsAggregation) {
|
||||
// For "All Hosts" mode, use system_statistics table
|
||||
trendsData = await prisma.system_statistics.findMany({
|
||||
where: {
|
||||
timestamp: {
|
||||
gte: startDate,
|
||||
lte: endDate,
|
||||
},
|
||||
};
|
||||
|
||||
// Add host filter if specified
|
||||
if (hostId && hostId !== "all" && hostId !== "undefined") {
|
||||
whereClause.host_id = hostId;
|
||||
}
|
||||
|
||||
// Get all update history records in the date range
|
||||
const trendsData = await prisma.update_history.findMany({
|
||||
where: whereClause,
|
||||
},
|
||||
select: {
|
||||
timestamp: true,
|
||||
unique_packages_count: true,
|
||||
unique_security_count: true,
|
||||
total_packages: true,
|
||||
total_hosts: true,
|
||||
hosts_needing_updates: true,
|
||||
},
|
||||
orderBy: {
|
||||
timestamp: "asc",
|
||||
},
|
||||
});
|
||||
} else {
|
||||
// For individual host, use update_history table
|
||||
trendsData = await prisma.update_history.findMany({
|
||||
where: {
|
||||
host_id: hostId,
|
||||
timestamp: {
|
||||
gte: startDate,
|
||||
lte: endDate,
|
||||
},
|
||||
},
|
||||
select: {
|
||||
timestamp: true,
|
||||
packages_count: true,
|
||||
@@ -592,9 +613,74 @@ router.get(
|
||||
timestamp: "asc",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Enhanced data validation and processing
|
||||
const processedData = trendsData
|
||||
// Process data based on source
|
||||
let processedData;
|
||||
let aggregatedArray;
|
||||
|
||||
if (needsAggregation) {
|
||||
// For "All Hosts" mode, data comes from system_statistics table
|
||||
// Already aggregated, just need to format it
|
||||
processedData = trendsData
|
||||
.filter((record) => {
|
||||
// Enhanced validation
|
||||
return (
|
||||
record.total_packages !== null &&
|
||||
record.total_packages >= 0 &&
|
||||
record.unique_packages_count >= 0 &&
|
||||
record.unique_security_count >= 0 &&
|
||||
record.unique_security_count <= record.unique_packages_count
|
||||
);
|
||||
})
|
||||
.map((record) => {
|
||||
const date = new Date(record.timestamp);
|
||||
let timeKey;
|
||||
|
||||
if (daysInt <= 1) {
|
||||
// For "Last 24 hours", use full timestamp for each data point
|
||||
// This allows plotting all individual data points
|
||||
timeKey = date.toISOString(); // Full ISO timestamp
|
||||
} else {
|
||||
// For daily view, group by day
|
||||
timeKey = date.toISOString().split("T")[0]; // YYYY-MM-DD
|
||||
}
|
||||
|
||||
return {
|
||||
timeKey,
|
||||
total_packages: record.total_packages,
|
||||
packages_count: record.unique_packages_count,
|
||||
security_count: record.unique_security_count,
|
||||
timestamp: record.timestamp,
|
||||
};
|
||||
});
|
||||
|
||||
if (daysInt <= 1) {
|
||||
// For "Last 24 hours", use all individual data points without grouping
|
||||
// Sort by timestamp
|
||||
aggregatedArray = processedData.sort(
|
||||
(a, b) => a.timestamp.getTime() - b.timestamp.getTime(),
|
||||
);
|
||||
} else {
|
||||
// For longer periods, group by timeKey and take the latest value for each period
|
||||
const aggregatedData = processedData.reduce((acc, item) => {
|
||||
if (
|
||||
!acc[item.timeKey] ||
|
||||
item.timestamp > acc[item.timeKey].timestamp
|
||||
) {
|
||||
acc[item.timeKey] = item;
|
||||
}
|
||||
return acc;
|
||||
}, {});
|
||||
|
||||
// Convert to array and sort
|
||||
aggregatedArray = Object.values(aggregatedData).sort((a, b) =>
|
||||
a.timeKey.localeCompare(b.timeKey),
|
||||
);
|
||||
}
|
||||
} else {
|
||||
// For individual host, data comes from update_history table
|
||||
processedData = trendsData
|
||||
.filter((record) => {
|
||||
// Enhanced validation
|
||||
return (
|
||||
@@ -602,17 +688,18 @@ router.get(
|
||||
record.total_packages >= 0 &&
|
||||
record.packages_count >= 0 &&
|
||||
record.security_count >= 0 &&
|
||||
record.security_count <= record.packages_count && // Security can't exceed outdated
|
||||
record.security_count <= record.packages_count &&
|
||||
record.status === "success"
|
||||
); // Only include successful reports
|
||||
);
|
||||
})
|
||||
.map((record) => {
|
||||
const date = new Date(record.timestamp);
|
||||
let timeKey;
|
||||
|
||||
if (daysInt <= 1) {
|
||||
// For hourly view, group by hour only (not minutes)
|
||||
timeKey = date.toISOString().substring(0, 13); // YYYY-MM-DDTHH
|
||||
// For "Last 24 hours", use full timestamp for each data point
|
||||
// This allows plotting all individual data points
|
||||
timeKey = date.toISOString(); // Full ISO timestamp
|
||||
} else {
|
||||
// For daily view, group by day
|
||||
timeKey = date.toISOString().split("T")[0]; // YYYY-MM-DD
|
||||
@@ -628,70 +715,14 @@ router.get(
|
||||
};
|
||||
});
|
||||
|
||||
// Determine if we need aggregation based on host filter
|
||||
const needsAggregation =
|
||||
!hostId || hostId === "all" || hostId === "undefined";
|
||||
|
||||
let aggregatedArray;
|
||||
|
||||
if (needsAggregation) {
|
||||
// For "All Hosts" mode, we need to calculate the actual total packages differently
|
||||
// Instead of aggregating historical data (which is per-host), we'll use the current total
|
||||
// and show that as a flat line, since total packages don't change much over time
|
||||
|
||||
// Get the current total packages count (unique packages across all hosts)
|
||||
const currentTotalPackages = await prisma.packages.count({
|
||||
where: {
|
||||
host_packages: {
|
||||
some: {}, // At least one host has this package
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// Aggregate data by timeKey when looking at "All Hosts" or no specific host
|
||||
const aggregatedData = processedData.reduce((acc, item) => {
|
||||
if (!acc[item.timeKey]) {
|
||||
acc[item.timeKey] = {
|
||||
timeKey: item.timeKey,
|
||||
total_packages: currentTotalPackages, // Use current total packages
|
||||
packages_count: 0,
|
||||
security_count: 0,
|
||||
record_count: 0,
|
||||
host_ids: new Set(),
|
||||
min_timestamp: item.timestamp,
|
||||
max_timestamp: item.timestamp,
|
||||
};
|
||||
}
|
||||
|
||||
// For outdated and security packages: SUM (these represent counts across hosts)
|
||||
acc[item.timeKey].packages_count += item.packages_count;
|
||||
acc[item.timeKey].security_count += item.security_count;
|
||||
|
||||
acc[item.timeKey].record_count += 1;
|
||||
acc[item.timeKey].host_ids.add(item.host_id);
|
||||
|
||||
// Track timestamp range
|
||||
if (item.timestamp < acc[item.timeKey].min_timestamp) {
|
||||
acc[item.timeKey].min_timestamp = item.timestamp;
|
||||
}
|
||||
if (item.timestamp > acc[item.timeKey].max_timestamp) {
|
||||
acc[item.timeKey].max_timestamp = item.timestamp;
|
||||
}
|
||||
|
||||
return acc;
|
||||
}, {});
|
||||
|
||||
// Convert to array and add metadata
|
||||
aggregatedArray = Object.values(aggregatedData)
|
||||
.map((item) => ({
|
||||
...item,
|
||||
host_count: item.host_ids.size,
|
||||
host_ids: Array.from(item.host_ids),
|
||||
}))
|
||||
.sort((a, b) => a.timeKey.localeCompare(b.timeKey));
|
||||
if (daysInt <= 1) {
|
||||
// For "Last 24 hours", use all individual data points without grouping
|
||||
// Sort by timestamp
|
||||
aggregatedArray = processedData.sort(
|
||||
(a, b) => a.timestamp.getTime() - b.timestamp.getTime(),
|
||||
);
|
||||
} else {
|
||||
// For specific host, show individual data points without aggregation
|
||||
// But still group by timeKey to handle multiple reports from same host in same time period
|
||||
// For longer periods, group by timeKey to handle multiple reports from same host in same time period
|
||||
const hostAggregatedData = processedData.reduce((acc, item) => {
|
||||
if (!acc[item.timeKey]) {
|
||||
acc[item.timeKey] = {
|
||||
@@ -729,9 +760,20 @@ router.get(
|
||||
}))
|
||||
.sort((a, b) => a.timeKey.localeCompare(b.timeKey));
|
||||
}
|
||||
}
|
||||
|
||||
// Handle sparse data by filling missing time periods
|
||||
const fillMissingPeriods = (data, daysInt) => {
|
||||
if (data.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
// For "Last 24 hours", return data as-is without filling gaps
|
||||
// This allows plotting all individual data points
|
||||
if (daysInt <= 1) {
|
||||
return data;
|
||||
}
|
||||
|
||||
const filledData = [];
|
||||
const startDate = new Date();
|
||||
startDate.setDate(startDate.getDate() - daysInt);
|
||||
@@ -741,43 +783,49 @@ router.get(
|
||||
const endDate = new Date();
|
||||
const currentDate = new Date(startDate);
|
||||
|
||||
// Find the last known values for interpolation
|
||||
// Sort data by timeKey to get chronological order
|
||||
const sortedData = [...data].sort((a, b) =>
|
||||
a.timeKey.localeCompare(b.timeKey),
|
||||
);
|
||||
|
||||
// Find the first actual data point (don't fill before this)
|
||||
const firstDataPoint = sortedData[0];
|
||||
const firstDataTimeKey = firstDataPoint?.timeKey;
|
||||
|
||||
// Track last known values as we iterate forward
|
||||
let lastKnownValues = null;
|
||||
if (data.length > 0) {
|
||||
lastKnownValues = {
|
||||
total_packages: data[0].total_packages,
|
||||
packages_count: data[0].packages_count,
|
||||
security_count: data[0].security_count,
|
||||
};
|
||||
}
|
||||
let hasSeenFirstDataPoint = false;
|
||||
|
||||
while (currentDate <= endDate) {
|
||||
let timeKey;
|
||||
if (daysInt <= 1) {
|
||||
timeKey = currentDate.toISOString().substring(0, 13); // Hourly
|
||||
currentDate.setHours(currentDate.getHours() + 1);
|
||||
} else {
|
||||
timeKey = currentDate.toISOString().split("T")[0]; // Daily
|
||||
// For daily view, group by day
|
||||
timeKey = currentDate.toISOString().split("T")[0]; // YYYY-MM-DD
|
||||
currentDate.setDate(currentDate.getDate() + 1);
|
||||
|
||||
// Skip periods before the first actual data point
|
||||
if (firstDataTimeKey && timeKey < firstDataTimeKey) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (dataMap.has(timeKey)) {
|
||||
const item = dataMap.get(timeKey);
|
||||
filledData.push(item);
|
||||
// Update last known values
|
||||
// Update last known values with actual data
|
||||
lastKnownValues = {
|
||||
total_packages: item.total_packages,
|
||||
packages_count: item.packages_count,
|
||||
security_count: item.security_count,
|
||||
total_packages: item.total_packages || 0,
|
||||
packages_count: item.packages_count || 0,
|
||||
security_count: item.security_count || 0,
|
||||
};
|
||||
hasSeenFirstDataPoint = true;
|
||||
} else {
|
||||
// For missing periods, use the last known values (interpolation)
|
||||
// This creates a continuous line instead of gaps
|
||||
// For missing periods AFTER the first data point, use forward-fill
|
||||
// Only fill if we have a last known value and we've seen the first data point
|
||||
if (lastKnownValues !== null && hasSeenFirstDataPoint) {
|
||||
filledData.push({
|
||||
timeKey,
|
||||
total_packages: lastKnownValues?.total_packages || 0,
|
||||
packages_count: lastKnownValues?.packages_count || 0,
|
||||
security_count: lastKnownValues?.security_count || 0,
|
||||
total_packages: lastKnownValues.total_packages,
|
||||
packages_count: lastKnownValues.packages_count,
|
||||
security_count: lastKnownValues.security_count,
|
||||
record_count: 0,
|
||||
host_count: 0,
|
||||
host_ids: [],
|
||||
@@ -786,6 +834,8 @@ router.get(
|
||||
isInterpolated: true, // Mark as interpolated for debugging
|
||||
});
|
||||
}
|
||||
// If we haven't seen the first data point yet, skip this period
|
||||
}
|
||||
}
|
||||
|
||||
return filledData;
|
||||
@@ -810,7 +860,7 @@ router.get(
|
||||
// Get current package state for offline fallback
|
||||
let currentPackageState = null;
|
||||
if (hostId && hostId !== "all" && hostId !== "undefined") {
|
||||
// Get current package counts for specific host
|
||||
// For individual host, get current package counts from host_packages
|
||||
const currentState = await prisma.host_packages.aggregate({
|
||||
where: {
|
||||
host_id: hostId,
|
||||
@@ -841,8 +891,29 @@ router.get(
|
||||
security_count: securityCount,
|
||||
};
|
||||
} else {
|
||||
// Get current package counts for all hosts
|
||||
// Total packages = count of unique packages installed on at least one host
|
||||
// For "All Hosts" mode, use the latest system_statistics record if available
|
||||
// Otherwise calculate from database
|
||||
const latestStats = await prisma.system_statistics.findFirst({
|
||||
orderBy: {
|
||||
timestamp: "desc",
|
||||
},
|
||||
select: {
|
||||
total_packages: true,
|
||||
unique_packages_count: true,
|
||||
unique_security_count: true,
|
||||
timestamp: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (latestStats) {
|
||||
// Use latest system statistics (collected by scheduled job)
|
||||
currentPackageState = {
|
||||
total_packages: latestStats.total_packages,
|
||||
packages_count: latestStats.unique_packages_count,
|
||||
security_count: latestStats.unique_security_count,
|
||||
};
|
||||
} else {
|
||||
// Fallback: calculate from database if no statistics collected yet
|
||||
const totalPackagesCount = await prisma.packages.count({
|
||||
where: {
|
||||
host_packages: {
|
||||
@@ -851,25 +922,34 @@ router.get(
|
||||
},
|
||||
});
|
||||
|
||||
// Get counts for boolean fields separately
|
||||
const outdatedCount = await prisma.host_packages.count({
|
||||
const uniqueOutdatedCount = await prisma.packages.count({
|
||||
where: {
|
||||
host_packages: {
|
||||
some: {
|
||||
needs_update: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const securityCount = await prisma.host_packages.count({
|
||||
const uniqueSecurityCount = await prisma.packages.count({
|
||||
where: {
|
||||
host_packages: {
|
||||
some: {
|
||||
needs_update: true,
|
||||
is_security_update: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
currentPackageState = {
|
||||
total_packages: totalPackagesCount,
|
||||
packages_count: outdatedCount,
|
||||
security_count: securityCount,
|
||||
packages_count: uniqueOutdatedCount,
|
||||
security_count: uniqueSecurityCount,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Format data for chart
|
||||
const chartData = {
|
||||
@@ -923,6 +1003,11 @@ router.get(
|
||||
chartData.datasets[2].data.push(item.security_count);
|
||||
});
|
||||
|
||||
// Replace the last label with "Now" to indicate current state
|
||||
if (chartData.labels.length > 0) {
|
||||
chartData.labels[chartData.labels.length - 1] = "Now";
|
||||
}
|
||||
|
||||
// Calculate data quality metrics
|
||||
const dataQuality = {
|
||||
totalRecords: trendsData.length,
|
||||
|
||||
@@ -13,6 +13,7 @@ const OrphanedPackageCleanup = require("./orphanedPackageCleanup");
|
||||
const DockerInventoryCleanup = require("./dockerInventoryCleanup");
|
||||
const DockerImageUpdateCheck = require("./dockerImageUpdateCheck");
|
||||
const MetricsReporting = require("./metricsReporting");
|
||||
const SystemStatistics = require("./systemStatistics");
|
||||
|
||||
// Queue names
|
||||
const QUEUE_NAMES = {
|
||||
@@ -23,6 +24,7 @@ const QUEUE_NAMES = {
|
||||
DOCKER_INVENTORY_CLEANUP: "docker-inventory-cleanup",
|
||||
DOCKER_IMAGE_UPDATE_CHECK: "docker-image-update-check",
|
||||
METRICS_REPORTING: "metrics-reporting",
|
||||
SYSTEM_STATISTICS: "system-statistics",
|
||||
AGENT_COMMANDS: "agent-commands",
|
||||
};
|
||||
|
||||
@@ -106,6 +108,9 @@ class QueueManager {
|
||||
this.automations[QUEUE_NAMES.METRICS_REPORTING] = new MetricsReporting(
|
||||
this,
|
||||
);
|
||||
this.automations[QUEUE_NAMES.SYSTEM_STATISTICS] = new SystemStatistics(
|
||||
this,
|
||||
);
|
||||
|
||||
console.log("✅ All automation classes initialized");
|
||||
}
|
||||
@@ -191,6 +196,15 @@ class QueueManager {
|
||||
workerOptions,
|
||||
);
|
||||
|
||||
// System Statistics Worker
|
||||
this.workers[QUEUE_NAMES.SYSTEM_STATISTICS] = new Worker(
|
||||
QUEUE_NAMES.SYSTEM_STATISTICS,
|
||||
this.automations[QUEUE_NAMES.SYSTEM_STATISTICS].process.bind(
|
||||
this.automations[QUEUE_NAMES.SYSTEM_STATISTICS],
|
||||
),
|
||||
workerOptions,
|
||||
);
|
||||
|
||||
// Agent Commands Worker
|
||||
this.workers[QUEUE_NAMES.AGENT_COMMANDS] = new Worker(
|
||||
QUEUE_NAMES.AGENT_COMMANDS,
|
||||
@@ -323,6 +337,7 @@ class QueueManager {
|
||||
await this.automations[QUEUE_NAMES.DOCKER_INVENTORY_CLEANUP].schedule();
|
||||
await this.automations[QUEUE_NAMES.DOCKER_IMAGE_UPDATE_CHECK].schedule();
|
||||
await this.automations[QUEUE_NAMES.METRICS_REPORTING].schedule();
|
||||
await this.automations[QUEUE_NAMES.SYSTEM_STATISTICS].schedule();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -358,6 +373,10 @@ class QueueManager {
|
||||
].triggerManual();
|
||||
}
|
||||
|
||||
async triggerSystemStatistics() {
|
||||
return this.automations[QUEUE_NAMES.SYSTEM_STATISTICS].triggerManual();
|
||||
}
|
||||
|
||||
async triggerMetricsReporting() {
|
||||
return this.automations[QUEUE_NAMES.METRICS_REPORTING].triggerManual();
|
||||
}
|
||||
|
||||
140
backend/src/services/automation/systemStatistics.js
Normal file
140
backend/src/services/automation/systemStatistics.js
Normal file
@@ -0,0 +1,140 @@
|
||||
const { prisma } = require("./shared/prisma");
|
||||
const { v4: uuidv4 } = require("uuid");
|
||||
|
||||
/**
|
||||
* System Statistics Collection Automation
|
||||
* Collects aggregated system-wide statistics every 30 minutes
|
||||
* for use in package trends charts
|
||||
*/
|
||||
class SystemStatistics {
|
||||
constructor(queueManager) {
|
||||
this.queueManager = queueManager;
|
||||
this.queueName = "system-statistics";
|
||||
}
|
||||
|
||||
/**
|
||||
* Process system statistics collection job
|
||||
*/
|
||||
async process(_job) {
|
||||
const startTime = Date.now();
|
||||
console.log("📊 Starting system statistics collection...");
|
||||
|
||||
try {
|
||||
// Calculate unique package counts across all hosts
|
||||
const uniquePackagesCount = await prisma.packages.count({
|
||||
where: {
|
||||
host_packages: {
|
||||
some: {
|
||||
needs_update: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const uniqueSecurityCount = await prisma.packages.count({
|
||||
where: {
|
||||
host_packages: {
|
||||
some: {
|
||||
needs_update: true,
|
||||
is_security_update: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// Calculate total unique packages installed on at least one host
|
||||
const totalPackages = await prisma.packages.count({
|
||||
where: {
|
||||
host_packages: {
|
||||
some: {}, // At least one host has this package
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// Calculate total hosts
|
||||
const totalHosts = await prisma.hosts.count({
|
||||
where: {
|
||||
status: "active",
|
||||
},
|
||||
});
|
||||
|
||||
// Calculate hosts needing updates (distinct hosts with packages needing updates)
|
||||
const hostsNeedingUpdates = await prisma.hosts.count({
|
||||
where: {
|
||||
status: "active",
|
||||
host_packages: {
|
||||
some: {
|
||||
needs_update: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// Store statistics in database
|
||||
await prisma.system_statistics.create({
|
||||
data: {
|
||||
id: uuidv4(),
|
||||
unique_packages_count: uniquePackagesCount,
|
||||
unique_security_count: uniqueSecurityCount,
|
||||
total_packages: totalPackages,
|
||||
total_hosts: totalHosts,
|
||||
hosts_needing_updates: hostsNeedingUpdates,
|
||||
timestamp: new Date(),
|
||||
},
|
||||
});
|
||||
|
||||
const executionTime = Date.now() - startTime;
|
||||
console.log(
|
||||
`✅ System statistics collection completed in ${executionTime}ms - Unique packages: ${uniquePackagesCount}, Security: ${uniqueSecurityCount}, Total hosts: ${totalHosts}`,
|
||||
);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
uniquePackagesCount,
|
||||
uniqueSecurityCount,
|
||||
totalPackages,
|
||||
totalHosts,
|
||||
hostsNeedingUpdates,
|
||||
executionTime,
|
||||
};
|
||||
} catch (error) {
|
||||
const executionTime = Date.now() - startTime;
|
||||
console.error(
|
||||
`❌ System statistics collection failed after ${executionTime}ms:`,
|
||||
error.message,
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Schedule recurring system statistics collection (every 30 minutes)
|
||||
*/
|
||||
async schedule() {
|
||||
const job = await this.queueManager.queues[this.queueName].add(
|
||||
"system-statistics",
|
||||
{},
|
||||
{
|
||||
repeat: { pattern: "*/30 * * * *" }, // Every 30 minutes
|
||||
jobId: "system-statistics-recurring",
|
||||
},
|
||||
);
|
||||
console.log("✅ System statistics collection scheduled (every 30 minutes)");
|
||||
return job;
|
||||
}
|
||||
|
||||
/**
|
||||
* Trigger manual system statistics collection
|
||||
*/
|
||||
async triggerManual() {
|
||||
const job = await this.queueManager.queues[this.queueName].add(
|
||||
"system-statistics-manual",
|
||||
{},
|
||||
{ priority: 1 },
|
||||
);
|
||||
console.log("✅ Manual system statistics collection triggered");
|
||||
return job;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = SystemStatistics;
|
||||
Reference in New Issue
Block a user