From a305fe23d38f7a1905ce9948d8ccc39f87caaa04 Mon Sep 17 00:00:00 2001 From: Muhammad Ibrahim Date: Tue, 7 Oct 2025 20:52:46 +0100 Subject: [PATCH] Fixed issues with the agent not sending apt data properly Added Indexing to the database for faster and efficient searching Fixed some filtering from the hosts page relating to packages that need updating Added buy me a coffee link (sorry and thank you <3) --- agents/patchmon-agent.sh | 60 ++++++++++++++++--- .../add_host_packages_indexes/migration.sql | 30 ++++++++++ backend/prisma/schema.prisma | 11 ++++ backend/src/routes/packageRoutes.js | 46 +++++++++++--- frontend/src/components/Layout.jsx | 28 +++++++-- frontend/src/pages/HostDetail.jsx | 6 +- frontend/src/pages/Hosts.jsx | 6 +- frontend/src/pages/Packages.jsx | 52 ++++++++++------ 8 files changed, 197 insertions(+), 42 deletions(-) create mode 100644 backend/prisma/migrations/add_host_packages_indexes/migration.sql diff --git a/agents/patchmon-agent.sh b/agents/patchmon-agent.sh index 8bd4c16..038cd10 100755 --- a/agents/patchmon-agent.sh +++ b/agents/patchmon-agent.sh @@ -605,8 +605,24 @@ get_apt_packages() { local -n packages_ref=$1 local -n first_ref=$2 - # Update package lists (use apt-get for older distros; quieter output) - apt-get update -qq + # Update package lists with retry logic for lock conflicts + local retry_count=0 + local max_retries=3 + local retry_delay=5 + + while [[ $retry_count -lt $max_retries ]]; do + if apt-get update -qq 2>/dev/null; then + break + else + retry_count=$((retry_count + 1)) + if [[ $retry_count -lt $max_retries ]]; then + warning "APT lock detected, retrying in ${retry_delay} seconds... (attempt $retry_count/$max_retries)" + sleep $retry_delay + else + warning "APT lock persists after $max_retries attempts, continuing without update..." + fi + fi + done # Determine upgradable packages using apt-get simulation (compatible with Ubuntu 18.04) # Example line format: @@ -626,6 +642,11 @@ get_apt_packages() { is_security_update=true fi + # Escape JSON special characters in package data + package_name=$(echo "$package_name" | sed 's/"/\\"/g' | sed 's/\\/\\\\/g') + current_version=$(echo "$current_version" | sed 's/"/\\"/g' | sed 's/\\/\\\\/g') + available_version=$(echo "$available_version" | sed 's/"/\\"/g' | sed 's/\\/\\\\/g') + if [[ "$first_ref" == true ]]; then first_ref=false else @@ -642,7 +663,11 @@ get_apt_packages() { while IFS=' ' read -r package_name version; do if [[ -n "$package_name" && -n "$version" ]]; then # Check if this package is not in the upgrade list - if ! echo "$upgradable" | grep -q "^$package_name/"; then + if ! echo "$upgradable_sim" | grep -q "^Inst $package_name "; then + # Escape JSON special characters in package data + package_name=$(echo "$package_name" | sed 's/"/\\"/g' | sed 's/\\/\\\\/g') + version=$(echo "$version" | sed 's/"/\\"/g' | sed 's/\\/\\\\/g') + if [[ "$first_ref" == true ]]; then first_ref=false else @@ -883,6 +908,15 @@ send_update() { local network_json=$(get_network_info) local system_json=$(get_system_info) + # Validate JSON before sending + if ! echo "$packages_json" | jq empty 2>/dev/null; then + error "Invalid packages JSON generated: $packages_json" + fi + + if ! echo "$repositories_json" | jq empty 2>/dev/null; then + error "Invalid repositories JSON generated: $repositories_json" + fi + info "Sending update to PatchMon server..." # Merge all JSON objects into one @@ -909,15 +943,27 @@ EOF # Merge the base payload with the system information local payload=$(echo "$base_payload $merged_json" | jq -s '.[0] * .[1]') + # Write payload to temporary file to avoid "Argument list too long" error + local temp_payload_file=$(mktemp) + echo "$payload" > "$temp_payload_file" + + # Debug: Show payload size + local payload_size=$(wc -c < "$temp_payload_file") + echo -e "${BLUE}ℹī¸ 📊 Payload size: $payload_size bytes${NC}" local response=$(curl $CURL_FLAGS -X POST \ -H "Content-Type: application/json" \ -H "X-API-ID: $API_ID" \ -H "X-API-KEY: $API_KEY" \ - -d "$payload" \ - "$PATCHMON_SERVER/api/$API_VERSION/hosts/update") + -d @"$temp_payload_file" \ + "$PATCHMON_SERVER/api/$API_VERSION/hosts/update" 2>&1) - if [[ $? -eq 0 ]]; then + local curl_exit_code=$? + + # Clean up temporary file + rm -f "$temp_payload_file" + + if [[ $curl_exit_code -eq 0 ]]; then if echo "$response" | grep -q "success"; then local packages_count=$(echo "$response" | grep -o '"packagesProcessed":[0-9]*' | cut -d':' -f2) success "Update sent successfully (${packages_count} packages processed)" @@ -953,7 +999,7 @@ EOF error "Update failed: $response" fi else - error "Failed to send update" + error "Failed to send update (curl exit code: $curl_exit_code): $response" fi } diff --git a/backend/prisma/migrations/add_host_packages_indexes/migration.sql b/backend/prisma/migrations/add_host_packages_indexes/migration.sql new file mode 100644 index 0000000..711bbcf --- /dev/null +++ b/backend/prisma/migrations/add_host_packages_indexes/migration.sql @@ -0,0 +1,30 @@ +-- Add indexes to host_packages table for performance optimization +-- These indexes will dramatically speed up queries filtering by host_id, package_id, needs_update, and is_security_update + +-- Index for queries filtering by host_id (very common - used when viewing packages for a specific host) +CREATE INDEX IF NOT EXISTS "host_packages_host_id_idx" ON "host_packages"("host_id"); + +-- Index for queries filtering by package_id (used when finding hosts for a specific package) +CREATE INDEX IF NOT EXISTS "host_packages_package_id_idx" ON "host_packages"("package_id"); + +-- Index for queries filtering by needs_update (used when finding outdated packages) +CREATE INDEX IF NOT EXISTS "host_packages_needs_update_idx" ON "host_packages"("needs_update"); + +-- Index for queries filtering by is_security_update (used when finding security updates) +CREATE INDEX IF NOT EXISTS "host_packages_is_security_update_idx" ON "host_packages"("is_security_update"); + +-- Composite index for the most common query pattern: host_id + needs_update +-- This is optimal for "show me outdated packages for this host" +CREATE INDEX IF NOT EXISTS "host_packages_host_id_needs_update_idx" ON "host_packages"("host_id", "needs_update"); + +-- Composite index for host_id + needs_update + is_security_update +-- This is optimal for "show me security updates for this host" +CREATE INDEX IF NOT EXISTS "host_packages_host_id_needs_update_security_idx" ON "host_packages"("host_id", "needs_update", "is_security_update"); + +-- Index for queries filtering by package_id + needs_update +-- This is optimal for "show me hosts where this package needs updates" +CREATE INDEX IF NOT EXISTS "host_packages_package_id_needs_update_idx" ON "host_packages"("package_id", "needs_update"); + +-- Index on last_checked for cleanup/maintenance queries +CREATE INDEX IF NOT EXISTS "host_packages_last_checked_idx" ON "host_packages"("last_checked"); + diff --git a/backend/prisma/schema.prisma b/backend/prisma/schema.prisma index b0fcd0f..7a603e9 100644 --- a/backend/prisma/schema.prisma +++ b/backend/prisma/schema.prisma @@ -44,6 +44,14 @@ model host_packages { packages packages @relation(fields: [package_id], references: [id], onDelete: Cascade) @@unique([host_id, package_id]) + @@index([host_id]) + @@index([package_id]) + @@index([needs_update]) + @@index([is_security_update]) + @@index([host_id, needs_update]) + @@index([host_id, needs_update, is_security_update]) + @@index([package_id, needs_update]) + @@index([last_checked]) } model host_repositories { @@ -108,6 +116,9 @@ model packages { created_at DateTime @default(now()) updated_at DateTime host_packages host_packages[] + + @@index([name]) + @@index([category]) } model repositories { diff --git a/backend/src/routes/packageRoutes.js b/backend/src/routes/packageRoutes.js index ee2186f..d944a1f 100644 --- a/backend/src/routes/packageRoutes.js +++ b/backend/src/routes/packageRoutes.js @@ -14,6 +14,7 @@ router.get("/", async (req, res) => { category = "", needsUpdate = "", isSecurityUpdate = "", + host = "", } = req.query; const skip = (parseInt(page, 10) - 1) * parseInt(limit, 10); @@ -33,8 +34,27 @@ router.get("/", async (req, res) => { : {}, // Category filter category ? { category: { equals: category } } : {}, - // Update status filters - needsUpdate + // Host filter - only return packages installed on the specified host + // Combined with update status filters if both are present + host + ? { + host_packages: { + some: { + host_id: host, + // If needsUpdate or isSecurityUpdate filters are present, apply them here + ...(needsUpdate + ? { needs_update: needsUpdate === "true" } + : {}), + ...(isSecurityUpdate + ? { is_security_update: isSecurityUpdate === "true" } + : {}), + }, + }, + } + : {}, + // Update status filters (only applied if no host filter) + // If host filter is present, these are already applied above + !host && needsUpdate ? { host_packages: { some: { @@ -43,7 +63,7 @@ router.get("/", async (req, res) => { }, } : {}, - isSecurityUpdate + !host && isSecurityUpdate ? { host_packages: { some: { @@ -84,24 +104,32 @@ router.get("/", async (req, res) => { // Get additional stats for each package const packagesWithStats = await Promise.all( packages.map(async (pkg) => { + // Build base where clause for this package + const baseWhere = { package_id: pkg.id }; + + // If host filter is specified, add host filter to all queries + const hostWhere = host ? { ...baseWhere, host_id: host } : baseWhere; + const [updatesCount, securityCount, packageHosts] = await Promise.all([ prisma.host_packages.count({ where: { - package_id: pkg.id, + ...hostWhere, needs_update: true, }, }), prisma.host_packages.count({ where: { - package_id: pkg.id, + ...hostWhere, needs_update: true, is_security_update: true, }, }), prisma.host_packages.findMany({ where: { - package_id: pkg.id, - needs_update: true, + ...hostWhere, + // If host filter is specified, include all packages for that host + // Otherwise, only include packages that need updates + ...(host ? {} : { needs_update: true }), }, select: { hosts: { @@ -112,6 +140,10 @@ router.get("/", async (req, res) => { os_type: true, }, }, + current_version: true, + available_version: true, + needs_update: true, + is_security_update: true, }, take: 10, // Limit to first 10 for performance }), diff --git a/frontend/src/components/Layout.jsx b/frontend/src/components/Layout.jsx index fdac292..35509ef 100644 --- a/frontend/src/components/Layout.jsx +++ b/frontend/src/components/Layout.jsx @@ -898,7 +898,25 @@ const Layout = ({ children }) => { )} - {/* 2) Roadmap */} + {/* 2) Buy Me a Coffee */} + + + Buy Me a Coffee + + + + {/* 3) Roadmap */} { > - {/* 3) Docs */} + {/* 4) Docs */} { > - {/* 4) Discord */} + {/* 5) Discord */} { > - {/* 5) Email */} + {/* 6) Email */} { > - {/* 6) YouTube */} + {/* 7) YouTube */} { {host.stats.total_packages}

- Total Packages + Total Installed

diff --git a/frontend/src/pages/Packages.jsx b/frontend/src/pages/Packages.jsx index ffac6bb..e3ac20c 100644 --- a/frontend/src/pages/Packages.jsx +++ b/frontend/src/pages/Packages.jsx @@ -115,8 +115,20 @@ const Packages = () => { refetch, isFetching, } = useQuery({ - queryKey: ["packages"], - queryFn: () => packagesAPI.getAll({ limit: 1000 }).then((res) => res.data), + queryKey: ["packages", hostFilter, updateStatusFilter], + queryFn: () => { + const params = { limit: 10000 }; // High limit to effectively get all packages + if (hostFilter && hostFilter !== "all") { + params.host = hostFilter; + } + // Pass update status filter to backend to pre-filter packages + if (updateStatusFilter === "needs-updates") { + params.needsUpdate = "true"; + } else if (updateStatusFilter === "security-updates") { + params.isSecurityUpdate = "true"; + } + return packagesAPI.getAll(params).then((res) => res.data); + }, staleTime: 5 * 60 * 1000, // Data stays fresh for 5 minutes refetchOnWindowFocus: false, // Don't refetch when window regains focus }); @@ -160,15 +172,13 @@ const Packages = () => { const matchesUpdateStatus = updateStatusFilter === "all-packages" || - updateStatusFilter === "needs-updates" || - (updateStatusFilter === "security-updates" && pkg.isSecurityUpdate) || - (updateStatusFilter === "regular-updates" && !pkg.isSecurityUpdate); - - // For "all-packages", we don't filter by update status - // For other filters, we only show packages that need updates - const matchesUpdateNeeded = - updateStatusFilter === "all-packages" || - (pkg.stats?.updatesNeeded || 0) > 0; + (updateStatusFilter === "needs-updates" && + (pkg.stats?.updatesNeeded || 0) > 0) || + (updateStatusFilter === "security-updates" && + (pkg.stats?.securityUpdates || 0) > 0) || + (updateStatusFilter === "regular-updates" && + (pkg.stats?.updatesNeeded || 0) > 0 && + (pkg.stats?.securityUpdates || 0) === 0); const packageHosts = pkg.packageHosts || []; const matchesHost = @@ -176,11 +186,7 @@ const Packages = () => { packageHosts.some((host) => host.hostId === hostFilter); return ( - matchesSearch && - matchesCategory && - matchesUpdateStatus && - matchesUpdateNeeded && - matchesHost + matchesSearch && matchesCategory && matchesUpdateStatus && matchesHost ); }); @@ -435,8 +441,16 @@ const Packages = () => { }); const uniquePackageHostsCount = uniquePackageHosts.size; - // Calculate total packages available - const totalPackagesCount = packages?.length || 0; + // Calculate total packages installed + // When filtering by host, count each package once (since it can only be installed once per host) + // When not filtering, sum up all installations across all hosts + const totalPackagesCount = + hostFilter && hostFilter !== "all" + ? packages?.length || 0 + : packages?.reduce( + (sum, pkg) => sum + (pkg.stats?.totalInstalls || 0), + 0, + ) || 0; // Calculate outdated packages const outdatedPackagesCount = @@ -517,7 +531,7 @@ const Packages = () => {

- Total Packages + Total Installed

{totalPackagesCount}