prometheus: Add a pushgateway.

Prometheus declares metrics to be "stale" if they go 5min without an
update[^1], and removes them -- which makes longer scrape intervals
impossible.

Add a pushgateway, which caches output from such jobs.

[^1]: https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness

puppet/kandra/manifests/prometheus/pushgateway.pp

@@ -0,0 +1,32 @@
+# @summary Pushgateway for cron jobs
+#
+class kandra::prometheus::pushgateway {
+  include kandra::prometheus::base
+  include zulip::supervisor
+
+  $version = $zulip::common::versions['pushgateway']['version']
+  $dir = "/srv/zulip-pushgateway-${version}"
+  $bin = "${dir}/pushgateway"
+
+  zulip::external_dep { 'pushgateway':
+    version        => $version,
+    url            => "https://github.com/prometheus/pushgateway/releases/download/v${version}/pushgateway-${version}.linux-${zulip::common::goarch}.tar.gz",
+    tarball_prefix => "pushgateway-${version}.linux-${zulip::common::goarch}",
+    bin            => [$bin],
+    cleanup_after  => [Service[supervisor]],
+  }
+
+  file { "${zulip::common::supervisor_conf_dir}/prometheus_pushgateway.conf":
+    ensure  => file,
+    require => [
+      User[zulip],
+      Package[supervisor],
+      File[$bin],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/prometheus_pushgateway.conf.template.erb'),
+    notify  => Service[supervisor],
+  }
+}

puppet/kandra/templates/prometheus/prometheus.yaml.template.erb

@@ -13,6 +13,12 @@ scrape_configs:
     static_configs:
       - targets: ["localhost:3000"]
 
+  - job_name: "pushgateway"
+    scrape_interval: 2m
+    honor_labels: true
+    static_configs:
+      - targets: ["localhost:9091"]
+
   - job_name: "node"
     ec2_sd_configs:
       - region: us-east-1

puppet/kandra/templates/supervisor/conf.d/prometheus_pushgateway.conf.template.erb

@@ -0,0 +1,8 @@
+[program:prometheus_pushgateway]
+command=<%= @bin %>
+priority=10
+autostart=true
+autorestart=true
+user=zulip
+redirect_stderr=true
+stdout_logfile=/var/log/zulip/pushgateway.log

puppet/zulip/manifests/common.pp

@@ -176,6 +176,15 @@ class zulip::common {
       },
     },
 
+    # https://github.com/prometheus/pushgateway/releases
+    'pushgateway' => {
+      'version' => '1.11.1',
+      'sha256'  => {
+        'amd64'   => '6ce6ffab84d0d71195036326640295c02165462abd12b8092b0fa93188f5ee37',
+        'aarch64' => 'b6dc1c1c46d1137e5eda253f6291247e39330d3065a839857b947e59b4f3e64b',
+      },
+    },
+
     # https://github.com/oliver006/redis_exporter/releases
     'redis_exporter' => {
       'version' => '1.79.0',