zulip_ops: Add configuration for Vector Akamai stats.

Akamai writes access logs to S3; we use an SQS events queue, combined
with Vector, to transform those into Prometheus statistics.

puppet/zulip/manifests/common.pp

@@ -162,5 +162,14 @@ class zulip::common {
         'aarch64' => 'ab7c9298d2fe5c5f58e3fe7c905929e93979d2b3b11c75eb8ba6ccc7a547238c',
       },
     },
+
+    # https://vector.dev/download/
+    'vector' => {
+      'version' => '0.34.0',
+      'sha256'  => {
+        'amd64'   => '3a39e712da43126262db878c3ae7647b23aac88834dea2763cc84269cb3c0206',
+        'aarch64' => '57acc628c495882e2ace787ac804e7bab57e42ae322fcc4e3861cd5a106a1323',
+      },
+    },
   }
 }

puppet/zulip_ops/manifests/prometheus/akamai.pp

@@ -0,0 +1,33 @@
+# @summary Prometheus monitoring of Akamai access logs
+#
+class zulip_ops::prometheus::akamai {
+  include zulip_ops::prometheus::base
+  include zulip_ops::vector
+  include zulip::supervisor
+
+  $bin = $zulip_ops::vector::bin
+  $conf = '/etc/vector.toml'
+  $sqs_url = zulipsecret('secrets', 'akamai_sqs_url', '')
+
+  file { $conf:
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('zulip_ops/vector.toml.template.erb'),
+  }
+  file { "${zulip::common::supervisor_conf_dir}/prometheus_akamai_exporter.conf":
+    ensure  => file,
+    require => [
+      User[zulip],
+      Package[supervisor],
+      File['/etc/vector.toml'],
+      Zulip::External_Dep['vector'],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('zulip_ops/supervisor/conf.d/prometheus_akamai_exporter.conf.template.erb'),
+    notify  => Service[supervisor],
+  }
+}

puppet/zulip_ops/manifests/vector.pp

@@ -0,0 +1,18 @@
+# @summary Installs Vector to transform Prometheus data
+#
+class zulip_ops::vector {
+  $version = $zulip::common::versions['vector']['version']
+  $dir = "/srv/zulip-vector-${version}"
+  $bin = "${dir}/bin/vector"
+
+  $arch = $::os['architecture'] ? {
+    'amd64'   => 'x86_64',
+    'aarch64' => 'aarch64',
+  }
+
+  zulip::external_dep { 'vector':
+    version        => $version,
+    url            => "https://packages.timber.io/vector/${version}/vector-${version}-${arch}-unknown-linux-gnu.tar.gz",
+    tarball_prefix => "vector-${arch}-unknown-linux-gnu",
+  }
+}

puppet/zulip_ops/templates/supervisor/conf.d/prometheus_akamai_exporter.conf.template.erb

@@ -0,0 +1,8 @@
+[program:prometheus_akamai_exporter]
+command=<%= @bin %> --config <%= @conf %>
+priority=10
+autostart=true
+autorestart=true
+user=zulip
+redirect_stderr=true
+stdout_logfile=/var/log/zulip/akamai_exporter.log

puppet/zulip_ops/templates/vector.toml.template.erb

@@ -0,0 +1,69 @@
+[sources.vector_metrics]
+  type = "internal_metrics"
+
+[sources.s3_akamai_static]
+  # Akamai Datastream2 logs all accesses into AWS S3:
+  # https://techdocs.akamai.com/datastream2/docs/stream-amazon-s3
+  #
+  # The S3 bucket is configured to send event notifications to the SQS
+  # queue, which this host is allowed to read from.  This consumer
+  # deletes the messages from the queue, and the S3 bucket is
+  # configured to purge old logs.
+  # https://vector.dev/docs/reference/configuration/sources/aws_s3/
+  type = "aws_s3"
+  region = "us-east-1"
+  compression = "gzip"
+  sqs.delete_message = true
+  sqs.poll_secs = 15
+  sqs.queue_url = "<%= @sqs_url %>"
+
+[transforms.parsing]
+  type = "remap"
+  inputs = ["s3_akamai_static"]
+  source = '''
+  . = parse_json!(string!(.message))
+  .turnAroundTimeSec = to_int!(.turnAroundTimeMSec) / 1000.0
+  '''
+
+[transforms.logs2metrics-requests]
+  type = "log_to_metric"
+  inputs = ["parsing"]
+
+  [[transforms.logs2metrics-requests.metrics]]
+    field = "cacheStatus"
+    name = "requests_cache_count"
+    namespace = "akamai_static"
+    type = "counter"
+      [transforms.logs2metrics-requests.metrics.tags]
+      status_code = "{{statusCode}}"
+      cached = "{{cacheStatus}}"
+      host = "{{reqHost}}"
+
+  [[transforms.logs2metrics-requests.metrics]]
+    field = "bytes"
+    name = "requests_bytes"
+    namespace = "akamai_static"
+    type = "counter"
+    increment_by_value = true
+      [transforms.logs2metrics-requests.metrics.tags]
+      status_code = "{{statusCode}}"
+      cached = "{{cacheStatus}}"
+      host = "{{reqHost}}"
+
+  [[transforms.logs2metrics-requests.metrics]]
+    field = "turnAroundTimeSec"
+    name = "turnaround_time_sec"
+    namespace = "akamai_static"
+    type = "histogram"
+      [transforms.logs2metrics-requests.metrics.tags]
+      status_code = "{{statusCode}}"
+      cached = "{{cacheStatus}}"
+      host = "{{reqHost}}"
+
+[sinks.prometheus_exporter]
+  type = "prometheus_exporter"
+  inputs = ["vector_metrics", "logs2metrics*"]
+  buckets = [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5]
+  address = "0.0.0.0:9081"
+  flush_period_secs = 120
+  suppress_timestamp = true