kandra: Add a grok exporter to parse nginx logfiles.

This provides access logging metrics to Prometheus.  For cardinality
reasons, we cannot (nor would we want to) put every request path into
its own label value -- but we do separate out the most-frequent access
paths (as well as some low-frequency but high-interest ones) into
their own label values.

In order to differentiate accesses to https://zulip.com/ from
https://example.zulipchat.com/ (both of which appear at path `/`), we
use a `grok_exporter.realm_names_regex` value in `zulip.conf`, which
is expected to be set to match the hostname of all possible realms.

puppet/kandra/manifests/prometheus/grok.pp

@@ -0,0 +1,45 @@
+# @summary Parses nginx access_log files
+#
+class kandra::prometheus::grok {
+  include kandra::prometheus::base
+  include zulip::supervisor
+
+  $version = $zulip::common::versions['grok_exporter']['version']
+  $dir = "/srv/zulip-grok_exporter-${version}"
+  $bin = "${dir}/grok_exporter"
+
+  zulip::external_dep { 'grok_exporter':
+    version        => $version,
+    url            => "https://github.com/fstab/grok_exporter/releases/download/v${version}/grok_exporter-${version}.linux-${zulip::common::goarch}.zip",
+    tarball_prefix => "grok_exporter-${version}.linux-${zulip::common::goarch}",
+    bin            => [$bin],
+    cleanup_after  => [Service[supervisor]],
+  }
+
+  $realm_names_regex = zulipconf('grok_exporter', 'realm_names_regex', '__impossible__')
+  $include_dir = "${dir}/patterns"
+  file { '/etc/grok_exporter.yaml':
+    ensure  => file,
+    owner   => zulip,
+    group   => zulip,
+    mode    => '0644',
+    content => template('kandra/prometheus/grok_exporter.yaml.template.erb'),
+    notify  => Service[supervisor],
+  }
+
+  kandra::firewall_allow { 'grok_exporter': port => '9144' }
+  file { "${zulip::common::supervisor_conf_dir}/prometheus_grok_exporter.conf":
+    ensure  => file,
+    require => [
+      User[zulip],
+      Package[supervisor],
+      File[$bin],
+      File['/etc/grok_exporter.yaml'],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/prometheus_grok_exporter.conf.template.erb'),
+    notify  => Service[supervisor],
+  }
+}

puppet/kandra/templates/prometheus/grok_exporter.yaml.template.erb

@@ -0,0 +1,68 @@
+global:
+  config_version: 3
+input:
+  type: file
+  path: /var/log/nginx/access.log
+  fail_on_missing_logfile: false
+  readall: false
+imports:
+- type: grok_patterns
+  dir: <%= @include_dir %>
+grok_patterns:
+  - 'NONQUERY [^? ]+'
+  - 'OPTIONALQUERY (?:\?%{NOTSPACE})?'
+  - 'APIPATH /+(api/v1|json)(?<apipath>/(events|users/me/presence|messages(/flags)?|remotes/push/(register|unregister|notify)|remotes/server/(register|analytics|analytics/status)|typing|register|server_settings))'
+  - 'EXTERNALPATH /+(?<external>api/v1/external/)[a-zA-Z0-9_-]+'
+  - 'ROOTPATH (?<rootpath>/+)'
+  - 'TOPPATH /+(?<toppath>(api/internal/(email_mirror_message|tusd)|compatibility|error_tracing))'
+  - 'PREFIXPATH /+(?<prefixpath>(static|user_uploads|avatar|api/v1/tus)/)\S+'
+  - 'ANYPATH (%{APIPATH}|%{EXTERNALPATH}|%{TOPPATH}|%{PREFIXPATH}|%{ROOTPATH}|%{NONQUERY:otherpath})%{OPTIONALQUERY}'
+  - 'OURHOSTNAME (?<hostname>((?<realm><%= @realm_names_regex %>)|%{NOTSPACE}))'
+  - 'OURLOG %{IPORHOST:clientip} - \S+ \[%{HTTPDATE:timestamp}\] "%{WORD:verb} %{ANYPATH} HTTP/%{NUMBER:httpversion}" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent} %{OURHOSTNAME} %{NUMBER:response_time}'
+metrics:
+- type: counter
+  name: nginx_http_response_codes_total
+  help: Total number of requests, by response code, normalized path, and HTTP method
+  match: '%{OURLOG}'
+  labels:
+    code: '{{.response}}'
+    method: '{{.verb}}'
+    path: '{{if .apipath}}/api/v1{{.apipath}}{{else if .external}}/api/v1/external/...{{else if .rootpath}}{{if .realm}}/{{else}}(other){{end}}{{else if .toppath}}/{{.toppath}}{{else if .prefixpath}}/{{.prefixpath}}...{{else}}(other){{end}}'
+- type: counter
+  name: nginx_http_response_bytes
+  help: Total number of bytes, by normalized path
+  match: '%{OURLOG}'
+  value: '{{.bytes}}'
+  labels:
+    path: '{{if .apipath}}/api/v1{{.apipath}}{{else if .external}}/api/v1/external/...{{else if .rootpath}}{{if .realm}}/{{else}}(other){{end}}{{else if .toppath}}/{{.toppath}}{{else if .prefixpath}}/{{.prefixpath}}...{{else}}(other){{end}}'
+- type: histogram
+  name: nginx_http_response_time_seconds
+  help: Response time in seconds, from first request byte to last response byte
+  match: '%{OURLOG}'
+  value: '{{.response_time}}'
+  buckets:
+    - 0.001
+    - 0.002
+    - 0.005
+    - 0.010
+    - 0.025
+    - 0.050
+    - 0.100
+    - 0.200
+    - 0.500
+    - 1.000
+    - 2.000
+    - 5.000
+    - 10.00
+    - 20.00
+    - 50.00
+    - 60.00
+    - 120.0
+  labels:
+    code: '{{.response}}'
+    method: '{{.verb}}'
+    path: '{{if .apipath}}/api/v1{{.apipath}}{{else if .external}}/api/v1/external/...{{else if .rootpath}}{{if .realm}}/{{else}}(other){{end}}{{else if .toppath}}/{{.toppath}}{{else if .prefixpath}}/{{.prefixpath}}...{{else}}(other){{end}}'
+
+server:
+  protocol: http
+  port: 9144

puppet/kandra/templates/prometheus/prometheus.yaml.template.erb

@@ -85,6 +85,29 @@ scrape_configs:
         regex: "(.+)"
         target_label: "instance"
 
+  - job_name: "access_logs"
+    ec2_sd_configs:
+      - region: us-east-1
+        port: 9144
+        filters:
+          - name: "tag:role"
+            values: ["prod_app_frontend", "staging_app_frontend"]
+          - name: instance-state-name
+            values: ["running"]
+    static_configs:
+      - targets: ["<%= @czo %>:9144"]
+        labels:
+          deploy: prod
+          instance: <%= @czo %>
+    relabel_configs:
+      - source_labels: ["__meta_ec2_tag_Name"]
+        regex: "(.+)"
+        target_label: "instance"
+      - source_labels: ["__meta_ec2_tag_role"]
+        regex: "(prod|staging)_app_frontend"
+        replacement: "${1}"
+        target_label: "deploy"
+
   - job_name: "uwsgi"
     ec2_sd_configs:
       - region: us-east-1

puppet/kandra/templates/supervisor/conf.d/prometheus_grok_exporter.conf.template.erb

@@ -0,0 +1,8 @@
+[program:prometheus_grok_exporter]
+command=<%= @bin %> -config /etc/grok_exporter.yaml
+priority=10
+autostart=true
+autorestart=true
+user=zulip
+redirect_stderr=true
+stdout_logfile=/var/log/zulip/grok_exporter.log

puppet/zulip/manifests/common.pp

@@ -122,6 +122,15 @@ class zulip::common {
       },
     },
 
+    # https://github.com/fstab/grok_exporter/tags
+    'grok_exporter' => {
+      'version' => '1.0.0.RC5',
+      'sha256'  => {
+        'amd64' => 'b8771a6d7ca8447c222548d6cb8b2f8ee058b55bfd1801c2f6eb739534df5ded',
+        # No aarch64 builds
+      },
+    },
+
     # https://prometheus.io/download/#node_exporter
     'node_exporter' => {
       'version' => '1.9.0',