integrations: Add AlertManager webhook.

This commit is contained in:
Chris Heald
2020-02-07 02:04:22 -07:00
committed by Tim Abbott
parent f247721a2d
commit 68628149db
9 changed files with 213 additions and 0 deletions

Binary file not shown.

After

Width:  |  Height:  |  Size: 27 KiB

View File

@@ -0,0 +1 @@
<svg height="64" viewBox="-4.649 -0.667 64 64" width="64" xmlns="http://www.w3.org/2000/svg"><path d="M27.35-.667c-17.673 0-32 14.326-32 32s14.328 32 32 32 32-14.327 32-32-14.328-32-32-32zm0 59.89c-5.028 0-9.105-3.36-9.105-7.5h18.2c0 4.142-4.077 7.5-9.105 7.5zM42.4 49.24H12.31v-5.454H42.4zm-.108-8.26H12.397l-.297-.344c-3.08-3.74-3.804-5.7-4.508-7.68-.012-.066 3.734.766 6.39 1.363 0 0 1.367.316 3.364.68-1.918-2.25-3.057-5.107-3.057-8.03 0-6.415 4.92-12.02 3.145-16.55 1.728.14 3.575 3.646 3.7 9.126 1.837-2.538 2.605-7.172 2.605-10.014 0-2.942 1.94-6.36 3.878-6.477-1.73 2.85.448 5.29 2.382 11.35.726 2.276.633 6.106 1.193 8.535.186-5.045 1.053-12.405 4.254-14.946-1.412 3.2.21 7.205 1.318 9.13 1.79 3.106 2.873 5.46 2.873 9.9 0 2.984-1.102 5.793-2.96 8 2.113-.397 3.572-.754 3.572-.754l6.862-1.34c0-.001-.997 4.1-4.828 8.05z" fill="#da4e31"/></svg>

After

Width:  |  Height:  |  Size: 852 B

View File

@@ -227,6 +227,12 @@ EMBEDDED_BOTS = [
WEBHOOK_INTEGRATIONS = [
WebhookIntegration('airbrake', ['monitoring']),
WebhookIntegration(
'alertmanager',
['monitoring'],
display_name='Prometheus AlertManager',
logo='images/integrations/logos/prometheus.svg'
),
WebhookIntegration('ansibletower', ['deployment'], display_name='Ansible Tower'),
WebhookIntegration('appfollow', ['customer-support'], display_name='AppFollow'),
WebhookIntegration('appveyor', ['continuous-integration'], display_name='AppVeyor'),

View File

View File

@@ -0,0 +1,23 @@
Get Zulip notifications from AlertManager!
1. {!create-stream.md!}
1. {!create-bot-construct-url-indented.md!}
Additionally, you may specify URL parameters named `name` and `desc` to specify which labels
or annotations will be used to construct the alert message. This allows you to use arbitrary labels
and annotations defined in your alerting rules.
{{ api_url }}{{ integration_url }}?api_key=abcdefgh&stream=stream%20name&name=host&desc=alertname
1. In your AlertManager config, set up a new webhook receiver, like so:
```
- name: ops-zulip
webhook_configs:
- url: "<the url constructed above>"
```
{!congrats.md!}
![](/static/images/integrations/alertmanager/001.png)

View File

@@ -0,0 +1,51 @@
{
"receiver": "ops-zulip",
"status": "firing",
"groupLabels": {},
"commonLabels": {
"alertname": "High CPU Temperature",
"severity": "critical"
},
"commonAnnotations": {
"summary": "High CPU core temperature"
},
"externalURL": "http://ops1:9093",
"version": "4",
"groupKey": "{}/{}:{}",
"alerts": [
{
"status": "firing",
"labels": {
"alertname": "High CPU Temperature",
"host": "andromeda",
"severity": "critical"
},
"annotations": {
"description": "CPU core temperature is 34.75C",
"topic": "andromeda",
"summary": "High CPU core temperature"
},
"startsAt": "2020-02-06T00:17:58.432911368Z",
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "http://cobalt:9090/graph?g0.expr=avg+by%28host%29+%28sensors_temp_input%7Bfeature%3D~%22core_%5B0-9%5D%2B%22%7D%29+%3E+15&g0.tab=1",
"fingerprint": "a1bc48475b20e73d"
},
{
"status": "firing",
"labels": {
"alertname": "High CPU Temperature",
"host": "andromeda",
"severity": "critical"
},
"annotations": {
"description": "CPU core temperature is 17.625C",
"topic": "andromeda",
"summary": "High CPU core temperature"
},
"startsAt": "2020-02-06T00:17:58.432911368Z",
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "http://cobalt:9090/graph?g0.expr=avg+by%28host%29+%28sensors_temp_input%7Bfeature%3D~%22core_%5B0-9%5D%2B%22%7D%29+%3E+15&g0.tab=1",
"fingerprint": "c2ea85182f28cbd9"
}
]
}

View File

@@ -0,0 +1,34 @@
{
"receiver": "ops-zulip",
"status": "resolved",
"groupLabels": {},
"commonLabels": {
"alertname": "High CPU Temperature",
"severity": "critical"
},
"commonAnnotations": {
"summary": "High CPU core temperature"
},
"externalURL": "http://ops1:9093",
"version": "4",
"groupKey": "{}/{}:{}",
"alerts": [
{
"status": "resolved",
"labels": {
"alertname": "High CPU Temperature",
"host": "andromeda",
"severity": "critical"
},
"annotations": {
"description": "CPU core temperature is 34.75C",
"topic": "andromeda",
"summary": "High CPU core temperature"
},
"startsAt": "2020-02-06T00:17:58.432911368Z",
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "http://cobalt:9090/graph?g0.expr=avg+by%28host%29+%28sensors_temp_input%7Bfeature%3D~%22core_%5B0-9%5D%2B%22%7D%29+%3E+15&g0.tab=1",
"fingerprint": "a1bc48475b20e73d"
}
]
}

View File

@@ -0,0 +1,36 @@
# -*- coding: utf-8 -*-
from zerver.lib.test_classes import WebhookTestCase
class AlertmanagerHookTests(WebhookTestCase):
STREAM_NAME = 'alertmanager'
URL_TEMPLATE = "/api/v1/external/alertmanager?&api_key={api_key}&stream={stream}&name=topic&desc=description"
FIXTURE_DIR_NAME = 'alertmanager'
def test_error_issue_message(self) -> None:
expected_topic = "andromeda"
expected_message = """
:alert: **FIRING**
* CPU core temperature is 34.75C ([graph](http://cobalt:9090/graph?g0.expr=avg+by%28host%29+%28sensors_temp_input%7Bfeature%3D~%22core_%5B0-9%5D%2B%22%7D%29+%3E+15&g0.tab=0))
* CPU core temperature is 17.625C ([graph](http://cobalt:9090/graph?g0.expr=avg+by%28host%29+%28sensors_temp_input%7Bfeature%3D~%22core_%5B0-9%5D%2B%22%7D%29+%3E+15&g0.tab=0))
""".strip()
self.send_and_test_stream_message(
'alert',
expected_topic,
expected_message,
"application/json"
)
def test_single_error_issue_message(self) -> None:
expected_topic = "andromeda"
expected_message = """
:squared_ok: **Resolved** CPU core temperature is 34.75C ([graph](http://cobalt:9090/graph?g0.expr=avg+by%28host%29+%28sensors_temp_input%7Bfeature%3D~%22core_%5B0-9%5D%2B%22%7D%29+%3E+15&g0.tab=0))
""".strip()
self.send_and_test_stream_message(
'single_alert',
expected_topic,
expected_message,
"application/json"
)

View File

@@ -0,0 +1,62 @@
# Webhooks for external integrations.
from typing import Any, Dict, List
from django.http import HttpRequest, HttpResponse
from zerver.decorator import api_key_only_webhook_view
from zerver.lib.request import REQ, has_request_variables
from zerver.lib.response import json_success
from zerver.lib.webhooks.common import check_send_webhook_message
from zerver.models import UserProfile
@api_key_only_webhook_view('AlertManager')
@has_request_variables
def api_alertmanager_webhook(request: HttpRequest, user_profile: UserProfile,
payload: Dict[str, Any] = REQ(argument_type='body')) -> HttpResponse:
name_field = request.GET.get("name", "instance")
desc_field = request.GET.get("desc", "alertname")
topics = {} # type: Dict[str, Dict[str, List[str]]]
for alert in payload["alerts"]:
labels = alert.get("labels", {})
annotations = alert.get("annotations", {})
name = labels.get(
name_field, annotations.get(name_field, "(unknown)"))
desc = labels.get(
desc_field, annotations.get(desc_field, "<missing field: {}>".format(desc_field)))
url = alert.get("generatorURL").replace("tab=1", "tab=0")
body = "{description} ([graph]({url}))".format(description=desc, url=url)
if name not in topics:
topics[name] = {"firing": [], "resolved": []}
topics[name][alert["status"]].append(body)
for topic, statuses in topics.items():
for status, messages in statuses.items():
if len(messages) == 0:
continue
if status == "firing":
icon = ":alert:"
title = "FIRING"
else:
title = "Resolved"
icon = ":squared_ok:"
if len(messages) == 1:
body = "{icon} **{title}** {message}".format(
icon=icon,
title=title,
message=messages[0])
else:
message_list = "\n".join(["* {}".format(m) for m in messages])
body = "{icon} **{title}**\n{messages}".format(
icon=icon,
title=title,
messages=message_list)
check_send_webhook_message(request, user_profile, topic, body)
return json_success()