integrations: Update GoCD integration.

Previously, the GoCD integration relied on GoCD's integration with bug
tracking and monitoring tools through the users' Config XML file [1].
However, this feature no longer works as expected, as it cannot send
HTTPS payloads to an external endpoint.

This commit updates our GoCD integration to use Sentry's WebHook
notifier GoCD plugin [2] to send webhook payloads from users GoCD server
to Zulip. We are using an older version of the plugin—v0.0.6—because the
newer version—v0.0.9—doesn't work at this time.

Additionally, this change reformats the notifications to include more
details on the GoCD event, such as lists of passed and failed jobs.

Fixes #21224.

Co-authored-by: Pieter CK <pieterceka123@gmail.com>

[1]:
https://docs.gocd.org/current/integration/#integration-with-bug-tracking-and-story-management-tools
[2]:
https://github.com/getsentry/gocd-webhook-notification-plugin/releases/tag/v0.0.6

zerver/lib/integrations.py

@@ -765,7 +765,7 @@ DOC_SCREENSHOT_CONFIG: dict[str, list[BaseScreenshotConfig]] = {
     "github": [ScreenshotConfig("push__1_commit.json")],
     "githubsponsors": [ScreenshotConfig("created.json")],
     "gitlab": [ScreenshotConfig("push_hook__push_local_branch_without_commits.json")],
-    "gocd": [ScreenshotConfig("pipeline.json")],
+    "gocd": [ScreenshotConfig("pipeline_with_mixed_job_result.json")],
     "gogs": [ScreenshotConfig("pull_request__opened.json")],
     "gosquared": [ScreenshotConfig("traffic_spike.json", image_name="000.png")],
     "grafana": [ScreenshotConfig("alert_values_v11.json")],

zerver/webhooks/gocd/doc.md

@@ -1,24 +1,41 @@
-Zulip supports integration with GoCD and can notify you of
+# Zulip GoCD integration
-your build statuses.
+
+Get GoCD notifications in Zulip!
+
+{start_tabs}
 
 1. {!create-channel.md!}
 
 1. {!create-an-incoming-webhook.md!}
 
-1. {!generate-integration-url.md!}
+1. {!generate-webhook-url-basic.md!}
 
-1. Add the following to your `Config.XML` file.
+1. [Download][1] and [install][2] Sentry's **GoCD WebHook Notification
+   plugin**.
 
-    ```
+    !!! warn ""
-    <pipeline name="mypipeline">
-        <trackingtool link="<URL constructed above>" regex="##(\d+)"/>
-        ...
-    </pipeline>
-    ```
 
-    Push this change to your repository. For further information,
+        **Note**: the GoCD WebHook Notification plugin will only send
-    see [GoCD's documentation](https://docs.gocd.org/current/integration/).
+        webhook payloads over HTTPS.
+
+1. In your GoCD server, go to **Admin > Server Configuration > Plugins**,
+   and click on the gear icon beside the **GoCD WebHook Notification
+   plugin** that you installed.
+
+1. Set **WebHook URL** to the URL generated above, and click **Save**.
+
+{end_tabs}
 
 {!congrats.md!}
 
 ![](/static/images/integrations/gocd/001.png)
+
+### Related Branches
+
+- [GoCD plugin user guide][3]
+
+{!webhooks-url-specification.md!}
+
+[1]: https://github.com/getsentry/gocd-webhook-notification-plugin/releases
+[2]: https://docs.gocd.org/current/extension_points/plugin_user_guide.html#installing-and-uninstalling-of-plugins
+[3]: https://docs.gocd.org/current/extension_points/plugin_user_guide.html

zerver/webhooks/gocd/fixtures/build_details.json

@@ -1,18 +0,0 @@
-{
-  "build_details": {
-    "_links": {
-      "job": {
-        "href": "https://ci.example.com/go/tab/build/detail/pipelineName/pipelineCounter/stageName/stageCounter/jobName"
-      },
-      "stage": {
-        "href": "https://ci.example.com/go/pipelines/pipelineName/pipelineCounter/stageName/stageCounter"
-      },
-      "pipeline": {
-        "href": "https://ci.example.com/go/tab/pipeline/history/pipelineName"
-      }
-    },
-    "pipeline_name": "pipelineName",
-    "stage_name": "stageName",
-    "job_name": "jobName"
-  }
-}

zerver/webhooks/gocd/fixtures/pipeline.json

@@ -1,59 +0,0 @@
-{
-  "build_cause": {
-    "approver": "",
-    "material_revisions": [
-      {
-        "modifications": [
-          {
-             "email_address": null,
-             "id": 7225,
-             "modified_time": 1435728005000,
-             "user_name": "Balaji B <balaji@example.com>",
-             "comment": "my hola mundo changes",
-             "revision": "a788f1876e2e1f6e5a1e91006e75cd1d467a0edb"
-          }
-        ],
-        "material": {
-          "description": "URL: https://github.com/gocd/gocd, Branch: master",
-          "fingerprint": "61e2da369d0207a7ef61f326eed837f964471b35072340a03f8f55d993afe01d",
-          "type": "Git",
-          "id": 4
-        },
-        "changed": true
-      }
-    ],
-    "trigger_forced": false,
-    "trigger_message": "modified by Balaji <balaji@example.com>"
-  },
-  "name": "PipelineName",
-  "natural_order": 1,
-  "can_run": false,
-  "comment": null,
-  "stages": [
-    {
-      "name": "stage1",
-      "approved_by": "changes",
-      "jobs": [
-        {
-          "name": "jsunit",
-          "result": "Passed",
-          "state": "Completed",
-          "id": 1,
-          "scheduled_date": 1398332981981
-        }
-      ],
-      "can_run": false,
-      "result": "Passed",
-      "approval_type": "success",
-      "counter": "1",
-      "id": 1,
-      "operate_permission": false,
-      "rerun_of_counter": null,
-      "scheduled": true
-    }
-  ],
-  "counter": 1,
-  "id": 1,
-  "preparing_to_schedule": false,
-  "label": "14.1.0.1-b14a81825d081411993853ea5ea45266ced578b4"
-}

zerver/webhooks/gocd/fixtures/pipeline_building.json

@@ -0,0 +1,47 @@
+{
+  "data": {
+      "pipeline": {
+          "name": "Pipeline",
+          "counter": "90",
+          "group": "defaultGroup",
+          "build-cause": [
+              {
+                  "material": {
+                      "git-configuration": {
+                          "shallow-clone": false,
+                          "branch": "main",
+                          "url": "https://github.com/swayam0322/Test"
+                      },
+                      "type": "git"
+                  },
+                  "changed": false,
+                  "modifications": [
+                      {
+                          "revision": "59f3c6e4540b6a89ad5505790e5efdf964e7b837",
+                          "modified-time": "Jan 31, 2024, 1:56:14 AM",
+                          "data": {}
+                      }
+                  ]
+              }
+          ],
+          "stage": {
+              "name": "Stage",
+              "counter": "1",
+              "approval-type": "success",
+              "approved-by": "anonymous",
+              "state": "Building",
+              "result": "Unknown",
+              "create-time": "Feb 1, 2024, 1:58:13 AM",
+              "jobs": [
+                  {
+                      "name": "Job",
+                      "schedule-time": "Feb 1, 2024, 1:58:13 AM",
+                      "state": "Scheduled",
+                      "result": "Unknown"
+                  }
+              ]
+          }
+      }
+  },
+  "type": "stage"
+}

zerver/webhooks/gocd/fixtures/pipeline_failed.json

@@ -1,59 +1,50 @@
 {
-  "build_cause": {
+    "data": {
-    "approver": "anonymous",
+      "pipeline": {
-    "material_revisions": [
+        "name": "pipeline-one",
-      {
+        "counter": "7",
-        "modifications": [
+        "group": "defaultGroup",
+        "build-cause": [
           {
-             "email_address": null,
+            "material": {
-             "id": 1998,
+              "git-configuration": {
-             "modified_time": 1434957613000,
+                "shallow-clone": false,
-             "user_name": "User Name <username123@example.com>",
+                "branch": "master",
-             "comment": "my hola mundo changes",
+                "url": "https://github.com/PieterCK/getting-started-repo.git"
-             "revision": "f6e7a3899c55e1682ffb00383bdf8f882bcee2141e79a8728254190a1fddcf4f"
+              },
+              "type": "git"
+            },
+            "changed": false,
+            "modifications": [
+              {
+                "revision": "963eb239c7b91eac2e15c727e06e908fd334e6f9",
+                "modified-time": "Dec 14, 2016, 5:09:14 AM",
+                "data": {}
+              }
+            ]
           }
         ],
-        "material": {
+        "stage": {
-          "description": "URL: https://github.com/gocd/gocd, Branch: master",
+          "name": "stage-two",
-          "fingerprint": "61e2da369d0207a7ef61f326eed837f964471b35072340a03f8f55d993afe01d",
+          "counter": "1",
-          "type": "Git",
+          "approval-type": "success",
-          "id": 4
+          "approved-by": "anonymous",
-        },
+          "state": "Failed",
-        "changed": true
-      }
-    ],
-    "trigger_forced": false,
-    "trigger_message": "modified by User Name <username123@example.com>"
-  },
-  "name": "PipelineName",
-  "natural_order": 8,
-  "can_run": false,
-  "comment": null,
-  "stages": [
-    {
-      "name": "stage1",
-      "approved_by": "changes",
-      "jobs": [
-        {
-          "name": "job123",
           "result": "Failed",
-          "state": "Completed",
+          "create-time": "Aug 28, 2024, 9:30:19 PM",
-          "id": 21,
+          "last-transition-time": "Aug 28, 2024, 9:31:00 PM",
-          "scheduled_date": 1436172201081
+          "jobs": [
+            {
+              "name": "task-two",
+              "schedule-time": "Aug 28, 2024, 9:30:19 PM",
+              "complete-time": "Aug 28, 2024, 9:31:00 PM",
+              "state": "Completed",
+              "result": "Failed",
+              "agent-uuid": "a35cf64c-7746-46ae-ba65-597d70898d50"
+            }
+          ]
         }
-      ],
+      }
-      "can_run": false,
+    },
-      "result": "Failed",
+    "type": "stage"
-      "approval_type": "success",
+  }
-      "counter": "1",
-      "id": 21,
-      "operate_permission": false,
-      "rerun_of_counter": null,
-      "scheduled": true
-    }
-  ],
-  "counter": 1,
-  "id": 21,
-  "preparing_to_schedule": false,
-  "label": "6"
-}

zerver/webhooks/gocd/fixtures/pipeline_passed.json

@@ -0,0 +1,50 @@
+{
+  "data": {
+    "pipeline": {
+      "name": "Pipeline",
+      "counter": "90",
+      "group": "defaultGroup",
+      "build-cause": [
+        {
+          "material": {
+            "git-configuration": {
+              "shallow-clone": false,
+              "branch": "main",
+              "url": "https://github.com/swayam0322/Test"
+            },
+            "type": "git"
+          },
+          "changed": false,
+          "modifications": [
+            {
+              "revision": "59f3c6e4540b6a89ad5505790e5efdf964e7b837",
+              "modified-time": "Jan 31, 2024, 1:56:14 AM",
+              "data": {}
+            }
+          ]
+        }
+      ],
+      "stage": {
+        "name": "Stage",
+        "counter": "1",
+        "approval-type": "success",
+        "approved-by": "anonymous",
+        "state": "Passed",
+        "result": "Passed",
+        "create-time": "Feb 1, 2024, 1:58:13 AM",
+        "last-transition-time": "Feb 1, 2024, 1:58:40 AM",
+        "jobs": [
+          {
+            "name": "Job",
+            "schedule-time": "Feb 1, 2024, 1:58:13 AM",
+            "complete-time": "Feb 1, 2024, 1:58:40 AM",
+            "state": "Completed",
+            "result": "Passed",
+            "agent-uuid": "95dde0d9-8da7-48ae-8572-5c7de18bff88"
+          }
+        ]
+      }
+    }
+  },
+  "type": "stage"
+}

zerver/webhooks/gocd/fixtures/pipeline_with_mixed_job_result.json

@@ -0,0 +1,75 @@
+
+{
+  "data": {
+    "pipeline": {
+      "name": "test-pipeline",
+      "counter": "6",
+      "group": "defaultGroup",
+      "build-cause": [
+        {
+          "material": {
+            "git-configuration": {
+              "shallow-clone": false,
+              "branch": "master",
+              "url": "https://github.com/PieterCK/getting-started-repo.git"
+            },
+            "type": "git"
+          },
+          "changed": false,
+          "modifications": [
+            {
+              "revision": "963eb239c7b91eac2e15c727e06e908fd334e6f9",
+              "modified-time": "Dec 14, 2016, 5:09:14 AM",
+              "data": {}
+            }
+          ]
+        }
+      ],
+      "stage": {
+        "name": "backend-tests",
+        "counter": "1",
+        "approval-type": "success",
+        "approved-by": "changes",
+        "state": "Failed",
+        "result": "Failed",
+        "create-time": "Aug 29, 2024, 3:59:18 PM",
+        "last-transition-time": "Aug 29, 2024, 4:00:15 PM",
+        "jobs": [
+          {
+            "name": "check-backend-lints",
+            "schedule-time": "Aug 29, 2024, 3:59:18 PM",
+            "complete-time": "Aug 29, 2024, 3:59:23 PM",
+            "state": "Completed",
+            "result": "Failed",
+            "agent-uuid": "a35cf64c-7746-46ae-ba65-597d70898d50"
+          },
+          {
+            "name": "check-backend-tests",
+            "schedule-time": "Aug 29, 2024, 3:59:18 PM",
+            "complete-time": "Aug 29, 2024, 3:59:47 PM",
+            "state": "Completed",
+            "result": "Passed",
+            "agent-uuid": "a35cf64c-7746-46ae-ba65-597d70898d50"
+          },
+          {
+            "name": "test-frontend-js",
+            "schedule-time": "Aug 29, 2024, 3:59:18 PM",
+            "complete-time": "Aug 29, 2024, 4:00:15 PM",
+            "state": "Completed",
+            "result": "Failed",
+            "agent-uuid": "a35cf64c-7746-46ae-ba65-597d70898d50"
+          },
+          {
+            "name": "zulip-ci-debian-12",
+            "schedule-time": "Aug 29, 2024, 3:59:18 PM",
+            "complete-time": "Aug 29, 2024, 4:00:11 PM",
+            "state": "Completed",
+            "result": "Passed",
+            "agent-uuid": "a35cf64c-7746-46ae-ba65-597d70898d50"
+          }
+        ]
+      }
+    }
+  },
+  "type": "stage"
+}

zerver/webhooks/gocd/tests.py

@@ -5,36 +5,44 @@ class GocdHookTests(WebhookTestCase):
     CHANNEL_NAME = "gocd"
     URL_TEMPLATE = "/api/v1/external/gocd?stream={stream}&api_key={api_key}"
     WEBHOOK_DIR_NAME = "gocd"
-    TOPIC_NAME = "https://github.com/gocd/gocd"
 
-    def test_gocd_message(self) -> None:
+    def test_building_pipeline(self) -> None:
-        expected_message = (
+        expected_topic = "Pipeline / Stage"
-            "Author: Balaji B <balaji@example.com>\n"
+        expected_message = """**Pipeline building**: Pipeline / Stage
-            "Build status: Passed :thumbs_up:\n"
+- **Commit**: [`59f3c6e4540`](https://github.com/swayam0322/Test/commit/59f3c6e4540) on branch `main`
-            "Details: [build log](https://ci.example.com"
+- **Started**: Feb 1, 2024, 1:58:13 AM"""
-            "/go/tab/pipeline/history/pipelineName)\n"
-            "Comment: my hola mundo changes"
-        )
 
         self.check_webhook(
-            "pipeline",
+            "pipeline_building",
-            self.TOPIC_NAME,
+            expected_topic,
             expected_message,
-            content_type="application/x-www-form-urlencoded",
         )
 
-    def test_failed_message(self) -> None:
+    def test_completed_pipeline_success(self) -> None:
-        expected_message = (
+        expected_topic = "Pipeline / Stage"
-            "Author: User Name <username123@example.com>\n"
+        expected_message = """:green_circle: **Build passed**: Pipeline / Stage
-            "Build status: Failed :thumbs_down:\n"
+- **Commit**: [`59f3c6e4540`](https://github.com/swayam0322/Test/commit/59f3c6e4540) on branch `main`
-            "Details: [build log](https://ci.example.com"
+- **Started**: Feb 1, 2024, 1:58:13 AM
-            "/go/tab/pipeline/history/pipelineName)\n"
+- **Finished**: Feb 1, 2024, 1:58:40 AM
-            "Comment: my hola mundo changes"
+- **Passed**: `Job`"""
-        )
 
-        self.check_webhook(
+        self.check_webhook("pipeline_passed", expected_topic, expected_message)
-            "pipeline_failed",
+
-            self.TOPIC_NAME,
+    def test_completed_pipeline_fail(self) -> None:
-            expected_message,
+        expected_topic = "pipeline-one / stage-two"
-            content_type="application/x-www-form-urlencoded",
+        expected_message = """:red_circle: **Build failed**: pipeline-one / stage-two
-        )
+- **Commit**: [`963eb239c7b`](https://github.com/PieterCK/getting-started-repo.git/commit/963eb239c7b) on branch `master`
+- **Started**: Aug 28, 2024, 9:30:19 PM
+- **Finished**: Aug 28, 2024, 9:31:00 PM
+- **Failed**: `task-two`"""
+        self.check_webhook("pipeline_failed", expected_topic, expected_message)
+
+    def test_completed_pipeline_with_mixed_result(self) -> None:
+        expected_topic = "test-pipeline / backend-tests"
+        expected_message = """:red_circle: **Build failed**: test-pipeline / backend-tests
+- **Commit**: [`963eb239c7b`](https://github.com/PieterCK/getting-started-repo.git/commit/963eb239c7b) on branch `master`
+- **Started**: Aug 29, 2024, 3:59:18 PM
+- **Finished**: Aug 29, 2024, 4:00:15 PM
+- **Failed**: `check-backend-lints`, `test-frontend-js`
+- **Passed**: `check-backend-tests`, `zulip-ci-debian-12`"""
+        self.check_webhook("pipeline_with_mixed_job_result", expected_topic, expected_message)

zerver/webhooks/gocd/view.py

@@ -1,6 +1,6 @@
 # Webhooks for external integrations.
-import json
+
-import os
+from collections import defaultdict
 
 from django.http import HttpRequest, HttpResponse
 
@@ -9,13 +9,24 @@ from zerver.lib.response import json_success
 from zerver.lib.typed_endpoint import JsonBodyPayload, typed_endpoint
 from zerver.lib.validator import WildValue, check_string
 from zerver.lib.webhooks.common import check_send_webhook_message
+from zerver.lib.webhooks.git import get_short_sha
 from zerver.models import UserProfile
 
-MESSAGE_TEMPLATE = """\
+COMMIT_INFO_TEMPLATE = """[`{commit_details}`]({commit_link}) on branch `{branch_name}`"""
-Author: {}
+TOPIC_TEMPLATE = "{pipeline} / {stage}"
-Build status: {} {}
+
-Details: [build log]({})
+SCHEDULED_BODY_TEMPLATE = """
-Comment: {}"""
+**Pipeline {status}**: {pipeline} / {stage}
+- **Commit**: {commit_details}
+- **Started**: {start_time}
+"""
+
+COMPLETED_BODY_TEMPLATE = """
+{emoji} **Build {status}**: {pipeline} / {stage}
+- **Commit**: {commit_details}
+- **Started**: {start_time}
+- **Finished**: {end_time}
+"""
 
 
 @webhook_view("Gocd")
@@ -26,31 +37,75 @@ def api_gocd_webhook(
     *,
     payload: JsonBodyPayload[WildValue],
 ) -> HttpResponse:
-    modifications = payload["build_cause"]["material_revisions"][0]["modifications"][0]
+    type = payload["type"].tame(check_string)
-    result = payload["stages"][0]["result"].tame(check_string)
+    if type == "stage":
-    material = payload["build_cause"]["material_revisions"][0]["material"]
+        body = get_body(payload)
-
+        topic_name = get_topic(payload)
-    if result == "Passed":
+        check_send_webhook_message(request, user_profile, topic_name, body)
-        emoji = ":thumbs_up:"
-    elif result == "Failed":
-        emoji = ":thumbs_down:"
-
-    build_details_file = os.path.join(os.path.dirname(__file__), "fixtures/build_details.json")
-
-    with open(build_details_file) as f:
-        contents = json.load(f)
-        build_link = contents["build_details"]["_links"]["pipeline"]["href"]
-
-    body = MESSAGE_TEMPLATE.format(
-        modifications["user_name"].tame(check_string),
-        result,
-        emoji,
-        build_link,
-        modifications["comment"].tame(check_string),
-    )
-    branch = material["description"].tame(check_string).split(",")
-    topic_name = branch[0].split(" ")[1]
-
-    check_send_webhook_message(request, user_profile, topic_name, body)
-
     return json_success(request)
+
+
+def get_topic(payload: WildValue) -> str:
+    return TOPIC_TEMPLATE.format(
+        pipeline=payload["data"]["pipeline"]["name"].tame(check_string),
+        stage=payload["data"]["pipeline"]["stage"]["name"].tame(check_string),
+    )
+
+
+def get_commit_details(payload: WildValue) -> str:
+    build = payload["data"]["pipeline"]["build-cause"][0]
+    material = build["material"]
+    url_base = material["git-configuration"]["url"].tame(check_string)
+    revision = build["modifications"][0]["revision"].tame(check_string)
+    commit_sha = get_short_sha(revision)
+    url = f"{url_base}/commit/{commit_sha}"
+    branch = material["git-configuration"]["branch"].tame(check_string)
+    return COMMIT_INFO_TEMPLATE.format(
+        commit_details=commit_sha,
+        commit_link=url,
+        branch_name=branch,
+    )
+
+
+def get_jobs_details(pipeline_data: WildValue) -> str:
+    job_dict_list = pipeline_data["stage"]["jobs"]
+    formatted_job_dict = defaultdict(list)
+    job_details_template = ""
+
+    for job in job_dict_list:
+        job_name = job["name"].tame(check_string)
+        job_result = job["result"].tame(check_string)
+        formatted_job_dict[job_result].append(f"`{job_name}`")
+
+    for key in formatted_job_dict:
+        formatted_job_list = ", ".join(formatted_job_dict[key])
+        job_details_template += f"- **{key}**: {formatted_job_list}\n"
+
+    return job_details_template
+
+
+def get_body(payload: WildValue) -> str:
+    pipeline_data = payload["data"]["pipeline"]
+    body_details = {
+        "commit_details": get_commit_details(payload),
+        "status": pipeline_data["stage"]["state"].tame(check_string).lower(),
+        "pipeline": pipeline_data["name"].tame(check_string),
+        "stage": pipeline_data["stage"]["name"].tame(check_string),
+        "start_time": pipeline_data["stage"]["create-time"].tame(check_string),
+    }
+
+    if body_details["status"] == "building":
+        return SCHEDULED_BODY_TEMPLATE.format(**body_details)
+
+    result = pipeline_data["stage"]["result"].tame(check_string)
+    body_details.update(
+        {
+            "result": result,
+            "emoji": ":green_circle:" if result == "Passed" else ":red_circle:",
+            "end_time": pipeline_data["stage"]["last-transition-time"].tame(check_string),
+        }
+    )
+    body = COMPLETED_BODY_TEMPLATE.format(**body_details)
+
+    body += get_jobs_details(pipeline_data)
+    return body