From 79586cc4668e82c63c67c48a631bc932bf6633fc Mon Sep 17 00:00:00 2001 From: Vishnu KS Date: Fri, 1 Jan 2021 23:13:34 +0530 Subject: [PATCH] github: Create action for generating DigitalOcean one click app image. --- .github/workflows/update-oneclick-apps.yml | 24 +++ docs/subsystems/release-checklist.md | 4 + tools/oneclickapps/README.md | 73 ++++++++ ...are_digital_ocean_one_click_app_release.py | 159 ++++++++++++++++++ 4 files changed, 260 insertions(+) create mode 100644 .github/workflows/update-oneclick-apps.yml create mode 100644 tools/oneclickapps/README.md create mode 100644 tools/oneclickapps/prepare_digital_ocean_one_click_app_release.py diff --git a/.github/workflows/update-oneclick-apps.yml b/.github/workflows/update-oneclick-apps.yml new file mode 100644 index 0000000000..2f4116deb8 --- /dev/null +++ b/.github/workflows/update-oneclick-apps.yml @@ -0,0 +1,24 @@ +name: Update one click apps +on: + release: + types: [published] +jobs: + update-digitalocean-oneclick-app: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Update DigitalOcean one click app + env: + DIGITALOCEAN_API_KEY: ${{ secrets.DIGITALOCEAN_API_KEY }} + ZULIP_API_KEY: ${{ secrets.ZULIP_BOT_API_KEY }} + ZULIP_EMAIL: ${{ secrets.ZULIP_BOT_EMAIL }} + ZULIP_SITE: ${{ secrets.ZULIP_ORGANIZATION_URL }} + ZULIP_STREAM_TO_SEND_BOT_MESSAGES: ${{ secrets.ZULIP_STREAM_TO_SEND_BOT_MESSAGES }} + PYTHON_DIGITALOCEAN_REQUEST_TIMEOUT_SEC: 30 + RELEASE_VERSION: ${{ github.event.release.tag_name }} + run: | + export PATH="$HOME/.local/bin:$PATH" + git clone https://github.com/zulip/marketplace-partners + pip3 install python-digitalocean zulip fab-classic + echo $PATH + python3 tools/oneclickapps/prepare_digital_ocean_one_click_app_release.py diff --git a/docs/subsystems/release-checklist.md b/docs/subsystems/release-checklist.md index 82b596616b..7925b66a7f 100644 --- a/docs/subsystems/release-checklist.md +++ b/docs/subsystems/release-checklist.md @@ -51,6 +51,10 @@ preparing a new release. * Post the release by [editing the latest tag on GitHub](https://github.com/zulip/zulip/tags); use the text from `changelog.md` for the release notes. + + **Note:** This will trigger the [GitHub action](https://github.com/zulip/zulip/blob/master/tools/oneclickapps/README.md) + for updating DigitalOcean one-click app image. The action uses the latest release + tarball published on `zulip.org` for creating the image. * Update the [Docker image](https://github.com/zulip/docker-zulip) and do a release of that. * Update the image of DigitalOcean one click app using diff --git a/tools/oneclickapps/README.md b/tools/oneclickapps/README.md new file mode 100644 index 0000000000..711b12bf38 --- /dev/null +++ b/tools/oneclickapps/README.md @@ -0,0 +1,73 @@ +# One click app release automation + +This directory contains scripts for automating the release of Zulip one click apps. + +## DigitalOcean 1-Click Application +`prepare_digital_ocean_one_click_app_release.py` creates the image of DigitalOcean 1-Click +app from the latest Zulip release (fetched from https://www.zulip.org/dist/releases). It will +also create a test droplet from the image and send the image and droplet +details to a pre-configured Zulip stream. Anyone, whose key is added to the +Zulip DigitalOcean team can SSH into the droplet for testing. + +### Running as GitHub action + +`.github/workflows/update-oneclick-apps.yml` is configured to invoke +`prepare_digital_ocean_one_click_app_release.py` as a GitHub action during each Zulip +server release. + +You also need to set the following secrets in your GitHub repository to make the action +work correctly. These secrets are passed as environment variables to the GitHub action. + +* `DIGITALOCEAN_API_KEY` - DigitalOcean API KEY used for creating droplets, snapshots etc. +* `ZULIP_ORGANIZATION_URL` - The URL of the Zulip organization where the message containing + droplet/image details should be sent. +* `ZULIP_BOT_API_KEY` - The API key of the Zulip Bot used for sending messages. +* `ZULIP_STREAM_TO_SEND_BOT_MESSAGES` - The stream to which the messages should be sent. +* `ZULIP_BOT_EMAIL` - The email of the Zulip bot. + +Also pass the following as environment variables in `.github/workflows/update-oneclick-apps.yml`. +* `PYTHON_DIGITALOCEAN_REQUEST_TIMEOUT_SEC` - This configures the maximum number of seconds + to wait before the requests made by `python-digitalocean` timeout. If not configured, it's + common for the requests to take 20+ minutes before getting timed out. + +### Verifying the one click app image +* The action will send the image name and test droplet details to the stream configured in the + above steps. +* SSH into the test droplet by following the instructions in the message. +* After logging into the test droplet, exit the installer and run the following script. + + https://raw.githubusercontent.com/digitalocean/marketplace-partners/master/scripts/img_check.sh + + This script checks whether the image created is valid. It is also run by the DigitalOcean team + before they approve the image submission in the one click app marketplace. +* If there are no errors (you can ignore most of the warnings), exit the SSH connection and + reconnect. +* Populate the details asked by the installer and verify that the installer completes successfully. + If there are errors see the section below. +* Use the link generated by the installer to create a new Zulip organization. Do some basic + testing like sending a bunch of messages and reloading the webpage. +* If there are no issues, submit the image in the + [DigitalOcean vendor portal](https://marketplace.digitalocean.com/vendorportal). You need to be + added to the vendor portal team for doing this. Ask Tim to add you if required. During the submission, + make sure to update the blog post URL if it's a major release. +* Keep checking the vendor portal for change in status of the submission. DigitalOcean does nominally send + emails when there are updates on the submission, but we have found the emails to not always arrive. +* If there are any issues with submission, rebuild the image by manually invoking the script and + resubmit. The issues we have seen mostly in the past are caused by the dependencies getting outdated + by the time the DigitalOcean team run the checks. In that case you have to just rebuild the image + by invoking the script. +* Delete the test droplet `oneclickapp-{release_version}-test` after you have completed testing + by going to the [DigitalOcean Zulip team account](https://cloud.digitalocean.com/droplets?i=0242e0). + If there are other existing test droplets with the same name format but with with older release versions + feel free to delete them as well. These droplets are also tagged with `github-action` and`temporary` + tags. + +**Errors** + +If there are any errors while setting up the one click app installer, you have three options +* Include the fix in the Fabric script that setups the installer. + [01-initial-setup](https://raw.githubusercontent.com/zulip/marketplace-partners/master/marketplace_docs/templates/Fabric/scripts/01-initial-setup) + file should be a good place to include the fix. See + [zulip/marketplace-partners#4](https://github.com/zulip/marketplace-partners/pull/4/files) for an + example fix. +* Wait for the next release to fix the error. diff --git a/tools/oneclickapps/prepare_digital_ocean_one_click_app_release.py b/tools/oneclickapps/prepare_digital_ocean_one_click_app_release.py new file mode 100644 index 0000000000..85f74a0f66 --- /dev/null +++ b/tools/oneclickapps/prepare_digital_ocean_one_click_app_release.py @@ -0,0 +1,159 @@ +import os +import subprocess +import time +from pathlib import Path +from typing import List + +import digitalocean +import zulip +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry + +manager = digitalocean.Manager(token=os.environ["DIGITALOCEAN_API_KEY"]) +zulip_client = zulip.Client() +TEST_DROPLET_SUBDOMAIN = "oneclicktest" + + +def generate_ssh_keys() -> None: + subprocess.call( + ["ssh-keygen", "-f", str(Path.home()) + "/.ssh/id_ed25519", "-P", "", "-t", "ed25519"] + ) + + +def get_public_ssh_key() -> str: + try: + with open(str(Path.home()) + "/.ssh/id_ed25519.pub") as f: + return f.read() + except FileNotFoundError: + return "" + + +def sleep_until_droplet_action_is_completed( + droplet: digitalocean.Droplet, action_type: str +) -> None: + incomplete = True + while incomplete: + for action in droplet.get_actions(): + action.load() + print(f"...[{action.type}]: {action.status}") + if action.type == action_type and action.status == "completed": + incomplete = False + break + if incomplete: + time.sleep(5) + droplet.load() + + +def set_api_request_retry_limits(api_object: digitalocean.baseapi.BaseAPI) -> None: + retry = Retry(connect=5, read=5, backoff_factor=0.1) + adapter = HTTPAdapter(max_retries=retry) + api_object._session.mount("https://", adapter) + + +def create_droplet( + name: str, ssh_keys: List[str], image: str = "ubuntu-18-04-x64" +) -> digitalocean.Droplet: + droplet = digitalocean.Droplet( + token=manager.token, + name=name, + region="nyc3", + size_slug="s-1vcpu-2gb", + image=image, + backups=False, + ssh_keys=ssh_keys, + tags=["github-action", "temporary"], + ) + set_api_request_retry_limits(droplet) + droplet.create() + sleep_until_droplet_action_is_completed(droplet, "create") + return droplet + + +def create_ssh_key(name: str, public_key: str) -> digitalocean.SSHKey: + action_public_ssh_key_object = digitalocean.SSHKey( + name=name, public_key=public_key, token=manager.token + ) + set_api_request_retry_limits(action_public_ssh_key_object) + action_public_ssh_key_object.create() + return action_public_ssh_key_object + + +def create_snapshot(droplet: digitalocean.Droplet, snapshot_name: str) -> None: + droplet.take_snapshot(snapshot_name, power_off=True) + droplet.load() + sleep_until_droplet_action_is_completed(droplet, "snapshot") + + +def create_dns_records(droplet: digitalocean.Droplet) -> None: + domain = digitalocean.Domain(token=manager.token, name="zulipdev.org") + set_api_request_retry_limits(domain) + domain.load() + + oneclick_test_app_record_names = [TEST_DROPLET_SUBDOMAIN, f"*.{TEST_DROPLET_SUBDOMAIN}"] + for record in domain.get_records(): + if ( + record.name in oneclick_test_app_record_names + and record.domain == "zulipdev.org" + and record.type == "A" + ): + record.destroy() + + domain.load() + for record_name in oneclick_test_app_record_names: + domain.create_new_domain_record(type="A", name=record_name, data=droplet.ip_address) + + +def setup_one_click_app_installer(droplet: digitalocean.Droplet) -> None: + subprocess.call( + [ + "fab", + "build_image", + "-H", + droplet.ip_address, + "--keepalive", + "5", + "--connection-attempts", + "10", + ], + cwd="marketplace-partners/marketplace_docs/templates/Fabric", + ) + + +def send_message(content: str) -> None: + request = { + "type": "stream", + "to": os.environ["ZULIP_STREAM_TO_SEND_BOT_MESSAGES"], + "topic": "DigitalOcean One Click App", + "content": content, + } + zulip_client.send_message(request) + + +if __name__ == "__main__": + release_version = os.environ["RELEASE_VERSION"] + + generate_ssh_keys() + action_public_ssh_key_object = create_ssh_key( + f"oneclickapp-{release_version}-image-generator-public-key", get_public_ssh_key() + ) + + image_generator_droplet = create_droplet( + f"oneclickapp-{release_version}-image-generator", [action_public_ssh_key_object] + ) + + setup_one_click_app_installer(image_generator_droplet) + + oneclick_image_name = f"oneclickapp-{release_version}" + create_snapshot(image_generator_droplet, oneclick_image_name) + snapshot = image_generator_droplet.get_snapshots()[0] + send_message(f"One click app image `{oneclick_image_name}` created.") + + image_generator_droplet.destroy() + action_public_ssh_key_object.destroy() + + test_droplet_name = f"oneclickapp-{release_version}-test" + test_droplet = create_droplet(test_droplet_name, manager.get_all_sshkeys(), image=snapshot.id) + create_dns_records(test_droplet) + send_message( + f"Test droplet `{test_droplet_name}` created. SSH as root to {TEST_DROPLET_SUBDOMAIN}.zulipdev.org for testing." + )