management: Skip hourly crons during deploys.

This is most important for `send_zulip_update_announcements`, which
can race with the version run as a post-deploy hook.  However, all of
these crons can tolerate being slightly delayed, and there's little
benefit to them taking CPU or possibly hitting odd borderline race
conditions when the deploy is in progress.

For safety, we only trust the deploy lockfile if it was created
within the last hour -- deploys should not take more than an hour, and
failing to ever run hourly crons is much worse than perhaps running
them during a real very-long deploy.

analytics/management/commands/update_analytics_counts.py

@@ -10,7 +10,7 @@ from django.utils.timezone import now as timezone_now
 from typing_extensions import override
 
 from analytics.lib.counts import ALL_COUNT_STATS, logger, process_count_stat
-from zerver.lib.management import ZulipBaseCommand, abort_unless_locked
+from zerver.lib.management import ZulipBaseCommand, abort_cron_during_deploy, abort_unless_locked
 from zerver.lib.remote_server import send_server_data_to_push_bouncer, should_send_analytics_data
 from zerver.lib.timestamp import floor_to_hour
 from zerver.models import Realm
@@ -38,6 +38,7 @@ class Command(ZulipBaseCommand):
         )
 
     @override
+    @abort_cron_during_deploy
     @abort_unless_locked
     def handle(self, *args: Any, **options: Any) -> None:
         self.run_update_analytics_counts(options)

puppet/zulip/templates/cron.template.erb

@@ -2,6 +2,7 @@ MAILTO=zulip
 PATH=/usr/local/bin:/usr/bin:/bin
 SHELL=/bin/bash
 USER=<%= @user %>
+RUNNING_UNDER_CRON=1
 <% if @proxy != '' -%>
 HTTP_proxy="<%= @proxy %>"
 HTTPS_proxy="<%= @proxy %>"

zerver/lib/management.py

@@ -2,6 +2,7 @@
 import logging
 import os
 import sys
+import time
 from argparse import ArgumentParser, BooleanOptionalAction, RawTextHelpFormatter, _ActionsContainer
 from dataclasses import dataclass
 from functools import reduce, wraps
@@ -14,6 +15,7 @@ from django.core.management.base import BaseCommand, CommandError, CommandParser
 from django.db.models import Q, QuerySet
 from typing_extensions import override
 
+from scripts.lib.zulip_tools import LOCK_DIR as DEPLOYMENT_LOCK_DIR
 from zerver.lib.context_managers import lockfile_nonblocking
 from zerver.lib.initial_password import initial_password
 from zerver.models import Client, Realm, UserProfile
@@ -66,6 +68,26 @@ def abort_unless_locked(handle_func: HandleMethod) -> HandleMethod:
     return our_handle
 
 
+def abort_cron_during_deploy(handle_func: HandleMethod) -> HandleMethod:
+    @wraps(handle_func)
+    def our_handle(self: BaseCommand, *args: Any, **kwargs: Any) -> None:
+        # For safety, we only trust the lock directory if it was
+        # created within the last hour -- otherwise, a spurious
+        # deploy lock could linger and block all hourly crons.
+        if (
+            os.environ.get("RUNNING_UNDER_CRON")
+            and os.path.exists(DEPLOYMENT_LOCK_DIR)
+            and time.time() - os.path.getctime(DEPLOYMENT_LOCK_DIR) < 3600
+        ):  # nocoverage
+            self.stdout.write(
+                self.style.ERROR("Deployment in process; aborting cron management command.")
+            )
+            sys.exit(1)
+        handle_func(self, *args, **kwargs)
+
+    return our_handle
+
+
 @dataclass
 class CreateUserParameters:
     email: str

zerver/management/commands/fetch_tor_exit_nodes.py

@@ -7,7 +7,7 @@ from django.conf import settings
 from typing_extensions import override
 from urllib3.util import Retry
 
-from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.management import ZulipBaseCommand, abort_cron_during_deploy
 from zerver.lib.outgoing_http import OutgoingSession
 
 
@@ -44,6 +44,7 @@ Does nothing unless RATE_LIMIT_TOR_TOGETHER is enabled.
         )
 
     @override
+    @abort_cron_during_deploy
     def handle(self, *args: Any, **options: Any) -> None:
         if not settings.RATE_LIMIT_TOR_TOGETHER:
             return

zerver/management/commands/promote_new_full_members.py

@@ -3,13 +3,14 @@ from typing import Any
 from typing_extensions import override
 
 from zerver.actions.user_groups import promote_new_full_members
-from zerver.lib.management import ZulipBaseCommand, abort_unless_locked
+from zerver.lib.management import ZulipBaseCommand, abort_cron_during_deploy, abort_unless_locked
 
 
 class Command(ZulipBaseCommand):
     help = """Add users to full members system group."""
 
     @override
+    @abort_cron_during_deploy
     @abort_unless_locked
     def handle(self, *args: Any, **options: Any) -> None:
         promote_new_full_members()

zerver/management/commands/send_zulip_update_announcements.py

@@ -4,7 +4,7 @@ from typing import Any
 from django.conf import settings
 from typing_extensions import override
 
-from zerver.lib.management import ZulipBaseCommand, abort_unless_locked
+from zerver.lib.management import ZulipBaseCommand, abort_cron_during_deploy, abort_unless_locked
 from zerver.lib.zulip_update_announcements import send_zulip_update_announcements
 from zerver.models import Realm
 
@@ -26,6 +26,7 @@ class Command(ZulipBaseCommand):
         )
 
     @override
+    @abort_cron_during_deploy
     @abort_unless_locked
     def handle(self, *args: Any, **options: Any) -> None:
         if options["reset_level"] is not None: