management: Provide a common lockfile dir, and a decorator for it.

Factor out the repeated pattern of taking a lock, or immediately
aborting with a message if it cannot be acquired.  The exit code in
that situation is changed to be exit code 1, rather than the successful
0; we are likely missing new work since that process started.

We move the lockfiles to a common directory under `/srv/zulip-locks`
rather than muddy up `/home/zulip/deployments`.

analytics/management/commands/update_analytics_counts.py

@@ -11,8 +11,7 @@ from django.utils.timezone import now as timezone_now
 from typing_extensions import override
 
 from analytics.lib.counts import ALL_COUNT_STATS, logger, process_count_stat
-from scripts.lib.zulip_tools import ENDC, WARNING
-from zerver.lib.context_managers import lockfile_nonblocking
+from zerver.lib.management import abort_unless_locked
 from zerver.lib.remote_server import send_server_data_to_push_bouncer
 from zerver.lib.timestamp import floor_to_hour
 from zerver.models import Realm
@@ -41,17 +40,9 @@ class Command(BaseCommand):
         )
 
     @override
+    @abort_unless_locked
     def handle(self, *args: Any, **options: Any) -> None:
-        with lockfile_nonblocking(
-            settings.ANALYTICS_LOCK_FILE,
-        ) as lock_acquired:
-            if lock_acquired:
         self.run_update_analytics_counts(options)
-            else:
-                print(
-                    f"{WARNING}Analytics lock {settings.ANALYTICS_LOCK_FILE} is unavailable;"
-                    f" exiting.{ENDC}"
-                )
 
     def run_update_analytics_counts(self, options: Dict[str, Any]) -> None:
         # installation_epoch relies on there being at least one realm; we

puppet/zulip/manifests/app_frontend_base.pp

@@ -208,6 +208,7 @@ class zulip::app_frontend_base {
     '/home/zulip/tornado',
     '/home/zulip/prod-static',
     '/home/zulip/deployments',
+    '/srv/zulip-locks',
     '/srv/zulip-emoji-cache',
     '/srv/zulip-uploaded-files-cache',
   ]:

zerver/lib/management.py

@@ -1,9 +1,11 @@
 # Library code for use in management commands
 import logging
+import os
+import sys
 from argparse import ArgumentParser, RawTextHelpFormatter, _ActionsContainer
 from dataclasses import dataclass
-from functools import reduce
-from typing import Any, Dict, Optional
+from functools import reduce, wraps
+from typing import Any, Dict, Optional, Protocol
 
 from django.conf import settings
 from django.core import validators
@@ -12,6 +14,7 @@ from django.core.management.base import BaseCommand, CommandError, CommandParser
 from django.db.models import Q, QuerySet
 from typing_extensions import override
 
+from zerver.lib.context_managers import lockfile_nonblocking
 from zerver.lib.initial_password import initial_password
 from zerver.models import Client, Realm, UserProfile
 from zerver.models.clients import get_client
@@ -38,6 +41,31 @@ def check_config() -> None:
         raise CommandError(f"Error: You must set {setting_name} in /etc/zulip/settings.py.")
 
 
+class HandleMethod(Protocol):
+    def __call__(self, *args: Any, **kwargs: Any) -> None: ...
+
+
+def abort_unless_locked(handle_func: HandleMethod) -> HandleMethod:
+    @wraps(handle_func)
+    def our_handle(self: BaseCommand, *args: Any, **kwargs: Any) -> None:
+        os.makedirs(settings.LOCKFILE_DIRECTORY, exist_ok=True)
+        # Trim out just the last part of the module name, which is the
+        # command name, to use as the lockfile name;
+        # `zerver.management.commands.send_zulip_update_announcements`
+        # becomes `/srv/zulip-locks/send_zulip_update_announcements.lock`
+        lockfile_name = handle_func.__module__.split(".")[-1]
+        lockfile_path = settings.LOCKFILE_DIRECTORY + "/" + lockfile_name + ".lock"
+        with lockfile_nonblocking(lockfile_path) as lock_acquired:
+            if not lock_acquired:  # nocoverage
+                self.stdout.write(
+                    self.style.ERROR(f"Lock {lockfile_path} is unavailable; exiting.")
+                )
+                sys.exit(1)
+            handle_func(self, *args, **kwargs)
+
+    return our_handle
+
+
 @dataclass
 class CreateUserParameters:
     email: str

zerver/management/commands/send_zulip_update_announcements.py

@@ -1,12 +1,9 @@
 from argparse import ArgumentParser
 from typing import Any
 
-from django.conf import settings
 from typing_extensions import override
 
-from scripts.lib.zulip_tools import ENDC, WARNING
-from zerver.lib.context_managers import lockfile_nonblocking
-from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.management import ZulipBaseCommand, abort_unless_locked
 from zerver.lib.zulip_update_announcements import send_zulip_update_announcements
 
 
@@ -22,14 +19,6 @@ class Command(ZulipBaseCommand):
         )
 
     @override
+    @abort_unless_locked
     def handle(self, *args: Any, **options: Any) -> None:
-        with lockfile_nonblocking(
-            settings.ZULIP_UPDATE_ANNOUNCEMENTS_LOCK_FILE,
-        ) as lock_acquired:
-            if lock_acquired:
         send_zulip_update_announcements(skip_delay=options["skip_delay"])
-            else:
-                print(
-                    f"{WARNING}Update announcements lock {settings.ZULIP_UPDATE_ANNOUNCEMENTS_LOCK_FILE} is unavailable;"
-                    f" exiting.{ENDC}"
-                )

zproject/computed_settings.py

@@ -687,7 +687,6 @@ QUEUE_ERROR_DIR = zulip_path("/var/log/zulip/queue_error")
 QUEUE_STATS_DIR = zulip_path("/var/log/zulip/queue_stats")
 DIGEST_LOG_PATH = zulip_path("/var/log/zulip/digest.log")
 ANALYTICS_LOG_PATH = zulip_path("/var/log/zulip/analytics.log")
-ANALYTICS_LOCK_FILE = zulip_path("/home/zulip/deployments/analytics-lock.lock")
 WEBHOOK_LOG_PATH = zulip_path("/var/log/zulip/webhooks_errors.log")
 WEBHOOK_ANOMALOUS_PAYLOADS_LOG_PATH = zulip_path("/var/log/zulip/webhooks_anomalous_payloads.log")
 WEBHOOK_UNSUPPORTED_EVENTS_LOG_PATH = zulip_path("/var/log/zulip/webhooks_unsupported_events.log")
@@ -699,8 +698,9 @@ AUTH_LOG_PATH = zulip_path("/var/log/zulip/auth.log")
 SCIM_LOG_PATH = zulip_path("/var/log/zulip/scim.log")
 
 ZULIP_WORKER_TEST_FILE = zulip_path("/var/log/zulip/zulip-worker-test-file")
-ZULIP_UPDATE_ANNOUNCEMENTS_LOCK_FILE = zulip_path(
-    "/home/zulip/deployments/zulip_update_announcements.lock"
+
+LOCKFILE_DIRECTORY = (
+    "/srv/zulip-locks" if not DEVELOPMENT else os.path.join(os.path.join(DEPLOY_ROOT, "var/locks"))
 )