zulip_tools.py: Move zulip_tools.py in scripts/lib.

This commit moves zulip_tools.py as part of cleaning the root directory
and organizing proejct into better directory structure.

scripts/deploy-zulip-from-git

@@ -24,7 +24,7 @@ except (configparser.NoSectionError, configparser.NoOptionError):
 os.environ["PYTHONUNBUFFERED"] = "y"
 
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
-from zulip_tools import DEPLOYMENTS_DIR, FAIL, WARNING, ENDC, make_deploy_path, \
+from scripts.lib.zulip_tools import DEPLOYMENTS_DIR, FAIL, WARNING, ENDC, make_deploy_path, \
     get_deployment_lock, release_deployment_lock, su_to_zulip
 
 logging.basicConfig(format="%(asctime)s update-deployment: %(message)s",

scripts/lib/create-production-venv

@@ -9,7 +9,7 @@ ZULIP_PATH = dirname(dirname(dirname(abspath(__file__))))
 if ZULIP_PATH not in sys.path:
     sys.path.append(ZULIP_PATH)
 
-from zulip_tools import run
+from scripts.lib.zulip_tools import run
 from scripts.lib.setup_venv import setup_virtualenv, VENV_DEPENDENCIES
 
 parser = argparse.ArgumentParser(description="Create a production virtualenv with caching")

scripts/lib/install

@@ -95,7 +95,7 @@ if [ "$has_postgres" = 0 ]; then
 fi
 
 if [ "$has_appserver" = 0 ]; then
-    deploy_path=$(/root/zulip/zulip_tools.py make_deploy_path)
+    deploy_path=$(/root/zulip/scripts/lib/zulip_tools.py make_deploy_path)
     mv /root/zulip "$deploy_path"
     ln -nsf /home/zulip/deployments/next /root/zulip
     ln -nsf "$deploy_path" /home/zulip/deployments/next

scripts/lib/setup_venv.py

@@ -4,7 +4,7 @@ import os
 import sys
 from os.path import dirname, abspath
 import subprocess
-from zulip_tools import run
+from scripts.lib.zulip_tools import run
 
 ZULIP_PATH = dirname(dirname(dirname(abspath(__file__))))
 VENV_CACHE_PATH = "/srv/zulip-venv-cache"

scripts/lib/unpack-zulip

@@ -10,7 +10,7 @@ import glob
 os.environ["PYTHONUNBUFFERED"] = "y"
 
 sys.path.append(os.path.join(os.path.dirname(__file__), '..', ".."))
-from zulip_tools import DEPLOYMENTS_DIR, FAIL, ENDC, make_deploy_path
+from scripts.lib.zulip_tools import DEPLOYMENTS_DIR, FAIL, ENDC, make_deploy_path
 
 if len(sys.argv) != 2:
     print(FAIL + "Usage: %s <tarball>" % (sys.argv[0],) + ENDC)

scripts/lib/upgrade-zulip

@@ -10,7 +10,7 @@ TARBALL_ARCHIVE_PATH="/home/zulip/archives"
 os.environ["PYTHONUNBUFFERED"] = "y"
 
 sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..'))
-from zulip_tools import DEPLOYMENTS_DIR, FAIL, WARNING, ENDC, \
+from scripts.lib.zulip_tools import DEPLOYMENTS_DIR, FAIL, WARNING, ENDC, \
     mkdir_p, su_to_zulip, get_deployment_lock, release_deployment_lock
 
 logging.basicConfig(format="%(asctime)s upgrade-zulip: %(message)s",

scripts/lib/upgrade-zulip-stage-2

@@ -14,7 +14,7 @@ import logging
 os.environ["PYTHONUNBUFFERED"] = "y"
 
 sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..'))
-from zulip_tools import DEPLOYMENTS_DIR, FAIL, WARNING, ENDC, su_to_zulip
+from scripts.lib.zulip_tools import DEPLOYMENTS_DIR, FAIL, WARNING, ENDC, su_to_zulip
 
 logging.basicConfig(format="%(asctime)s upgrade-zulip-stage-2: %(message)s",
                     level=logging.INFO)

scripts/lib/zulip_tools.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python
+from __future__ import print_function
+import datetime
+import errno
+import os
+import pwd
+import shutil
+import subprocess
+import sys
+import time
+
+if False:
+    from typing import Sequence
+
+DEPLOYMENTS_DIR = "/home/zulip/deployments"
+LOCK_DIR = os.path.join(DEPLOYMENTS_DIR, "lock")
+TIMESTAMP_FORMAT = '%Y-%m-%d-%H-%M-%S'
+
+# Color codes
+OKBLUE = '\033[94m'
+OKGREEN = '\033[92m'
+WARNING = '\033[93m'
+FAIL = '\033[91m'
+ENDC = '\033[0m'
+
+def subprocess_text_output(args):
+    # type: (Sequence[str]) -> str
+    return subprocess.check_output(args, universal_newlines=True).strip()
+
+def su_to_zulip():
+    # type: () -> None
+    pwent = pwd.getpwnam("zulip")
+    os.setgid(pwent.pw_gid)
+    os.setuid(pwent.pw_uid)
+    os.environ['HOME'] = os.path.abspath(os.path.join(DEPLOYMENTS_DIR, '..'))
+
+def make_deploy_path():
+    # type: () -> str
+    timestamp = datetime.datetime.now().strftime(TIMESTAMP_FORMAT)
+    return os.path.join(DEPLOYMENTS_DIR, timestamp)
+
+if __name__ == '__main__':
+    cmd = sys.argv[1]
+    if cmd == 'make_deploy_path':
+        print(make_deploy_path())
+
+def mkdir_p(path):
+    # type: (str) -> None
+    # Python doesn't have an analog to `mkdir -p` < Python 3.2.
+    try:
+        os.makedirs(path)
+    except OSError as e:
+        if e.errno == errno.EEXIST and os.path.isdir(path):
+            pass
+        else:
+            raise
+
+def get_deployment_lock(error_rerun_script):
+    # type: (str) -> None
+    start_time = time.time()
+    got_lock = False
+    while time.time() - start_time < 300:
+        try:
+            os.mkdir(LOCK_DIR)
+            got_lock = True
+            break
+        except OSError:
+            print(WARNING + "Another deployment in progress; waiting for lock... "
+                  + "(If no deployment is running, rmdir %s)" % (LOCK_DIR,) + ENDC)
+            sys.stdout.flush()
+            time.sleep(3)
+
+    if not got_lock:
+        print(FAIL + "Deployment already in progress.  Please run\n"
+              + "  %s\n" % (error_rerun_script,)
+              + "manually when the previous deployment finishes, or run\n"
+              + "  rmdir %s\n"  % (LOCK_DIR,)
+              + "if the previous deployment crashed."
+              + ENDC)
+        sys.exit(1)
+
+def release_deployment_lock():
+    # type: () -> None
+    shutil.rmtree(LOCK_DIR)
+
+def run(args):
+    # type: (Sequence[str]) -> int
+    # Output what we're doing in the `set -x` style
+    print("+ %s" % (" ".join(args)))
+    process = subprocess.Popen(args)
+    rc = process.wait()
+    if rc:
+        raise subprocess.CalledProcessError(rc, args) # type: ignore # https://github.com/python/typeshed/pull/329
+    return 0

scripts/purge-old-deployments

@@ -6,7 +6,7 @@ import datetime
 import shutil
 
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
-from zulip_tools import DEPLOYMENTS_DIR, TIMESTAMP_FORMAT
+from scripts.lib.zulip_tools import DEPLOYMENTS_DIR, TIMESTAMP_FORMAT
 
 logging.basicConfig(format="%(asctime)s purge-deployments: %(message)s",
                     level=logging.INFO)

scripts/restart-server

@@ -8,7 +8,7 @@ import logging
 import time
 
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
-from zulip_tools import ENDC, OKGREEN, DEPLOYMENTS_DIR
+from scripts.lib.zulip_tools import ENDC, OKGREEN, DEPLOYMENTS_DIR
 
 logging.basicConfig(format="%(asctime)s restart-server: %(message)s",
                     level=logging.INFO)