upgrade-zulip: Move static asset compilation before shutdown.

This saves about a minute of downtime when using
upgrade-zulip-from-git in the default configuration.

It should also save several seconds of downtime when upgrading to a
production release tarball as well.

scripts/lib/upgrade-zulip-stage-2

@@ -54,9 +54,33 @@ if not os.path.exists((os.path.join(deploy_path, "zproject/prod_settings"))):
 if os.path.islink((os.path.join(deploy_path, "zproject/local_settings.py"))):
     subprocess.check_call(["rm", os.path.join(deploy_path, "zproject/local_settings.py")])
 
+# Now we should have an environment setup where we can run our tools;
+# first, creating the production venv.
 subprocess.check_call([os.path.join(deploy_path, "scripts", "lib", "create-production-venv"),
                        deploy_path])
 
+# And then, building/installing the static assets.
+if args.from_git:
+    # Note: The fact that this is before we apply puppet changes means
+    # that we don't support adding new puppet dependencies of
+    # update-prod-static with the git upgrade process.  But it'll fail
+    # safely; this seems like a worthwhile tradeoff to minimize downtime.
+    logging.info("Building static assets...")
+    subprocess.check_call(["./tools/update-prod-static", "--prev-deploy",
+                           os.path.join(DEPLOYMENTS_DIR, 'current')],
+                          preexec_fn=su_to_zulip)
+else:
+    # Since this doesn't do any actual work, it's likely safe to have
+    # this run before we apply puppet changes (saving a bit of downtime).
+    logging.info("Installing static assets...")
+    subprocess.check_call(["cp", "-rT", os.path.join(deploy_path, 'prod-static/serve'),
+                           '/home/zulip/prod-static'], preexec_fn=su_to_zulip)
+    # Sets up npm cache
+    setup_node_modules(npm_args=['--production'], copy_modules=True)
+
+# Our next optimization is to check whether any migrations are needed
+# before we start the critical section of the restart.  This saves
+# about 1s of downtime in a no-op upgrade.
 migrations_needed = False
 if not args.skip_migrations:
     logging.info("Checking for needed migrations")
@@ -66,11 +90,15 @@ if not args.skip_migrations:
         if ln.startswith("[ ]"):
             migrations_needed = True
 
+# Now we start shutting down services; we start with
+# process-fts-updates, which isn't on the critical serving path.
 if os.path.exists("/etc/supervisor/conf.d/zulip_db.conf"):
     subprocess.check_call(["supervisorctl", "stop", "process-fts-updates"], preexec_fn=su_to_zulip)
 
 if not args.skip_puppet or migrations_needed:
-    # If we're running either puppet or migrations, we shut the server down for the duration
+    # By default, we shut down the service to apply migrations and
+    # puppet changes, to minimize risk of issues due to inconsistent
+    # state.
     logging.info("Stopping Zulip...")
     subprocess.check_call(["supervisorctl", "stop", "zulip-workers:*", "zulip-django",
                            "zulip-tornado", "zulip-senders:*"], preexec_fn=su_to_zulip)
@@ -84,17 +112,6 @@ if migrations_needed:
     logging.info("Applying database migrations...")
     subprocess.check_call(["./manage.py", "migrate", "--noinput"], preexec_fn=su_to_zulip)
 
-if args.from_git:
-    logging.info("Building static assets...")
-    subprocess.check_call(["./tools/update-prod-static", "--prev-deploy",
-                           os.path.join(DEPLOYMENTS_DIR, 'current')],
-                          preexec_fn=su_to_zulip)
-else:
-    subprocess.check_call(["cp", "-rT", os.path.join(deploy_path, 'prod-static/serve'),
-                           '/home/zulip/prod-static'], preexec_fn=su_to_zulip)
-    # Sets up npm cache
-    setup_node_modules(npm_args=['--production'], copy_modules=True)
-
 logging.info("Restarting Zulip...")
 subprocess.check_output(["./scripts/restart-server"], preexec_fn=su_to_zulip)