zulip/tools/minify-js

#!/usr/bin/env python3

# Minifies JavaScripts, creating source maps


import os
import subprocess
import argparse
import sys
import shutil

parser = argparse.ArgumentParser()
parser.add_argument('--prev-deploy', metavar='DIR',
                    help='a previous deploy from which to reuse files if possible')
args = parser.parse_args()
prev_deploy = args.prev_deploy

# We have to pull out JS_SPECS, defined in our settings file, so we know what
# JavaScript source files to minify (and what output files to create).
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
import scripts.lib.setup_path_on_import
from typing import Any, Dict, Optional, Set

os.environ['DJANGO_SETTINGS_MODULE'] = 'zproject.settings'
from django.conf import settings

os.chdir(settings.DEPLOY_ROOT)

STATIC_PATH = 'static/'

# Compile Handlebars templates
subprocess.check_call(['tools/compile-handlebars-templates'])

# Create webpack bundle
subprocess.check_call(['tools/webpack'])

def get_changed_source_files(other_checkout):
    # type: (str) -> Optional[Set[str]]
    """ Get list of changed static files since other_checkout.
    If git fails to return a reasonable looking list, this returns None,
    in which case it should be assumed no files can be reused from
    other_checkout. """

    try:
        git_dir = os.path.join(other_checkout, '.git')
        old_commit_sha1 = subprocess.check_output(['git', 'rev-parse', 'HEAD'],
                                                  env={'GIT_DIR': git_dir}, universal_newlines=True)
        old_commit_sha1 = old_commit_sha1.rstrip()

        git_diff = subprocess.check_output(['git', 'diff', '--name-only',
                                            old_commit_sha1], universal_newlines=True)
    except subprocess.CalledProcessError:
        # If git returned an error, assume we can't reuse any files, and
        # regenerate everything.
        print("Warning: git returned an error when comparing to the previous")
        print("deploy in %s. Will re-minify JavaScript instead of reusing"
              % (other_checkout,))
        return None

    changed = set()  # type: Set[str]
    for filename in git_diff.split('\n'):
        if filename in ["package.json", "zproject/settings.py", "tools/minify-js"]:
            print("Changed a core JS pipeline file; not reusing cached minification results")
            return None
        if not filename.startswith(STATIC_PATH):
            continue  # Ignore non-static files.

        if filename.endswith('.handlebars'):
            continue

        changed.add(filename)

    return changed


changed_files = set()  # type: Set[str]
if prev_deploy:
    changed_files_tmp = get_changed_source_files(prev_deploy)
    if changed_files_tmp is None:
        prev_deploy = None
    else:
        changed_files = changed_files_tmp

# Always use the newly compiled handlebars templates and webpack bundle.
if prev_deploy:
    changed_files.add(os.path.join(STATIC_PATH, 'templates/compiled.js'))

JS_SPECS = settings.JS_SPECS
CLOSURE_BINARY = '/usr/bin/closure-compiler'
if not os.path.exists(CLOSURE_BINARY):
    CLOSURE_BINARY = 'tools/closure-compiler/run'
    if not os.path.exists(CLOSURE_BINARY):
        print("closure-compiler not installed; the Vagrant tools/provision installs it via apt "
              "or you can manually unpack http://dl.google.com/closure-compiler/compiler-latest.zip to "
              "tools/closure-compiler")
        sys.exit(1)

# Where to put minified JS and source maps
MIN_DIR = os.path.join(STATIC_PATH, 'min/')
MAP_DIR = os.path.join(STATIC_PATH, 'source-map/')
os.makedirs(MIN_DIR, exist_ok=True)
os.makedirs(MAP_DIR, exist_ok=True)

for js_group_filespec_pair in JS_SPECS.items():
    # JS_SPECS is not typed, so forcefully type keys and values being read from JS_SPECS
    js_group = js_group_filespec_pair[0]  # type: str
    filespec = js_group_filespec_pair[1]  # type: Dict[str, Any]
    #      JS_SPECS look like        'js/foobar.js'.
    # changed_files look like 'static/js/foobar.js'.
    # So we prepend 'static/' to the JS_SPECS so these match up.
    in_files = [os.path.join(STATIC_PATH, filename)
                for filename in filespec['source_filenames']]

    min_files = [os.path.join(STATIC_PATH, filename)
                 for filename in filespec.get('minifed_source_filenames', [])]

    out_file = os.path.join(MIN_DIR, os.path.basename(filespec['output_filename']))
    map_file = os.path.join(MAP_DIR, os.path.basename(filespec['output_filename']) +
                            '.map')

    if ('force_minify' not in filespec) and \
       (prev_deploy and len(set(in_files) & changed_files) == 0):
        # Try to reuse the output file from previous deploy
        try:
            for dest in [out_file, map_file]:
                src = os.path.join(prev_deploy, dest)
                os.path.getsize(src)  # Just to throw error if it doesn't exist.
                if os.path.abspath(src) != os.path.abspath(dest):
                    shutil.copyfile(src, dest)
            continue  # Copy succeeded, so go on to next file.
        except (subprocess.CalledProcessError, OSError):
            pass  # Copy failed, so fall through to minification instead.

    # No previous deploy, or a source file has changed, or copying was
    # supposed to work but failed.  Thus, minify the JS anew.
    # (N.B. we include STATIC_HEADER_FILE before the JavaScripts.
    #  This way it doesn't throw off the source map.)
    cmd = '%s --language_in ECMASCRIPT5 --create_source_map %s %s %s' % (
        CLOSURE_BINARY, map_file,
        settings.STATIC_HEADER_FILE, ' '.join(in_files))
    js = subprocess.check_output(cmd, shell=True)

    # Write out the JS
    with open(out_file, 'wb') as fp:
        # Minified source files (most likely libraries) should be loaded
        # first to prevent any dependency errors.
        for file in min_files:
            with open(file, 'rb') as f:
                fp.write(f.read())

        fp.write(js)