Make stack trace annotation work

The code now unminifies all calls in the stack, including those outside
of app.js.

This requires the Python package sourcemap, recently added as a
dependency.

(imported from commit 550c73ad5bfe78a2c7169c11da0c95cbaac238d7)

zephyr/lib/unminify.py

@@ -1,129 +1,44 @@
 from __future__ import absolute_import
 
 import re
-import bisect
+import os.path
-import ujson
+import sourcemap
-import collections
-from os import path
 
-from django.conf import settings
-
-
-## Un-concatenating source files
-
-class LineToFile(object):
-    '''Map line numbers in the concatencated source files to
-       individual file/line pairs.'''
-    def __init__(self):
-        self._names             = []
-        self._cumulative_counts = []
-
-        total = 0
-        for filename in settings.JS_SPECS['app']['source_filenames']:
-            self._names.append(filename)
-            self._cumulative_counts.append(total)
-            with open(path.join('zephyr/static', filename), 'r') as fil:
-                total += sum(1 for ln in fil) + 1
-
-    def __call__(self, total):
-        i = bisect.bisect_right(self._cumulative_counts, total) - 1
-        return (self._names[i], total - self._cumulative_counts[i])
-
-line_to_file = LineToFile()
-
-
-## Parsing source maps
-
-# Mapping from Base64 digits to numerical value
-digits = dict(zip(
-    'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/',
-    range(64)))
-
-def parse_base64_vlq(input_str):
-    '''Interpret a sequence of Base64 digits as sequence of integers
-       in VLQ encoding.'''
-    accum, shift = 0, 0
-    for digit in input_str:
-        value = digits[digit]
-
-        # Low 5 bits provide the next 5 more significant
-        # bits of the output value.
-        accum |= (value & 0b11111) << shift
-        shift += 5
-
-        # Top bit is cleared if this is the last digit
-        # for this output value.
-        if not value & 0b100000:
-            # Bottom bit of the result is sign.
-            sign = -1 if accum & 1 else 1
-            yield sign * (accum >> 1)
-            accum, shift = 0, 0
-
-Link = collections.namedtuple('Link',
-    ['src_line', 'src_col', 'gen_line', 'gen_col'])
-
-def parse_mapping(mapstr):
-    '''Parse a source map v3 mapping string into a sequence of
-       'links' between source and generated code.'''
-
-    fields = [0,0,0,0,0]
-    for genline_no, group in enumerate(mapstr.split(';')):
-        # The first field (generated code starting column)
-        # resets for every group.
-        fields[0] = 0
-        for segment in group.split(','):
-            # Each segment contains VLQ-encoded deltas to the fields.
-            delta = list(parse_base64_vlq(segment))
-            delta += [0] * (5-len(delta))
-            fields = [x+y for x,y in zip(fields, delta)]
-
-            # fields[1] indicates which source file produced this
-            # code, but Pipeline concatenates all files together,
-            # so this field is always 0.
-
-            # Lines and columns are numbered from zero.
-            yield Link(src_line=fields[2],  src_col=fields[3],
-                       gen_line=genline_no, gen_col=fields[0])
-
-
-## Performing the lookup
 
 class SourceMap(object):
-    '''Map (line,column) pairs from generated to source file.'''
+    '''Map (line, column) pairs from generated to source file.'''
-    def __init__(self, sourcemap_file):
-        with open(sourcemap_file, 'r') as fil:
-            sourcemap = ujson.load(fil)
 
-        # Pair each link with a sort / search key
+    def __init__(self, sourcemap_dir):
-        self._links = [ ((link.gen_line, link.gen_col), link)
+        self._dir = sourcemap_dir
-            for link in parse_mapping(sourcemap['mappings']) ]
+        self._indices = {}
-        self._links.sort(key = lambda p: p[0])
-        self._keys = [p[0] for p in self._links]
 
-    def _map_position(self, gen_line, gen_col):
+    def _index_for(self, minified_src):
-        i = bisect.bisect_right(self._keys, (gen_line, gen_col))
+        '''Return the source map index for minified_src, loading it if not
-        if not i:
+           already loaded.'''
-            # Zero index indicates no match
+        if minified_src not in self._indices:
-            return None
+            with open(os.path.join(self._dir, minified_src + '.map')) as fp:
+                self._indices[minified_src] = sourcemap.load(fp)
 
-        link = self._links[i-1][1]
+        return self._indices[minified_src]
-        filename, src_line = line_to_file(link.src_line)
-        src_col = link.src_col + (gen_col - link.gen_col)
-
-        return (filename, src_line, src_col)
 
     def annotate_stacktrace(self, stacktrace):
         out = ''
         for ln in stacktrace.splitlines():
             out += ln + '\n'
-            match = re.search(r'/static/min/app(\.[0-9a-f]+)?\.js:(\d+):(\d+)', ln)
+            match = re.search(r'/static/min/(.+)(\.[0-9a-f]+)\.js:(\d+):(\d+)', ln)
             if match:
-                gen_line, gen_col = map(int, match.groups()[1:3])
+                # Get the appropriate source map for the minified file.
-                result = self._map_position(gen_line-1, gen_col-1)
+                minified_src = match.groups()[0] + '.js'
-                if result:
+                index = self._index_for(minified_src)
-                    filename, src_line, src_col = result
+
+                gen_line, gen_col = map(int, match.groups()[2:4])
+                # The sourcemap lib is 0-based, so subtract 1 from line and col.
+                try:
+                    result = index.lookup(line=gen_line-1, column=gen_col-1)
                     out += ('       = %s line %d column %d\n' %
-                        (filename, src_line+1, src_col+1))
+                        (result.src, result.src_line+1, result.src_col+1))
+                except IndexError:
+                    out +=  '       [Unable to look up in source map]'
 
             if ln.startswith('    at'):
                 out += '\n'

zephyr/management/commands/explain_js_error.py

@@ -28,14 +28,14 @@ def get_full_paste():
         os.system('stty cooked echo')
 
 class Command(BaseCommand):
-    args = '<source map file>'
+    args = '<source map directory>'
     help = '''Add source locations to a stack backtrace generated by minified code.
 
 The currently checked out code should match the version that generated the error.'''
 
     def handle(self, *args, **options):
         if len(args) != 1:
-            raise CommandError('No source map file specified')
+            raise CommandError('No source map directory specified')
 
         source_map = SourceMap(args[0])
 

zephyr/views.py

@@ -1873,7 +1873,7 @@ def json_get_active_statuses(request, user_profile):
 js_source_map = None
 if not (settings.DEBUG or settings.TEST_SUITE):
     js_source_map = SourceMap(path.join(
-        settings.DEPLOY_ROOT, 'prod-static/source-map/app.js.map'))
+        settings.DEPLOY_ROOT, 'prod-static/source-map'))
 
 @authenticated_json_post_view
 @has_request_variables