diff --git a/zephyr/lib/unminify.py b/zephyr/lib/unminify.py new file mode 100644 index 0000000000..77c0903108 --- /dev/null +++ b/zephyr/lib/unminify.py @@ -0,0 +1,128 @@ +import re +import bisect +import simplejson +import collections +from os import path + +from django.conf import settings + + +## Un-concatenating source files + +class LineToFile(object): + '''Map line numbers in the concatencated source files to + individual file/line pairs.''' + def __init__(self): + self._names = [] + self._cumulative_counts = [] + + total = 0 + for filename in settings.PIPELINE_JS['app']['source_filenames']: + self._names.append(filename) + self._cumulative_counts.append(total) + with open(path.join('zephyr/static', filename), 'r') as fil: + total += sum(1 for ln in fil) + 1 + + def __call__(self, total): + i = bisect.bisect_right(self._cumulative_counts, total) - 1 + return (self._names[i], total - self._cumulative_counts[i]) + +line_to_file = LineToFile() + + +## Parsing source maps + +# Mapping from Base64 digits to numerical value +digits = dict(zip( + 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/', + range(64))) + +def parse_base64_vlq(input_str): + '''Interpret a sequence of Base64 digits as sequence of integers + in VLQ encoding.''' + accum, shift = 0, 0 + for digit in input_str: + value = digits[digit] + + # Low 5 bits provide the next 5 more significant + # bits of the output value. + accum |= (value & 0b11111) << shift + shift += 5 + + # Top bit is cleared if this is the last digit + # for this output value. + if not value & 0b100000: + # Bottom bit of the result is sign. + sign = -1 if accum & 1 else 1 + yield sign * (accum >> 1) + accum, shift = 0, 0 + +Link = collections.namedtuple('Link', + ['src_line', 'src_col', 'gen_line', 'gen_col']) + +def parse_mapping(mapstr): + '''Parse a source map v3 mapping string into a sequence of + 'links' between source and generated code.''' + + fields = [0,0,0,0,0] + for genline_no, group in enumerate(mapstr.split(';')): + # The first field (generated code starting column) + # resets for every group. + fields[0] = 0 + for segment in group.split(','): + # Each segment contains VLQ-encoded deltas to the fields. + delta = list(parse_base64_vlq(segment)) + delta += [0] * (5-len(delta)) + fields = [x+y for x,y in zip(fields, delta)] + + # fields[1] indicates which source file produced this + # code, but Pipeline concatenates all files together, + # so this field is always 0. + + # Lines and columns are numbered from zero. + yield Link(src_line=fields[2], src_col=fields[3], + gen_line=genline_no, gen_col=fields[0]) + + +## Performing the lookup + +class SourceMap(object): + '''Map (line,column) pairs from generated to source file.''' + def __init__(self, sourcemap_file): + with open(sourcemap_file, 'r') as fil: + sourcemap = simplejson.load(fil) + + # Pair each link with a sort / search key + self._links = [ ((link.gen_line, link.gen_col), link) + for link in parse_mapping(sourcemap['mappings']) ] + self._links.sort(key = lambda p: p[0]) + self._keys = [p[0] for p in self._links] + + def _map_position(self, gen_line, gen_col): + i = bisect.bisect_right(self._keys, (gen_line, gen_col)) + if not i: + # Zero index indicates no match + return None + + link = self._links[i-1][1] + filename, src_line = line_to_file(link.src_line) + src_col = link.src_col + (gen_col - link.gen_col) + + return (filename, src_line, src_col) + + def annotate_stacktrace(self, stacktrace): + out = '' + for ln in stacktrace.splitlines(): + out += ln + '\n' + match = re.search(r'/static/min/app(\.[0-9a-f]+)?\.js:(\d+):(\d+)', ln) + if match: + gen_line, gen_col = map(int, match.groups()[1:3]) + result = self._map_position(gen_line-1, gen_col-1) + if result: + filename, src_line, src_col = result + out += (' = %s line %d column %d\n' % + (filename, src_line+1, src_col+1)) + + if ln.startswith(' at'): + out += '\n' + return out diff --git a/zephyr/management/commands/explain_js_error.py b/zephyr/management/commands/explain_js_error.py index da1bafc22a..0cfcc745bc 100644 --- a/zephyr/management/commands/explain_js_error.py +++ b/zephyr/management/commands/explain_js_error.py @@ -1,121 +1,10 @@ -import re import os import sys -import bisect import select -import simplejson -import collections -from os import path -from django.conf import settings from django.core.management.base import BaseCommand, CommandError - -## Un-concatenating source files - -class LineToFile(object): - '''Map line numbers in the concatencated source files to - individual file/line pairs.''' - def __init__(self): - self._names = [] - self._cumulative_counts = [] - - total = 0 - for filename in settings.PIPELINE_JS['app']['source_filenames']: - self._names.append(filename) - self._cumulative_counts.append(total) - with open(path.join('zephyr/static', filename), 'r') as fil: - total += sum(1 for ln in fil) + 1 - - def __call__(self, total): - i = bisect.bisect_right(self._cumulative_counts, total) - 1 - return (self._names[i], total - self._cumulative_counts[i]) - -line_to_file = LineToFile() - - -## Parsing source maps - -# Mapping from Base64 digits to numerical value -digits = dict(zip( - 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/', - range(64))) - -def parse_base64_vlq(input_str): - '''Interpret a sequence of Base64 digits as sequence of integers - in VLQ encoding.''' - accum, shift = 0, 0 - for digit in input_str: - value = digits[digit] - - # Low 5 bits provide the next 5 more significant - # bits of the output value. - accum |= (value & 0b11111) << shift - shift += 5 - - # Top bit is cleared if this is the last digit - # for this output value. - if not value & 0b100000: - # Bottom bit of the result is sign. - sign = -1 if accum & 1 else 1 - yield sign * (accum >> 1) - accum, shift = 0, 0 - -Link = collections.namedtuple('Link', - ['src_line', 'src_col', 'gen_line', 'gen_col']) - -def parse_mapping(mapstr): - '''Parse a source map v3 mapping string into a sequence of - 'links' between source and generated code.''' - - fields = [0,0,0,0,0] - for genline_no, group in enumerate(mapstr.split(';')): - # The first field (generated code starting column) - # resets for every group. - fields[0] = 0 - for segment in group.split(','): - # Each segment contains VLQ-encoded deltas to the fields. - delta = list(parse_base64_vlq(segment)) - delta += [0] * (5-len(delta)) - fields = [x+y for x,y in zip(fields, delta)] - - # fields[1] indicates which source file produced this - # code, but Pipeline concatenates all files together, - # so this field is always 0. - - # Lines and columns are numbered from zero. - yield Link(src_line=fields[2], src_col=fields[3], - gen_line=genline_no, gen_col=fields[0]) - - -## Performing the lookup - -class GenToSrc(object): - '''Map (line,column) pairs from generated to source file.''' - def __init__(self, sourcemap_file): - with open(sourcemap_file, 'r') as fil: - sourcemap = simplejson.load(fil) - - # Pair each link with a sort / search key - self._links = [ ((link.gen_line, link.gen_col), link) - for link in parse_mapping(sourcemap['mappings']) ] - self._links.sort(key = lambda p: p[0]) - self._keys = [p[0] for p in self._links] - - def __call__(self, gen_line, gen_col): - i = bisect.bisect_right(self._keys, (gen_line, gen_col)) - if not i: - # Zero index indicates no match - return None - - link = self._links[i-1][1] - filename, src_line = line_to_file(link.src_line) - src_col = link.src_col + (gen_col - link.gen_col) - - return (filename, src_line, src_col) - - -## UI +from zephyr.lib.unminify import SourceMap # Wait for the user to paste text, then time out quickly and # return it. Disable echo so that we can re-echo the same @@ -146,28 +35,13 @@ The currently checked out code should match the version that generated the error if len(args) != 1: raise CommandError('No source map file specified') - gen_to_src = GenToSrc(args[0]) - - write = sys.stdout.write + source_map = SourceMap(args[0]) if os.isatty(sys.stdin.fileno()): - write('Paste stacktrace:\n\n') + sys.stdout.write('Paste stacktrace:\n\n') sys.stdout.flush() - lines = get_full_paste().splitlines() + stacktrace = get_full_paste() else: - lines = sys.stdin.readlines() + stacktrace = sys.stdin.read() - for ln in lines: - ln = ln.rstrip() - write(ln + '\n') - match = re.search(r'/static/min/app(\.[0-9a-f]+)?\.js:(\d+):(\d+)', ln) - if match: - gen_line, gen_col = map(int, match.groups()[1:3]) - result = gen_to_src(gen_line-1, gen_col-1) - if result: - filename, src_line, src_col = result - write(' = %s line %d column %d\n' % - (filename, src_line+1, src_col+1)) - - if ln.startswith(' at'): - write('\n') + sys.stdout.write(source_map.annotate_stacktrace(stacktrace))