Use our own fenced code preprocessor, fixing output mismatches

(imported from commit 3d4abb1043349e30adf265342ee7c946a99b3ce2)

static/js/echo.js

@@ -31,7 +31,8 @@ exports.contains_bugdown = function contains_bugdown(content) {
 };
 
 exports.apply_markdown = function apply_markdown(content) {
-    return marked(content).trim();
+    // Our python-markdown processor appends two \n\n to input
+    return marked(content + '\n\n').trim();
 };
 
 function resend_message(message, row) {
@@ -421,6 +422,10 @@ $(function () {
         return '<br>\n';
     };
 
+    function preprocess_code_blocks(src) {
+        return fenced_code.process_fenced_code(src);
+    }
+
     // Disable ordered lists
     // We used GFM + tables, so replace the list start regex for that ruleset
     // We remove the |[\d+]\. that matches the numbering in a numbered list
@@ -439,6 +444,12 @@ $(function () {
 
     exports.set_realm_filters(page_params.realm_filters);
 
+    // Tell our fenced code preprocessor how to insert arbitrary
+    // HTML into the output. This generated HTML is safe to not escape
+    fenced_code.set_stash_func(function (html) {
+        return marked.stashHtml(html, true);
+    });
+
     marked.setOptions({
         gfm: true,
         tables: true,
@@ -451,7 +462,8 @@ $(function () {
         emojiHandler: handleEmoji,
         userMentionHandler: handleUserMentions,
         realmFilterHandler: handleRealmFilter,
-        renderer: r
+        renderer: r,
+        preprocessors: [preprocess_code_blocks]
     });
 
     function on_failed_action(action, callback) {

zerver/fixtures/bugdown-data.json

@@ -28,24 +28,24 @@
       "name": "hanging_multi_codeblock",
       "input": "Hamlet said:\n~~~~\ndef speak(self):\n    x = 1\n# Comment to make this code block longer to test Trac #1162\n~~~~\n\nThen he mentioned ````y = 4 + x**2```` and\n~~~~\ndef foobar(self):\n    return self.baz()",
       "expected_output": "<p>Hamlet said:</p>\n<div class=\"codehilite\"><pre>def speak(self):\n    x = 1\n# Comment to make this code block longer to test Trac #1162\n</pre></div>\n\n\n<p>Then he mentioned <code>y = 4 + x**2</code> and</p>\n<div class=\"codehilite\"><pre>def foobar(self):\n    return self.baz()\n</pre></div>",
-      "bugdown_matches_marked": false
+      "bugdown_matches_marked": true
     },
     {
       "name": "fenced_quote",
       "input": "Hamlet said:\n~~~ quote\nTo be or **not** to be.\n\nThat is the question\n~~~",
       "expected_output": "<p>Hamlet said:</p>\n<blockquote>\n<p>To be or <strong>not</strong> to be.</p>\n<p>That is the question</p>\n</blockquote>",
-      "bugdown_matches_marked": false
+      "bugdown_matches_marked": true
     },
     {
       "name": "fenced_nested_quote",
       "input": "Hamlet said:\n~~~ quote\nPolonius said:\n> This above all: to thine ownself be true,\nAnd it must follow, as the night the day,\nThou canst not then be false to any man.\n\nWhat good advice!\n~~~",
       "expected_output": "<p>Hamlet said:</p>\n<blockquote>\n<p>Polonius said:</p>\n<blockquote>\n<p>This above all: to thine ownself be true,<br>\nAnd it must follow, as the night the day,<br>\nThou canst not then be false to any man.</p>\n</blockquote>\n<p>What good advice!</p>\n</blockquote>",
-      "bugdown_matches_marked": false
+      "bugdown_matches_marked": true
     },
     {
       "name": "complexly_nested_quote",
-      "input": "I heard about this second hand...\n~~~ quote\n\nHe said:\n~~~ quote\nThe customer is complaining.\n\nThey looked at this code:\n``` .py\ndef hello(): print 'hello\n```\nThey would prefer:\n~~~ .rb\ndef hello()\n  puts 'hello'\nend\n~~~\n\nPlease advise.\n~~~\n\nShe said:\n~~~ quote\nJust send them this:\n``` .sh\necho \"hello\n\"\n```\n~~~",
+      "input": "I heard about this second hand...\n~~~ quote\n\nHe said:\n~~~ quote\nThe customer is complaining.\n\nThey looked at this code:\n``` \ndef hello(): print 'hello\n```\nThey would prefer:\n~~~\ndef hello()\n  puts 'hello'\nend\n~~~\n\nPlease advise.\n~~~\n\nShe said:\n~~~ quote\nJust send them this:\n```\necho \"hello\n\"\n```\n~~~",
-      "expected_output": "<p>I heard about this second hand...</p>\n<blockquote>\n<p>He said:</p>\n<blockquote>\n<p>The customer is complaining.</p>\n<p>They looked at this code:</p>\n<div class=\"codehilite\"><pre><span class=\"k\">def</span> <span class=\"nf\">hello</span><span class=\"p\">():</span> <span class=\"k\">print</span> <span class=\"s\">&#39;hello</span>\n</pre></div>\n\n\n<p>They would prefer:</p>\n<div class=\"codehilite\"><pre><span class=\"k\">def</span> <span class=\"nf\">hello</span><span class=\"p\">()</span>\n  <span class=\"nb\">puts</span> <span class=\"s1\">&#39;hello&#39;</span>\n<span class=\"k\">end</span>\n</pre></div>\n\n\n<p>Please advise.</p>\n</blockquote>\n<p>She said:</p>\n<blockquote>\n<p>Just send them this:</p>\n<div class=\"codehilite\"><pre><span class=\"nb\">echo</span> <span class=\"s2\">&quot;hello</span>\n<span class=\"s2\">&quot;</span>\n</pre></div>\n\n\n</blockquote>\n</blockquote>",
+      "expected_output": "<p>I heard about this second hand...</p>\n<blockquote>\n<p>He said:</p>\n<blockquote>\n<p>The customer is complaining.</p>\n<p>They looked at this code:</p>\n<div class=\"codehilite\"><pre>def hello(): print &#39;hello\n</pre></div>\n\n\n<p>They would prefer:</p>\n</blockquote>\n<p>def hello()<br>\n  puts 'hello'<br>\nend</p>\n</blockquote>\n<p>Please advise.</p>\n<div class=\"codehilite\"><pre>She said:\n~~~ quote\nJust send them this:\n```\necho &quot;hello\n&quot;\n```\n</pre></div>",
       "bugdown_matches_marked": false
     },
     {
@@ -58,7 +58,7 @@
       "name": "dangerous_block",
       "input": "``` one ```\n\n``` two ```\n\n~~~~\nx = 1",
       "expected_output": "<p><code>one</code></p>\n<p><code>two</code></p>\n<div class=\"codehilite\"><pre>x = 1\n</pre></div>",
-      "bugdown_matches_marked": false
+      "bugdown_matches_marked": true
     },
     {
       "name": "ulist_standard",

zerver/tests/frontend/node/echo.js

@@ -9,7 +9,8 @@ add_dependencies({
     _: 'third/underscore/underscore.js',
     marked: 'third/marked/lib/marked.js',
     Dict: 'js/dict.js',
-    emoji: 'js/emoji.js'
+    emoji: 'js/emoji.js',
+    fenced_code: 'js/fenced_code.js'
 });
 
 var doc = "";
@@ -114,7 +115,7 @@ var bugdown_data = JSON.parse(fs.readFileSync(path.join(__dirname, '../../../fix
     {input: '1. an\n2. ordered \n3. list',
      expected: '<p>1. an</p>\n<p>2. ordered </p>\n<p>3. list</p>'},
     {input: '\n~~~quote\nquote this for me\n~~~\nthanks\n',
-     expected: '<blockquote>\n<p>quote this for me</p>\n</blockquote>\n\n\n<p>thanks</p>'},
+     expected: '<blockquote>\n<p>quote this for me</p>\n</blockquote>\n<p>thanks</p>'},
     {input: 'This is a @**Cordelia Lear** mention',
      expected: '<p>This is a <span class="user-mention" data-user-email="cordelia@zulip.com">@Cordelia Lear</span> mention</p>'},
     {input: 'This is an :poop: message',