markdown: Allow setting a default language for code blocks.

This adds a new realm setting: default_code_block_language.

This PR also adds a new widget to specify a language, which
behaves somewhat differently from other widgets of the same
kind; instead of exposing methods to the whole module, we
just create a single IIFE that handles all the interactions
with the DOM for the widget.

We also move the code for remapping languages to format_code
function since we want to preserve the original language to
decide if we override it using default_code_clock_language.

Fixes #14404.

frontend_tests/node_tests/settings_org.js

@@ -789,6 +789,8 @@ run_test('set_up', () => {
 
     const stub_render_notifications_stream_ui = settings_org.render_notifications_stream_ui;
     settings_org.render_notifications_stream_ui = noop;
+    const stub_language_render = settings_org.default_code_language_widget.render;
+    settings_org.default_code_language_widget.render = noop;
     $("#id_realm_message_content_edit_limit_minutes").set_parent($.create('<stub edit limit parent>'));
     $("#id_realm_message_content_delete_limit_minutes").set_parent($.create('<stub delete limit parent>'));
     $("#message_content_in_email_notifications_label").set_parent($.create('<stub in-content setting checkbox>'));
@@ -826,6 +828,7 @@ run_test('set_up', () => {
     test_discard_changes_button(discard_changes);
 
     settings_org.render_notifications_stream_ui = stub_render_notifications_stream_ui;
+    settings_org.default_code_language_widget.render = stub_language_render;
 });
 
 run_test('test get_organization_settings_options', () => {

static/js/admin.js

@@ -16,6 +16,7 @@ const admin_settings_label = {
     realm_message_content_allowed_in_email_notifications:
         i18n.t("Allow message content in missed message emails"),
     realm_digest_emails_enabled: i18n.t("Send weekly digest emails to inactive users"),
+    realm_default_code_language: i18n.t("Default language for code blocks:"),
 
     // Organization permissions
     realm_name_changes_disabled: i18n.t("Prevent users from changing their name"),

static/js/settings_org.js

@@ -3,11 +3,104 @@ const render_settings_admin_auth_methods_list = require('../templates/settings/a
 const render_settings_admin_realm_domains_list = require("../templates/settings/admin_realm_domains_list.hbs");
 const render_settings_admin_realm_dropdown_stream_list = require("../templates/settings/admin_realm_dropdown_stream_list.hbs");
 const render_settings_organization_settings_tip = require("../templates/settings/organization_settings_tip.hbs");
+const pygments_data = require("../generated/pygments_data.json");
 
 const meta = {
     loaded: false,
 };
 
+exports.default_code_language_widget = (function (element_id) {
+
+    const render_language_list = require("../templates/settings/admin_realm_dropdown_code_languages_list.hbs");
+    const language_list = Object.keys(pygments_data.langs).map(x => {
+        return {
+            name: x,
+            priority: pygments_data.langs[x],
+        };
+    });
+
+    const setup = () => {
+        // populate the dropdown
+        const dropdown_list_body = $(`#${element_id} .dropdown-list-body`).expectOne();
+        const search_input = $(`#${element_id} .dropdown-search > input[type=text]`);
+        list_render.create(dropdown_list_body, language_list, {
+            name: "admin-realm-default-code-language-dropdown-list",
+            modifier: function (item) {
+                return render_language_list({ language: item });
+            },
+            filter: {
+                element: search_input,
+                predicate: function (item, value) {
+                    return item.name.toLowerCase().includes(value);
+                },
+            },
+        }).init();
+        $(`#${element_id} .dropdown-search`).click(function (e) {
+            e.stopPropagation();
+        });
+
+        $(`#${element_id} .dropdown-toggle`).click(function () {
+            search_input.val("").trigger("input");
+        });
+    };
+
+    const render = (name) => {
+        $(`#${element_id} #id_realm_default_code_block_language`).data("language", name);
+
+        const elem = $(`#${element_id} #realm_default_code_block_language_name`);
+
+        if (!name) {
+            elem.text(i18n.t("No language set"));
+            elem.addClass("text-warning");
+            elem.closest('.input-group').find('.default_code_block_language_unset').hide();
+            return;
+        }
+
+        // Happy path
+        elem.text(name);
+        elem.removeClass('text-warning');
+        elem.closest('.input-group').find('.default_code_block_language_unset').show();
+    };
+
+    const update = (lang, save_discard_widget_status_handler) => {
+        render(lang);
+        save_discard_widget_status_handler($('#org-other-settings'));
+    };
+
+    const register_event_handlers = (save_discard_widget_status_handler) => {
+        $(`#${element_id} .dropdown-list-body`).on("click keypress", ".lang_name", function (e) {
+            const setting_elem = $(this).closest(".realm_default_code_block_language_setting");
+            if (e.type === "keypress") {
+                if (e.which === 13) {
+                    setting_elem.find(".dropdown-menu").dropdown("toggle");
+                } else {
+                    return;
+                }
+            }
+            const lang = $(this).attr('data-language');
+            update(lang, save_discard_widget_status_handler);
+        });
+        $(`#${element_id} .default_code_block_language_unset`).click(function () {
+            update(null, save_discard_widget_status_handler);
+        });
+    };
+
+    const value = () => {
+        let val = $(`#${element_id} #id_realm_default_code_block_language`).data('language');
+        if (val === null) {
+            val = '';
+        }
+        return val;
+    };
+
+    return {
+        setup,
+        render,
+        register_event_handlers,
+        value,
+    };
+}('realm_default_code_block_language_widget'));
+
 exports.reset = function () {
     meta.loaded = false;
 };
@@ -431,6 +524,8 @@ function discard_property_element_changes(elem) {
         exports.render_notifications_stream_ui(property_value, "notifications");
     } else if (property_name === 'realm_signup_notifications_stream') {
         exports.render_notifications_stream_ui(property_value, "signup_notifications");
+    } else if (property_name === 'realm_default_code_block_language') {
+        exports.default_code_language_widget.render(property_value);
     } else if (typeof property_value === 'boolean') {
         elem.prop('checked', property_value);
     } else if (typeof property_value === 'string' || typeof property_value === 'number') {
@@ -557,9 +652,11 @@ exports.build_page = function () {
         const streams = stream_data.get_streams_for_settings_page();
         exports.populate_notifications_stream_dropdown(streams);
         exports.populate_signup_notifications_stream_dropdown(streams);
+        exports.default_code_language_widget.setup();
     }
     exports.render_notifications_stream_ui(page_params.realm_notifications_stream_id, 'notifications');
     exports.render_notifications_stream_ui(page_params.realm_signup_notifications_stream_id, 'signup_notifications');
+    exports.default_code_language_widget.render(page_params.realm_default_code_block_language);
 
     // Populate realm domains
     exports.populate_realm_domains(page_params.realm_domains);
@@ -604,6 +701,8 @@ exports.build_page = function () {
             changed_val = parseInt($("#id_realm_notifications_stream").data('stream-id'), 10);
         } else if (property_name === 'realm_signup_notifications_stream') {
             changed_val = parseInt($("#id_realm_signup_notifications_stream").data('stream-id'), 10);
+        } else if (property_name === 'realm_default_code_block_language') {
+            changed_val = exports.default_code_language_widget.value();
         } else if (typeof current_val === 'boolean') {
             changed_val = elem.prop('checked');
         } else if (typeof current_val === 'string') {
@@ -727,6 +826,9 @@ exports.build_page = function () {
                 new_message_retention_days = "";
             }
 
+            const code_block_language_value = exports.default_code_language_widget.value();
+            data.default_code_block_language = JSON.stringify(code_block_language_value);
+
             data.message_retention_days = new_message_retention_days !== "" ?
                 JSON.stringify(parseInt(new_message_retention_days, 10)) : null;
         } else if (subsection === 'other_permissions') {
@@ -960,6 +1062,9 @@ exports.build_page = function () {
         save_discard_widget_status_handler($('#org-notifications'));
     }
 
+    exports.default_code_language_widget.register_event_handlers(
+        save_discard_widget_status_handler);
+
     $(".notifications-stream-setting .dropdown-list-body").on("click keypress", ".stream_name", function (e) {
         const notifications_stream_setting_elem = $(this).closest(".notifications-stream-setting");
         if (e.type === "keypress") {

static/styles/settings.scss

@@ -710,6 +710,7 @@ input[type=checkbox].inline-block {
     border: none;
 }
 
+#realm_default_code_block_language_widget button,
 #realm_notifications_stream_label > button,
 #realm_signup_notifications_stream_label > button {
     margin: 0px 5px;
@@ -1614,11 +1615,13 @@ body:not(.night-mode) #settings_page .custom_user_field .datepicker {
     top: 5px;
 }
 
+#realm_default_code_block_language_widget .dropdown-search > input[type=text],
 #id_realm_notifications_stream .dropdown-search > input[type=text],
 #id_realm_signup_notifications_stream .dropdown-search > input[type=text] {
     margin: 9px;
 }
 
+#realm_default_code_block_language_widget .dropdown-list-body,
 #id_realm_notifications_stream .dropdown-list-body,
 #id_realm_signup_notifications_stream .dropdown-list-body {
     position: relative;

static/templates/settings/admin_realm_dropdown_code_languages_list.hbs

@@ -0,0 +1,7 @@
+{{#with language}}
+<li class="lang_name" role="presentation" data-language="{{name}}">
+    <a role="menuitem" tabindex="0">
+        {{name}}
+    </a>
+</li>
+{{/with}}

static/templates/settings/default_code_language_settings_widget.hbs

@@ -0,0 +1,21 @@
+<div class="input-group" id="realm_default_code_block_language_widget">
+    <label for="realm_default_code_block_language" id="realm_default_code_block_language_label" class="inline-block">
+        {{ label }}
+        <span class="realm_default_code_block_language_setting dropup actual-dropdown-menu prop-element" id="id_realm_default_code_block_language"
+          name="realm_default_code_block_language" aria-labelledby="realm_default_code_block_language_label">
+            <button class="button small rounded dropdown-toggle" data-toggle="dropdown">
+                <span id="realm_default_code_block_language_name"></span>
+                <i class="fa fa-pencil"></i>
+            </button>
+            <ul class="dropdown-menu modal-bg" role="menu">
+                <li class="dropdown-search" role="presentation">
+                    <input class="no-input-change-detection" type="text" role="menuitem" placeholder="{{t 'Filter languages' }}" autofocus/>
+                </li>
+                <span class="dropdown-list-body" data-simplebar></span>
+            </ul>
+        </span>
+    </label>
+    {{#if is_admin }}
+    <a class="default_code_block_language_unset" id="default_code_block_language_unset">{{t "[Unset]" }}</a>
+    {{/if}}
+</div>

static/templates/settings/organization_settings_admin.hbs

@@ -192,6 +192,9 @@
                     </div>
                 </div>
 
+                {{> default_code_language_settings_widget
+                  label=admin_settings_label.realm_default_code_language }}
+
                 {{> settings_checkbox
                   setting_name="realm_message_content_allowed_in_email_notifications"
                   prefix="id_"

templates/zerver/for/open-source.md

@@ -131,6 +131,9 @@ Import your existing organization from [Slack](/help/import-from-slack),
 syntax highlighting, makes it easy to discuss code, paste an error message,
 or explain a complicated point. Full LaTeX support as well.
 
+If your community primarily uses a single programming language,
+consider setting a default language for syntax highlighting.
+
 ### Permalink to conversations.
 
 Zulip makes it easy to get a [permanent link to a

templates/zerver/help/format-your-message-using-markdown.md

@@ -111,6 +111,10 @@ typeahead will pop up when you start typing after the ` ``` `. If you can't
 find your language, search for it [here](https://pygments.org/docs/lexers/)
 and try the **short names** listed for the lexers for your language.
 
+Organization administrators can also configure a default syntax
+highlighting language.  In this configuration, one can use ````text`
+to display content without any syntax highlighting.
+
 ## Latex
 ~~~
 Inline: $$O(n^2)$$

zerver/lib/bugdown/fenced_code.py

@@ -177,8 +177,6 @@ def check_for_new_fence(processor: Any, output: MutableSequence[str], line: str,
         fence = m.group('fence')
         lang = m.group('lang')
 
-        lang = remap_language(lang)
-
         handler = generic_handler(processor, output, fence, lang, run_content_validators)
         processor.push(handler)
     else:
@@ -315,6 +313,12 @@ class FencedBlockPreprocessor(markdown.preprocessors.Preprocessor):
         return output
 
     def format_code(self, lang: str, text: str) -> str:
+        if not lang:
+            try:
+                lang = self.md.zulip_realm.default_code_block_language
+            except AttributeError:
+                pass
+        lang = remap_language(lang)
         if lang:
             langclass = LANG_TAG % (lang,)
         else:

zerver/migrations/0272_realm_default_code_block_language.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.10 on 2020-03-31 00:21
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('zerver', '0271_huddle_set_recipient_column_values'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='realm',
+            name='default_code_block_language',
+            field=models.TextField(default=None, null=True),
+        ),
+    ]

zerver/models.py

@@ -320,6 +320,8 @@ class Realm(models.Model):
     zoom_api_key = models.TextField(default="")
     zoom_api_secret = models.TextField(default="")
 
+    default_code_block_language = models.TextField(null=True, default=None)  # type: Optional[str]
+
     # Define the types of the various automatically managed properties
     property_types = dict(
         add_emoji_by_admins_only=bool,
@@ -356,6 +358,7 @@ class Realm(models.Model):
         digest_weekday=int,
         private_message_policy=int,
         user_group_edit_policy=int,
+        default_code_block_language=(str, type(None)),
     )  # type: Dict[str, Union[type, Tuple[type, ...]]]
 
     DIGEST_WEEKDAY_VALUES = [0, 1, 2, 3, 4, 5, 6]

zerver/tests/test_bugdown.py

@@ -7,6 +7,7 @@ from zerver.lib.actions import (
     do_set_user_display_setting,
     do_remove_realm_emoji,
     do_set_alert_words,
+    do_set_realm_property,
 )
 from zerver.lib.alert_words import get_alert_word_automaton
 from zerver.lib.create_user import create_user
@@ -120,7 +121,7 @@ class FencedBlockPreprocessorTest(TestCase):
             'weirdchar()',
             '```',
             '',
-            '``` none',
+            '```',
             'no-highlight()',
             '```',
             ''
@@ -1342,6 +1343,34 @@ class BugdownTest(ZulipTestCase):
         # Only hamlet has alert-word 'issue124' present in the message content
         self.assertEqual(msg.user_ids_with_alert_words, expected_user_ids)
 
+    def test_default_code_block_language(self) -> None:
+        realm = get_realm('zulip')
+        self.assertEqual(realm.default_code_block_language, None)
+        text = "```{}\nconsole.log('Hello World');\n```\n"
+
+        # Render without default language
+        msg_with_js_before = bugdown_convert(text.format('js'))
+        msg_with_python_before = bugdown_convert(text.format('python'))
+        msg_without_language_before = bugdown_convert(text.format(''))
+
+        # Render with default=javascript
+        do_set_realm_property(realm, 'default_code_block_language', 'javascript')
+        msg_without_language_after = bugdown_convert(text.format(''))
+        msg_with_python_after = bugdown_convert(text.format('python'))
+
+        # Render with default=python
+        do_set_realm_property(realm, 'default_code_block_language', 'python')
+        msg_without_language_later = bugdown_convert(text.format(''))
+        msg_with_none_later = bugdown_convert(text.format('none'))
+
+        # Render without default language
+        do_set_realm_property(realm, 'default_code_block_language', None)
+        msg_without_language_final = bugdown_convert(text.format(''))
+
+        self.assertTrue(msg_with_js_before == msg_without_language_after)
+        self.assertTrue(msg_with_python_before == msg_with_python_after == msg_without_language_later)
+        self.assertTrue(msg_without_language_before == msg_with_none_later == msg_without_language_final)
+
     def test_mention_wildcard(self) -> None:
         user_profile = self.example_user('othello')
         msg = Message(sender=user_profile, sending_client=get_client("test"))

zerver/tests/test_events.py

@@ -1626,7 +1626,8 @@ class EventsRegisterTest(ZulipTestCase):
             google_hangouts_domain=[u"zulip.com", u"zulip.org"],
             zoom_api_secret=[u"abc", u"xyz"],
             zoom_api_key=[u"abc", u"xyz"],
-            zoom_user_id=[u"example@example.com", u"example@example.org"]
+            zoom_user_id=[u"example@example.com", u"example@example.org"],
+            default_code_block_language=[u'python', u'javascript']
         )  # type: Dict[str, Any]
 
         vals = test_values.get(name)
@@ -1636,6 +1637,8 @@ class EventsRegisterTest(ZulipTestCase):
             vals = bool_tests
         elif property_type is str:
             validator = check_string
+        elif property_type == (str, type(None)):
+            validator = check_string
         elif property_type is int:
             validator = check_int
         elif property_type == (int, type(None)):

zerver/tests/test_home.py

@@ -134,6 +134,7 @@ class HomeTest(ZulipTestCase):
             "realm_bot_domain",
             "realm_bots",
             "realm_create_stream_policy",
+            "realm_default_code_block_language",
             "realm_default_external_accounts",
             "realm_default_language",
             "realm_default_stream_groups",

zerver/tests/test_realm.py

@@ -710,6 +710,7 @@ class RealmAPITest(ZulipTestCase):
         bool_tests = [False, True]  # type: List[bool]
         test_values = dict(
             default_language=[u'de', u'en'],
+            default_code_block_language=[u'javascript', u''],
             description=[u'Realm description', u'New description'],
             digest_weekday=[0, 1, 2],
             message_retention_days=[10, 20],

zerver/views/realm.py

@@ -78,6 +78,7 @@ def update_realm(
         zoom_user_id: Optional[str]=REQ(validator=check_string, default=None),
         zoom_api_key: Optional[str]=REQ(validator=check_string, default=None),
         zoom_api_secret: Optional[str]=REQ(validator=check_string, default=None),
+        default_code_block_language: Optional[str]=REQ(validator=check_string, default=None),
         digest_weekday: Optional[int]=REQ(validator=check_int_in(Realm.DIGEST_WEEKDAY_VALUES), default=None),
 ) -> HttpResponse:
     realm = user_profile.realm
@@ -196,6 +197,13 @@ def update_realm(
                                                      signup_notifications_stream_id)
             data['signup_notifications_stream_id'] = signup_notifications_stream_id
 
+    if default_code_block_language is not None:
+        # Migrate '', used in the API to encode the default/None behavior of this feature.
+        if default_code_block_language == '':
+            data['default_code_block_language'] = None
+        else:
+            data['default_code_block_language'] = default_code_block_language
+
     return json_success(data)
 
 @require_realm_admin