docker: Set has_appserver=0 correctly.

Most of the recommendations in this guide were written by Rishi
Gupta (rishig), with a few contributions from Eeshan Garg.

.circleci/config.yml

@@ -19,12 +19,13 @@ jobs:
               dirs=(/srv/zulip-{npm,venv}-cache)
               sudo mkdir -p "${dirs[@]}"
               sudo chown -R circleci "${dirs[@]}"
+
       - restore_cache:
           keys:
-          - v1-npm-base.trusty.1
+          - v1-npm-base.trusty-{{ checksum "package.json" }}-{{ checksum "yarn.lock" }}
       - restore_cache:
           keys:
-          - v1-venv-base.trusty.1
+          - v1-venv-base.trusty-{{ checksum "requirements/thumbor.txt" }}-{{ checksum "requirements/dev.txt" }}
 
       - run:
           name: install dependencies
@@ -50,11 +51,11 @@ jobs:
       - save_cache:
           paths:
             - /srv/zulip-npm-cache
-          key: v1-npm-base.trusty.1
+          key: v1-npm-base.trusty-{{ checksum "package.json" }}-{{ checksum "yarn.lock" }}
       - save_cache:
           paths:
             - /srv/zulip-venv-cache
-          key: v1-venv-base.trusty.1
+          key: v1-venv-base.trusty-{{ checksum "requirements/thumbor.txt" }}-{{ checksum "requirements/dev.txt" }}
       # TODO: in Travis we also cache ~/zulip-emoji-cache, ~/node, ~/misc
 
       # The moment of truth!  Run the tests.
@@ -99,16 +100,18 @@ jobs:
               dirs=(/srv/zulip-{npm,venv}-cache)
               sudo mkdir -p "${dirs[@]}"
               sudo chown -R circleci "${dirs[@]}"
+
       - restore_cache:
           keys:
-          - v1-npm-base.xenial.1
+          - v1-npm-base.xenial-{{ checksum "package.json" }}-{{ checksum "yarn.lock" }}
       - restore_cache:
           keys:
-          - v1-venv-base.xenial.1
+          - v1-venv-base.xenial-{{ checksum "requirements/thumbor.txt" }}-{{ checksum "requirements/dev.txt" }}
 
       - run:
           name: install dependencies
           command: |
+            sudo apt-get update
             sudo apt-get install -y moreutils
             rm -f /home/circleci/.gitconfig
             mispipe "tools/travis/setup-backend" ts
@@ -116,11 +119,11 @@ jobs:
       - save_cache:
           paths:
             - /srv/zulip-npm-cache
-          key: v1-npm-base.xenial.1
+          key: v1-npm-base.xenial-{{ checksum "package.json" }}-{{ checksum "yarn.lock" }}
       - save_cache:
           paths:
             - /srv/zulip-venv-cache
-          key: v1-venv-base.xenial.1
+          key: v1-venv-base.xenial-{{ checksum "requirements/thumbor.txt" }}-{{ checksum "requirements/dev.txt" }}
 
       - run:
           name: run backend tests

.eslintrc.json

@@ -14,14 +14,11 @@
         "Handlebars": false,
         "XDate": false,
         "zxcvbn": false,
-        "LazyLoad": false,
-        "Dropbox": false,
         "SockJS": false,
         "marked": false,
         "md5": false,
         "moment": false,
         "i18n": false,
-        "DynamicText": false,
         "LightboxCanvas": false,
         "bridge": false,
         "page_params": false,
@@ -33,6 +30,7 @@
         "server_events": false,
         "server_events_dispatch": false,
         "message_scroll": false,
+        "keydown_util": false,
         "info_overlay": false,
         "ui": false,
         "ui_report": false,
@@ -47,11 +45,11 @@
         "user_groups": false,
         "navigate": false,
         "toMarkdown": false,
+        "settings_toggle": false,
         "settings_account": false,
         "settings_display": false,
         "settings_notifications": false,
         "settings_muting": false,
-        "settings_lab": false,
         "settings_bots": false,
         "settings_sections": false,
         "settings_emoji": false,
@@ -99,6 +97,7 @@
         "flatpickr": false,
         "pointer": false,
         "util": false,
+        "MessageListData": false,
         "MessageListView": false,
         "blueslip": false,
         "rows": false,
@@ -107,6 +106,7 @@
         "Socket": false,
         "channel": false,
         "components": false,
+        "scroll_util": false,
         "message_viewport": false,
         "upload_widget": false,
         "avatar": false,
@@ -142,6 +142,10 @@
         "tab_bar": false,
         "emoji": false,
         "presence": false,
+        "user_search": false,
+        "buddy_data": false,
+        "buddy_list": false,
+        "list_cursor": false,
         "activity": false,
         "invite": false,
         "colorspace": false,
@@ -167,7 +171,7 @@
         "emoji_codes": false,
         "drafts": false,
         "katex": false,
-        "Clipboard": false,
+        "ClipboardJS": false,
         "emoji_picker": false,
         "hotspots": false,
         "compose_ui": false,
@@ -202,6 +206,15 @@
         "eqeqeq": 2,
         "func-style": [ "off", "expression" ],
         "guard-for-in": 2,
+        "indent": ["error", 4, {
+            "ArrayExpression": "first",
+            "outerIIFEBody": 0,
+            "ObjectExpression": "first",
+            "SwitchCase": 0,
+            "CallExpression": {"arguments": "first"},
+            "FunctionExpression": {"parameters": "first"},
+            "FunctionDeclaration": {"parameters": "first"}
+        }],
         "keyword-spacing": [ "error",
             {
                 "before": true,

.gitattributes

@@ -10,4 +10,5 @@
 *.png binary
 *.otf binary
 *.tif binary
+*.ogg binary
 yarn.lock binary

.isort.cfg

@@ -6,3 +6,5 @@ known_third_party = django, ujson, sqlalchemy
 known_first_party = zerver, zproject, version, confirmation, zilencer, analytics, frontend_tests, scripts, corporate
 sections = FUTURE, STDLIB, THIRDPARTY, FIRSTPARTY, LOCALFOLDER
 lines_after_imports = 1
+# See the comment related to ioloop_logging for why this is skipped.
+skip = zerver/management/commands/runtornado.py

README.md

@@ -58,7 +58,7 @@ You might be interested in:
 * **Running a Zulip server**. Setting up a server takes just a couple of
   minutes. Zulip runs on Ubuntu 16.04 Xenial and Ubuntu 14.04 Trusty. The
   installation process is
-  [documented here](https://zulip.readthedocs.io/en/1.7.1/prod.html).
+  [documented here](https://zulip.readthedocs.io/en/stable/prod.html).
   Commercial support is available; see <https://zulipchat.com/plans> for
   details.
 

Vagrantfile

@@ -121,15 +121,19 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   end
 
   config.vm.network "forwarded_port", guest: 9991, host: host_port, host_ip: host_ip_addr
+  config.vm.network "forwarded_port", guest: 9994, host: host_port + 3, host_ip: host_ip_addr
   # Specify LXC provider before VirtualBox provider so it's preferred.
   config.vm.provider "lxc" do |lxc|
     if command? "lxc-ls"
       LXC_VERSION = `lxc-ls --version`.strip unless defined? LXC_VERSION
-      if LXC_VERSION >= "1.1.0"
+      if LXC_VERSION >= "1.1.0" and LXC_VERSION < "3.0.0"
         # Allow start without AppArmor, otherwise Box will not Start on Ubuntu 14.10
         # see https://github.com/fgrehm/vagrant-lxc/issues/333
         lxc.customize 'aa_allow_incomplete', 1
       end
+      if LXC_VERSION >= "3.0.0"
+        lxc.customize 'apparmor.allow_incomplete', 1
+      end
       if LXC_VERSION >= "2.0.0"
         lxc.backingstore = 'dir'
       end
@@ -167,7 +171,7 @@ Welcome to the Zulip development environment!  Popular commands:
 * tools/lint - Run the linter (quick and catches many problmes)
 * tools/test-* - Run tests (use --help to learn about options)
 
-Read https://zulip.readthedocs.io/en/latest/testing.html to learn
+Read https://zulip.readthedocs.io/en/latest/testing/testing.html to learn
 how to run individual test suites so that you can get a fast debug cycle.
 
 EndOfMessage'

analytics/lib/counts.py

@@ -3,7 +3,7 @@ from collections import OrderedDict, defaultdict
 from datetime import datetime, timedelta
 import logging
 from typing import Any, Callable, Dict, List, \
-    Optional, Text, Tuple, Type, Union
+    Optional, Tuple, Type, Union
 
 from django.conf import settings
 from django.db import connection, models
@@ -48,7 +48,7 @@ class CountStat:
         else:  # frequency == CountStat.DAY
             self.interval = timedelta(days=1)
 
-    def __str__(self) -> Text:
+    def __str__(self) -> str:
         return "<CountStat: %s>" % (self.property,)
 
 class LoggingCountStat(CountStat):

analytics/management/commands/populate_analytics_db.py

@@ -1,6 +1,6 @@
 
 from datetime import datetime, timedelta
-from typing import Any, Dict, List, Mapping, Optional, Text, Type, Union
+from typing import Any, Dict, List, Mapping, Optional, Type, Union
 
 from django.core.management.base import BaseCommand
 from django.utils.timezone import now as timezone_now
@@ -20,8 +20,8 @@ class Command(BaseCommand):
     DAYS_OF_DATA = 100
     random_seed = 26
 
-    def create_user(self, email: Text,
-                    full_name: Text,
+    def create_user(self, email: str,
+                    full_name: str,
                     is_staff: bool,
                     date_joined: datetime,
                     realm: Realm) -> UserProfile:

analytics/models.py

@@ -1,5 +1,5 @@
 import datetime
-from typing import Any, Dict, Optional, Text, Tuple, Union
+from typing import Any, Dict, Optional, Tuple, Union
 
 from django.db import models
 
@@ -7,7 +7,7 @@ from zerver.lib.timestamp import floor_to_day
 from zerver.models import Realm, Recipient, Stream, UserProfile
 
 class FillState(models.Model):
-    property = models.CharField(max_length=40, unique=True)  # type: Text
+    property = models.CharField(max_length=40, unique=True)  # type: str
     end_time = models.DateTimeField()  # type: datetime.datetime
 
     # Valid states are {DONE, STARTED}
@@ -17,7 +17,7 @@ class FillState(models.Model):
 
     last_modified = models.DateTimeField(auto_now=True)  # type: datetime.datetime
 
-    def __str__(self) -> Text:
+    def __str__(self) -> str:
         return "<FillState: %s %s %s>" % (self.property, self.end_time, self.state)
 
 # The earliest/starting end_time in FillState
@@ -36,17 +36,17 @@ def last_successful_fill(property: str) -> Optional[datetime.datetime]:
 
 # would only ever make entries here by hand
 class Anomaly(models.Model):
-    info = models.CharField(max_length=1000)  # type: Text
+    info = models.CharField(max_length=1000)  # type: str
 
-    def __str__(self) -> Text:
+    def __str__(self) -> str:
         return "<Anomaly: %s... %s>" % (self.info, self.id)
 
 class BaseCount(models.Model):
     # Note: When inheriting from BaseCount, you may want to rearrange
     # the order of the columns in the migration to make sure they
     # match how you'd like the table to be arranged.
-    property = models.CharField(max_length=32)  # type: Text
-    subgroup = models.CharField(max_length=16, null=True)  # type: Optional[Text]
+    property = models.CharField(max_length=32)  # type: str
+    subgroup = models.CharField(max_length=16, null=True)  # type: Optional[str]
     end_time = models.DateTimeField()  # type: datetime.datetime
     value = models.BigIntegerField()  # type: int
     anomaly = models.ForeignKey(Anomaly, on_delete=models.SET_NULL, null=True)  # type: Optional[Anomaly]
@@ -59,7 +59,7 @@ class InstallationCount(BaseCount):
     class Meta:
         unique_together = ("property", "subgroup", "end_time")
 
-    def __str__(self) -> Text:
+    def __str__(self) -> str:
         return "<InstallationCount: %s %s %s>" % (self.property, self.subgroup, self.value)
 
 class RealmCount(BaseCount):
@@ -69,7 +69,7 @@ class RealmCount(BaseCount):
         unique_together = ("realm", "property", "subgroup", "end_time")
         index_together = ["property", "end_time"]
 
-    def __str__(self) -> Text:
+    def __str__(self) -> str:
         return "<RealmCount: %s %s %s %s>" % (self.realm, self.property, self.subgroup, self.value)
 
 class UserCount(BaseCount):
@@ -82,7 +82,7 @@ class UserCount(BaseCount):
         # aggregating from users to realms
         index_together = ["property", "realm", "end_time"]
 
-    def __str__(self) -> Text:
+    def __str__(self) -> str:
         return "<UserCount: %s %s %s %s>" % (self.user, self.property, self.subgroup, self.value)
 
 class StreamCount(BaseCount):
@@ -95,6 +95,6 @@ class StreamCount(BaseCount):
         # aggregating from streams to realms
         index_together = ["property", "realm", "end_time"]
 
-    def __str__(self) -> Text:
+    def __str__(self) -> str:
         return "<StreamCount: %s %s %s %s %s>" % (
             self.stream, self.property, self.subgroup, self.value, self.id)

analytics/tests/test_counts.py

@@ -1,6 +1,6 @@
 
 from datetime import datetime, timedelta
-from typing import Any, Dict, List, Optional, Text, Tuple, Type, Union
+from typing import Any, Dict, List, Optional, Tuple, Type, Union
 
 import ujson
 from django.apps import apps
@@ -91,8 +91,8 @@ class AnalyticsTestCase(TestCase):
         return Message.objects.create(**kwargs)
 
     # kwargs should only ever be a UserProfile or Stream.
-    def assertCountEquals(self, table: Type[BaseCount], value: int, property: Optional[Text]=None,
-                          subgroup: Optional[Text]=None, end_time: datetime=TIME_ZERO,
+    def assertCountEquals(self, table: Type[BaseCount], value: int, property: Optional[str]=None,
+                          subgroup: Optional[str]=None, end_time: datetime=TIME_ZERO,
                           realm: Optional[Realm]=None, **kwargs: models.Model) -> None:
         if property is None:
             property = self.current_property

analytics/tests/test_views.py

@@ -24,6 +24,24 @@ class TestStatsEndpoint(ZulipTestCase):
         # Check that we get something back
         self.assert_in_response("Zulip analytics for", result)
 
+    def test_stats_for_realm(self) -> None:
+        user_profile = self.example_user('hamlet')
+        self.login(user_profile.email)
+
+        result = self.client_get('/stats/realm/zulip/')
+        self.assertEqual(result.status_code, 302)
+
+        user_profile = self.example_user('hamlet')
+        user_profile.is_staff = True
+        user_profile.save(update_fields=['is_staff'])
+
+        result = self.client_get('/stats/realm/not_existing_realm/')
+        self.assertEqual(result.status_code, 302)
+
+        result = self.client_get('/stats/realm/zulip/')
+        self.assertEqual(result.status_code, 200)
+        self.assert_in_response("Zulip analytics for", result)
+
 class TestGetChartData(ZulipTestCase):
     def setUp(self) -> None:
         self.realm = get_realm('zulip')
@@ -233,6 +251,28 @@ class TestGetChartData(ZulipTestCase):
                                      {'chart_name': 'number_of_humans'})
         self.assert_json_error_contains(result, 'No analytics data available')
 
+    def test_get_chart_data_for_realm(self) -> None:
+        user_profile = self.example_user('hamlet')
+        self.login(user_profile.email)
+
+        result = self.client_get('/json/analytics/chart_data/realm/zulip/',
+                                 {'chart_name': 'number_of_humans'})
+        self.assert_json_error(result, "Must be an server administrator", 400)
+
+        user_profile = self.example_user('hamlet')
+        user_profile.is_staff = True
+        user_profile.save(update_fields=['is_staff'])
+        stat = COUNT_STATS['realm_active_humans::day']
+        self.insert_data(stat, [None], [])
+
+        result = self.client_get('/json/analytics/chart_data/realm/not_existing_realm',
+                                 {'chart_name': 'number_of_humans'})
+        self.assert_json_error(result, 'Invalid organization', 400)
+
+        result = self.client_get('/json/analytics/chart_data/realm/zulip',
+                                 {'chart_name': 'number_of_humans'})
+        self.assert_json_success(result)
+
 class TestGetChartDataHelpers(ZulipTestCase):
     # last_successful_fill is in analytics/models.py, but get_chart_data is
     # the only function that uses it at the moment

analytics/urls.py

@@ -11,6 +11,8 @@ i18n_urlpatterns = [
         name='analytics.views.get_realm_activity'),
     url(r'^user_activity/(?P<email>[\S]+)/$', analytics.views.get_user_activity,
         name='analytics.views.get_user_activity'),
+    url(r'^stats/realm/(?P<realm_str>[\S]+)/$', analytics.views.stats_for_realm,
+        name='analytics.views.stats_for_realm'),
 
     # User-visible stats page
     url(r'^stats$', analytics.views.stats,
@@ -29,6 +31,8 @@ v1_api_and_json_patterns = [
     # get data for the graphs at /stats
     url(r'^analytics/chart_data$', rest_dispatch,
         {'GET': 'analytics.views.get_chart_data'}),
+    url(r'^analytics/chart_data/realm/(?P<realm_str>[\S]+)$', rest_dispatch,
+        {'GET': 'analytics.views.get_chart_data_for_realm'}),
 ]
 
 i18n_urlpatterns += [

analytics/views.py

@@ -7,7 +7,7 @@ import time
 from collections import defaultdict
 from datetime import datetime, timedelta
 from typing import Any, Callable, Dict, List, \
-    Optional, Set, Text, Tuple, Type, Union
+    Optional, Set, Tuple, Type, Union
 
 import pytz
 from django.conf import settings
@@ -26,9 +26,10 @@ from analytics.lib.counts import COUNT_STATS, CountStat, process_count_stat
 from analytics.lib.time_utils import time_range
 from analytics.models import BaseCount, InstallationCount, \
     RealmCount, StreamCount, UserCount, last_successful_fill
-from zerver.decorator import require_server_admin, \
+from zerver.decorator import require_server_admin, require_server_admin_api, \
     to_non_negative_int, to_utc_datetime, zulip_login_required
 from zerver.lib.exceptions import JsonableError
+from zerver.lib.json_encoder_for_html import JSONEncoderForHTML
 from zerver.lib.request import REQ, has_request_variables
 from zerver.lib.response import json_success
 from zerver.lib.timestamp import ceiling_to_day, \
@@ -36,17 +37,48 @@ from zerver.lib.timestamp import ceiling_to_day, \
 from zerver.models import Client, get_realm, Realm, \
     UserActivity, UserActivityInterval, UserProfile
 
-@zulip_login_required
-def stats(request: HttpRequest) -> HttpResponse:
+def render_stats(request: HttpRequest, realm: Realm) -> HttpRequest:
+    page_params = dict(
+        is_staff        = request.user.is_staff,
+        stats_realm     = realm.string_id,
+        debug_mode      = False,
+    )
+
     return render(request,
                   'analytics/stats.html',
-                  context=dict(realm_name = request.user.realm.name))
+                  context=dict(target_realm_name=realm.name,
+                               page_params=JSONEncoderForHTML().encode(page_params)))
+
+@zulip_login_required
+def stats(request: HttpRequest) -> HttpResponse:
+    realm = request.user.realm
+    return render_stats(request, realm)
+
+@require_server_admin
+@has_request_variables
+def stats_for_realm(request: HttpRequest, realm_str: str) -> HttpResponse:
+    realm = get_realm(realm_str)
+    if realm is None:
+        return HttpResponseNotFound("Realm %s does not exist" % (realm_str,))
+
+    return render_stats(request, realm)
+
+@require_server_admin_api
+@has_request_variables
+def get_chart_data_for_realm(request: HttpRequest, user_profile: UserProfile,
+                             realm_str: str, **kwargs: Any) -> HttpResponse:
+    realm = get_realm(realm_str)
+    if realm is None:
+        raise JsonableError(_("Invalid organization"))
+
+    return get_chart_data(request=request, user_profile=user_profile, realm=realm, **kwargs)
 
 @has_request_variables
-def get_chart_data(request: HttpRequest, user_profile: UserProfile, chart_name: Text=REQ(),
+def get_chart_data(request: HttpRequest, user_profile: UserProfile, chart_name: str=REQ(),
                    min_length: Optional[int]=REQ(converter=to_non_negative_int, default=None),
                    start: Optional[datetime]=REQ(converter=to_utc_datetime, default=None),
-                   end: Optional[datetime]=REQ(converter=to_utc_datetime, default=None)) -> HttpResponse:
+                   end: Optional[datetime]=REQ(converter=to_utc_datetime, default=None),
+                   realm: Optional[Realm]=None) -> HttpResponse:
     if chart_name == 'number_of_humans':
         stat = COUNT_STATS['realm_active_humans::day']
         tables = [RealmCount]
@@ -62,10 +94,10 @@ def get_chart_data(request: HttpRequest, user_profile: UserProfile, chart_name:
     elif chart_name == 'messages_sent_by_message_type':
         stat = COUNT_STATS['messages_sent:message_type:day']
         tables = [RealmCount, UserCount]
-        subgroup_to_label = {'public_stream': 'Public streams',
-                             'private_stream': 'Private streams',
-                             'private_message': 'Private messages',
-                             'huddle_message': 'Group private messages'}
+        subgroup_to_label = {'public_stream': _('Public streams'),
+                             'private_stream': _('Private streams'),
+                             'private_message': _('Private messages'),
+                             'huddle_message': _('Group private messages')}
         labels_sort_function = lambda data: sort_by_totals(data['realm'])
         include_empty_subgroups = True
     elif chart_name == 'messages_sent_by_client':
@@ -88,7 +120,8 @@ def get_chart_data(request: HttpRequest, user_profile: UserProfile, chart_name:
         raise JsonableError(_("Start time is later than end time. Start: %(start)s, End: %(end)s") %
                             {'start': start, 'end': end})
 
-    realm = user_profile.realm
+    if realm is None:
+        realm = user_profile.realm
     if start is None:
         start = realm.date_created
     if end is None:
@@ -435,6 +468,7 @@ def realm_summary_table(realm_minutes: Dict[str, float]) -> str:
 
     # formatting
     for row in rows:
+        row['stats_link'] = realm_stats_link(row['string_id'])
         row['string_id'] = realm_activity_link(row['string_id'])
 
     # Count active sites
@@ -456,6 +490,7 @@ def realm_summary_table(realm_minutes: Dict[str, float]) -> str:
 
     rows.append(dict(
         string_id='Total',
+        stats_link = '',
         date_created_day='',
         realm_admin_email='',
         dau_count=total_dau_count,
@@ -901,6 +936,12 @@ def realm_activity_link(realm_str: str) -> mark_safe:
     realm_link = '<a href="%s">%s</a>' % (url, realm_str)
     return mark_safe(realm_link)
 
+def realm_stats_link(realm_str: str) -> mark_safe:
+    url_name = 'analytics.views.stats_for_realm'
+    url = reverse(url_name, kwargs=dict(realm_str=realm_str))
+    stats_link = '<a href="{}"><i class="fa fa-pie-chart"></i></a>'.format(url, realm_str)
+    return mark_safe(stats_link)
+
 def realm_client_table(user_summaries: Dict[str, Dict[str, Dict[str, Any]]]) -> str:
     exclude_keys = [
         'internal',
@@ -972,7 +1013,7 @@ def user_activity_summary_table(user_summary: Dict[str, Dict[str, Any]]) -> str:
     return make_table(title, cols, rows)
 
 def realm_user_summary_table(all_records: List[QuerySet],
-                             admin_emails: Set[Text]) -> Tuple[Dict[str, Dict[str, Any]], str]:
+                             admin_emails: Set[str]) -> Tuple[Dict[str, Dict[str, Any]], str]:
     user_records = {}
 
     def by_email(record: QuerySet) -> str:

confirmation/models.py

@@ -22,7 +22,7 @@ from zerver.models import PreregistrationUser, EmailChangeStatus, MultiuseInvite
     UserProfile, Realm
 from random import SystemRandom
 import string
-from typing import Any, Dict, Optional, Text, Union
+from typing import Any, Dict, Optional, Union
 
 class ConfirmationKeyException(Exception):
     WRONG_LENGTH = 1
@@ -102,7 +102,7 @@ class Confirmation(models.Model):
     REALM_CREATION = 7
     type = models.PositiveSmallIntegerField()  # type: int
 
-    def __str__(self) -> Text:
+    def __str__(self) -> str:
         return '<Confirmation: %s>' % (self.content_object,)
 
 class ConfirmationType:
@@ -145,7 +145,7 @@ def validate_key(creation_key: Optional[str]) -> Optional['RealmCreationKey']:
         raise RealmCreationKey.Invalid()
     return key_record
 
-def generate_realm_creation_url(by_admin: bool=False) -> Text:
+def generate_realm_creation_url(by_admin: bool=False) -> str:
     key = generate_key()
     RealmCreationKey.objects.create(creation_key=key,
                                     date_created=timezone_now(),

docs/README.md

@@ -0,0 +1,6 @@
+# Zulip markdown documentation hosted elsewhere
+
+The markdown files in this directory ( /zulip/docs ) are not intended
+to be read on GitHub. Instead, visit our
+[ReadTheDocs](https://zulip.readthedocs.io/en/latest/index.html) to
+read the Zulip documentation.

docs/THIRDPARTY

@@ -149,10 +149,6 @@ Files: static/third/jquery-throttle-debounce/*
 Copyright: 2010 "Cowboy" Ben Alman
 License: Expat or GPL
 
-Files: src/zulip/static/third/lazyload/*
-Copyright: 2011 Ryan Grove
-License: Expat
-
 Files: static/third/marked/*
 Copyright: 2011-2013, Christopher Jeffrey
 License: Expat

docs/conf.py

@@ -44,7 +44,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = 'Zulip'
-copyright = '2015-2017, The Zulip Team'
+copyright = '2015-2018, The Zulip Team'
 author = 'The Zulip Team'
 
 # The version info for the project you're documenting, acts as replacement for
@@ -52,9 +52,9 @@ author = 'The Zulip Team'
 # built documents.
 #
 # The short X.Y version.
-version = '1.7+git'
+version = '1.8+git'
 # The full version, including alpha/beta/rc tags.
-release = '1.7.1+git'
+release = '1.8.1+git'
 
 # This allows us to insert a warning that appears only on an unreleased
 # version, e.g. to say that something is likely to have changed.

docs/contributing/accessibility.md

@@ -60,7 +60,7 @@ Problems with Zulip's accessibility should be reported as
 label.  This label can be added by entering the following text in a separate
 comment on the issue:
 
-    @zulipbot label "accessibility"
+    @zulipbot label "area: accessibility"
 
 If you want to help make Zulip more accessible, here is a list of the
 [currently open accessibility issues][accessibility-issues].

docs/contributing/mypy.md

@@ -22,15 +22,11 @@ You can learn more about it at:
 
 * The
   [mypy cheat sheet for Python 3](http://mypy.readthedocs.io/en/latest/cheat_sheet_py3.html)
-  (and its
-  [python 2 version](https://github.com/python/mypy/blob/master/docs/source/cheat_sheet.rst))
-  are the best resources for quickly understanding how to write the
-  PEP 484 type annotations used by mypy correctly.
+  is the best resource for quickly understanding how to write the PEP
+  484 type annotations used by mypy correctly.
 
-* The [Python 2 type annotation syntax spec in PEP
-  484](https://www.python.org/dev/peps/pep-0484/#suggested-syntax-for-python-2-7-and-straddling-code)
-
-* [Using mypy with Python 2 code](http://mypy.readthedocs.io/en/latest/python2.html)
+* The
+  [Python type annotation spec in PEP 484](https://www.python.org/dev/peps/pep-0484/)
 
 The mypy type checker is run automatically as part of Zulip's Travis
 CI testing process in the `backend` build.
@@ -73,13 +69,6 @@ because a list can have many elements, which would make the output too large.
 Similarly in dicts, one key's type and the corresponding value's type are printed.
 So `{1: 'a', 2: 'b', 3: 'c'}` will be printed as `{int: str, ...}`.
 
-## Zulip goals
-
-Zulip is hoping to reach 100% of the codebase annotated with mypy
-static types, and then enforce that it stays that way.  Our current
-coverage is shown in
-[Codecov](https://codecov.io/gh/zulip/zulip).
-
 ## Installing mypy
 
 If you installed Zulip's development environment correctly, mypy
@@ -112,31 +101,6 @@ test.py: note: In function "test":
 test.py:200: error: Incompatible types in assignment (expression has type "str", variable has type "int")
 ```
 
-If you need help interpreting or debugging mypy errors, please feel
-free to mention @sharmaeklavya2 or @timabbott on your pull request (or
-ask in [chat.zulip.org](https://chat.zulip.org)) to get help; we'd love to both
-build a great troubleshooting guide in this doc and also help
-contribute improvements to error messages upstream.
-
-Since mypy is a new tool under rapid development and occasionally
-makes breaking changes, Zulip is using a pinned version of mypy from
-its [git repository](https://github.com/python/mypy) rather than
-tracking the (older) latest mypy release on PyPI.
-
-## Excluded files
-
-Since several Python files in Zulip's code don't pass mypy's checks
-(even for unannotated code) right now, a list of files to be excluded
-from the check for CI is present in `tools/run-mypy`.
-
-To run mypy on all Python files, ignoring the exclude list, you can
-pass the `--all` option to `tools/run-mypy`.
-
-    tools/run-mypy --all
-
-If you type annotate some of those files so that they pass without
-errors, please remove them from the exclude list.
-
 ## Mypy is there to find bugs in Zulip before they impact users
 
 For the purposes of Zulip development, you can treat `mypy` like a
@@ -164,48 +128,3 @@ developers by opening an issue on [Zulip's GitHub
 repository](https://github.com/zulip/zulip/issues) or posting on
 [zulip-devel](https://groups.google.com/d/forum/zulip-devel).  If it's
 indeed a mypy bug, we can help with reporting it upstream.
-
-## Annotating strings
-
-In Python 3, strings can have non-ASCII characters without any problems.
-Such characters are required to support languages which use non-latin
-scripts like Japanese and Hindi.  They are also needed to support special
-characters like mathematical symbols, musical symbols, etc.
-In Python 2, however, `str` generally doesn't work well with non-ASCII
-characters.  That's why `unicode` was introduced in Python 2.
-
-But there are problems with the `unicode` and `str` system.  Implicit
-conversions between `str` and `unicode` use the `ascii` codec, which
-fails on strings containing non-ASCII characters.  Such errors are hard
-to detect by people who always write in English.  To minimize such
-implicit conversions, we should have a strict separation between `str`
-and `unicode` in Python 2.  It might seem that using `unicode` everywhere
-will solve all problems, but unfortunately it doesn't.  This is because
-some parts of the standard library and the Python language (like keyword
-argument unpacking) insist that parameters passed to them are `str`.
-
-To make our code work correctly in Python 2, we have to identify strings
-which contain data which could come from non-ASCII sources like stream
-names, people's names, domain names, content of messages, emails, etc.
-These strings should be `unicode`.  We also have to identify strings
-which should be `str` like Exception names, attribute names, parameter
-names, etc.
-
-Mypy can help with this.  We just have to annotate each string as either
-`str` or `unicode` and mypy's static type checking will tell us if we
-are incorrectly mixing the two.  However, `unicode` is not defined in
-Python 3.  We want our code to be Python 3 compatible in the future.
-This can be achieved using 'typing.Text', a Python 2 and 3 compatibility type.
-
-`typing.Text` is defined as `str` in Python 3 and as `unicode` in
-Python 2.  We'll be using `Text` (instead of `unicode`) and `str`
-to annotate strings in Zulip's code.  We follow the style of doing
-`from typing import Text` and using `Text` for annotation instead
-of doing `import typing` and using `typing.Text` for annotation, because
-`Text` is used so extensively for type annotations that we don't
-need to be that verbose.
-
-Sometimes you'll find that you have to convert strings from one type to
-another.  `zerver/lib/str_utils.py` has utility functions to help with that.
-It also has documentation (in docstrings) which explains the right way
-to use them.

docs/development/overview.md

@@ -13,8 +13,9 @@ internet connection throughout the entire installation processes.** You can
 
 ## Recommended setup (Vagrant)
 
-**For first-time contributors on macOS, Windows, and Ubuntu, we recommend using
-the [Vagrant development environment][install-vagrant]**.
+**For first-time contributors on macOS, Windows, and most Debian-based distros
+(like Ubuntu), we recommend using the [Vagrant development
+environment][install-vagrant]**.
 
 This method creates a virtual machine (for Windows and macOS) or a Linux
 container (for Ubuntu) inside which the Zulip server and all related services
@@ -29,8 +30,9 @@ want to or can't use Vagrant, Zulip supports a wide range of ways to install
 the Zulip development environment on **macOS and Linux (Ubuntu
 recommended)**:
 
-* On **Ubuntu** 16.04 Xenial and 14.04 Trusty, you can easily **[install
-  without using Vagrant][install-direct]**.
+* On **Ubuntu** 16.04 Xenial and 14.04 Trusty and **Debian** 9
+  Stretch, you can easily
+  **[install without using Vagrant][install-direct]**.
 * On **other Linux** distributions, you'll need to follow slightly different
   instructions to **[install manually][install-generic]**.
 * On **macOS and Linux** (Ubuntu recommended), you can install **[using

docs/development/setup-advanced.md

@@ -2,24 +2,25 @@
 
 Contents:
 
-* [Installing directly on Ubuntu](#installing-directly-on-ubuntu)
+* [Installing directly on Ubuntu or Debian](#installing-directly-on-ubuntu-or-debian)
 * [Installing manually on Linux](#installing-manually-on-linux)
 * [Installing directly on cloud9](#installing-on-cloud9)
 * [Using Docker (experimental)](#using-docker-experimental)
 
-## Installing directly on Ubuntu
+## Installing directly on Ubuntu or Debian
 
 Start by [cloning your fork of the Zulip repository][zulip-rtd-git-cloning]
 and [connecting the Zulip upstream repository][zulip-rtd-git-connect]:
 
 ```
 git clone --config pull.rebase https://github.com/YOURUSERNAME/zulip.git
+cd zulip
 git remote add -f upstream https://github.com/zulip/zulip.git
 ```
 
 If you'd like to install a Zulip development environment on a computer
-that's already running Ubuntu 16.04 Xenial or Ubuntu 14.04 Trusty, you
-can do that by just running:
+that's already running Ubuntu 16.04 Xenial, Ubuntu 14.04 Trusty, or
+Debian 9 Stretch, you can do that by just running:
 
 ```
 # From a clone of zulip.git
@@ -91,6 +92,8 @@ sudo apt-get install postgresql-9.3-pgroonga
 sudo apt-get install postgresql-9.5-pgroonga
 # On 17.04 or 17.10
 sudo apt-get install postgresql-9.6-pgroonga
+# On 18.04
+sudo apt-get install postgresql-10-pgroonga
 
 # If using Debian, follow the instructions here: http://pgroonga.github.io/install/debian.html
 
@@ -103,6 +106,8 @@ sudo apt-get update
 sudo apt-get install postgresql-9.3-tsearch-extras
 # On 16.04
 sudo apt-get install postgresql-9.5-tsearch-extras
+# On 18.04
+sudo apt-get install postgresql-10-tsearch-extras
 
 
 # Otherwise, you can download a .deb directly
@@ -118,9 +123,14 @@ sudo dpkg -i postgresql-9.3-tsearch-extras_0.1.3_amd64.deb
 wget https://dl.dropboxusercontent.com/u/283158365/zuliposs/postgresql-9.4-tsearch-extras_0.1_amd64.deb
 sudo dpkg -i postgresql-9.4-tsearch-extras_0.1_amd64.deb
 
-# If on 16.04 or stretch
-wget https://launchpad.net/~tabbott/+archive/ubuntu/zulip/+files/postgresql-9.5-tsearch-extras_0.2_amd64.deb
-sudo dpkg -i postgresql-9.5-tsearch-extras_0.3_amd64.deb
+# If on 16.04
+wget https://launchpad.net/~tabbott/+archive/ubuntu/zulip/+files/postgresql-9.5-tsearch-extras_0.4_amd64.deb
+sudo dpkg -i postgresql-9.5-tsearch-extras_0.4_amd64.deb
+
+# If on Stretch
+wget --content-disposition \
+  https://packagecloud.io/zulip/server/packages/debian/stretch/postgresql-9.6-tsearch-extras_0.4_amd64.deb/download.deb
+sudo dpkg -i postgresql-9.6-tsearch-extras_0.4_amd64.deb
 ```
 
 Alternatively, you can always build the package from [tsearch-extras

docs/development/setup-vagrant.md

@@ -1,7 +1,7 @@
 ## Vagrant environment setup tutorial
 
 This section guides first-time contributors through installing the
-Zulip development environment on Windows, macOS, and Ubuntu.
+Zulip development environment on Windows, macOS, Ubuntu and Debian.
 
 The recommended method for installing the Zulip development environment is to use
 Vagrant with VirtualBox on Windows and macOS, and Vagrant with LXC on

docs/overview/changelog.md

@@ -7,111 +7,95 @@ All notable changes to the Zulip server are documented in this file.
 This section lists notable unreleased changes; it is generally updated
 in bursts.
 
-**Highlights:**
+### 1.8.1 -- 2018-05-07
 
-- Added a user setting to choose the emoji set used in Zulip: Google,
-  Twitter, Apple, or Emoji One.
-- Added a video call integration powered by Jitsi.
+- Added an automated tool (`manage.py register_server`) to sign up for
+  the [mobile push notifications service](../production/mobile-push-notifications.html).
+- Improved rendering of block quotes in mobile push notifications.
+- Improved some installer error messages.
+- Fixed several minor bugs with the new Slack import feature.
+- Fixed several visual bugs with the new compose input pills.
+- Fixed several minor visual bugs with night mode.
+- Fixed bug with visual clipping of "g" in the left sidebar.
+- Fixed an issue with the LDAP backend users' Organization Unit (OU)
+  being cached, resulting in trouble logging in after a user was moved
+  between OUs.
+- Fixed a couple subtle bugs with muting.
+
+### 1.8.0 -- 2018-04-17
+
+**Highlights:**
 - Dramatically simplified the server installation process; it's now possible
   to install Zulip without first setting up outgoing email.
-- Added certbot support to the installer for getting certificates.
-- Added support for mentioning groups of users.
+- Added experimental support for importing an organization's history
+  from Slack.
 - Added a new "night mode" theme for dark environments.
-- Added experimental support for importing an organization from Slack.
-- Overhauled our settings system to eliminate the ugly "save changes"
-  button system.
-- Rewrote our API documentation to be much more friendly and
-  expansive; it now covers most important endpoints, with nice examples.
+- Added a video call integration powered by Jitsi.
+- Lots of visual polish improvements.
+- Countless small bugfixes both in the backend and the UI.
+
+
+**Security and privacy:**
+- Several important security fixes since 1.7.0, which were released
+  already in 1.7.1 and 1.7.2.
 - The security model for private streams has changed.  Now
   organization administrators can remove users, edit descriptions, and
   rename private streams they are not subscribed to.  See Zulip's
   security model documentation for details.
-- Lots of visual polish improvements.
-
-**Full feature changelog:**
-
-- New integrations: ErrBot, GoCD, Google Code-In, Opbeat, Groove, Raygun,
-  Insping, Dropbox, Front, Intercom, Statuspage.io, Flock and Beeminder.
-- The local uploads backend now does the same security checks that the
-  S3 backend did before serving files to users.
-- Added support for users in multiple realms having the same email.
-- Added support for embedded interactive bots.
-- Added inline preview + player for Vimeo videos.
-- Added a setting to allow users to delete their messages.
+- On Xenial, the local uploads backend now does the same security
+  checks that the S3 backend did before serving files to users.
+  Ubuntu Trusty's version of nginx is too old to support this and so
+  the legacy model is the default; we recommend upgrading.
 - Added an organization setting to limit creation of bots.
+- Refactored the authentication backends codebase to be much easier to
+  verify.
+- Added a user setting to control whether email notifications include
+  message content (or just the fact that there are new messages).
+
+
+**Visual and UI:**
+- Added a user setting to translate emoticons/smileys to emoji.
+- Added a user setting to choose the emoji set used in Zulip: Google,
+  Twitter, Apple, or Emoji One.
+- Expanded setting for displaying emoji as text to cover all display
+  settings (previously only affected reactions).
+- Overhauled our settings system to eliminate the old "save changes"
+  button system.
+- Redesigned the "uploaded files" UI.
+- Redesigned the "account settings" UI.
+- Redesigned error pages for the various email confirmation flows.
+- Our emoji now display at full resolution on retina displays.
+- Improved placement of text when inserting emoji via picker.
+- Improved the descriptions and UI for many settings.
+- Improved visual design of the help center (/help/).
+
+
+**Core chat experience:**
+- Added support for mentioning groups of users.
+- Added a setting to allow users to delete their messages.
 - Added support for uploading files in the message-edit UI.
-- Added new event types to several webhook integrations.
-- Added a display for whether the user is logged-in in logged-out
-  pages.
-- Added support for hosting multiple domains, not all as subdomains of
-  the same base domain.
-- Added a new /team/ page explaining the team, with a nice
-  visualization of our contributors.
-- Added support for default bots to receive messages when they're
-mentioned, even if they are not subscribed.
-- Added support for inviting a new user as an administrator.
-- Added a new organization settings page for managing invites.
-- Added a user setting to control whether the organization's name is
-  included in email subject lines.
-- Added support for clicking on a mention to see a user's profile.
-- Added new compose features for pasting HTML.
 - Redesigned the compose are for private messages to use pretty pills
   rather than raw email addresses to display recipients.
 - Added new ctrl+B, ctrl+I, ctrl+L compose shortcuts for inserting
   common syntax.
 - Added warning when linking to a private stream via typeahead.
-- Added rate-limiting on inviting users to join a realm (prevents spam).
 - Added support for automatically-numbered markdown lists.
 - Added a big warning when posting to #announce.
-- Added a user setting to control whether email notifications include
-  message content (or just the fact that there are new messages).
 - Added a notification when drafts are saved, to make them more
-discoverable.
+  discoverable.
 - Added a fast local echo to emoji reactions.
-- Added new "basics" section to keyboard shortcuts documentation.
-- Added a new ">" keyboard shortcut for quote-and-reply.
-- Added a new "p" keyboard shortcut to just to next unread PM thread.
-- Added support for overriding the topic is all incoming webhook integrations.
-- Added a new nagios check for the Zulip analytics state.
-- Added a menu item to mark all messages as read.
-- Added support for logging into the mobile apps with RemoteUserBackend.
-- Added an organization setting to disable welcome emails to new users.
-- Added traffic statistics (messages/week) to the "Manage streams" UI.
-- Added a display setting to translate emoticons/smileys to emoji.
-- Added an organization setting to ban disposable email addresses
-  (I.e.. those from sites like mailinator.com).
-- Added a server setting to control whether digest emails are sent.
-- Links to logged-in content in Zulip now take the user to the
-  appropriate upload or view after a user logs in.
-- Incoming webhooks now send a private message to the bot owner for
-  more convenient testing.
-- Rewrote documentation for many integrations to use a cleaner
-  numbered-list format.
-- Renamed "Home" to "All messages", to avoid users clicking on it too
-  early in using Zulip.
 - Messages containing just a link to an image (or an uploaded image)
   now don't clutter the feed with the URL: we just display the image.
-- Refactored the authentication backends codebase to be much easier to
-  verify.
-- Expanded setting for displaying emoji as text to cover all display
-  settings (previously only affected reactions).
 - Redesigned the API for emoji reactions to support the full range of
   how emoji reactions are used.
-- Migrated the codebase to use the nice Python 3 typing syntax.
-- Optimized how user avatar URLs are transmitted over the wire.
-- Optimized message sending performance a bit more.
-- Split the Notifications Stream setting in two settings, one for new
-  users, the other for new streams.
-- Fixed numerous issues in the "stream settings" UI.
 - Fixed most of the known (mostly obscure) bugs in how messages are
   formatted in Zulip.
 - Fixed "more topics" to correctly display all historical topics for
   public streams, even though from before a user subscribed.
-- Fixed several bugs around interacting with deactivated users.
+- Added a menu item to mark all messages as read.
 - Fixed image upload file pickers offering non-image files.
 - Fixed some subtle bugs with full-text search and unicode.
 - Fixed bugs in the "edit history" HTML rendering process.
-- Fixed several hotkeys scope bugs.
 - Fixed popovers being closed when new messages come in.
 - Fixed unexpected code blocks when using the email mirror.
 - Fixed clicking on links to a narrow opening a new window.
@@ -119,47 +103,132 @@ discoverable.
 - Fixed layering issues with mobile Safari.
 - Fixed several obscure real-time synchronization bugs.
 - Fixed handling of messages with a very large HTML rendering.
-- Fixed buggy APNs logic that could cause extra exception emails.
+- Fixed several bugs around interacting with deactivated users.
 - Fixed interaction bugs with unread counts and deleting messages.
 - Fixed support for replacing deactivated custom emoji.
-- Fixed a missing dependency for the localhost_sso auth backend.
-- Fixed uploading user avatars encoded using the CMYK mode.
 - Fixed scrolling downwards in narrows.
-- Fixed numerous subtle bugs with the stream creation UI.
-- Dramatically improved organization of developer docs.
-- Statistics on translation percentages now include mobile apps.
+- Optimized how user avatar URLs are transmitted over the wire.
+- Optimized message sending performance a bit more.
+- Fixed a subtle and hard-to-reproduce bug that resulted in every
+  message being condensed ([More] appearing on every message).
 - Improved typeahead's handling of editing an already-completed mention.
 - Improved syntax for inline LaTeX to be more convenient.
-- Improve keyboard navigation of left and right sidebars with arrow keys.
-- Changes the URL scheme for stream narrows to encode the stream ID,
-  so that they can be robust to streams being renamed.  The change is
-  backwards-compatible; existing narrow URLs still work.
-- APIs for fetching messages now provide more metadata to help clients.
-- Clarified instructions for server settings (especially LDAP auth).
-- Redesigned the "uploaded files" UI.
-- Redesigned the "account settings" UI.
-- Redesigned error pages for the various email confirmation flows.
-- Added missing information on requesting user in many exception emails.
-- Our emoji now display at full resolution on retina displays.
-- Improved placement of text when inserting emoji via picker.
-- Improved the password reset flow to be less confusing if you don't
-  have an account.
+- Improved syntax for permanent links to streams in Zulip.
 - Improved behavior of copy-pasting a large number of messages.
-- Improved Tornado retry logic for connecting to RabbitMQ.
-- Improved the descriptions and UI for many settings.
 - Improved handling of browser undo in compose.
-- Improved mobile notifications to support narrowing when one click a
-  mobile push notification.
-- Improved visual design of the help center (/help/).
 - Improved saved drafts system to garbage-collect old drafts and sort
   by last modification, not creation.
 - Removed the legacy "Zulip labs" autoscroll_forever setting.  It was
   enabled mostly by accident.
 - Removed some long-deprecated markdown syntax for mentions.
+- Added support for clicking on a mention to see a user's profile.
+- Links to logged-in content in Zulip now take the user to the
+  appropriate upload or view after a user logs in.
+- Renamed "Home" to "All messages", to avoid users clicking on it too
+  early in using Zulip.
+- Added a user setting to control whether the organization's name is
+  included in email subject lines.
+- Fixed uploading user avatars encoded using the CMYK mode.
+
+
+**User accounts and invites:**
+- Added support for users in multiple realms having the same email.
+- Added a display for whether the user is logged-in in logged-out
+  pages.
+- Added support for inviting a new user as an administrator.
+- Added a new organization settings page for managing invites.
+- Added rate-limiting on inviting users to join a realm (prevents spam).
+- Added an organization setting to disable welcome emails to new users.
+- Added an organization setting to ban disposable email addresses
+  (I.e.. those from sites like mailinator.com).
+- Improved the password reset flow to be less confusing if you don't
+  have an account.
+- Split the Notifications Stream setting in two settings, one for new
+  users, the other for new streams.
+
+
+**Stream subscriptions and settings:**
+- Added traffic statistics (messages/week) to the "Manage streams" UI.
+- Fixed numerous issues in the "stream settings" UI.
+- Fixed numerous subtle bugs with the stream creation UI.
+- Changes the URL scheme for stream narrows to encode the stream ID,
+  so that they can be robust to streams being renamed.  The change is
+  backwards-compatible; existing narrow URLs still work.
+
+
+**API, bots, and integrations:**
+- Rewrote our API documentation to be much more friendly and
+  expansive; it now covers most important endpoints, with nice examples.
+- New integrations: ErrBot, GoCD, Google Code-In, Opbeat, Groove,
+  Raygun, Insping, Dialogflow, Dropbox, Front, Intercom,
+  Statuspage.io, Flock and Beeminder.
+- Added support for embedded interactive bots.
+- Added inline preview + player for Vimeo videos.
+- Added new event types and fixed bugs in several webhook integrations.
+- Added support for default bots to receive messages when they're
+  mentioned, even if they are not subscribed.
+- Added support for overriding the topic is all incoming webhook integrations.
+- Incoming webhooks now send a private message to the bot owner for
+  more convenient testing if a stream is not specified.
+- Rewrote documentation for many integrations to use a cleaner
+  numbered-list format.
+- APIs for fetching messages now provide more metadata to help clients.
+
+
+**Keyboard shortcuts:**
+- Added new "basics" section to keyboard shortcuts documentation.
+- Added a new ">" keyboard shortcut for quote-and-reply.
+- Added a new "p" keyboard shortcut to jump to next unread PM thread.
+- Fixed several hotkeys scope bugs.
+- Changed the hotkey for compose-private-message from "C" to "x".
+- Improve keyboard navigation of left and right sidebars with arrow keys.
+
+
+**Mobile apps backend:**
+- Added support for logging into the mobile apps with RemoteUserBackend.
+- Improved mobile notifications to support narrowing when one clicks a
+  mobile push notification.
 - Statistics on the fraction of strings that are translated now
   include strings in the mobile apps as well.
+
+
+**For server admins:**
+- Added certbot support to the installer for getting certificates.
+- Added support for hosting multiple domains, not all as subdomains of
+  the same base domain.
+- Added a new nagios check for the Zulip analytics state.
+- Fixed buggy APNs logic that could cause extra exception emails.
+- Fixed a missing dependency for the localhost_sso auth backend.
+- Fixed subtle bugs in garbage-collection of old node_modules versions.
+- Clarified instructions for server settings (especially LDAP auth).
+- Added missing information on requesting user in many exception emails.
+- Improved Tornado retry logic for connecting to RabbitMQ.
+- Added a server setting to control whether digest emails are sent.
+
+
+**For Zulip developers:**
+- Migrated the codebase to use the nice Python 3 typing syntax.
+- Added a new /team/ page explaining the team, with a nice
+  visualization of our contributors.
+- Dramatically improved organization of developer docs.
 - Backend test coverage is now 95%.
-- Countless other little bug fixes both in the backend and the UI.
+
+
+### 1.7.2 -- 2018-04-12
+
+This is a security release, with a handful of cherry-picked changes
+since 1.7.1.  All Zulip server admins are encouraged to upgrade
+promptly.
+
+- CVE-2018-9986: Fix XSS issues with frontend markdown processor.
+- CVE-2018-9987: Fix XSS issue with muting notifications.
+- CVE-2018-9990: Fix XSS issue with stream names in topic typeahead.
+- CVE-2018-9999: Fix XSS issue with user uploads.  The fix for this
+  adds a Content-Security-Policy for the `LOCAL_UPLOADS_DIR` storage
+  backend for user-uploaded files.
+
+Thanks to Suhas Sunil Gaikwad for reporting CVE-2018-9987 and w2w for
+reporting CVE-2018-9986 and CVE-2018-9990.
 
 ### 1.7.1 -- 2017-11-21
 

docs/overview/directory-structure.md

@@ -46,7 +46,7 @@ See [our docs](../subsystems/html-templates.html) for details on Zulip's
 templating systems.
 
 * `templates/zerver/` For [Jinja2](http://jinja.pocoo.org/) templates
-  for the backend (for zerver app).
+  for the backend (for zerver app; logged-in content is in `templates/zerver/app`).
 
 * `static/templates/` [Handlebars](http://handlebarsjs.com/) templates for the frontend.
 

docs/overview/gsoc-ideas.md

@@ -61,7 +61,7 @@ contributors to the project today).
 
 ### Expectations for GSoC students
 
-[Our guide for having a great summer with Zulip](../contributing/summer-with-zulip)
+[Our guide for having a great summer with Zulip](../contributing/summer-with-zulip.html)
 is focused on what one should know once doing a summer project with
 Zulip.  But it has a lot of useful advice on how we expect students to
 interact, above and beyond what is discussed in Google's materials.

docs/production/email.md

@@ -1,30 +1,58 @@
 # Outgoing email
 
-This page documents everything you need to know about setting up
-outgoing email in a Zulip production environment.  It's pretty simple
-if you already have an outgoing SMTP provider; just start reading from
-[the configuration section](#configuration).
+Zulip needs to be able to send email so it can confirm new users'
+email addresses and send notifications.
+
+## How to configure
+
+1. Identify an outgoing email (SMTP) account where you can have Zulip
+   send mail.  If you don't already have one you want to use, see
+   [Email services](#email-services) below.
+
+2. Fill out the section of `/etc/zulip/settings.py` headed "Outgoing
+   email (SMTP) settings".  This includes the hostname and typically
+   the port to reach your SMTP provider, and the username to log into
+   it as.
+
+3. Put the password for the SMTP user account in
+   `/etc/zulip/zulip-secrets.conf` by setting `email_password`. For
+   example: `email_password = abcd1234`.
+
+   Like any other change to the Zulip configuration, be sure to
+   [restart the server](settings.html) to make your changes take
+   effect.
+
+4. Test that your configuration is working.  See the test command in
+   the [Troubleshooting](#troubleshooting) section below.  If it's not
+   working, see the suggestions in that section.
+
+## Email services
 
 ### Free outgoing email services
 
 For sending outgoing email from your Zulip server, we highly recommend
 using a "transactional email" service like
-[Mailgun](https://documentation.mailgun.com/en/latest/quickstart-sending.html#send-via-smtp)
-or for AWS users,
+[SendGrid](https://sendgrid.com/docs/API_Reference/SMTP_API/integrating_with_the_smtp_api.html),
+[Mailgun](https://documentation.mailgun.com/en/latest/quickstart-sending.html#send-via-smtp),
+or, for AWS users,
 [Amazon SES](http://docs.aws.amazon.com/ses/latest/DeveloperGuide/send-email-smtp.html).
 These services are designed to send email from servers, and are by far
 the easiest way to get outgoing email working reliably.
 
 If you don't have an existing outgoing SMTP provider, don't worry!
-Both of the options we recommend above (as well as dozens of other
-services) have free options; we recommend Mailgun as the easiest to
-get setup with.  Once you've signed up, you'll want to find the
-service's provided "SMTP credentials", and configure Zulip as follows:
+Each of the options we recommend above (as well as dozens of other
+services) have free options.  Once you've signed up, you'll want to
+find the service's provided "SMTP credentials", and configure Zulip as
+follows:
 
-* The hostname as `EMAIL_HOST = 'smtp.mailgun.org'` in `/etc/zulip/settings.py`
-* The username as `EMAIL_HOST_USER = 'username@example.com` in
+* The hostname like `EMAIL_HOST = 'smtp.mailgun.org'` in `/etc/zulip/settings.py`
+* The username like `EMAIL_HOST_USER = 'username@example.com` in
   `/etc/zulip/settings.py`.
-* The password as `email_password = abcd1234` in `/etc/zulip/zulip-secrets.conf`.
+* The TLS setting as `EMAIL_USE_TLS = True` in
+  `/etc/zulip/settings.py`, for most providers
+* The port as `EMAIL_PORT = 587` in `/etc/zulip/settings.py`, for most
+  providers
+* The password like `email_password = abcd1234` in `/etc/zulip/zulip-secrets.conf`.
 
 ### Using Gmail for outgoing email
 
@@ -44,38 +72,28 @@ how to make it work:
 * Note also that the rate limits for Gmail are also quite low
   (e.g. 100 / day), so it's easy to get rate-limited if your server
   has significant traffic.  For more active servers, we recommend
-  moving to a free account from a transaction email service.
+  moving to a free account on a transactional email service.
 
 ### Logging outgoing email to a file for prototyping
 
-If for prototyping, you don't want to bother setting up an email
-provider, you can add to `/etc/zulip/settings.py` the following:
+For prototyping, you might want to proceed without setting up an email
+provider.  If you want to see the emails Zulip would have sent, you
+can log them to a file instead.
+
+To do so, add these lines to `/etc/zulip/settings.py`:
 
 ```
 EMAIL_BACKEND = 'django.core.mail.backends.filebased.EmailBackend'
 EMAIL_FILE_PATH = '/var/log/zulip/emails'
 ```
 
-Outgoing emails that Zulip would have sent will just be written to
-files in `/var/log/zulip/emails/`.  This is enough to get you through
-initial user registration without an SMTP provider.
+Then outgoing emails that Zulip would have sent will just be written
+to files in `/var/log/zulip/emails/`.
 
-Remember to delete this configuration and restart the server if you
-later setup a real SMTP provider!
+Remember to delete this configuration (and restart the server) if you
+later set up a real SMTP provider!
 
-### Configuration
-
-To configure outgoing SMTP, you will need to complete the following steps:
-
-1. Fill out the outgoing email sending configuration block in
-`/etc/zulip/settings.py`, including `EMAIL_HOST`, and
-`EMAIL_HOST_USER`.  You may also need to set `EMAIL_PORT` if your
-provider doesn't use the standard SMTP submission port (587).
-
-2. Put the SMTP password for `EMAIL_HOST_USER` in
-`/etc/zulip/zulip-secrets.conf` as `email_password = yourPassword`.
-
-#### Testing and troubleshooting
+## Troubleshooting
 
 You can quickly test your outgoing email configuration using:
 
@@ -87,42 +105,50 @@ su zulip
 If it doesn't throw an error, it probably worked; you can confirm by
 checking your email.
 
-It's important to test, because outgoing email often doesn't work the
-first time.  Common causes of failures are:
+If it doesn't work, check these common failure causes:
 
-* Your hosting provider blocking outgoing SMTP traffic in its
-default firewall rules.  Check whether `EMAIL_PORT` is blocked in your
-hosting provider's firewall.
-* Forgetting to put the password in `/etc/zulip/zulip-secrets.conf`.
-* Typos in transcribing the username or password.
+* Your hosting provider may block outgoing SMTP traffic in its default
+  firewall rules.  Check whether the port `EMAIL_PORT` is blocked in
+  your hosting provider's firewall.
 
-Once you have it working from the management command, remember to
-restart your Zulip server using
-`/home/zulip/deployments/current/scripts/restart-server` so that the running
-server is using the latest configuration.
+* Make sure you set the password in `/etc/zulip/zulip-secrets.conf`.
 
-#### Advanced troubleshooting
+* Check the username and password for typos.
+
+* Be sure to restart your Zulip server after editing either
+  `settings.py` or `zulip-secrets.conf`, using
+  `/home/zulip/deployments/current/scripts/restart-server` .
+  Note that the `manage.py` command above will read the latest
+  configuration from the config files, even if the server is still
+  running with an old configuration.
+
+### Advanced troubleshooting
 
 Here are a few final notes on what to look at when debugging why you
 aren't receiving emails from Zulip:
 
 * Most transactional email services have an "outgoing email" log where
-  you can inspect the emails that reached the service, whether it was
-  flagged as spam, etc.
+  you can inspect the emails that reached the service, whether an
+  email was flagged as spam, etc.
+
 * Starting with Zulip 1.7, Zulip logs an entry in
   `/var/log/zulip/send_email.log` whenever it attempts to send an
-  email, including whether the request succeeded or failed.
+  email.  The log entry includes whether the request succeeded or failed.
+
 * If attempting to send an email throws an exception, a traceback
   should be in `/var/log/zulip/errors.log`, along with any other
   exceptions Zulip encounters.
+
 * Zulip's email sending configuration is based on the standard Django
-  [SMTP backend](https://docs.djangoproject.com/en/1.10/topics/email/#smtp-backend)
+  [SMTP backend](https://docs.djangoproject.com/en/2.0/topics/email/#smtp-backend)
   configuration.  So if you're having trouble getting your email
   provider working, you may want to search for documentation related
-  to using your email provider with Django.  The one thing we've
-  changed from the defaults is reading the email password from the
-  `email_password` entry in the Zulip secrets file, as part of our
-  policy of not having any secret information in the
-  `/etc/zulip/settings.py` file.  In other words, if Django
-  documentation references setting `EMAIL_HOST_PASSWORD`, you should
-  instead set `email_password` in `/etc/zulip/zulip-secrets.conf`.
+  to using your email provider with Django.
+
+  The one thing we've changed from the Django defaults is that we read
+  the email password from the `email_password` entry in the Zulip
+  secrets file, as part of our policy of not having any secret
+  information in the `/etc/zulip/settings.py` file.  In other words,
+  if Django documentation references setting `EMAIL_HOST_PASSWORD`,
+  you should instead set `email_password` in
+  `/etc/zulip/zulip-secrets.conf`.

docs/production/install-existing-server.md

@@ -16,7 +16,7 @@ recommended, and you may break your server. Make sure you have backups
 and a provisioning script ready to go to wipe and restore your
 existing services if (when) your server goes down.
 
-These instructions are only for experts.  If you're not an experiecned
+These instructions are only for experts.  If you're not an experienced
 Linux sysadmin, you will have a much better experience if you get a
 dedicated VM to install Zulip on instead (or [use zulipchat.com](https://zulipchat.com).
 

docs/production/mobile-push-notifications.md

@@ -18,28 +18,28 @@ support forwarding push notifications to a central push notification
 forwarding service.  You can enable this for your Zulip server as
 follows:
 
-1. First, contact support@zulipchat.com with the `zulip_org_id` and
-   `zulip_org_key` values from your `/etc/zulip/zulip-secrets.conf` file, as
-   well as a hostname and contact email address you'd like us to use in case
-   of any issues (we hope to have a nice web flow available for this soon).
-
-2. We'll enable push notifications for your server on our end. Look for a
-   reply from Zulipchat support within 24 hours.
-
-3. Uncomment the `PUSH_NOTIFICATION_BOUNCER_URL = "https://push.zulipchat.com"`
-   line in your `/etc/zulip/settings.py` file, and
+1. Uncomment the `PUSH_NOTIFICATION_BOUNCER_URL =
+   'https://push.zulipchat.com'` line in your `/etc/zulip/settings.py`
+   file (i.e. remove the `#` at the start of the line), and
    [restart your Zulip server](../production/maintain-secure-upgrade.html#updating-settings).
-   Note that if you installed Zulip older than 1.6, you'll need to add
-   the line (it won't be there to uncomment).
+   If you installed your Zulip server with a version older than 1.6,
+   you'll need to add the line (it won't be there to uncomment).
 
-4. If you or your users have already set up the Zulip mobile app,
+1. If you're running Zulip 1.8.1 or newer, you can run `manage.py
+   register_server` from `/home/zulip/deployments/current`.  This
+   command will print the registration data it would send to the
+   mobile push notifications service, ask you to accept the terms of
+   service, and if you accept, register your server.  Otherwise, see
+   the [legacy signup instructions](#legacy-signup).
+
+1. If you or your users have already set up the Zulip mobile app,
    you'll each need to log out and log back in again in order to start
    getting push notifications.
 
-That should be all you need to do!
+Congratulations!  You've successful setup the service.
 
-If you'd like to verify the full pipeline, you can do the following.
-Please follow the instructions carefully:
+If you'd like to verify that everything is working, you can do the
+following.  Please follow the instructions carefully:
 
 * [Configure mobile push notifications to always be sent][notification-settings]
   (normally they're only sent if you're idle, which isn't ideal for
@@ -57,9 +57,19 @@ in the Android notification area.
 
 [notification-settings]: https://zulipchat.com/help/configure-mobile-notifications
 
-Note that use of the push notification bouncer is subject to the
-[Zulipchat Terms of Service](https://zulipchat.com/terms/). By using push
-notifications, you agree to those terms.
+## Updating your server's registration
+
+Your server's registration includes the server's hostname and contact
+email address (from `EXTERNAL_HOST` and `ZULIP_ADMINISTRATOR` in
+`/etc/zulip/settings.py`, aka the `--hostname` and `--email` options
+in the installer).  You can update your server's registration data by
+running `manage.py register_server` again.
+
+If you'd like to rotate your server's API key for this service
+(`zulip_org_key`), you need to use `manage.py register_server
+--rotate-key` option; it will automatically generate a new
+`zulip_org_key` and store that new key in
+`/etc/zulip/zulip-secrets.conf`.
 
 ## Why this is necessary
 
@@ -77,11 +87,22 @@ notification forwarding service, which allows registered Zulip servers
 to send push notifications to the Zulip app indirectly (through the
 forwarding service).
 
-## Security and privacy implications
+## Security and privacy
+
+Use of the push notification bouncer is subject to the
+[Zulipchat Terms of Service](https://zulipchat.com/terms/). By using
+push notifications, you agree to those terms.
 
 We've designed this push notification bouncer service with security
 and privacy in mind:
 
+* A central design goal of the the Push Notification Service is to
+  avoid any message content being stored or logged by the service,
+  even in error cases.  We store only the necessary metadata for
+  delivering the notifications.  This includes the tokens needed to
+  push notifications to the devices, and user ID numbers generated by
+  your Zulip server.  These user ID numbers are are opaque to the Push
+  Notification Service, since it has no other data about those users.
 * All of the network requests (both from Zulip servers to the Push
   Notification Service and from the Push Notification Service to the
   relevant Google and Apple services) are encrypted over the wire with
@@ -89,17 +110,69 @@ and privacy in mind:
 * The code for the push notification forwarding service is 100% open
   source and available as part of the
   [Zulip server project on GitHub](https://github.com/zulip/zulip).
-  The Push Notification Service is designed to avoid any message
-  content being stored or logged, even in error cases.
 * The push notification forwarding servers are professionally managed
-  by a small team of security experts.
-* There's a `PUSH_NOTIFICATION_REDACT_CONTENT` setting available to
-  disable any message content being sent via the push notification
-  bouncer (i.e. message content will be replaced with
-  `***REDACTED***`).  Note that this setting makes push notifications
-  significantly less usable.  We plan to
+  by a small team of security expert engineers.
+* If you'd like an extra layer of protection, there's a
+  `PUSH_NOTIFICATION_REDACT_CONTENT` setting available to disable any
+  message content being sent via the push notification bouncer
+  (i.e. message content will be replaced with `***REDACTED***`).  Note
+  that this setting makes push notifications significantly less
+  usable.  We plan to
   [replace this feature with end-to-end encryption](https://github.com/zulip/zulip/issues/6954)
   which would eliminate that usability tradeoff.
 
 If you have any questions about the security model, contact
 support@zulipchat.com.
+
+## Legacy signup
+
+Here are the legacy instructions for signing a server up for push
+notifications:
+
+1. First, contact support@zulipchat.com with the `zulip_org_id` and
+   `zulip_org_key` values from your `/etc/zulip/zulip-secrets.conf` file, as
+   well as a `hostname` and `contact email` address you'd like us to use in case
+   of any issues (we hope to have a nice web flow available for this soon).
+
+2. We'll enable push notifications for your server on our end. Look for a
+   reply from Zulipchat support within 24 hours.
+## Sending push notifications directly from your server
+
+As we discussed above, it is impossible for a single app in their
+stores to receive push notifications from multiple, mutually
+untrusted, servers.  The Mobile Push Notification Service is one of
+the possible solutions to this problem.  The other possible solution
+is for an individual Zulip server's administrators to build and
+distribute their own copy of the Zulip mobile apps, hardcoding a key
+that they possess.
+
+This solution is possible with Zulip, but it requires the server
+administrators to publish their own copies of
+the Zulip mobile apps (and there's nothing the Zulip team can do to
+eliminate this onorous requirement).
+
+The main work is distributing your own copies of the Zulip mobile apps
+configured to use APNS/GCM keys that you generate.  This is not for
+the faint of heart!  If you haven't done this before, be warned that
+one can easily spend hundreds of dollars (on things like a DUNS number
+registration) and a week struggling through the hoops Apple requires
+to build and distribute an app through the Apple app store, even if
+you're making no code modifications to an app already present in the
+store (as would be the case here). The Zulip mobile app also gets
+frequent updates that you will have to either forgo or republish to
+the app stores yourself.
+
+If you've done that work, the Zulip server configuration for sending
+push notifications through the new app is quite straightforward:
+* Create a
+  [GCM push notifications](https://developers.google.com/cloud-messaging/android/client)
+  key in the Google Developer console and set `android_gcm_api_key` in
+  `/etc/zulip/zulip-secrets.conf` to that key.
+* Register for a
+  [mobile push notification certificate][apple-docs]
+  from Apple's developer console.  Set `APNS_SANDBOX=False` and
+  `APNS_CERT_FILE` to be the path of your APNS certificate file in
+  `/etc/zulip/settings.py`.
+* Restart the Zulip server.
+
+[apple-docs]: https://developer.apple.com/library/content/documentation/NetworkingInternet/Conceptual/RemoteNotificationsPG/APNSOverview.html

docs/production/ssl-certificates.md

@@ -13,10 +13,14 @@ If you already have an SSL certificate, just install (or symlink) its
 files into place at the following paths:
 * `/etc/ssl/private/zulip.key` for the private key
 * `/etc/ssl/certs/zulip.combined-chain.crt` for the certificate.
-  Because Zulip uses nginx as its web server, this should be in the
-  format of a [chained certificate bundle][nginx-https].
 
-[nginx-https]: http://nginx.org/en/docs/http/configuring_https_servers.html
+Your certificate file should contain not only your own certificate but
+its full chain, including any intermediate certificates used by your
+CA.  See the [nginx documentation][nginx-chains] for details on what
+this means and how to do it and test it.  If you're missing part of
+the chain, your server may work with some browsers but not others.
+
+[nginx-chains]: http://nginx.org/en/docs/http/configuring_https_servers.html#chains
 
 ## Certbot (recommended)
 

docs/subsystems/email.md

@@ -13,7 +13,7 @@ with only a few things you need to know to get started.
 
 * All email templates are in `templates/zerver/emails/`. Each email has three
   template files: `<template_prefix>.subject`, `<template_prefix>.txt`, and
-  `<template_prefix>.html`. Email templates, along with all other templates
+  `<template_prefix>.source.html`. Email templates, along with all other templates
   in the `templates/` directory, are Jinja2 templates.
 * Most of the CSS and HTML layout for emails is in `email_base.html`. Note
   that email has to ship with all of its CSS and HTML, so nothing in
@@ -79,3 +79,68 @@ backend. The `locmem` backend stores messages in a special attribute
 of the django.core.mail module, "outbox". The outbox attribute is
 created when the first message is sent. It’s a list with an
 EmailMessage instance for each message that would be sent.
+
+Other notes:
+* After changing any HTML email or `email_base.html`, you need to run
+  `tools/inline-email-css` for the changes to be reflected in the dev
+  environment. The script generates files like
+  `templates/zerver/emails/compiled/<template_prefix>.html`.
+## Email templates
+
+Zulip's email templates live under `templates/zerver/emails`.  Email
+templates are a messy problem, because on the one hand, you want nice,
+readable markup and styling, but on the other, email clients have very
+limited CSS support and generaly require us to inject any CSS we're
+using in the emails into the email as inline styles.  And then you
+also need both plain-text and HTML emails.  We solve these problems
+using a combination of the
+[premailer](https://github.com/peterbe/premailer) library and having
+two copies of each email (plain-text and HTML).
+
+So for each email, there are two source templates: the `.txt` version
+(for plain-text format) as well as a `.source.html` template.  The
+`.txt` version is used directly; while the `.source.html` template is
+processed by `tools/inline-email-css` (generating a `.html` template
+under `templates/zerver/emails/compiled`); that tool (powered by
+`premailer`) injects the CSS we use for styling our emails
+(`templates/zerver/emails/email.css`) into the templates inline.
+
+What this means is that when you're editing emails, **you need to run
+`tools/inline-email-css`** after making changes to see the changes
+take effect.  Our tooling automatically runs this as part of
+`tools/provision` and production deployments; but you should bump
+`PROVISION_VERSION` when making changes to emails that change test
+behavior, or other developers will get test failures until they
+provision.
+
+While this model is great for the markup side, it isn't ideal for
+[translations](../translating/translating.html).  The Django
+translation system works with exact strings, and having different new
+markup can require translators to re-translate strings, which can
+result in problems like needing 2 copies of each string (one for
+plain-text, one for HTML) and/or needing to re-translate a bunch of
+strings after making a CSS tweak.  Re-translating these strings is
+relatively easy in Transifex, but annoying.
+
+So when writing email templates, we try to translate individual
+sentences that are shared between the plain-text and HTML content
+rather than larger blocks that might contain markup; this allows
+translators to not have to deal with multiple versions of each string
+in our emails.
+
+One can test whether you did the translating part right by running
+`tools/inline-email-css && manage.py makemessages` and then searching
+for the strings in `static/locale/en/LC_MESSAGES/django.po`; if there
+are multiple copies or they contain CSS colors, you did it wrong.
+
+A final note for translating emails is that strings that are sent to
+user accounts (where we know the user's language) are higher-priority
+to translate than things sent to an email address (where we don't).
+E.g. for password reset emails, it makes sense for the code path for
+people with an actual account can be tagged for translation, while the
+code path for the "you don't have an account email" might not be,
+since we might not know what language to use in the second case.
+
+Future work in this space could be to actually generate the plain-text
+versions of emails from the `.source.html` markup, so that we don't
+need to maintain two copies of each email's text.

docs/subsystems/guest-users.md

@@ -0,0 +1,27 @@
+# Guest users
+
+This page documents how guest users work in Zulip.  This documentation
+attempts to cover the current state first, and then the
+to-be-implemented restrictions.
+
+Guest users are like normal users in Zulip, except that they cannot do
+the following:
+
+* Join public streams without being added by another user.
+* Access message history on public streams.
+* Create streams (public or private)
+* Create or own bots users
+
+For many of these limitations, we haven't yet hidden the relevant
+section(s) of the UI, and the prototype guest user experience will
+feel buggy until we hide those features.
+
+Limitations to be implemented:
+* Manage streams (e.g. Add other users to a stream they are subscribed
+  to)
+* See streams they are not subscribed to.
+* Interact with user groups
+* Send private messages to users not in a stream with them.
+
+(And more, this is an initial high-level TODO list).
+

docs/subsystems/html-templates.md

@@ -32,8 +32,8 @@ renders the template. For example, if you want to find the context
 passed to `index.html`, you can do:
 
 ```
-$ git grep zerver/index.html '*.py'
-zerver/views/home.py:    response = render(request, 'zerver/index.html',
+$ git grep zerver/app/index.html '*.py'
+zerver/views/home.py:    response = render(request, 'zerver/app/index.html',
 ```
 
 The next line in the code being the context definition.
@@ -75,6 +75,14 @@ static/js/invite.js:    $('#streams_to_add').html(templates.render('invite_subsc
 
 The second argument to `templates.render` is the context.
 
+### Toolchain
+
+Handlebars is in our `package.json` and thus ends up in
+`node_modules`; and then we have a script,
+`tools/compile-handlebars-templates`, which is responsible for
+compiling the templates, both in production and as they change in a
+development environment.
+
 ### Translation
 
 All user-facing strings (excluding pages only visible to sysadmins or

docs/subsystems/index.rst

@@ -30,11 +30,13 @@ Subsystems Documentation
    logging
    typing-indicators
    users
+   guest-users
    release-checklist
    api-release-checklist
    swagger-api-docs
    documentation
    conversion
    input-pills
+   presence
    unread_messages
    user-docs

docs/subsystems/markdown.md

@@ -38,7 +38,7 @@ The Python-Markdown implementation is tested by
 `frontend_tests/node_tests/markdown.js`.
 
 A shared set of fixed test data ("test fixtures") is present in
-`zerver/fixtures/markdown_test_cases.json`, and is automatically used
+`zerver/tests/fixtures/markdown_test_cases.json`, and is automatically used
 by both test suites; as a result, it is the preferred place to add new
 tests for Zulip's markdown system.  Some important notes on reading
 this file:
@@ -99,8 +99,8 @@ places:
   `static/third/marked/lib/marked.js`), or `markdown.contains_backend_only_syntax` if
   your changes won't be supported in the frontend processor.
 * If desired, the typeahead logic in `static/js/composebox_typeahead.js`.
-* The test suite, probably via adding entries to `zerver/fixtures/markdown_test_cases.json`.
-* The in-app markdown documentation (`templates/zerver/markdown_help.html`).
+* The test suite, probably via adding entries to `zerver/tests/fixtures/markdown_test_cases.json`.
+* The in-app markdown documentation (`templates/zerver/app/markdown_help.html`).
 * The list of changes to markdown at the end of this document.
 
 Important considerations for any changes are:

docs/subsystems/pointer.md

@@ -36,7 +36,7 @@ what you clicked on, and in fact the message you clicked on stays at
 exactly the same scroll position in the window after the narrowing as
 it was at before.
 
-### Search or sidebar click: unread/recent matching narrow
+### Search, sidebar click, or new narrowed tab: unread/recent matching narrow
 
 If you instead narrow by clicking on something in the left sidebar or
 typing some terms into the search box, Zulip will instead select
@@ -71,16 +71,6 @@ streams in your All messages view, this can lag.
 We plan to change this to automatically advance the pointer in a way
 similar to the unnarrow logic.
 
-### Narrow in a new tab: closest to pointer
-
-When you load a new browser tab or window to a narrowed view, Zulip
-will select the message closest to your pointer, which is what you
-would have got had you loaded the browser window to your All messages view and
-then clicked on the nearest message matching your narrow (which might
-have been offscreen).
-
-We plan to change this to match the Search/sidebar behavior.
-
 ### Forced reload: state preservation
 
 When the server forces a reload of a browser that's otherwise caught

docs/subsystems/presence.md

@@ -0,0 +1,55 @@
+# Presence
+
+This document explains the model for Zulip's presence.
+
+In a chat tool like Zulip, users expect to see the “presence” status
+of other users: is the person I want to talk to currently online? If
+not, were they last online 5 minutes ago, or more like an hour ago, or
+a week?  Presence helps set expectations for whether someone is likely
+to respond soon.  To a user, this feature can seem like a simple thing
+that should be easy.  But presence is actually one of the hardest
+scalability problems for a team chat tool like Zulip.
+
+There's a lot of performance-related details in the backend and
+network protocol design that we won't get into here.  The focus of
+this is what one needs to know to correctly implement a Zulip client's
+presence implementation (e.g. webapp, mobile app, terminal client, or
+other tool that's intended to represent whether a user is online and
+using Zulip).
+
+A client should report to the server every minute a `POST` request to
+`/users/me/presence`, containing the current user's status.  The
+requests contains a few parameters.  The most important is "status",
+which had 2 valid values:
+
+* "active" -- this means the user has interacted with the client
+  recently.  We use this for the "green" state in the webapp.
+* "idle" -- the user has not interacted with the client recently.
+  This is important for the case where a user left a Zulip tab open on
+  their desktop at work and went home for the weekend.  We use this
+  for the "orange" state in the webapp.
+
+The client receives in the response to that request a data set that,
+for each user, contains their status and timestamp that we last heard
+from that client.  There are a few important details to understand
+about that data structure:
+
+* It's really important that the timestamp is the last time we heard
+  from the client.  A client can only interpret the status to display
+  about another user by doing a simple computation using the (status,
+  timestamp) pair.  E.g. a user who last used Zulip 1 week ago will
+  have a timestamp of 1 week ago and a status of "active".  Why?
+  Because this correctly handles the race conditions.  For example, if
+  the threshhold for displaying a user as "offline" was 5 minutes
+  since the user was last online, the client can at any time
+  accurately compute whether that user is offline (even if the last
+  data from the server was 45 seconds ago, and the user was last
+  online 4:30 before the client received that server data).
+* The `status_from_timestamp` function in `static/js/presence.js` is
+  useful sample code; the `OFFLINE_THRESHOLD_SECS` check is critical
+  to correct output.
+* We provide the data for e.g. whether the user was online on their
+  desktop or the mobile app, but for a basic client, you will likely
+  only want to parse the "aggregated" key, which shows the summary
+  answer for "is this user online".
+

docs/subsystems/release-checklist.md

@@ -53,6 +53,4 @@ preparing a new release.
   to master.
 * Update `ZULIP_VERSION` in `version.py`, and `release` and `version` in
  `docs/conf.py`, to e.g. `1.6.0+git`.
-* Update the handful of places where we link to docs for the latest
-  release, rather than for master.  See `git grep 'zulip.readthedocs.io/en/[0-9]'`.
 * Consider removing a few old releases from ReadTheDocs.

docs/subsystems/user-docs.md

@@ -249,9 +249,9 @@ always create a new macro by adding a new file to that folder.
 
 ### **Organization settings** `{!admin.md!}` macro
 
-* **About:** Links to the **Organization settings** documentation.
-Usually preceded by the [**Go to the** macro](#go-to-the-go-to-the-md-macro)
-and a link to a particular section on the **Organization settings** page.
+* **About:** Links to the **Organization settings** documentation. Usually
+preceded by a link to a particular section on the **Organization settings**
+page.
 
 * **Contents:**
     ```md
@@ -260,7 +260,7 @@ and a link to a particular section on the **Organization settings** page.
 
 * **Example usage and rendering:**
     ```md
-    {!go-to-the.md!} [Organization settings](/#organization/organization-settings)
+    1. Go to the [Organization settings](/#organization/organization-settings)
     {!admin.md!}
     ```
     ```md
@@ -284,7 +284,7 @@ immediately after the title.
     ```md
     {!admin-only.md!}
 
-    {!follow-steps.md!} change who can join your stream by changing the stream's
+    Follow the following steps to change who can join your stream by changing the stream's
     accessibility.
     ```
     ```md
@@ -348,27 +348,6 @@ macro](#message-actions-message-actions-md-macro).
     down chevron (<i class="fa fa-chevron-down"></i>) icon to reveal an actions dropdown.
     ```
 
-### **Go to the** `{!go-to-the.md}` macro
-
-* **About:** Usually precedes the [**Settings** macro](#settings-settings-md-macro)
-or the [**Organization settings** macro](#organization-settings-admin-md-macro). Transforms
-following content into a step.
-
-* **Contents:**
-    ```md
-    1. Go to the
-    ```
-
-* **Example usage and rendering:**
-    ```md
-    {!go-to-the.md!} [Notifications](/#settings/notifications)
-    {!settings.md!}
-    ```
-    ```md
-    1. Go to the [Notifications](/#settings/notifications) tab on the
-    [Settings](/help/edit-settings) page.
-    ```
-
 ### **Filter streams** `{!filter-streams.md!}` macro
 
 * **About:** Explains how to search for specific streams in the
@@ -392,24 +371,6 @@ following content into a step.
     name of the stream in the **Filter streams** input.
     ```
 
-### **Follow steps** `{!follow-steps.md!}` macro
-
-* **About:** Prepends phrases with instructions to follow the following steps.
-
-* **Contents:**
-    ```md
-    Follow the following steps to
-    ```
-
-* **Example usage and rendering:**
-    ```md
-    {!follow-steps.md!}  change your mobile notification settings.
-    ```
-    ```md
-    Follow the following steps to change your mobile notification
-    settings.
-    ```
-
 ### **Message actions** `{!message-actions.md!}` macro
 
 * **About:** Explains how to view the actions of message. Usually followed by an instruction
@@ -456,8 +417,7 @@ describing the settings they modified.
 ### **Settings** `{!settings.md!}` macro
 
 * **About:** Links to the **Edit Settings** documentation. Usually preceded by
-the [**Go to the** macro](#go-to-the-go-to-the-md-macro) and a link to a
-particular section on the **Settings** page.
+a link to a particular section on the **Settings** page.
 
 * **Contents:**
     ```md
@@ -466,7 +426,7 @@ particular section on the **Settings** page.
 
 * **Example usage and rendering:**
     ```md
-    {!go-to-the.md!} [Notifications](/#settings/notifications)
+    1. Go to the [Notifications](/#settings/notifications)
     {!settings.md!}
     ```
     ```md

docs/testing/testing-with-django.md

@@ -319,7 +319,7 @@ reads a bunch of sample inputs from a JSON fixture file, feeds them
 to our GitHub integration code, and then verifies the output against
 expected values from the same JSON fixture file.
 
-Our fixtures live in `zerver/fixtures`.
+Our fixtures live in `zerver/tests/fixtures`.
 
 ### Mocks and stubs
 

docs/testing/testing-with-node.md

@@ -3,9 +3,10 @@
 As an alternative to the black-box whole-app testing, you can unit test
 individual JavaScript files.
 
-If you are writing JavaScript code that manipulates data (as opposed
-to coordinating UI changes), then you probably modify existing unit test
-modules to ensure the quality of your code and prevent regressions.
+You can run tests as follow:
+```
+    tools/test-js-with-node
+```
 
 The JS unit tests are written to work with node.  You can find them
 in `frontend_tests/node_tests`.  Here is an example test from
@@ -35,34 +36,19 @@ see if there are corresponding test in `frontend_tests/node_tests`.  If
 there are, you should strive to follow the patterns of the existing tests
 and add your own tests.
 
-## HTML output
-
-The JavaScript unit tests can generate output to be viewed in the
-browser.  The best examples of this are in `frontend_tests/node_tests/templates.js`.
-
-The main use case for this mechanism is to be able to unit test
-templates and see how they are rendered without the complications
-of the surrounding app.  (Obviously, you still need to test the
-app itself!)  The HTML output can also help to debug the unit tests.
-
-Each test calls a method named `write_handlebars_output` after it
-renders a template with similar data.  This API is still evolving,
-but you should be able to look at existing code for patterns.
-
-When you run `tools/test-js-with-node`, it will present you with a
-message like "To see more output, open var/test-js-with-node/index.html."
-Basically, you just need to open the file in the browser.  (If you are
-running a VM, this might require switching to another terminal window
-to launch the `open` command.)
-
 ## Coverage reports
 
 You can automatically generate coverage reports for the JavaScript unit
 tests like this:
 
->     tools/test-js-with-node --coverage
+```
+    tools/test-js-with-node --coverage
+```
 
-Then open `coverage/lcov-report/js/index.html` in your browser. Modules
+If tests pass, you will get instructions to view coverage reports
+in your browser.
+
+Note that modules that
 we don't test *at all* aren't listed in the report, so this tends to
 overstate how good our overall coverage is, but it's accurate for
 individual files. You can also click a filename to see the specific
@@ -75,12 +61,7 @@ good goal.
 The following scheme helps avoid tests leaking globals between each
 other.
 
-First, if you can avoid globals, do it, and the code that is directly
-under test can simply be handled like this:
-
->     var search = require('js/search_suggestion.js');
-
-For deeper dependencies, you want to categorize each module as follows:
+You want to categorize each module as follows:
 
 -   Exercise the module's real code for deeper, more realistic testing?
 -   Stub out the module's interface for more control, speed, and
@@ -102,7 +83,7 @@ like this:
 >     });
 >
 >     // then maybe further down
->     global.page_params.email = 'alice@zulip.com';
+>     page_params.email = 'alice@zulip.com';
 
 Finally, there's the hybrid situation, where you want to borrow some of
 a module's real functionality but stub out other pieces. Obviously, this
@@ -110,51 +91,28 @@ is a pretty strong smell that the other module might be lacking in
 cohesion, but that code might be outside your jurisdiction. The pattern
 here is this:
 
->     // Use real versions of parse/unparse
->     var narrow = require('js/narrow.js');
->     set_global('narrow', {
->         parse: narrow.parse,
->         unparse: narrow.unparse
->     });
+>     // Import real code.
+>     zrequire('narrow');
 >
->     // But later, I want to stub the stream without having to call super-expensive
->     // real code like narrow.activate().
->     global.narrow.stream = function () {
+>     // And later...
+>     narrow.stream = function () {
 >         return 'office';
 >     };
 
 ## Creating new test modules
 
+The test runner (`index.js`) automatically runs all .js files in the
+`frontend_tests/node directory`, so you can simply start editing a file
+in that directory to create a new test.
+
 The nodes tests rely on JS files that use the module pattern. For example, to
-test the `foobar.js` file, you would first add the following to the
-bottom of `foobar.js`:
+test the `foobar.js` file, you would first ensure that code like below
+is at the bottom of `foobar.js`:
 
 >     if (typeof module !== 'undefined') {
 >         module.exports = foobar;
 >     }
 
-This makes `foobar.js` follow the CommonJS module pattern, so it can be
+This means `foobar.js` follow the CommonJS module pattern, so it can be
 required in Node.js, which runs our tests.
 
-Now create `frontend_tests/node_tests/foobar.js`. At the top, require
-the [Node.js assert module](http://nodejs.org/api/assert.html), and the
-module you're testing, like so:
-
->     var assert = require('assert');
->     var foobar = require('js/foobar.js');
-
-And of course, if the module you're testing depends on other modules,
-or modifies global state, you may need to review the
-[section on handling dependencies](#handling-dependencies-in-unit-tests) above.
-
-Define and call some tests using the [assert
-module](http://nodejs.org/api/assert.html). Note that for "equal"
-asserts, the *actual* value comes first, the *expected* value second.
-
->     (function test_somefeature() {
->         assert.strictEqual(foobar.somefeature('baz'), 'quux');
->         assert.throws(foobar.somefeature('Invalid Input'));
->     }());
-
-The test runner (`index.js`) automatically runs all .js files in the
-frontend\_tests/node directory.

docs/testing/testing.md

@@ -9,11 +9,12 @@ important components are documented in depth in their own sections:
 - [Casper](../testing/testing-with-casper.html): end-to-end UI tests
 - [Node](../testing/testing-with-node.html): unit tests for JS front end code
 - [Linters](../testing/linters.html): Our parallel linter suite
-- [Travis CI details](travis.html): How all of these run in Travis CI
+- [CI details](travis.html): How all of these run in CI
+- [Other test suites](#other-test-suites): Our various smaller test suites.
 
 This document covers more general testing issues, such as how to run the
 entire test suite, how to troubleshoot database issues, how to manually
-test the front end, and how to plan for the future upgrade to Python3.
+test the front end, etc.
 
 We also document [how to manually test the app](manual-testing.html).
 
@@ -53,6 +54,50 @@ if you're working on new database migrations.  To do this, run:
 ./tools/do-destroy-rebuild-test-database
 ```
 
+## Other test suites
+
+Zulip also has about a dozen smaller tests suites:
+
+- `tools/test-migrations`: Checks whether the `zerver/migrations`
+  migration content the models defined in `zerver/models.py`.  See our
+  [schema migration documentation](../subsystems/schema-migrations.html)
+  for details on how to do database migrations correctly.
+- `tools/test-documentation`: Checks for broken links in this
+  ReadTheDocs documentation site.
+- `tools/test-help-documentation`: Checks for broken links in the
+  `/help` user documentation site, and related pages.
+- `tools/test-api`: Tests that the API documentation at `/api`
+  actually works; the actual code for this is defined in
+  `zerver/lib/api_test_helpers.py`.
+- `test-locked-requirements`: Verifies that developers didn't forget
+  to run `tools/update-locked-requirements` after modifying
+  `requirements/*.in`.  See
+  [our dependency documentation](../subsystems/dependencies.html) for
+  details on the system this is verifying.
+- `tools/check-capitalization`: Checks whether translated strings (aka
+  user-facing strings) correctly follow Zulip's capitalization
+  conventions.  This requires some maintainance of an exclude list of
+  proper nouns mentioned in the Zulip project, but helps a lot in
+  avoiding new strings being added that don't match our style.
+- `tools/check-frontend-i18n`: Checks for a common bug in Handlebars
+  templates, of using the wrong syntax for translating blocks
+  containing variables.
+- `./tools/test-run-dev`: Checks that `run-dev.py` starts properly;
+  this helps prevent bugs that break the development environment.
+- `./tools/test-queue-worker-reload`: Verifies that Zulip's queue
+  processors properly reload themselves after code changes.
+- `./tools/optimize-svg`: Checks whether all SVG files for integration
+  logos are properly optimized for size (since we're not going to edit
+  third-party logos, this helps keep the Zulip codebase from getting huge).
+- `./tools/test-tools`: Automated tests for various parts of our
+  development tooling (mostly various linters) that are not used in
+  production.
+
+Each of these has a reason (usually, performance or a need to do messy
+things to the environment) why they are not part of the handful of
+major test suites like `test-backend`, but they all contribute
+something valuable to helping keep Zulip bug-free.
+
 ### Possible testing issues
 
 - When running the test suite, if you get an error like this:

docs/tutorials/new-feature-tutorial.md

@@ -121,7 +121,7 @@ feature requires UI changes, you may need to add additional CSS to this
 file.
 
 **Templates:** The initial page structure is rendered via Jinja2
-templates located in `templates/zerver`. For JavaScript, Zulip uses
+templates located in `templates/zerver/app`. For JavaScript, Zulip uses
 Handlebars templates located in `static/templates`. Templates are
 precompiled as part of the build/deploy process.
 
@@ -483,8 +483,9 @@ to be added to the admin page (and its value added to the data sent back
 to server when a realm is updated) and the change event needs to be
 handled on the client.
 
-To add the checkbox to the admin page, modify the relevant template,
-`static/templates/settings/organization-permissions-admin.handlebars`
+To add the checkbox to the admin page, modify the relevant template in
+`static/templates/settings/`, which can be
+`organization-permissions-admin.handlebars` or `organization-settings-admin.handlebars`
 (omitted here since it is relatively straightforward).
 
 Then add the new form control in `static/js/admin.js`.
@@ -507,66 +508,81 @@ function _setup_page() {
 The JavaScript code for organization settings and permissions can be found in
 `static/js/settings_org.js`.
 
-There is a front-end version of `property_types`, which reduces the code
-needed on the front end for a new feature.
+In frontend, we have split the `property_types` into three objects:
 
-Add the new feature to the `property_types` object in `settings_org.js`.
-The key should be the setting name and the value should be an object with
-the following keys:
+- `org_profile`: This contains properties for the "organization
+    profile" settings page.
 
-* type
-* checked_msg (what message the user sees when they enable the setting)
-* unchecked_msg (what message the user sees when they disable the setting)
+- `org_settings`: This contains properties for the "organization
+    settings" page. Settings belonging to this section generally
+    decide what features should be available to a user like deleting a
+    message, message edit history etc.  Our `mandatory_topics` feature
+    belongs in this section.
+
+- `org_permissions`: This contains properties for the "organization
+    permissions" section. These properties control security controls
+    like who can join the organization and whether normal users can
+    create streams or upload custom emoji.
+
+Once you've determined wheter the new setting belongs, the next step
+is to find the right subsection of that page to put the setting
+in. For example in this case of `mandatory_topics` it will lie in
+"Message feed" (`msg_feed`) subsection.
+
+*If you're not sure in which section your feature belongs, it's is
+better to discuss it in the [community](https://chat.zulip.org/)
+before implementing it.*
+
+When defining the property, you'll also need to specify the property
+field type (i.e. whether it's a `bool`, `integer` or `text`).
 
 ``` diff
 
 // static/js/settings_org.js
-
-var property_types = {
-    settings: {
+var org_settings = {
+    msg_editing: {
         // ...
     },
-    permissions: { // ...
+    msg_feed: {
+        // ...
 +       mandatory_topics: {
 +           type: 'bool',
-+           checked_msg: i18n.t("Topics are required in messages to streams"),
-+           unchecked_msg: i18n.t("Topics are not required in messages to streams"),
-        },
++       },
     },
 };
+
 ```
 
-Additionally, any code needed to update the UI when the setting is changed
-should be written in a function inside `settings_org.js`.
-For example, when a realm description is updated, that value change should
-occur in other windows where the description field is visible:
+Note that some settings, like `realm_create_stream_permission`,
+reuqire special treatment, because they don't match the common
+pattern.  We can't extract the property name and compare the value of
+such input elements with those in `page_params`, so we have to
+manually handle such situations in a couple key functions:
 
-    # static/js/settings_org.js
+- `settings_org.get_property_value`: This processes the property name
+    when it doesn't match a corresponding key in `page_params`, and
+    returns the current value of that property, which we can use to
+    compare and set the values of corresponding DOM element.
 
-    exports.update_realm_description = function () {
-        if (!meta.loaded) {
-            return;
-        }
-
-        $('#id_realm_description').val(page_params.realm_description);
-    };
-
-
-This ensures the appropriate code will run even if the
-changes are made in another browser window.
-
-In the example of updating a `mandatory_topics` setting, most of the changes
-are on the backend, so no UI updates are required.
+- `settings_org.update_dependent_subsettings`: This handles settings
+    whose value and state depend on other elements.  For example,
+    `realm_waiting_period_threshold` is only shown for with the right
+    state of `realm_create_stream_permission`.
 
 Finally, update `server_events_dispatch.js` to handle related events coming from
 the server. There is an object, `realm_settings`, in the function
 `dispatch_normal_event`. The keys in this object are setting names and the
 values are the UI updating functions to run when an event has occurred.
 
-If there is no relevant UI change to make, the value should be `noop`
-(this is the case for `mandatory_topics`). However, if you had written
-a function in `settings_org.js` to update UI, that function should
-be the value in the `realm_settings` object.
+If there is no relevant UI change to make other than in settings page
+itself, the value should be `noop` (this is the case for
+`mandatory_topics`, since this setting only has an effect on the
+backend, so no UI updates are required.).
+
+However, if you had written a function to update the UI after a given
+setting has changed, your function should be referenced in the
+`realm_settings` of `server_events_dispatch.js`.  See for example
+`settings_emoji.update_custom_emoji_ui`.
 
 ``` diff
 
@@ -585,10 +601,35 @@ function dispatch_normal_event(event) {
       };
 ```
 
+Checkboxes and other common input elements handle the UI updates
+automatically through the logic in `settings_org.sync_realm_settings`.
+
 The rest of the `dispatch_normal_events` function updates the state of the
 application if an update event has occurred on a realm property and runs
 the associated function to update the application's UI, if necessary.
 
+Here are few important cases you should consider when testing your changes:
+
+- For organization settings where we have a "save/discard" model, make
+  sure both the "Save" and "Discard changes" buttons are working
+  properly.
+
+- If your setting is dependent on another setting, carefully check
+  that both are properly synchronized.  For example, the input element
+  for `realm_waiting_period_threshold` is shown only when we have
+  selected the custom time limit option in the
+  `realm_create_stream_permission` dropdown.
+
+- Do some manual testing for the real-time synchronization of input
+  elements across the browsers and just like "Discard changes" button,
+  check whether dependent settings are synchronized properly (this is
+  easy to do by opening two browser windows to the settings page, and
+  making changes in one while watching the other).
+
+- Each subsection has independent "Save" and "Discard changes"
+  buttons, so changes and saving in one subsection shouldn't affect
+  the others.
+
 ### Front End Tests
 
 A great next step is to write front end tests. There are two types of

docs/tutorials/writing-views.md

@@ -101,7 +101,7 @@ def home(request):
 ### Writing a template
 
 Templates for the main website are found in
-[templates/zerver](https://github.com/zulip/zulip/blob/master/templates/zerver).
+[templates/zerver/app](https://github.com/zulip/zulip/blob/master/templates/zerver/app).
 
 
 ## Writing API REST endpoints
@@ -275,7 +275,7 @@ and in [zerver/lib/actions.py](https://github.com/zulip/zulip/blob/master/zerver
 
 ```py
 def do_set_realm_name(realm, name):
-    # type: (Realm, Text) -> None
+    # type: (Realm, str) -> None
     realm.name = name
     realm.save(update_fields=['name'])
     event = dict(

frontend_tests/.eslintrc.json

@@ -1,7 +1,9 @@
 {
     "env": {
-        "shared-node-browser": true
+        "shared-node-browser": true,
+        "es6": true
     },
+    "parserOptions": { "ecmaVersion": 2018 },
     "globals": {
         "assert": false,
         "casper": false,
@@ -11,6 +13,7 @@
         "zrequire": false
     },
     "rules": {
-        "no-sync": 0
+        "no-sync": 0,
+        "prefer-const": "error"
     }
 }

frontend_tests/casper_tests/00-realm-creation.js

@@ -17,8 +17,8 @@ casper.then(function () {
     });
     // Make sure confirmation email is send
     this.waitWhileVisible('form[action^="/new/"]', function () {
-         var regex = new RegExp('^http://[^/]+/accounts/send_confirm/' + email);
-         this.test.assertUrlMatch(regex, 'Confirmation mail send');
+        var regex = new RegExp('^http://[^/]+/accounts/send_confirm/' + email);
+        this.test.assertUrlMatch(regex, 'Confirmation mail send');
     });
 });
 

frontend_tests/casper_tests/02-site.js

@@ -19,14 +19,14 @@ casper.then(function () {
 
     msg.headings.forEach(function (heading) {
         casper.test.assertMatch(common.normalize_spaces(heading),
-            /(^You and )|( )/,
-            'Heading is well-formed');
+                                /(^You and )|( )/,
+                                'Heading is well-formed');
     });
 
     msg.bodies.forEach(function (body) {
         casper.test.assertMatch(body,
-            /^(<p>(.|\n)*<\/p>)?$/,
-            'Body is well-formed');
+                                /^(<p>(.|\n)*<\/p>)?$/,
+                                'Body is well-formed');
     });
 
     casper.test.info('Sending messages');

frontend_tests/casper_tests/03-narrow.js

@@ -337,30 +337,36 @@ casper.waitForSelector('#stream_filters .highlighted_stream', function () {
 
 // Use arrow keys to navigate through suggestions
 casper.then(function () {
-    // Down: Denmark -> Scotland
-    casper.sendKeys('.stream-list-filter', casper.page.event.key.Down, {keepFocus: true});
-    // Up: Scotland -> Denmark
-    casper.sendKeys('.stream-list-filter', casper.page.event.key.Up, {keepFocus: true});
-    // Up: Denmark -> Verona
-    casper.sendKeys('.stream-list-filter', casper.page.event.key.Up, {keepFocus: true});
+    function arrow(key) {
+        casper.sendKeys('.stream-list-filter',
+                        casper.page.event.key[key],
+                        {keepFocus: true});
+    }
+    arrow('Down'); // Denmark -> Scotland
+    arrow('Up'); // Scotland -> Denmark
+    arrow('Up'); // Denmark -> Denmark
+    arrow('Down'); // Denmark -> Scotland
 });
 
-casper.waitForSelector('#stream_filters [data-stream-name="Verona"].highlighted_stream', function () {
+casper.waitForSelector('#stream_filters [data-stream-name="Scotland"].highlighted_stream', function () {
     casper.test.info('Suggestion highlighting - after arrow key navigation');
-    casper.test.assertDoesntExist('#stream_filters [data-stream-name="Denmark"].highlighted_stream',
-                                  'Stream Denmark is not highlighted');
-    casper.test.assertDoesntExist('#stream_filters [data-stream-name="Scotland"].highlighted_stream',
-                                  'Stream Scotland is not highlighted');
-    casper.test.assertExist('#stream_filters [data-stream-name="Verona"].highlighted_stream',
-                            'Stream Verona is highlighted');
+    casper.test.assertDoesntExist(
+        '#stream_filters [data-stream-name="Denmark"].highlighted_stream',
+        'Stream Denmark is not highlighted');
+    casper.test.assertExist(
+        '#stream_filters [data-stream-name="Scotland"].highlighted_stream',
+        'Stream Scotland is  highlighted');
+    casper.test.assertDoesntExist(
+        '#stream_filters [data-stream-name="Verona"].highlighted_stream',
+        'Stream Verona is not highlighted');
 });
 
-// We search for the beginning of "Verona", not case sensitive
+// We search for the beginning of "Scotland", not case sensitive
 casper.then(function () {
     casper.evaluate(function () {
         $('.stream-list-filter').expectOne()
             .focus()
-            .val('ver')
+            .val('sCoT')
             .trigger($.Event('input'))
             .trigger($.Event('click'));
     });
@@ -373,16 +379,16 @@ casper.waitWhileVisible('#stream_filters [data-stream-name="Denmark"]', function
     casper.test.assertDoesntExist('#stream_filters [data-stream-name="Denmark"]',
                                   'Filtered stream list does not contain Denmark');
 });
-casper.waitWhileVisible('#stream_filters [data-stream-name="Scotland"]', function () {
-    casper.test.assertDoesntExist('#stream_filters [data-stream-name="Scotland"]',
-                                  'Filtered stream list does not contain Scotland');
+casper.waitWhileVisible('#stream_filters [data-stream-name="Verona"]', function () {
+    casper.test.assertDoesntExist('#stream_filters [data-stream-name="Verona"]',
+                                  'Filtered stream list does not contain Verona');
 });
 
 casper.then(function () {
-    casper.test.assertExists('#stream_filters [data-stream-name="Verona"]',
-                             'Filtered stream list does contain Verona');
-    casper.test.assertExists('#stream_filters [data-stream-name="Verona"].highlighted_stream',
-                             'Stream Verona is highlighted');
+    casper.test.assertExists('#stream_filters [data-stream-name="Scotland"]',
+                             'Filtered stream list does contain Scotland');
+    casper.test.assertExists('#stream_filters [data-stream-name="Scotland"].highlighted_stream',
+                             'Stream Scotland is highlighted');
 });
 
 // Clearing the list should give us back all the streams in the list
@@ -470,22 +476,23 @@ casper.waitForSelector('#user_presences .highlighted_user', function () {
 
 // Use arrow keys to navigate through suggestions
 casper.then(function () {
-    // Down: Cordelia -> Hamlet
-    casper.sendKeys('.user-list-filter', casper.page.event.key.Down, {keepFocus: true});
-    // Up: Hamlet -> Cordelia
-    casper.sendKeys('.user-list-filter', casper.page.event.key.Up, {keepFocus: true});
-    // Up: Cordelia -> aaron
-    casper.sendKeys('.user-list-filter', casper.page.event.key.Up, {keepFocus: true});
+    function arrow(key) {
+        casper.sendKeys('.user-list-filter',
+                        casper.page.event.key[key],
+                        {keepFocus: true});
+    }
+    arrow('Down'); // Cordelia -> Hamlet
+    arrow('Up'); // Hamlet -> Cordelia
+    arrow('Up'); // already at top
+    arrow('Down'); // Cordelia -> Hamlet
 });
 
-casper.waitForSelector('#user_presences li.highlighted_user [data-name="aaron"]', function () {
+casper.waitForSelector('#user_presences li.highlighted_user [data-name="King Hamlet"]', function () {
     casper.test.info('Suggestion highlighting - after arrow key navigation');
     casper.test.assertDoesntExist('#user_presences li.highlighted_user [data-name="Cordelia Lear"]',
-        'User Cordelia Lear not is selected');
-    casper.test.assertDoesntExist('#user_presences li.highlighted_user [data-name="King Hamlet"]',
-        'User King Hamlet is not selected');
-    casper.test.assertExist('#user_presences li.highlighted_user [data-name="aaron"]',
-        'User aaron is selected');
+                                  'User Cordelia Lear not is selected');
+    casper.test.assertExist('#user_presences li.highlighted_user [data-name="King Hamlet"]',
+                            'User King Hamlet is selected');
 });
 
 common.then_log_out();

frontend_tests/casper_tests/05-subscriptions.js

@@ -160,8 +160,8 @@ casper.then(function () {
 casper.then(function () {
     casper.fill('form#add_new_subscription', {stream_name: 'was'});
     casper.evaluate(function () {
-      $('#add_new_subscription input[type="text"]').expectOne()
-        .trigger($.Event('input'));
+        $('#add_new_subscription input[type="text"]').expectOne()
+            .trigger($.Event('input'));
     });
 });
 casper.waitForSelectorTextChange('form#add_new_subscription', function () {

frontend_tests/casper_tests/06-settings.js

@@ -166,15 +166,15 @@ casper.then(function () {
         var form_sel = '.edit_bot_form[data-email="' + bot_email + '"]';
         casper.test.info('Testing edit bot form values');
 
-    //     casper.test.assertEqual(
-    //         common.get_form_field_value(form_sel + ' [name=bot_name]'),
-    //         'Bot 1');
-    //     casper.test.assertEqual(
-    //         common.get_form_field_value(form_sel + ' [name=bot_default_sending_stream]'),
-    //         'Denmark');
-    //     casper.test.assertEqual(
-    //         common.get_form_field_value(form_sel + ' [name=bot_default_events_register_stream]'),
-    //         'Rome');
+        // casper.test.assertEqual(
+        //     common.get_form_field_value(form_sel + ' [name=bot_name]'),
+        //     'Bot 1');
+        // casper.test.assertEqual(
+        //     common.get_form_field_value(form_sel + ' [name=bot_default_sending_stream]'),
+        //     'Denmark');
+        // casper.test.assertEqual(
+        //     common.get_form_field_value(form_sel + ' [name=bot_default_events_register_stream]'),
+        //     'Rome');
         casper.test.assertEqual(
             common.get_form_field_value(form_sel + ' [name=bot_name]'),
             'Bot 1');
@@ -302,7 +302,7 @@ casper.thenClick('a[data-code="en"]');
  * Changing the language back to English so that subsequent tests pass.
  */
 casper.waitUntilVisible('#language-settings-status a', function () {
-    casper.test.assertSelectorHasText('#language-settings-status', 'Saved. Please reload for the change to take effect.');
+    casper.test.assertSelectorHasText('#language-settings-status', 'Gespeichert. Bitte lade die Seite neu um die Änderungen zu aktivieren.');
 });
 
 casper.thenOpen("http://zulip.zulipdev.com:9981/");

frontend_tests/casper_tests/07-stars.js

@@ -19,9 +19,9 @@ casper.then(function () {
 });
 
 common.then_send_message('stream', {
-        stream:  'Verona',
-        subject: 'stars',
-        content: 'test star',
+    stream:  'Verona',
+    subject: 'stars',
+    content: 'test star',
 });
 
 casper.then(function () {
@@ -37,10 +37,15 @@ casper.then(function () {
 
     // Clicking on a message star stars it.
     toggle_last_star();
-    casper.test.assertEquals(star_count(), 1,
-                             "Got expected single star count.");
+});
 
-    casper.click('a[href^="#narrow/is/starred"]');
+casper.then(function () {
+    casper.waitUntilVisible('#zhome .icon-vector-star', function () {
+        casper.test.assertEquals(star_count(), 1,
+                                 "Got expected single star count.");
+
+        casper.click('a[href^="#narrow/is/starred"]');
+    });
 });
 
 casper.waitUntilVisible('#zfilt', function () {

frontend_tests/casper_tests/10-admin.js

@@ -193,16 +193,17 @@ casper.then(function () {
             name: 'Teams',
             field_type: '1',
         });
-        casper.click('form.admin-profile-field-form button.button');
+        casper.click("form.admin-profile-field-form button[type='submit']");
     });
 });
 
 casper.then(function () {
-    casper.waitUntilVisible('div#admin-profile-field-status', function () {
-        casper.test.assertSelectorHasText('div#admin-profile-field-status',
-                                          'Custom profile field added!');
+    casper.waitUntilVisible('#admin-profile-field-status img', function () {
+        casper.test.assertSelectorHasText('div#admin-profile-field-status', 'Saved');
+    });
+    casper.waitUntilVisible('.profile-field-row span.profile_field_name', function () {
         casper.test.assertSelectorHasText('.profile-field-row span.profile_field_name', 'Teams');
-        casper.test.assertSelectorHasText('.profile-field-row span.profile_field_type', 'Short Text');
+        casper.test.assertSelectorHasText('.profile-field-row span.profile_field_type', 'Short text');
         casper.click('.profile-field-row button.open-edit-form');
     });
 });
@@ -217,19 +218,19 @@ casper.then(function () {
 });
 
 casper.then(function () {
-    casper.waitUntilVisible('div#admin-profile-field-status', function () {
-        casper.test.assertSelectorHasText('div#admin-profile-field-status',
-                                          'Custom profile field updated!');
+    casper.waitUntilVisible('#admin-profile-field-status img', function () {
+        casper.test.assertSelectorHasText('div#admin-profile-field-status', 'Saved');
+    });
+    casper.waitForSelectorTextChange('.profile-field-row span.profile_field_name', function () {
         casper.test.assertSelectorHasText('.profile-field-row span.profile_field_name', 'team');
-        casper.test.assertSelectorHasText('.profile-field-row span.profile_field_type', 'Short Text');
+        casper.test.assertSelectorHasText('.profile-field-row span.profile_field_type', 'Short text');
         casper.click('.profile-field-row button.delete');
     });
 });
 
 casper.then(function () {
-    casper.waitUntilVisible('div#admin-profile-field-status', function () {
-        casper.test.assertSelectorHasText('div#admin-profile-field-status',
-                                          'Custom profile field deleted!');
+    casper.waitUntilVisible('#admin-profile-field-status img', function () {
+        casper.test.assertSelectorHasText('div#admin-profile-field-status', 'Saved');
     });
 });
 
@@ -286,9 +287,9 @@ function get_suggestions(str) {
     casper.then(function () {
         casper.evaluate(function (str) {
             $('.create_default_stream')
-            .focus()
-            .val(str)
-            .trigger($.Event('keyup', { which: 0 }));
+                .focus()
+                .val(str)
+                .trigger($.Event('keyup', { which: 0 }));
         }, str);
     });
 }
@@ -349,7 +350,7 @@ casper.then(function () {
         // Hack: Rather than submitting the form, we just fill the
         // form and then trigger a click event by clicking the button.
         casper.fill('form.admin-realm-form', {
-                realm_icon_file_input: 'static/images/logo/zulip-icon-128x128.png',
+            realm_icon_file_input: 'static/images/logo/zulip-icon-128x128.png',
         }, false);
         casper.click("#realm_icon_upload_button");
         casper.waitWhileVisible("#upload_icon_spinner .loading_indicator_spinner", function () {

frontend_tests/casper_tests/11-mention.js

@@ -47,7 +47,8 @@ casper.then(function () {
 
 casper.then(function () {
     common.expected_messages('zhome', ['Verona > Test mention all'],
-     ["<p><span class=\"user-mention user-mention-me\" data-user-id=\"*\">@all</span></p>"]);
+                             ["<p><span class=\"user-mention user-mention-me\" " +
+                              "data-user-id=\"*\">@all</span></p>"]);
 });
 
 

frontend_tests/casper_tests/12-toggle-message-editing.js

@@ -6,22 +6,6 @@ function heading(heading_str) {
     });
 }
 
-function submit_checked() {
-    casper.then(function () {
-        casper.waitUntilVisible('input:checked[type="checkbox"][id="id_realm_allow_message_editing"] + span', function () {
-            casper.click('#org-submit-msg-editing');
-        });
-    });
-}
-
-function submit_unchecked() {
-    casper.then(function () {
-        casper.waitUntilVisible('input:not(:checked)[type="checkbox"][id="id_realm_allow_message_editing"] + span', function () {
-            casper.click('#org-submit-msg-editing');
-        });
-    });
-}
-
 common.start_and_log_in();
 
 // For clarity these should be different than what 08-edit uses, until
@@ -94,23 +78,34 @@ casper.then(function () {
     });
 });
 
+function submit_edit_limit_changed() {
+    casper.test.assertSelectorHasText('#org-submit-msg-editing', "Save");
+    casper.click('#org-submit-msg-editing');
+}
+
 // DEACTIVATE
 
 heading("DEACTIVATE");
 common.then_click("li[data-section='organization-settings']");
 
 // deactivate "allow message editing"
-common.then_click('input[type="checkbox"][id="id_realm_allow_message_editing"] + span');
-
-submit_unchecked();
+casper.then(function () {
+    casper.test.info("Changing message edit limit setting");
+    casper.waitUntilVisible("#id_realm_msg_edit_limit_setting", function () {
+        casper.evaluate(function () {
+            $("#id_realm_msg_edit_limit_setting").val("never").change();
+        });
+        submit_edit_limit_changed();
+    });
+});
 
 casper.then(function () {
     casper.waitUntilVisible('#org-submit-msg-editing[data-status="saved"]', function () {
         casper.test.assertSelectorHasText('#org-submit-msg-editing',
                                           'Saved');
         casper.test.assertEval(function () {
-            return !(document.querySelector('input[type="checkbox"][id="id_realm_allow_message_editing"]').checked);
-        }, 'Allow message editing Setting de-activated');
+            return (document.querySelector('#id_realm_msg_edit_limit_setting').value === "never");
+        }, 'Message editing Setting disabled');
     });
 });
 
@@ -155,76 +150,47 @@ heading("REACTIVATE");
 common.then_click('#settings-dropdown');
 common.then_click('a[href^="#organization"]');
 common.then_click("li[data-section='organization-settings']");
-common.then_click('input[type="checkbox"][id="id_realm_allow_message_editing"] + span');
-submit_checked();
 
 casper.then(function () {
-    casper.waitUntilVisible('#org-submit-msg-editing[data-status="saved"]', function () {
-        casper.test.assertSelectorHasText('#org-submit-msg-editing',
-                                          'Saved');
-        casper.test.assertEval(function () {
-            return document.querySelector('input[type="checkbox"][id="id_realm_allow_message_editing"]').checked;
-        }, 'Allow message editing Setting re-activated');
-    });
-});
-
-// DEACTIVATE
-
-heading("DEACTIVATE");
-
-// go to admin page
-casper.then(function () {
-    casper.test.info('Organization page');
-    casper.click('a[href^="#organization"]');
-    casper.test.assertUrlMatch(/^http:\/\/[^\/]+\/#organization/, 'URL suggests we are on organization page');
-    casper.test.assertExists('#settings_overlay_container.show', 'Organization page is active');
-});
-
-casper.then(function () {
-    casper.waitUntilVisible('form.admin-realm-form button.button');
-});
-
-// deactivate message editing
-casper.then(function () {
-    casper.waitUntilVisible('input[type="checkbox"][id="id_realm_allow_message_editing"] + span', function () {
+    casper.test.info("Changing message edit limit setting");
+    casper.waitUntilVisible("#id_realm_msg_edit_limit_setting", function () {
         casper.evaluate(function () {
-            $('input[type="text"][id="id_realm_message_content_edit_limit_minutes"]').val('4');
+            $("#id_realm_msg_edit_limit_setting").val("upto_ten_min").change();
         });
+        submit_edit_limit_changed();
     });
 });
 
-common.then_click('input[type="checkbox"][id="id_realm_allow_message_editing"] + span');
-submit_unchecked();
-
 casper.then(function () {
     casper.waitUntilVisible('#org-submit-msg-editing[data-status="saved"]', function () {
         casper.test.assertSelectorHasText('#org-submit-msg-editing',
                                           'Saved');
         casper.test.assertEval(function () {
-            return !(document.querySelector('input[type="checkbox"][id="id_realm_allow_message_editing"]').checked);
-        }, 'Allow message editing Setting de-activated');
-        casper.test.assertEval(function () {
-            return $('input[type="text"][id="id_realm_message_content_edit_limit_minutes"]').val() === '4';
-        }, 'Message content edit limit now 4');
+            return (document.querySelector('#id_realm_msg_edit_limit_setting').value === "upto_ten_min");
+        }, 'Allow message editing Setting re-activated and set to 10 minutes');
     });
 });
 
-// REACTIVATE
-heading("REACTIVATE");
+// SET LIMIT TO 1 WEEK
+heading("LIMIT TO 1 WEEK");
 
-common.then_click('input[type="checkbox"][id="id_realm_allow_message_editing"] + span');
-submit_checked();
+casper.then(function () {
+    casper.test.info("Changing message edit limit setting");
+    casper.waitUntilVisible("#id_realm_msg_edit_limit_setting", function () {
+        casper.evaluate(function () {
+            $("#id_realm_msg_edit_limit_setting").val("upto_one_week").change();
+        });
+        submit_edit_limit_changed();
+    });
+});
 
 casper.then(function () {
     casper.waitUntilVisible('#org-submit-msg-editing[data-status="saved"]', function () {
         casper.test.assertSelectorHasText('#org-submit-msg-editing',
                                           'Saved');
         casper.test.assertEval(function () {
-            return document.querySelector('input[type="checkbox"][id="id_realm_allow_message_editing"]').checked;
-        }, 'Allow message editing Setting activated');
-        casper.test.assertEval(function () {
-            return $('input[type="text"][id="id_realm_message_content_edit_limit_minutes"]').val() === '4';
-        }, 'Message content edit limit still 4');
+            return (document.querySelector('#id_realm_msg_edit_limit_setting').value === "upto_one_week");
+        }, 'Message edit limit set to one week');
     });
 });
 
@@ -232,52 +198,78 @@ casper.then(function () {
 heading("NO LIMIT");
 
 casper.then(function () {
-    // allow arbitrary message editing
-    casper.waitUntilVisible('input[type="checkbox"][id="id_realm_allow_message_editing"] + span', function () {
+    casper.test.info("Changing message edit limit setting");
+    casper.waitUntilVisible("#id_realm_msg_edit_limit_setting", function () {
         casper.evaluate(function () {
-            $('input[type="text"][id="id_realm_message_content_edit_limit_minutes"]').val('0');
+            $("#id_realm_msg_edit_limit_setting").val("any_time").change();
         });
+        submit_edit_limit_changed();
     });
 });
-submit_checked();
 
 casper.then(function () {
     casper.waitUntilVisible('#org-submit-msg-editing[data-status="saved"]', function () {
         casper.test.assertSelectorHasText('#org-submit-msg-editing',
                                           'Saved');
         casper.test.assertEval(function () {
-            return document.querySelector('input[type="checkbox"][id="id_realm_allow_message_editing"]').checked;
-        }, 'Allow message editing Setting still activated');
-        casper.test.assertEval(function () {
-            return $('input[type="text"][id="id_realm_message_content_edit_limit_minutes"]').val() === '0';
-        }, 'Message content edit limit is 0');
+            return (document.querySelector('#id_realm_msg_edit_limit_setting').value === "any_time");
+        }, 'Message can be edited any time');
     });
 });
 
-// ILLEGAL LIMIT
-heading("ILLEGAL LIMIT");
+// CUSTOM LIMIT
+heading("CUSTOM LIMIT");
 
 casper.then(function () {
-    // disallow message editing, with illegal edit limit value. should be fixed by admin.js
-    casper.waitUntilVisible('input[type="checkbox"][id="id_realm_allow_message_editing"] + span', function () {
+    casper.test.info("Changing message edit limit setting");
+    casper.waitUntilVisible("#id_realm_msg_edit_limit_setting", function () {
         casper.evaluate(function () {
-            $('input[type="text"][id="id_realm_message_content_edit_limit_minutes"]').val('moo');
+            $("#id_realm_msg_edit_limit_setting").val("custom_limit").change();
         });
     });
+    casper.waitUntilVisible('#id_realm_message_content_edit_limit_minutes', function () {
+        casper.evaluate(function () {
+            $('#id_realm_message_content_edit_limit_minutes').val("100");
+        });
+        submit_edit_limit_changed();
+    });
 });
-common.then_click('input[type="checkbox"][id="id_realm_allow_message_editing"] + span');
-submit_unchecked();
 
 casper.then(function () {
     casper.waitUntilVisible('#org-submit-msg-editing[data-status="saved"]', function () {
         casper.test.assertSelectorHasText('#org-submit-msg-editing',
                                           'Saved');
         casper.test.assertEval(function () {
-            return !(document.querySelector('input[type="checkbox"][id="id_realm_allow_message_editing"]').checked);
-        }, 'Allow message editing Setting de-activated');
+            return $('#id_realm_msg_edit_limit_setting').val() === "custom_limit";
+        }, 'Custom message edit limit set');
         casper.test.assertEval(function () {
-            return $('input[type="text"][id="id_realm_message_content_edit_limit_minutes"]').val() === '10';
-        }, 'Message content edit limit has been reset to its default');
+            return $('#id_realm_message_content_edit_limit_minutes').val() === "100";
+        }, 'Message edit limit set to 100 minutes');
+    });
+});
+
+// INVALID LIMIT
+heading("INVALID LIMIT");
+
+casper.then(function () {
+    casper.test.info("Changing message edit limit setting");
+    casper.waitUntilVisible("#id_realm_msg_edit_limit_setting", function () {
+        casper.evaluate(function () {
+            $("#id_realm_msg_edit_limit_setting").val("custom_limit").change();
+        });
+    });
+    casper.waitUntilVisible('#id_realm_message_content_edit_limit_minutes', function () {
+        casper.evaluate(function () {
+            $('#id_realm_message_content_edit_limit_minutes').val("-100");
+        });
+        submit_edit_limit_changed();
+    });
+});
+
+casper.then(function () {
+    casper.waitUntilVisible('.admin-realm-failed-change-status', function () {
+        casper.test.assertSelectorHasText('#org-submit-msg-editing',
+                                          'Save');
     });
 });
 

frontend_tests/node_tests/.eslintrc.json

@@ -0,0 +1,12 @@
+{
+    "rules": {
+        "indent": ["error", 4, {
+            "ArrayExpression": "first",
+            "ObjectExpression": "first",
+            "SwitchCase": 0,
+            "CallExpression": {"arguments": "first"},
+            "FunctionExpression": {"parameters": "first"},
+            "FunctionDeclaration": {"parameters": "first"}
+        }]
+    }
+}

frontend_tests/node_tests/alert_words.js

@@ -18,30 +18,62 @@ global.people.add({
 global.people.initialize_current_user(42);
 
 
-var regular_message = { sender_email: 'another@zulip.com', content: '<p>a message</p>'};
-var own_message = { sender_email: 'tester@zulip.com', content: '<p>hey this message alertone</p>',
-                    alerted: true };
-var other_message = { sender_email: 'another@zulip.com', content: '<p>another alertone message</p>',
-                      alerted: true };
-var caps_message = { sender_email: 'another@zulip.com', content: '<p>another ALERTtwo message</p>',
-                     alerted: true };
-var alertwordboundary_message = { sender_email: 'another@zulip.com',
-                                  content: '<p>another alertthreemessage</p>', alerted: false };
-var multialert_message = { sender_email: 'another@zulip.com', content:
-                           '<p>another alertthreemessage alertone and then alerttwo</p>',
-                           alerted: true };
-var unsafe_word_message = { sender_email: 'another@zulip.com', content: '<p>gotta al*rt.*s all</p>',
-                            alerted: true };
-var alert_in_url_message = { sender_email: 'another@zulip.com', content: '<p>http://www.google.com/alertone/me</p>',
-                            alerted: true };
-var question_word_message = { sender_email: 'another@zulip.com', content: '<p>still alertone? me</p>',
-                            alerted: true };
+const regular_message = {
+    sender_email: 'another@zulip.com',
+    content: '<p>a message</p>',
+};
+const own_message = {
+    sender_email: 'tester@zulip.com',
+    content: '<p>hey this message alertone</p>',
+    alerted: true,
+};
+const other_message = {
+    sender_email: 'another@zulip.com',
+    content: '<p>another alertone message</p>',
+    alerted: true,
+};
+const caps_message = {
+    sender_email: 'another@zulip.com',
+    content: '<p>another ALERTtwo message</p>',
+    alerted: true,
+};
+const alertwordboundary_message = {
+    sender_email: 'another@zulip.com',
+    content: '<p>another alertthreemessage</p>',
+    alerted: false,
+};
+const multialert_message = {
+    sender_email: 'another@zulip.com',
+    content: '<p>another alertthreemessage alertone and then alerttwo</p>',
+    alerted: true,
+};
+const unsafe_word_message = {
+    sender_email: 'another@zulip.com',
+    content: '<p>gotta al*rt.*s all</p>',
+    alerted: true,
+};
+const alert_in_url_message = {
+    sender_email: 'another@zulip.com',
+    content: '<p>http://www.google.com/alertone/me</p>',
+    alerted: true,
+};
+const question_word_message = {
+    sender_email: 'another@zulip.com',
+    content: '<p>still alertone? me</p>',
+    alerted: true,
+};
 
-var alert_domain_message = { sender_email: 'another@zulip.com', content: '<p>now with link <a href="http://www.alerttwo.us/foo/bar" target="_blank" title="http://www.alerttwo.us/foo/bar">www.alerttwo.us/foo/bar</a></p>',
-                     alerted: true };
+const alert_domain_message = {
+    sender_email: 'another@zulip.com',
+    content: '<p>now with link <a href="http://www.alerttwo.us/foo/bar" target="_blank" title="http://www.alerttwo.us/foo/bar">www.alerttwo.us/foo/bar</a></p>',
+    alerted: true,
+};
 // This test ensure we are not mucking up rendered HTML content.
-var message_with_emoji = { sender_email: 'another@zulip.com', content: '<p>I <img alt=":heart:" class="emoji" src="/static/generated/emoji/images/emoji/unicode/2764.png" title="heart"> emoji!</p>',
-                           alerted: true };
+const message_with_emoji = {
+    sender_email: 'another@zulip.com',
+    content: '<p>I <img alt=":heart:" class="emoji" src="/static/generated/emoji/images/emoji/unicode/2764.png" title="heart"> emoji!</p>',
+    alerted: true,
+};
 
 (function test_notifications() {
     assert(!alert_words.notifies(regular_message));
@@ -56,7 +88,7 @@ var message_with_emoji = { sender_email: 'another@zulip.com', content: '<p>I <im
 }());
 
 (function test_munging() {
-    var saved_content = regular_message.content;
+    let saved_content = regular_message.content;
     alert_words.process_message(regular_message);
     assert.equal(saved_content, regular_message.content);
 
@@ -82,8 +114,8 @@ var message_with_emoji = { sender_email: 'another@zulip.com', content: '<p>I <im
     assert.equal(question_word_message.content, "<p>still <span class='alert-word'>alertone</span>? me</p>");
 
     alert_words.process_message(alert_domain_message);
-    assert.equal(alert_domain_message.content, '<p>now with link <a href="http://www.alerttwo.us/foo/bar" target="_blank" title="http://www.alerttwo.us/foo/bar">www.<span class=\'alert-word\'>alerttwo</span>.us/foo/bar</a></p>');
+    assert.equal(alert_domain_message.content, `<p>now with link <a href="http://www.alerttwo.us/foo/bar" target="_blank" title="http://www.alerttwo.us/foo/bar">www.<span class='alert-word'>alerttwo</span>.us/foo/bar</a></p>`);
 
     alert_words.process_message(message_with_emoji);
-    assert.equal(message_with_emoji.content, '<p>I <img alt=":heart:" class="emoji" src="/static/generated/emoji/images/emoji/unicode/2764.png" title="heart"> <span class=\'alert-word\'>emoji</span>!</p>');
+    assert.equal(message_with_emoji.content, `<p>I <img alt=":heart:" class="emoji" src="/static/generated/emoji/images/emoji/unicode/2764.png" title="heart"> <span class='alert-word'>emoji</span>!</p>`);
 }());

frontend_tests/node_tests/bot_data.js

@@ -1,19 +1,12 @@
-var patched_underscore = _.clone(_);
-patched_underscore.debounce = function (f) { return f; };
-global.patch_builtin('_', patched_underscore);
-
 zrequire('people');
 zrequire('bot_data');
 
-set_global('$', function (f) {
-    if (f) {
-        return f();
-    }
-    return {trigger: function () {}};
+set_global('$', () => {
+    return {trigger: () => {}};
 });
 set_global('document', null);
 
-var page_params = {
+const page_params = {
     realm_bots: [{email: 'bot0@zulip.com', user_id: 42, full_name: 'Bot 0'},
                  {email: 'outgoingwebhook@zulip.com', user_id: 314, full_name: "Outgoing webhook",
                   services: [{base_url: "http://foo.com", interface: 1}]}],
@@ -35,7 +28,7 @@ assert.equal(bot_data.get(42).full_name, 'Bot 0');
 assert.equal(bot_data.get(314).full_name, 'Outgoing webhook');
 
 (function () {
-    var test_bot = {
+    const test_bot = {
         email: 'bot1@zulip.com',
         user_id: 43,
         avatar_url: '',
@@ -44,7 +37,7 @@ assert.equal(bot_data.get(314).full_name, 'Outgoing webhook');
         extra: 'Not in data',
     };
 
-    var test_embedded_bot = {
+    const test_embedded_bot = {
         email: 'embedded-bot@zulip.com',
         user_id: 143,
         avatar_url: '',
@@ -56,8 +49,8 @@ assert.equal(bot_data.get(314).full_name, 'Outgoing webhook');
     (function test_add() {
         bot_data.add(test_bot);
 
-        var bot = bot_data.get(43);
-        var services = bot_data.get_services(43);
+        const bot = bot_data.get(43);
+        const services = bot_data.get_services(43);
         assert.equal('Bot 1', bot.full_name);
         assert.equal('http://bar.com', services[0].base_url);
         assert.equal(1, services[0].interface);
@@ -65,18 +58,15 @@ assert.equal(bot_data.get(314).full_name, 'Outgoing webhook');
     }());
 
     (function test_update() {
-        var bot;
-        var services;
-
         bot_data.add(test_bot);
 
-        bot = bot_data.get(43);
+        let bot = bot_data.get(43);
         assert.equal('Bot 1', bot.full_name);
         bot_data.update(43, {full_name: 'New Bot 1',
                              services: [{interface: 2,
                                          base_url: 'http://baz.com'}]});
         bot = bot_data.get(43);
-        services = bot_data.get_services(43);
+        const services = bot_data.get_services(43);
         assert.equal('New Bot 1', bot.full_name);
         assert.equal(2, services[0].interface);
         assert.equal('http://baz.com', services[0].base_url);
@@ -84,17 +74,17 @@ assert.equal(bot_data.get(314).full_name, 'Outgoing webhook');
 
     (function test_embedded_bot_update() {
         bot_data.add(test_embedded_bot);
-        var bot_id = 143;
-        var services = bot_data.get_services(bot_id);
+        const bot_id = 143;
+        const services = bot_data.get_services(bot_id);
         assert.equal('12345678', services[0].config_data.key);
         bot_data.update(bot_id, {services: [{config_data: {key: '87654321'}}]});
         assert.equal('87654321', services[0].config_data.key);
     }());
 
     (function test_remove() {
-        var bot;
+        let bot;
 
-        bot_data.add(_.extend({}, test_bot, {is_active: true}));
+        bot_data.add({ ...test_bot, is_active: true });
 
         bot = bot_data.get(43);
         assert.equal('Bot 1', bot.full_name);
@@ -105,9 +95,9 @@ assert.equal(bot_data.get(314).full_name, 'Outgoing webhook');
     }());
 
     (function test_delete() {
-        var bot;
+        let bot;
 
-        bot_data.add(_.extend({}, test_bot, {is_active: true}));
+        bot_data.add({ ...test_bot, is_active: true });
 
         bot = bot_data.get(43);
         assert.equal('Bot 1', bot.full_name);
@@ -118,37 +108,36 @@ assert.equal(bot_data.get(314).full_name, 'Outgoing webhook');
     }());
 
     (function test_owner_can_admin() {
-        var bot;
+        let bot;
 
-        bot_data.add(_.extend({owner: 'owner@zulip.com'}, test_bot));
+        bot_data.add({owner: 'owner@zulip.com', ...test_bot});
 
         bot = bot_data.get(43);
         assert(bot.can_admin);
 
-        bot_data.add(_.extend({owner: 'notowner@zulip.com'}, test_bot));
+        bot_data.add({owner: 'notowner@zulip.com', ...test_bot});
 
         bot = bot_data.get(43);
         assert.equal(false, bot.can_admin);
     }());
 
     (function test_admin_can_admin() {
-        var bot;
         page_params.is_admin = true;
 
         bot_data.add(test_bot);
 
-        bot = bot_data.get(43);
+        const bot = bot_data.get(43);
         assert(bot.can_admin);
 
         page_params.is_admin = false;
     }());
 
     (function test_get_editable() {
-        var can_admin;
+        let can_admin;
 
-        bot_data.add(_.extend({}, test_bot, {user_id: 44, owner: 'owner@zulip.com', is_active: true}));
-        bot_data.add(_.extend({}, test_bot, {user_id: 45, email: 'bot2@zulip.com', owner: 'owner@zulip.com', is_active: true}));
-        bot_data.add(_.extend({}, test_bot, {user_id: 46, email: 'bot3@zulip.com', owner: 'not_owner@zulip.com', is_active: true}));
+        bot_data.add({...test_bot ,user_id: 44, owner: 'owner@zulip.com', is_active: true});
+        bot_data.add({...test_bot, user_id: 45, email: 'bot2@zulip.com', owner: 'owner@zulip.com', is_active: true});
+        bot_data.add({...test_bot, user_id: 46, email: 'bot3@zulip.com', owner: 'not_owner@zulip.com', is_active: true});
 
         can_admin = _.pluck(bot_data.get_editable(), 'email');
         assert.deepEqual(['bot1@zulip.com', 'bot2@zulip.com'], can_admin);
@@ -160,7 +149,7 @@ assert.equal(bot_data.get(314).full_name, 'Outgoing webhook');
     }());
 
     (function test_get_all_bots_for_current_user() {
-        var bots = bot_data.get_all_bots_for_current_user();
+        const bots = bot_data.get_all_bots_for_current_user();
 
         assert.equal(bots.length, 2);
         assert.equal(bots[0].email, 'bot1@zulip.com');

frontend_tests/node_tests/buddy_data.js

@@ -0,0 +1,76 @@
+zrequire('people');
+zrequire('presence');
+zrequire('util');
+zrequire('buddy_data');
+
+// The buddy_data module is mostly tested indirectly through
+// activity.js, but we should feel free to add direct tests
+// here.
+
+
+set_global('page_params', {});
+
+(function make_people() {
+    _.each(_.range(1000, 2000), (i) => {
+        const person = {
+            user_id: i,
+            full_name: `Human ${i}`,
+            email: `person${i}@example.com`,
+        };
+        people.add_in_realm(person);
+    });
+}());
+
+(function activate_people() {
+    const server_time = 9999;
+    const info = {
+        website: {
+            status: "active",
+            timestamp: server_time,
+        },
+    };
+
+    // Make 400 of the users active
+    _.each(_.range(1000, 1400), (user_id) => {
+        presence.set_user_status(user_id, info, server_time);
+    });
+
+    // And then 300 not active
+    _.each(_.range(1400, 1700), (user_id) => {
+        presence.set_user_status(user_id, {}, server_time);
+    });
+}());
+
+(function test_user_ids() {
+    var user_ids;
+
+    // Even though we have 1000 users, we only get the 400 active
+    // users.  This is a consequence of buddy_data.maybe_shrink_list.
+    user_ids = buddy_data.get_filtered_and_sorted_user_ids();
+    assert.equal(user_ids.length, 400);
+
+    user_ids = buddy_data.get_filtered_and_sorted_user_ids('');
+    assert.equal(user_ids.length, 400);
+
+    // We don't match on "s", because it's not at the start of a
+    // word in the name/email.
+    user_ids = buddy_data.get_filtered_and_sorted_user_ids('s');
+    assert.equal(user_ids.length, 0);
+
+    // We match on "h" for the first name, and the result limit
+    // is relaxed for searches.
+    user_ids = buddy_data.get_filtered_and_sorted_user_ids('h');
+    assert.equal(user_ids.length, 1000);
+
+    // We match on "p" for the email.
+    user_ids = buddy_data.get_filtered_and_sorted_user_ids('p');
+    assert.equal(user_ids.length, 1000);
+
+
+    // Make our shrink limit higher, and go back to an empty search.
+    // We won't get all 1000 users, just the present ones.
+    buddy_data.max_size_before_shrinking = 50000;
+
+    user_ids = buddy_data.get_filtered_and_sorted_user_ids('');
+    assert.equal(user_ids.length, 700);
+}());

frontend_tests/node_tests/buddy_list.js

@@ -0,0 +1,14 @@
+set_global('$', global.make_zjquery());
+zrequire('buddy_list');
+
+(function test_get_items() {
+    const alice_li = $.create('alice stub');
+    const sel = 'li.user_sidebar_entry';
+
+    buddy_list.container.set_find_results(sel, {
+        map: (f) => [f(0, alice_li)],
+    });
+    const items = buddy_list.get_items();
+
+    assert.deepEqual(items, [alice_li]);
+}());

frontend_tests/node_tests/channel.js

@@ -2,19 +2,23 @@ zrequire('channel');
 
 set_global('$', {});
 set_global('reload', {});
-set_global('blueslip', {});
+set_global('blueslip', global.make_zblueslip());
 
-var default_stub_xhr = 'default-stub-xhr';
+const default_stub_xhr = 'default-stub-xhr';
 
 function test_with_mock_ajax(test_params) {
-    var ajax_called;
-    var ajax_options;
+    const {
+        xhr = default_stub_xhr,
+        run_code,
+        check_ajax_options,
+    } = test_params;
 
+    let ajax_called;
+    let ajax_options;
     $.ajax = function (options) {
         $.ajax = undefined;
         ajax_called = true;
         ajax_options = options;
-        var xhr = test_params.xhr || default_stub_xhr;
 
         options.simulate_success = function (data, text_status) {
             options.success(data, text_status, xhr);
@@ -27,9 +31,9 @@ function test_with_mock_ajax(test_params) {
         return xhr;
     };
 
-    test_params.run_code();
+    run_code();
     assert(ajax_called);
-    test_params.check_ajax_options(ajax_options);
+    check_ajax_options(ajax_options);
 }
 
 
@@ -113,15 +117,15 @@ function test_with_mock_ajax(test_params) {
 }());
 
 (function test_normal_post() {
-    var data = {
+    const data = {
         s: 'some_string',
         num: 7,
         lst: [1, 2, 4, 8],
     };
 
-    var orig_success_called;
-    var orig_error_called;
-    var stub_xhr = 'stub-xhr-normal-post';
+    let orig_success_called;
+    let orig_error_called;
+    const stub_xhr = 'stub-xhr-normal-post';
 
     test_with_mock_ajax({
         xhr: stub_xhr,
@@ -158,9 +162,9 @@ function test_with_mock_ajax(test_params) {
 }());
 
 (function test_patch_with_form_data() {
-    var appended;
+    let appended;
 
-    var data = {
+    const data = {
         append: function (k, v) {
             assert.equal(k, 'method');
             assert.equal(v, 'PATCH');
@@ -200,7 +204,7 @@ function test_with_mock_ajax(test_params) {
         },
 
         check_ajax_options: function (options) {
-            var reload_initiated;
+            let reload_initiated;
             reload.initiate = function (options) {
                 reload_initiated = true;
                 assert.deepEqual(options, {
@@ -229,15 +233,10 @@ function test_with_mock_ajax(test_params) {
         },
 
         check_ajax_options: function (options) {
-            var has_error;
-            blueslip.error = function (msg) {
-                assert.equal(msg, 'Unexpected 403 response from server');
-                has_error = true;
-            };
-
+            blueslip.set_test_data('error', 'Unexpected 403 response from server');
             options.simulate_error();
-
-            assert(has_error);
+            assert.equal(blueslip.get_test_logs('error').length, 1);
+            blueslip.clear_test_data();
         },
     });
 }());
@@ -252,12 +251,6 @@ function test_with_mock_ajax(test_params) {
         },
 
         check_ajax_options: function (options) {
-            var logged;
-            blueslip.log = function (msg) {
-                // Our log formatting is a bit broken.
-                assert.equal(msg, 'Retrying idempotent[object Object]');
-                logged = true;
-            };
             global.patch_builtin('setTimeout', function (f, delay) {
                 assert.equal(delay, 0);
                 f();
@@ -273,39 +266,35 @@ function test_with_mock_ajax(test_params) {
                 },
             });
 
-            assert(logged);
+            assert.equal(blueslip.get_test_logs('log').length, 1);
+            assert.equal(blueslip.get_test_logs('log')[0], 'Retrying idempotent[object Object]');
+            blueslip.clear_test_data();
         },
     });
 }());
 
 (function test_too_many_pending() {
     $.ajax = function () {
-        var xhr = 'stub';
+        const xhr = 'stub';
         return xhr;
     };
 
-    var warned;
-    blueslip.warn = function (msg) {
-        assert.equal(
-            msg,
-            'The length of pending_requests is over 50. Most likely they are not being correctly removed.'
-        );
-        warned = true;
-    };
-
+    blueslip.set_test_data('warn',
+                           'The length of pending_requests is over 50. ' +
+                           'Most likely they are not being correctly removed.');
     _.times(50, function () {
         channel.post({});
     });
-
-    assert(warned);
+    assert.equal(blueslip.get_test_logs('warn').length, 1);
+    blueslip.clear_test_data();
 }());
 
 (function test_xhr_error_message() {
-    var xhr = {
+    let xhr = {
         status: '200',
         responseText: 'does not matter',
     };
-    var msg = 'data added';
+    let msg = 'data added';
     assert.equal(channel.xhr_error_message(msg, xhr), 'data added');
 
     xhr = {

frontend_tests/node_tests/components.js

@@ -1,9 +1,12 @@
 set_global('i18n', global.stub_i18n);
 
+zrequire('keydown_util');
 zrequire('components');
 
-var LEFT_KEY = { which: 37 };
-var RIGHT_KEY = { which: 39 };
+var noop = function () {};
+
+var LEFT_KEY = { which: 37, preventDefault: noop };
+var RIGHT_KEY = { which: 39, preventDefault: noop };
 
 (function test_basics() {
     var keydown_f;
@@ -118,8 +121,10 @@ var RIGHT_KEY = { which: 39 };
         }
     });
 
-    var widget = components.toggle({
-        name: "info-overlay-toggle",
+    var callback_value;
+
+    var widget;
+    widget = components.toggle({
         selected: 0,
         values: [
             { label: i18n.t("Keyboard shortcuts"), key: "keyboard-shortcuts" },
@@ -129,6 +134,12 @@ var RIGHT_KEY = { which: 39 };
         callback: function (name, key) {
             assert.equal(callback_args, undefined);
             callback_args = [name, key];
+
+            // The subs code tries to get a widget value in the middle of a
+            // callback, which can lead to obscure bugs.
+            if (widget) {
+                callback_value = widget.value();
+            }
         },
     });
 
@@ -145,7 +156,7 @@ var RIGHT_KEY = { which: 39 };
 
     callback_args = undefined;
 
-    components.toggle.lookup("info-overlay-toggle").goto('markdown-help');
+    widget.goto('markdown-help');
     assert.equal(focused_tab, 1);
     assert.equal(tabs[0].class, 'first');
     assert.equal(tabs[1].class, 'middle selected');
@@ -162,6 +173,7 @@ var RIGHT_KEY = { which: 39 };
     assert.equal(tabs[2].class, 'last selected');
     assert.deepEqual(callback_args, ['translated: Search operators', 'search-operators']);
     assert.equal(widget.value(), 'translated: Search operators');
+    assert.equal(widget.value(), callback_value);
 
     // try to crash the key handler
     keydown_f.call(tabs[focused_tab], RIGHT_KEY);

frontend_tests/node_tests/compose.js

@@ -21,7 +21,9 @@ set_global('templates', {});
 
 var noop = function () {};
 
-set_global('blueslip', {});
+set_global('blueslip', global.make_zblueslip({
+    error: false, // Ignore errors. We only check for warnings in this module.
+}));
 set_global('drafts', {
     delete_draft_after_send: noop,
 });
@@ -43,6 +45,7 @@ set_global('notifications', {
     notify_above_composebox: noop,
     clear_compose_notifications: noop,
 });
+set_global('subs', {});
 
 // Setting these up so that we can test that links to uploads within messages are
 // automatically converted to server relative links.
@@ -101,8 +104,12 @@ people.add(bob);
 
     sub.subscribed = false;
     stream_data.add_sub('social', sub);
+    templates.render = function (template_name) {
+        assert.equal(template_name, 'compose_not_subscribed');
+        return 'compose_not_subscribed_stub';
+    };
     assert(!compose.validate_stream_message_address_info('social'));
-    assert.equal($('#compose-error-msg').html(), "translated: <p>You're not subscribed to the stream <b>social</b>.</p><p>Manage your subscriptions <a href='#streams/all'>on your Streams page</a>.</p>");
+    assert.equal($('#compose-error-msg').html(), 'compose_not_subscribed_stub');
 
     global.page_params.narrow_stream = false;
     channel.post = function (payload) {
@@ -121,7 +128,7 @@ people.add(bob);
         payload.success(payload.data);
     };
     assert(!compose.validate_stream_message_address_info('Frontend'));
-    assert.equal($('#compose-error-msg').html(), "translated: <p>You're not subscribed to the stream <b>Frontend</b>.</p><p>Manage your subscriptions <a href='#streams/all'>on your Streams page</a>.</p>");
+    assert.equal($('#compose-error-msg').html(), 'compose_not_subscribed_stub');
 
     channel.post = function (payload) {
         assert.equal(payload.data.stream, 'Frontend');
@@ -290,9 +297,6 @@ people.add(bob);
 }());
 
 (function test_markdown_shortcuts() {
-    blueslip.error = noop;
-    blueslip.log = noop;
-
     var queryCommandEnabled = true;
     var event = {
         keyCode: 66,
@@ -451,8 +455,6 @@ people.add(bob);
 }());
 
 (function test_send_message_success() {
-    blueslip.error = noop;
-    blueslip.log = noop;
     $("#compose-textarea").val('foobarfoobar');
     $("#compose-textarea").blur();
     $("#compose-send-status").show();
@@ -514,18 +516,18 @@ people.add(bob);
         };
         transmit.send_message = function (payload, success) {
             var single_msg = {
-              type: 'private',
-              content: '[foobar](/user_uploads/123456)',
-              sender_id: 101,
-              queue_id: undefined,
-              stream: '',
-              subject: '',
-              to: '["alice@example.com"]',
-              reply_to: 'alice@example.com',
-              private_message_recipient: 'alice@example.com',
-              to_user_ids: '31',
-              local_id: 1,
-              locally_echoed: true,
+                type: 'private',
+                content: '[foobar](/user_uploads/123456)',
+                sender_id: 101,
+                queue_id: undefined,
+                stream: '',
+                subject: '',
+                to: '["alice@example.com"]',
+                reply_to: 'alice@example.com',
+                private_message_recipient: 'alice@example.com',
+                to_user_ids: '31',
+                local_id: 1,
+                locally_echoed: true,
             };
             assert.deepEqual(payload, single_msg);
             payload.id = stub_state.local_id_counter;
@@ -622,7 +624,7 @@ people.add(bob);
         assert(!echo_error_msg_checked);
         assert.equal($("#compose-send-button").prop('disabled'), false);
         assert.equal($('#compose-error-msg').html(),
-                       'Error sending message: Server says 408');
+                     'Error sending message: Server says 408');
         assert.equal($("#compose-textarea").val(), 'foobarfoobar');
         assert($("#compose-textarea").is_focused());
         assert($("#compose-send-status").visible());
@@ -631,6 +633,8 @@ people.add(bob);
     }());
 }());
 
+set_global('document', 'document-stub');
+
 (function test_enter_with_preview_open() {
     // Test sending a message with content.
     compose_state.set_message_type('stream');
@@ -697,12 +701,10 @@ people.add(bob);
         };
 
         var compose_finished_event_checked = false;
-        $.stub_selector(document, {
-            trigger: function (e) {
-                assert.equal(e.name, 'compose_finished.zulip');
-                compose_finished_event_checked = true;
-            },
-        });
+        $(document).trigger = function (e) {
+            assert.equal(e.name, 'compose_finished.zulip');
+            compose_finished_event_checked = true;
+        };
         var send_message_called = false;
         compose.send_message = function () {
             send_message_called = true;
@@ -866,27 +868,27 @@ function test_raw_file_drop(raw_drop_func) {
     var keyup_handler_func = $(selector).get_on_handler('keyup');
 
     var set_focused_recipient_checked = false;
-    var update_faded_messages_checked = false;
+    var update_all_called = false;
 
     global.compose_fade = {
         set_focused_recipient: function (msg_type) {
             assert.equal(msg_type, 'private');
             set_focused_recipient_checked = true;
         },
-        update_faded_messages: function () {
-            update_faded_messages_checked = true;
+        update_all: function () {
+            update_all_called = true;
         },
     };
 
     compose_state.set_message_type(false);
     keyup_handler_func();
     assert(!set_focused_recipient_checked);
-    assert(!update_faded_messages_checked);
+    assert(!update_all_called);
 
     compose_state.set_message_type('private');
     keyup_handler_func();
     assert(set_focused_recipient_checked);
-    assert(update_faded_messages_checked);
+    assert(update_all_called);
 }());
 
 (function test_trigger_submit_compose_form() {
@@ -915,7 +917,7 @@ function test_raw_file_drop(raw_drop_func) {
 
         var data = {
             mentioned: {
-              email: 'foo@bar.com',
+                email: 'foo@bar.com',
             },
         };
 
@@ -942,10 +944,10 @@ function test_raw_file_drop(raw_drop_func) {
         var checks = [
             (function () {
                 var called;
-                compose_fade.would_receive_message = function (email) {
+                compose.needs_subscribe_warning = function (email) {
                     called = true;
                     assert.equal(email, 'foo@bar.com');
-                    return false;
+                    return true;
                 };
                 return function () { assert(called); };
             }()),
@@ -975,8 +977,8 @@ function test_raw_file_drop(raw_drop_func) {
 
         data = {
             mentioned: {
-              email: 'foo@bar.com',
-              full_name: 'Foo Barson',
+                email: 'foo@bar.com',
+                full_name: 'Foo Barson',
             },
         };
 
@@ -1033,7 +1035,7 @@ function test_raw_file_drop(raw_drop_func) {
 
     (function test_compose_all_everyone_confirm_clicked() {
         var handler = $("#compose-all-everyone")
-                      .get_on_handler('click', '.compose-all-everyone-confirm');
+            .get_on_handler('click', '.compose-all-everyone-confirm');
 
         setup_parents_and_mock_remove('compose-all-everyone',
                                       'compose-all-everyone',
@@ -1057,7 +1059,7 @@ function test_raw_file_drop(raw_drop_func) {
 
     (function test_compose_invite_users_clicked() {
         var handler = $("#compose_invite_users")
-                      .get_on_handler('click', '.compose_invite_link');
+            .get_on_handler('click', '.compose_invite_link');
         var subscription = {
             stream_id: 102,
             name: 'test',
@@ -1082,9 +1084,7 @@ function test_raw_file_drop(raw_drop_func) {
         assert(!container_removed);
 
         // !sub will result false here and we check the failure code path.
-        blueslip.warn = function (err_msg) {
-            assert.equal(err_msg, 'Stream no longer exists: no-stream');
-        };
+        blueslip.set_test_data('warn', 'Stream no longer exists: no-stream');
         $('#stream').val('no-stream');
         container.data = function (field) {
             assert.equal(field, 'useremail');
@@ -1101,6 +1101,8 @@ function test_raw_file_drop(raw_drop_func) {
         assert(target.attr('disabled'));
         assert(!invite_user_to_stream_called);
         assert(!container_removed);
+        assert.equal(blueslip.get_test_logs('warn').length, 1);
+        blueslip.clear_test_data();
 
         // !sub will result in true here and we check the success code path.
         stream_data.add_sub('test', subscription);
@@ -1122,7 +1124,7 @@ function test_raw_file_drop(raw_drop_func) {
 
     (function test_compose_invite_close_clicked() {
         var handler = $("#compose_invite_users")
-                        .get_on_handler('click', '.compose_invite_close');
+            .get_on_handler('click', '.compose_invite_close');
 
         setup_parents_and_mock_remove('compose_invite_users_close',
                                       'compose_invite_close',
@@ -1142,6 +1144,51 @@ function test_raw_file_drop(raw_drop_func) {
         assert(!$("#compose_invite_users").visible());
     }());
 
+    (function test_compose_not_subscribed_clicked() {
+        var handler = $("#compose-send-status")
+            .get_on_handler('click', '.sub_unsub_button');
+        var subscription = {
+            stream_id: 102,
+            name: 'test',
+            subscribed: false,
+        };
+        var compose_not_subscribed_called = false;
+        subs.sub_or_unsub = function () {
+            compose_not_subscribed_called = true;
+        };
+
+        setup_parents_and_mock_remove('compose-send-status',
+                                      'sub_unsub_button',
+                                      '.compose_not_subscribed');
+
+        handler(event);
+
+        assert(compose_not_subscribed_called);
+
+        stream_data.add_sub('test', subscription);
+        $('#stream').val('test');
+        $("#compose-send-status").show();
+
+        handler(event);
+
+        assert(!$("#compose-send-status").visible());
+    }());
+
+    (function test_compose_not_subscribed_close_clicked() {
+        var handler = $("#compose-send-status")
+            .get_on_handler('click', '#compose_not_subscribed_close');
+
+        setup_parents_and_mock_remove('compose_user_not_subscribed_close',
+                                      'compose_not_subscribed_close',
+                                      '.compose_not_subscribed');
+
+        $("#compose-send-status").show();
+
+        handler(event);
+
+        assert(!$("#compose-send-status").visible());
+    }());
+
     event = {
         preventDefault: noop,
     };
@@ -1175,33 +1222,33 @@ function test_raw_file_drop(raw_drop_func) {
         compose_state.set_message_type('stream');
 
         var checks = [
-           (function () {
-               var called;
-               templates.render = function (template_name, context) {
-                   called = true;
-                   assert.equal(template_name, 'compose_private_stream_alert');
-                   assert.equal(context.stream_name, 'Denmark');
-                   return 'fake-compose_private_stream_alert-template';
-               };
-               return function () { assert(called); };
-           }()),
+            (function () {
+                var called;
+                templates.render = function (template_name, context) {
+                    called = true;
+                    assert.equal(template_name, 'compose_private_stream_alert');
+                    assert.equal(context.stream_name, 'Denmark');
+                    return 'fake-compose_private_stream_alert-template';
+                };
+                return function () { assert(called); };
+            }()),
 
-           (function () {
-              var called;
-              $("#compose_private_stream_alert").append = function (html) {
-                  called = true;
-                  assert.equal(html, 'fake-compose_private_stream_alert-template');
-              };
-              return function () { assert(called); };
-           }()),
+            (function () {
+                var called;
+                $("#compose_private_stream_alert").append = function (html) {
+                    called = true;
+                    assert.equal(html, 'fake-compose_private_stream_alert-template');
+                };
+                return function () { assert(called); };
+            }()),
         ];
 
         data = {
-           stream: {
-               invite_only: true,
-               name: 'Denmark',
-               subscribers: Dict.from_array([1]),
-           },
+            stream: {
+                invite_only: true,
+                name: 'Denmark',
+                subscribers: Dict.from_array([1]),
+            },
         };
 
         handler({}, data);
@@ -1213,7 +1260,7 @@ function test_raw_file_drop(raw_drop_func) {
 
     (function test_attach_files_compose_clicked() {
         var handler = $("#compose")
-                        .get_on_handler("click", "#attach_files");
+            .get_on_handler("click", "#attach_files");
         $('#file_input').clone = function (param) {
             assert(param);
         };
@@ -1287,7 +1334,7 @@ function test_raw_file_drop(raw_drop_func) {
         function test_post_error(error_callback) {
             error_callback();
             assert.equal($("#preview_content").html(),
-                            'translated: Failed to generate preview');
+                         'translated: Failed to generate preview');
         }
 
         function mock_channel_post(msg) {
@@ -1317,7 +1364,7 @@ function test_raw_file_drop(raw_drop_func) {
         }
 
         var handler = $("#compose")
-                        .get_on_handler("click", "#markdown_preview");
+            .get_on_handler("click", "#markdown_preview");
 
         // Tests start here
         $("#compose-textarea").val('');
@@ -1326,7 +1373,7 @@ function test_raw_file_drop(raw_drop_func) {
         handler(event);
 
         assert.equal($("#preview_content").html(),
-                      'translated: Nothing to preview');
+                     'translated: Nothing to preview');
         assert_visibilities();
 
         var make_indicator_called = false;
@@ -1362,12 +1409,12 @@ function test_raw_file_drop(raw_drop_func) {
         assert(apply_markdown_called);
         assert_visibilities();
         assert.equal($("#preview_content").html(),
-                      'Server: foobarfoobar');
+                     'Server: foobarfoobar');
     }());
 
     (function test_undo_markdown_preview_clicked() {
         var handler = $("#compose")
-                        .get_on_handler("click", "#undo_markdown_preview");
+            .get_on_handler("click", "#undo_markdown_preview");
 
         $("#compose-textarea").hide();
         $("#undo_markdown_preview").show();

frontend_tests/node_tests/compose_actions.js

@@ -3,17 +3,13 @@ var return_false = function () { return false; };
 var return_true = function () { return true; };
 
 set_global('document', {
-    location: {
-    },
+    location: {}, // we need this to load compose.js
 });
 
 set_global('page_params', {
     use_websockets: false,
 });
 
-set_global('$', function () {
-});
-
 set_global('$', global.make_zjquery());
 
 set_global('compose_pm_pill', {
@@ -26,6 +22,8 @@ zrequire('util');
 zrequire('compose_state');
 zrequire('compose_actions');
 
+set_global('document', 'document-stub');
+
 var start = compose_actions.start;
 var cancel = compose_actions.cancel;
 var get_focus_area = compose_actions._get_focus_area;
@@ -71,7 +69,7 @@ set_global('narrow_state', {
 });
 
 set_global('unread_ops', {
-    mark_message_as_read: noop,
+    notify_server_message_read: noop,
 });
 
 set_global('common', {

frontend_tests/node_tests/compose_fade.js

@@ -1,5 +1,3 @@
-set_global('$', function () {
-});
 set_global('blueslip', {});
 global.blueslip.warn = function () {};
 
@@ -26,11 +24,11 @@ var bob = {
     full_name: 'Bob',
 };
 
-people.add(me);
+people.add_in_realm(me);
 people.initialize_current_user(me.user_id);
 
-people.add(alice);
-people.add(bob);
+people.add_in_realm(alice);
+people.add_in_realm(bob);
 
 
 (function test_set_focused_recipient() {
@@ -62,9 +60,10 @@ people.add(bob);
 
     compose_fade.set_focused_recipient('stream');
 
-    assert(compose_fade.would_receive_message('me@example.com'));
-    assert(compose_fade.would_receive_message('alice@example.com'));
-    assert(!compose_fade.would_receive_message('bob@example.com'));
+    assert.equal(compose_fade.would_receive_message('me@example.com'), true);
+    assert.equal(compose_fade.would_receive_message('alice@example.com'), true);
+    assert.equal(compose_fade.would_receive_message('bob@example.com'), false);
+    assert.equal(compose_fade.would_receive_message('nonrealmuser@example.com'), true);
 
     var good_msg = {
         type: 'stream',

frontend_tests/node_tests/compose_pm_pill.js

@@ -0,0 +1,144 @@
+zrequire('compose_pm_pill');
+zrequire('input_pill');
+zrequire('user_pill');
+
+set_global('$', global.make_zjquery());
+set_global('people', {});
+var pills = {
+    pill: {},
+};
+
+(function test_pills() {
+    var othello = {
+        user_id: 1,
+        email: 'othello@example.com',
+        full_name: 'Othello',
+    };
+
+    var iago = {
+        email: 'iago@zulip.com',
+        user_id: 2,
+        full_name: 'Iago',
+    };
+
+    var hamlet = {
+        email: 'hamlet@example.com',
+        user_id: 3,
+        full_name: 'Hamlet',
+    };
+
+    people.get_realm_persons = function () {
+        return [iago, othello, hamlet];
+    };
+
+    var recipient_stub = $("#private_message_recipient");
+    var pill_container_stub = $('.pill-container[data-before="You and"]');
+    recipient_stub.set_parent(pill_container_stub);
+    var create_item_handler;
+
+    var all_pills = {};
+
+    pills.appendValidatedData = function (item) {
+        var id = item.user_id;
+        assert.equal(all_pills[id], undefined);
+        all_pills[id] = item;
+    };
+    pills.items = function () {
+        return _.values(all_pills);
+    };
+
+    var text_cleared;
+    pills.clear_text = function () {
+        text_cleared = true;
+    };
+
+    var pills_cleared;
+    pills.clear = function () {
+        pills_cleared = true;
+        pills = {
+            pill: {},
+        };
+        all_pills= {};
+    };
+
+    var appendValue_called;
+    pills.appendValue = function (value) {
+        appendValue_called = true;
+        assert.equal(value, 'othello@example.com');
+        this.appendValidatedData(othello);
+    };
+
+    var get_by_email_called = false;
+    people.get_by_email = function (user_email) {
+        get_by_email_called = true;
+        if (user_email === iago.email) {
+            return iago;
+        }
+        if (user_email === othello.email) {
+            return othello;
+        }
+    };
+
+    var get_person_from_user_id_called = false;
+    people.get_person_from_user_id = function (id) {
+        get_person_from_user_id_called = true;
+        if (id === othello.user_id) {
+            return othello;
+        }
+        assert.equal(id, 3);
+        return hamlet;
+    };
+
+    function test_create_item(handler) {
+        (function test_rejection_path() {
+            var item = handler(othello.email, pills.items());
+            assert(get_by_email_called);
+            assert.equal(item, undefined);
+        }());
+
+        (function test_success_path() {
+            get_by_email_called = false;
+            var res = handler(iago.email, pills.items());
+            assert(get_by_email_called);
+            assert.equal(typeof(res), 'object');
+            assert.equal(res.user_id, iago.user_id);
+            assert.equal(res.display_value, iago.full_name);
+        }());
+    }
+
+    function input_pill_stub(opts) {
+        assert.equal(opts.container, pill_container_stub);
+        create_item_handler = opts.create_item_from_text;
+        assert(create_item_handler);
+        return pills;
+    }
+
+    set_global('input_pill', {
+        create: input_pill_stub,
+    });
+
+    compose_pm_pill.initialize();
+    assert(compose_pm_pill.my_pill);
+
+    compose_pm_pill.set_from_typeahead(othello);
+    compose_pm_pill.set_from_typeahead(hamlet);
+
+    var user_ids = compose_pm_pill.get_user_ids();
+    assert.deepEqual(user_ids, [othello.user_id, hamlet.user_id]);
+
+    var emails = compose_pm_pill.get_emails();
+    assert.equal(emails, 'othello@example.com,hamlet@example.com');
+
+    var items = compose_pm_pill.get_typeahead_items();
+    assert.deepEqual(items, [{email: 'iago@zulip.com', user_id: 2, full_name: 'Iago'}]);
+
+    test_create_item(create_item_handler);
+
+    compose_pm_pill.set_from_emails('othello@example.com');
+    assert(compose_pm_pill.my_pill);
+
+    assert(get_person_from_user_id_called);
+    assert(pills_cleared);
+    assert(appendValue_called);
+    assert(text_cleared);
+}());

frontend_tests/node_tests/composebox_typeahead.js

@@ -1,6 +1,8 @@
+set_global('i18n', global.stub_i18n);
 zrequire('compose_state');
 zrequire('ui_util');
 zrequire('pm_conversations');
+zrequire('emoji_picker');
 zrequire('util');
 zrequire('Handlebars', 'handlebars');
 zrequire('templates');
@@ -11,6 +13,9 @@ zrequire('stream_data');
 zrequire('user_pill');
 zrequire('compose_pm_pill');
 zrequire('composebox_typeahead');
+set_global('md5', function (s) {
+    return 'md5-' + s;
+});
 
 var ct = composebox_typeahead;
 var noop = function () {};
@@ -18,34 +23,57 @@ var noop = function () {};
 var emoji_stadium = {
     emoji_name: 'stadium',
     emoji_url: 'TBD',
+    codepoint: '1f3df',
 };
 var emoji_tada = {
     emoji_name: 'tada',
     emoji_url: 'TBD',
+    codepoint: '1f389',
 };
 var emoji_moneybag = {
     emoji_name: 'moneybag',
     emoji_url: 'TBD',
+    codepoint: '1f4b0',
 };
 var emoji_japanese_post_office = {
     emoji_name: 'japanese_post_office',
     emoji_url: 'TBD',
+    codepoint: '1f3e3',
 };
 var emoji_panda_face = {
     emoji_name: 'panda_face',
     emoji_url: 'TBD',
+    codepoint: '1f43c',
 };
 var emoji_see_no_evil = {
     emoji_name: 'see_no_evil',
     emoji_url: 'TBD',
+    codepoint: '1f648',
 };
 var emoji_thumbs_up = {
-    emoji_name: '+1',
+    emoji_name: 'thumbs_up',
     emoji_url: 'TBD',
+    codepoint: '1f44d',
+};
+var emoji_thermometer = {
+    emoji_name: 'thermometer',
+    emoji_url: 'TBD',
+    codepoint: '1f321',
+};
+var emoji_heart = {
+    emoji_name: 'heart',
+    emoji_url: 'TBD',
+    codepoint: '2764',
+};
+var emoji_headphones = {
+    emoji_name: 'headphones',
+    emoji_url: 'TBD',
+    codepoint: '1f3a7',
 };
 
 var emoji_list = [emoji_tada, emoji_moneybag, emoji_stadium, emoji_japanese_post_office,
-                  emoji_panda_face, emoji_see_no_evil, emoji_thumbs_up];
+                  emoji_panda_face, emoji_see_no_evil, emoji_thumbs_up, emoji_thermometer,
+                  emoji_heart, emoji_headphones];
 var stream_list = ['Denmark', 'Sweden', 'The Netherlands'];
 var sweden_stream = {
     name: 'Sweden',
@@ -218,6 +246,8 @@ user_pill.get_user_ids = function () {
     expected_value = '{ :octopus: ';
     assert.equal(actual_value, expected_value);
 
+
+
     // mention
     fake_this.completing = 'mention';
     var document_stub_trigger1_called = false;
@@ -339,8 +369,10 @@ user_pill.get_user_ids = function () {
         assert.deepEqual(actual_value, expected_value);
 
         // options.highlighter()
-        options.query = 'De';  // Beginning of "Denmark", one of the streams
-                               // provided in stream_list through .source().
+
+        // Beginning of "Denmark", one of the streams
+        // provided in stream_list through .source().
+        options.query = 'De';
         actual_value = options.highlighter('Denmark');
         expected_value = '<strong>Denmark</strong>';
         assert.equal(actual_value, expected_value);
@@ -446,17 +478,17 @@ user_pill.get_user_ids = function () {
         // corresponding parts in bold.
         options.query = 'oth';
         actual_value = options.highlighter(othello);
-        expected_value = '<strong>Othello, the Moor of Venice</strong>&nbsp;&nbsp;\n<small class="autocomplete_secondary">othello@zulip.com</small>\n';
+        expected_value = '        <img class="typeahead-image" src="https://secure.gravatar.com/avatar/md5-othello@zulip.com?d&#x3D;identicon&amp;s&#x3D;50" />\n<strong>Othello, the Moor of Venice</strong>&nbsp;&nbsp;\n<small class="autocomplete_secondary">othello@zulip.com</small>\n';
         assert.equal(actual_value, expected_value);
 
         options.query = 'Lear';
         actual_value = options.highlighter(cordelia);
-        expected_value = '<strong>Cordelia Lear</strong>&nbsp;&nbsp;\n<small class="autocomplete_secondary">cordelia@zulip.com</small>\n';
+        expected_value = '        <img class="typeahead-image" src="https://secure.gravatar.com/avatar/md5-cordelia@zulip.com?d&#x3D;identicon&amp;s&#x3D;50" />\n<strong>Cordelia Lear</strong>&nbsp;&nbsp;\n<small class="autocomplete_secondary">cordelia@zulip.com</small>\n';
         assert.equal(actual_value, expected_value);
 
         options.query = 'othello@zulip.com, co';
         actual_value = options.highlighter(cordelia);
-        expected_value = '<strong>Cordelia Lear</strong>&nbsp;&nbsp;\n<small class="autocomplete_secondary">cordelia@zulip.com</small>\n';
+        expected_value = '        <img class="typeahead-image" src="https://secure.gravatar.com/avatar/md5-cordelia@zulip.com?d&#x3D;identicon&amp;s&#x3D;50" />\n<strong>Cordelia Lear</strong>&nbsp;&nbsp;\n<small class="autocomplete_secondary">cordelia@zulip.com</small>\n';
         assert.equal(actual_value, expected_value);
 
         // options.matcher()
@@ -561,12 +593,12 @@ user_pill.get_user_ids = function () {
         // content_highlighter.
         fake_this = { completing: 'mention', token: 'othello' };
         actual_value = options.highlighter.call(fake_this, othello);
-        expected_value = '<strong>Othello, the Moor of Venice</strong>&nbsp;&nbsp;\n<small class="autocomplete_secondary">othello@zulip.com</small>\n';
+        expected_value = '        <img class="typeahead-image" src="https://secure.gravatar.com/avatar/md5-othello@zulip.com?d&#x3D;identicon&amp;s&#x3D;50" />\n<strong>Othello, the Moor of Venice</strong>&nbsp;&nbsp;\n<small class="autocomplete_secondary">othello@zulip.com</small>\n';
         assert.equal(actual_value, expected_value);
 
         fake_this = { completing: 'mention', token: 'hamletcharacters' };
         actual_value = options.highlighter.call(fake_this, hamletcharacters);
-        expected_value = '<strong>hamletcharacters</strong>&nbsp;&nbsp;\n<small class="autocomplete_secondary">Characters of Hamlet</small>\n';
+        expected_value = '        <i class="typeahead-image icon icon-vector-group"></i>\n<strong>hamletcharacters</strong>&nbsp;&nbsp;\n<small class="autocomplete_secondary">Characters of Hamlet</small>\n';
         assert.equal(actual_value, expected_value);
 
         // options.matcher()
@@ -599,6 +631,16 @@ user_pill.get_user_ids = function () {
         expected_value = [emoji_tada, emoji_stadium];
         assert.deepEqual(actual_value, expected_value);
 
+        fake_this = { completing: 'emoji', token: 'th' };
+        actual_value = options.sorter.call(fake_this, [emoji_thermometer, emoji_thumbs_up]);
+        expected_value = [emoji_thumbs_up, emoji_thermometer];
+        assert.deepEqual(actual_value, expected_value);
+
+        fake_this = { completing: 'emoji', token: 'he' };
+        actual_value = options.sorter.call(fake_this, [emoji_headphones, emoji_heart]);
+        expected_value = [emoji_heart, emoji_headphones];
+        assert.deepEqual(actual_value, expected_value);
+
         fake_this = { completing: 'mention', token: 'co' };
         actual_value = options.sorter.call(fake_this, [othello, cordelia]);
         expected_value = [cordelia, othello];
@@ -711,9 +753,9 @@ user_pill.get_user_ids = function () {
         pm_recipient_blur_called = true;
     };
 
-    page_params.enter_sends = false;  // We manually specify it the first
-                                      // time because the click_func
-                                      // doesn't exist yet.
+    page_params.enter_sends = false;
+    // We manually specify it the first time because the click_func
+    // doesn't exist yet.
     $("#stream").select(noop);
     $("#subject").select(noop);
     $("#private_message_recipient").select(noop);
@@ -892,20 +934,14 @@ user_pill.get_user_ids = function () {
         assert.deepEqual(returned, reference);
     }
 
-    var all_items = [
-        {
-            special_item_text: 'all (Notify everyone)',
-            email: 'all',
+    var all_items = _.map(['all', 'everyone', 'stream'], function (mention) {
+        return {
+            special_item_text: 'translated: ' + mention +" (Notify stream)",
+            email: mention,
             pm_recipient_count: Infinity,
-            full_name: 'all',
-        },
-        {
-            special_item_text: 'everyone (Notify everyone)',
-            email: 'everyone',
-            pm_recipient_count: Infinity,
-            full_name: 'everyone',
-        },
-    ];
+            full_name: mention,
+        };
+    });
 
     var people_with_all = global.people.get_realm_persons().concat(all_items);
     var all_mentions = people_with_all.concat(global.user_groups.get_realm_user_groups());
@@ -1090,20 +1126,14 @@ user_pill.get_user_ids = function () {
 }());
 
 (function test_typeahead_results() {
-    var all_items = [
-        {
-            special_item_text: 'all (Notify everyone)',
-            email: 'all',
+    var all_items = _.map(['all', 'everyone', 'stream'], function (mention) {
+        return {
+            special_item_text: 'translated: ' + mention +" (Notify stream)",
+            email: mention,
             pm_recipient_count: Infinity,
-            full_name: 'all',
-        },
-        {
-            special_item_text: 'everyone (Notify everyone)',
-            email: 'everyone',
-            pm_recipient_count: Infinity,
-            full_name: 'everyone',
-        },
-    ];
+            full_name: mention,
+        };
+    });
     var people_with_all = global.people.get_realm_persons().concat(all_items);
     var all_mentions = people_with_all.concat(global.user_groups.get_realm_user_groups());
     var stream_list = [denmark_stream, sweden_stream, netherland_stream];
@@ -1135,14 +1165,14 @@ user_pill.get_user_ids = function () {
         assert.deepEqual(returned, expected);
     }
 
-    assert_emoji_matches('da',[{emoji_name: "tada", emoji_url: "TBD"},
-        {emoji_name: "panda_face", emoji_url: "TBD"}]);
+    assert_emoji_matches('da',[{emoji_name: "tada", emoji_url: "TBD", codepoint: "1f389"},
+                               {emoji_name: "panda_face", emoji_url: "TBD", codepoint: "1f43c"}]);
     assert_emoji_matches('da_', []);
     assert_emoji_matches('da ', []);
-    assert_emoji_matches('panda ', [{emoji_name: "panda_face", emoji_url: "TBD"}]);
-    assert_emoji_matches('panda_', [{emoji_name: "panda_face", emoji_url: "TBD"}]);
-    assert_emoji_matches('japanese_post_', [{emoji_name: "japanese_post_office", emoji_url: "TBD"}]);
-    assert_emoji_matches('japanese post ', [{emoji_name: "japanese_post_office", emoji_url: "TBD"}]);
+    assert_emoji_matches('panda ', [{emoji_name: "panda_face", emoji_url: "TBD", codepoint: "1f43c"}]);
+    assert_emoji_matches('panda_', [{emoji_name: "panda_face", emoji_url: "TBD", codepoint: "1f43c"}]);
+    assert_emoji_matches('japanese_post_', [{emoji_name: "japanese_post_office", emoji_url: "TBD", codepoint: "1f3e3"}]);
+    assert_emoji_matches('japanese post ', [{emoji_name: "japanese_post_office", emoji_url: "TBD", codepoint: "1f3e3"}]);
     assert_emoji_matches('notaemoji', []);
     // Autocomplete user mentions by user name.
     assert_mentions_matches('cordelia', [cordelia]);

frontend_tests/node_tests/copy_and_paste.js

@@ -1,5 +1,9 @@
 global.stub_out_jquery();
 
+set_global('page_params', {
+    development: true,
+});
+
 var jsdom = require("jsdom");
 global.document = jsdom.jsdom('<!DOCTYPE html><p>Hello world</p>');
 var window = jsdom.jsdom().defaultView;
@@ -37,4 +41,12 @@ var copy_and_paste = zrequire('copy_and_paste');
     input = '<meta http-equiv="content-type" content="text/html; charset=utf-8"><i style="box-sizing: inherit; color: rgb(0, 0, 0); font-family: Verdana, sans-serif; font-size: 15px; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: start; text-indent: 0px; text-transform: none; white-space: normal; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; background-color: rgb(255, 255, 255); text-decoration-style: initial; text-decoration-color: initial;">This text is italic</i>';
     assert.equal(copy_and_paste.paste_handler_converter(input),
                  '*This text is italic*');
+
+    input = '<div class="preview-content"><div class="comment"><div class="comment-body markdown-body js-preview-body" style="min-height: 131px;"><p>Test List:</p><ul><li>Item 1</li><li>Item 2</li></ul></div></div></div>';
+    assert.equal(copy_and_paste.paste_handler_converter(input),
+                 'Test List:\n*   Item 1\n*   Item 2');
+
+    input = '<div class="ace-line gutter-author-d-iz88z86z86za0dz67zz78zz78zz74zz68zjz80zz71z9iz90za3z66zs0z65zz65zq8z75zlaz81zcz66zj6g2mz78zz76zmz66z22z75zfcz69zz66z ace-ltr focused-line" dir="auto" id="editor-3-ace-line-41"><span>Test List:</span></div><div class="ace-line gutter-author-d-iz88z86z86za0dz67zz78zz78zz74zz68zjz80zz71z9iz90za3z66zs0z65zz65zq8z75zlaz81zcz66zj6g2mz78zz76zmz66z22z75zfcz69zz66z line-list-type-bullet ace-ltr" dir="auto" id="editor-3-ace-line-42"><ul class="listtype-bullet listindent1 list-bullet1"><li><span class="ace-line-pocket-zws" data-faketext="" data-contentcollector-ignore-space-at="end"></span><span class="ace-line-pocket" data-faketext="" contenteditable="false"></span><span class="ace-line-pocket-zws" data-faketext="" data-contentcollector-ignore-space-at="start"></span><span>Item 1</span></li></ul></div><div class="ace-line gutter-author-d-iz88z86z86za0dz67zz78zz78zz74zz68zjz80zz71z9iz90za3z66zs0z65zz65zq8z75zlaz81zcz66zj6g2mz78zz76zmz66z22z75zfcz69zz66z line-list-type-bullet ace-ltr" dir="auto" id="editor-3-ace-line-43"><ul class="listtype-bullet listindent1 list-bullet1"><li><span class="ace-line-pocket-zws" data-faketext="" data-contentcollector-ignore-space-at="end"></span><span class="ace-line-pocket" data-faketext="" contenteditable="false"></span><span class="ace-line-pocket-zws" data-faketext="" data-contentcollector-ignore-space-at="start"></span><span>Item 2</span></li></ul></div>';
+    assert.equal(copy_and_paste.paste_handler_converter(input),
+                 'Test List:\n*   Item 1\n*   Item 2');
 }());

frontend_tests/node_tests/dict.js

@@ -1,4 +1,4 @@
-set_global('blueslip', {});
+set_global('blueslip', global.make_zblueslip());
 
 (function test_basic() {
     var d = new Dict();
@@ -57,9 +57,7 @@ set_global('blueslip', {});
 }());
 
 (function test_undefined_keys() {
-    global.blueslip.error = function (msg) {
-        assert.equal(msg, "Tried to call a Dict method with an undefined key.");
-    };
+    blueslip.set_test_data('error', 'Tried to call a Dict method with an undefined key.');
 
     var d = new Dict();
 
@@ -70,6 +68,9 @@ set_global('blueslip', {});
 
     assert.equal(d.has(undefined), false);
     assert.strictEqual(d.get(undefined), undefined);
+    assert.equal(blueslip.get_test_logs('error').length, 4);
+
+    blueslip.clear_test_data();
 }());
 
 (function test_restricted_keys() {

frontend_tests/node_tests/dispatch.js

@@ -360,6 +360,7 @@ var event_fixtures = {
                 name: 'devel',
                 stream_id: 42,
                 subscribers: ['alice@example.com', 'bob@example.com'],
+                email_address: 'devel+0138515295f4@zulipdev.com:9991',
                 // etc.
             },
         ],
@@ -491,6 +492,36 @@ var event_fixtures = {
             {id: 2, name: 'hobbies', type: 1},
         ],
     },
+    user_group__add: {
+        type: 'user_group',
+        op: 'add',
+        group: {
+            name: 'Mobile',
+            id: '1',
+            members: [1],
+        },
+    },
+    user_group__add_members: {
+        type: 'user_group',
+        op: 'add_members',
+        group_id: 1,
+        user_ids: [2],
+    },
+    user_group__remove_members: {
+        type: 'user_group',
+        op: 'remove_members',
+        group_id: 3,
+        user_ids: [99, 100],
+    },
+    user_group__update: {
+        type: 'user_group',
+        op: 'update',
+        group_id: 3,
+        data: {
+            name: 'Frontend',
+            description: 'All Frontend people',
+        },
+    },
 };
 
 function assert_same(actual, expected) {
@@ -511,6 +542,46 @@ with_overrides(function (override) {
 
 });
 
+with_overrides(function (override) {
+    // User groups
+    var event = event_fixtures.user_group__add;
+    override('settings_user_groups.reload', noop);
+    global.with_stub(function (stub) {
+        override('user_groups.add', stub.f);
+        dispatch(event);
+        var args = stub.get_args('group');
+        assert_same(args.group, event.group);
+    });
+
+    event = event_fixtures.user_group__add_members;
+    global.with_stub(function (stub) {
+        override('user_groups.add_members', stub.f);
+        dispatch(event);
+        var args = stub.get_args('group_id', 'user_ids');
+        assert_same(args.group_id, event.group_id);
+        assert_same(args.user_ids, event.user_ids);
+    });
+
+    event = event_fixtures.user_group__remove_members;
+    global.with_stub(function (stub) {
+        override('user_groups.remove_members', stub.f);
+        dispatch(event);
+        var args = stub.get_args('group_id', 'user_ids');
+        assert_same(args.group_id, event.group_id);
+        assert_same(args.user_ids, event.user_ids);
+    });
+
+    event = event_fixtures.user_group__update;
+    global.with_stub(function (stub) {
+        override('user_groups.update', stub.f);
+        dispatch(event);
+        var args = stub.get_args('event');
+        assert_same(args.event.group_id, event.group_id);
+        assert_same(args.event.data.name, event.data.name);
+        assert_same(args.event.data.description, event.data.description);
+    });
+});
+
 with_overrides(function (override) {
     // custom profile fields
     var event = event_fixtures.custom_profile_fields;
@@ -749,6 +820,7 @@ with_overrides(function (override) {
     event = event_fixtures.realm_user__remove;
     global.with_stub(function (stub) {
         override('people.deactivate', stub.f);
+        override('stream_data.remove_deactivated_user_from_all_streams', noop);
         dispatch(event);
         var args = stub.get_args('person');
         assert_same(args.person, event.person);
@@ -801,15 +873,21 @@ with_overrides(function (override) {
     });
 
     var event = event_fixtures.subscription__add;
-    global.with_stub(function (stub) {
-        override('stream_data.get_sub_by_id', function (stream_id) {
-            return {stream_id: stream_id};
+    global.with_stub(function (subscription_stub) {
+        global.with_stub(function (stream_email_stub) {
+            override('stream_data.get_sub_by_id', function (stream_id) {
+                return {stream_id: stream_id};
+            });
+            override('stream_events.mark_subscribed', subscription_stub.f);
+            override('stream_data.update_stream_email_address', stream_email_stub.f);
+            dispatch(event);
+            var args = subscription_stub.get_args('sub', 'subscribers');
+            assert_same(args.sub.stream_id, event.subscriptions[0].stream_id);
+            assert_same(args.subscribers, event.subscriptions[0].subscribers);
+            args = stream_email_stub.get_args('sub', 'email_address');
+            assert_same(args.email_address, event.subscriptions[0].email_address);
+            assert_same(args.sub.stream_id, event.subscriptions[0].stream_id);
         });
-        override('stream_events.mark_subscribed', stub.f);
-        dispatch(event);
-        var args = stub.get_args('sub', 'subscribers');
-        assert_same(args.sub.stream_id, event.subscriptions[0].stream_id);
-        assert_same(args.subscribers, event.subscriptions[0].subscribers);
     });
 
     event = event_fixtures.subscription__peer_add;
@@ -935,7 +1013,7 @@ with_overrides(function (override) {
     });
 });
 
-// mark_message_as_read requires message_store and these dependencies.
+// notify_server_message_read requires message_store and these dependencies.
 zrequire('unread_ops');
 zrequire('unread');
 zrequire('topic_data');

frontend_tests/node_tests/drafts.js

@@ -95,33 +95,33 @@ var draft_2 = {
 
     localStorage.clear();
     (function test_addDraft() {
-         stub_timestamp(1, function () {
-             var expected = _.clone(draft_1);
-             expected.updatedAt = 1;
-             var id = draft_model.addDraft(_.clone(draft_1));
+        stub_timestamp(1, function () {
+            var expected = _.clone(draft_1);
+            expected.updatedAt = 1;
+            var id = draft_model.addDraft(_.clone(draft_1));
 
-             assert.deepEqual(ls.get("drafts")[id], expected);
-         });
+            assert.deepEqual(ls.get("drafts")[id], expected);
+        });
     }());
 
     localStorage.clear();
     (function test_editDraft() {
-         stub_timestamp(2, function () {
-             ls.set("drafts", { id1: draft_1 });
-             var expected = _.clone(draft_2);
-             expected.updatedAt = 2;
-             draft_model.editDraft("id1", _.clone(draft_2));
+        stub_timestamp(2, function () {
+            ls.set("drafts", { id1: draft_1 });
+            var expected = _.clone(draft_2);
+            expected.updatedAt = 2;
+            draft_model.editDraft("id1", _.clone(draft_2));
 
-             assert.deepEqual(ls.get("drafts").id1, expected);
-         });
+            assert.deepEqual(ls.get("drafts").id1, expected);
+        });
     }());
 
     localStorage.clear();
     (function test_deleteDraft() {
-         ls.set("drafts", { id1: draft_1 });
-         draft_model.deleteDraft("id1");
+        ls.set("drafts", { id1: draft_1 });
+        draft_model.deleteDraft("id1");
 
-         assert.deepEqual(ls.get("drafts"), {});
+        assert.deepEqual(ls.get("drafts"), {});
     }());
 }());
 

frontend_tests/node_tests/emoji.js

@@ -3,6 +3,7 @@ set_global('page_params', {
     emojiset: 'google',
 });
 set_global('upload_widget', {});
+set_global('blueslip', global.make_zblueslip());
 
 zrequire('emoji_codes', 'generated/emoji/emoji_codes');
 zrequire('emoji');
@@ -66,15 +67,10 @@ zrequire('util');
     canonical_name = emoji.get_canonical_name('+1');
     assert.equal(canonical_name, '+1');
 
-    var errored = false;
-    set_global('blueslip', {
-        error: function (error) {
-            assert.equal(error, "Invalid emoji name: non_existent");
-            errored = true;
-        },
-    });
+    blueslip.set_test_data('error', 'Invalid emoji name: non_existent');
     emoji.get_canonical_name('non_existent');
-    assert(errored);
+    assert.equal(blueslip.get_test_logs('error').length, 1);
+    blueslip.clear_test_data();
 }());
 
 (function test_translate_emoticons_to_names() {
@@ -99,15 +95,16 @@ zrequire('util');
         {name: 'between symbols', original: 'Hello.<original>! World.', expected: 'Hello.<original>! World.'},
         {name: 'before end of sentence', original: 'Hello <original>!', expected: 'Hello <converted>!'},
     ];
-    Object.keys(emoji.EMOTICON_CONVERSIONS).forEach(key => {
-        testcases.forEach(t => {
-            var converted_value = `:${emoji.EMOTICON_CONVERSIONS[key]}:`;
-            t = Object.assign({}, t); // circumvent copy by reference.
-            t.original = t.original.replace(/(<original>)/g, key);
-            t.expected = t.expected.replace(/(<original>)/g, key);
-            t.expected = t.expected.replace(/(<converted>)/g, converted_value);
-            var result = emoji.translate_emoticons_to_names(t.original);
-            assert.equal(result, t.expected);
+    _.each(emoji.EMOTICON_CONVERSIONS, (full_name, shortcut) => {
+        _.each(testcases, (t) => {
+            var converted_value = ':' + full_name + ':';
+            var original = t.original;
+            var expected = t.expected;
+            original = original.replace(/(<original>)/g, shortcut);
+            expected = expected.replace(/(<original>)/g, shortcut)
+                .replace(/(<converted>)/g, converted_value);
+            var result = emoji.translate_emoticons_to_names(original);
+            assert.equal(result, expected);
         });
     });
 }());

frontend_tests/node_tests/fetch_status.js

@@ -22,6 +22,14 @@ function blocked_older() {
     assert.equal(fetch_status.can_load_older_messages(), false);
 }
 
+function has_found_newest() {
+    assert.equal(fetch_status.has_found_newest(), true);
+}
+
+function has_not_found_newest() {
+    assert.equal(fetch_status.has_found_newest(), false);
+}
+
 (function test_basics() {
     reset();
 
@@ -29,12 +37,14 @@ function blocked_older() {
 
     blocked_newer();
     blocked_older();
+    has_not_found_newest();
 
     fetch_status.finish_initial_narrow({
         found_oldest: true,
         found_newest: true,
     });
 
+    has_found_newest();
     blocked_newer();
     blocked_older();
 

frontend_tests/node_tests/filter.js

@@ -2,8 +2,10 @@ zrequire('util');
 zrequire('unread');
 zrequire('stream_data');
 zrequire('people');
+zrequire('Handlebars', 'handlebars');
 zrequire('Filter', 'js/filter');
 
+set_global('message_store', {});
 set_global('page_params', {});
 set_global('feature_flags', {});
 
@@ -77,6 +79,7 @@ function assert_same_operators(result, terms) {
 
     assert(filter.is_search());
     assert(! filter.can_apply_locally());
+    assert(! filter.is_exactly('stream'));
 
     // If our only stream operator is negated, then for all intents and purposes,
     // we don't consider ourselves to have a stream operator, because we don't
@@ -132,6 +135,7 @@ function assert_same_operators(result, terms) {
     var filter = new Filter(operators);
 
     assert.deepEqual(filter.operands('stream'), ['foo']);
+    assert(filter.is_exactly('stream'));
 }());
 
 (function test_public_operators() {
@@ -143,6 +147,7 @@ function assert_same_operators(result, terms) {
 
     var filter = new Filter(operators);
     assert_same_operators(filter.public_operators(), operators);
+    assert(!filter.is_exactly('stream'));
 
     global.page_params.narrow_stream = 'default';
     operators = [
@@ -585,7 +590,7 @@ function make_sub(name, stream_id) {
         {operator: 'stream', operand: 'devel'},
         {operator: 'topic', operand: 'JS'},
     ];
-    string = 'stream devel > JS';
+    string = 'stream devel > JS';
     assert.equal(Filter.describe(narrow), string);
 
     narrow = [
@@ -665,6 +670,175 @@ function make_sub(name, stream_id) {
     assert.equal(Filter.describe(narrow), string);
 }());
 
+(function test_is_functions() {
+    var terms = [
+        {operator: 'stream', operand: 'My Stream'},
+    ];
+    var filter = new Filter(terms);
+    assert.equal(filter.is_exactly('stream'), true);
+    assert.equal(filter.is_exactly('stream', 'topic'), false);
+    assert.equal(filter.is_exactly('pm-with'), false);
+
+    terms = [
+        // try a non-orthodox ordering
+        {operator: 'topic', operand: 'My Topic'},
+        {operator: 'stream', operand: 'My Stream'},
+    ];
+    filter = new Filter(terms);
+    assert.equal(filter.can_bucket_by('stream'), true);
+    assert.equal(filter.can_bucket_by('stream', 'topic'), true);
+    assert.equal(filter.is_exactly('stream'), false);
+    assert.equal(filter.is_exactly('stream', 'topic'), true);
+    assert.equal(filter.is_exactly('pm-with'), false);
+    assert.equal(filter.can_bucket_by('pm-with'), false);
+
+    terms = [
+        {operator: 'stream', operand: 'My Stream', negated: true},
+        {operator: 'topic', operand: 'My Topic'},
+    ];
+    filter = new Filter(terms);
+    assert.equal(filter.can_bucket_by('stream'), false);
+    assert.equal(filter.can_bucket_by('stream', 'topic'), false);
+    assert.equal(filter.is_exactly('stream'), false);
+    assert.equal(filter.is_exactly('stream', 'topic'), false);
+    assert.equal(filter.is_exactly('pm-with'), false);
+
+    terms = [
+        {operator: 'pm-with', operand: 'foo@example.com', negated: true},
+    ];
+    filter = new Filter(terms);
+    assert.equal(filter.is_exactly('stream'), false);
+    assert.equal(filter.is_exactly('stream', 'topic'), false);
+    assert.equal(filter.is_exactly('pm-with'), false);
+
+    terms = [
+        {operator: 'pm-with', operand: 'foo@example.com,bar@example.com'},
+    ];
+    filter = new Filter(terms);
+    assert.equal(filter.is_exactly('stream'), false);
+    assert.equal(filter.is_exactly('stream', 'topic'), false);
+    assert.equal(filter.is_exactly('pm-with'), true);
+    assert.equal(filter.is_exactly('is-mentioned'), false);
+    assert.equal(filter.is_exactly('is-private'), false);
+
+    terms = [
+        {operator: 'is', operand: 'private'},
+    ];
+    filter = new Filter(terms);
+    assert.equal(filter.is_exactly('is-mentioned'), false);
+    assert.equal(filter.is_exactly('is-private'), true);
+
+    terms = [
+        {operator: 'is', operand: 'mentioned'},
+    ];
+    filter = new Filter(terms);
+    assert.equal(filter.is_exactly('is-mentioned'), true);
+    assert.equal(filter.is_exactly('is-private'), false);
+
+    terms = [
+        {operator: 'is', operand: 'mentioned'},
+        {operator: 'is', operand: 'starred'},
+    ];
+    filter = new Filter(terms);
+    assert.equal(filter.is_exactly('is-mentioned'), false);
+    assert.equal(filter.is_exactly('is-private'), false);
+    assert.equal(filter.can_bucket_by('is-mentioned'), true);
+    assert.equal(filter.can_bucket_by('is-private'), false);
+
+    // The call below returns false for somewhat arbitrary
+    // reasons -- we say is-private has precedence over
+    // is-starred.
+    assert.equal(filter.can_bucket_by('is-starred'), false);
+
+    terms = [
+        {operator: 'is', operand: 'mentioned', negated: true},
+    ];
+    filter = new Filter(terms);
+    assert.equal(filter.is_exactly('is-mentioned'), false);
+    assert.equal(filter.is_exactly('is-private'), false);
+}());
+
+(function test_term_type() {
+    function assert_term_type(term, expected_term_type) {
+        assert.equal(Filter.term_type(term), expected_term_type);
+    }
+
+    function term(operator, operand, negated) {
+        return {
+            operator: operator,
+            operand: operand,
+            negated: negated,
+        };
+    }
+
+    assert_term_type(term('stream', 'whatever'), 'stream');
+    assert_term_type(term('pm-with', 'whomever'), 'pm-with');
+    assert_term_type(term('pm-with', 'whomever', true), 'not-pm-with');
+    assert_term_type(term('is', 'private'), 'is-private');
+    assert_term_type(term('has', 'link'), 'has-link');
+    assert_term_type(term('has', 'attachment', true), 'not-has-attachment');
+
+    function assert_term_sort(in_terms, expected) {
+        const sorted_terms = Filter.sorted_term_types(in_terms);
+        assert.deepEqual(sorted_terms, expected);
+    }
+
+    assert_term_sort(
+        ['topic', 'stream', 'sender'],
+        ['stream', 'topic', 'sender']
+    );
+
+    assert_term_sort(
+        ['has-link', 'near', 'is-unread', 'pm-with'],
+        ['pm-with', 'near', 'is-unread', 'has-link']
+    );
+
+    assert_term_sort(
+        ['bogus', 'stream', 'topic'],
+        ['stream', 'topic', 'bogus']
+    );
+    assert_term_sort(
+        ['stream', 'topic', 'stream'],
+        ['stream', 'stream', 'topic']
+    );
+
+    const terms = [
+        {operator: 'topic', operand: 'lunch'},
+        {operator: 'sender', operand: 'steve@foo.com'},
+        {operator: 'stream', operand: 'Verona'},
+    ];
+    const filter = new Filter(terms);
+    const term_types = filter.sorted_term_types();
+
+    assert.deepEqual(term_types, ['stream', 'topic', 'sender']);
+}());
+
+(function test_first_valid_id_from() {
+    const terms = [
+        {operator: 'is', operand: 'alerted'},
+    ];
+
+    const filter = new Filter(terms);
+
+    const messages = {
+        5: { id: 5, alerted: true },
+        10: { id: 10 },
+        20: { id: 20, alerted: true },
+        30: { id: 30, type: 'stream' },
+        40: { id: 40, alerted: false },
+    };
+
+    const msg_ids = [10, 20, 30, 40];
+
+    message_store.get = () => {};
+
+    assert.equal(filter.first_valid_id_from(msg_ids), undefined);
+
+    message_store.get = (msg_id) => messages[msg_id];
+
+    assert.equal(filter.first_valid_id_from(msg_ids), 20);
+}());
+
 (function test_update_email() {
     var terms = [
         {operator: 'pm-with', operand: 'steve@foo.com'},

frontend_tests/node_tests/general.js

@@ -0,0 +1,727 @@
+// This is a general tour of how to write node tests that
+// may also give you some quick insight on how the Zulip
+// browser app is constructed.  Let's start with testing
+// a function from util.js.
+//
+// The most basic unit tests load up code, call functions,
+// and assert truths:
+
+zrequire('util');
+assert(!util.is_all_or_everyone_mentioned('boring text'));
+assert(util.is_all_or_everyone_mentioned('mention @**everyone**'));
+
+// Let's test with people.js next.  We'll show this technique:
+//  * get a false value
+//  * change the data
+//  * get a true value
+
+zrequire('people');
+const isaac = {
+    email: 'isaac@example.com',
+    user_id: 30,
+    full_name: 'Isaac Newton',
+};
+
+assert(!people.is_known_user_id(isaac.user_id));
+people.add(isaac);
+assert(people.is_known_user_id(isaac.user_id));
+
+// The global.people object is a very fundamental object in the
+// Zulip app.  You can learn a lot more about it by reading
+// the tests in people.js in the same directory as this file.
+// Let's create the current user, which some future tests will
+// require.
+
+var me = {
+    email: 'me@example.com',
+    user_id: 31,
+    full_name: 'Me Myself',
+};
+people.add(me);
+people.initialize_current_user(me.user_id);
+
+// Let's look at stream_data next, and we will start by putting
+// some data at module scope (since it may be useful for future
+// tests):
+
+const denmark_stream = {
+    color: 'blue',
+    name: 'Denmark',
+    stream_id: 101,
+    subscribed: false,
+};
+
+// We often use IIFEs (immediately invoked function expressions)
+// to make our tests more self-containted.
+
+zrequire('stream_data');
+
+(function test_stream_data() {
+    assert.equal(stream_data.get_sub_by_name('Denmark'), undefined);
+    stream_data.add_sub('Denmark', denmark_stream);
+    const sub = stream_data.get_sub_by_name('Denmark');
+    assert.equal(sub.color, 'blue');
+}());
+
+// Hopefully the basic patterns for testing data-oriented modules
+// are starting to become apparent.  To reinforce that, we will present
+// few more examples that also expose you to some of our core
+// data objects.  Also, we start testing some objects that have
+// deeper dependencies.
+
+const messages = {
+    isaac_to_denmark_stream: {
+        id: 400,
+        sender_id: isaac.user_id,
+        stream_id: denmark_stream.stream_id,
+        type: 'stream',
+        flags: ['has_alert_word'],
+        subject: 'copenhagen',
+        // note we don't have every field that a "real" message
+        // would have, and that can be fine
+    },
+};
+
+// We are going to test a core module called messages_store.js.  It
+// depends on some code that we aren't really interested in testing,
+// so let's create some stub functions that do nothing.
+
+const noop = () => undefined;
+
+set_global('alert_words', {});
+set_global('composebox_typeahead', {});
+
+alert_words.process_message = noop;
+composebox_typeahead.add_topic = noop;
+
+// We can also bring in real code:
+zrequire('recent_senders');
+zrequire('topic_data');
+
+// And finally require the module that we will test directly:
+zrequire('message_store');
+
+(function test_message_store() {
+    // Our test runner automatically sets _ for us.
+    // See http://underscorejs.org/ for help on that library.
+    var in_message = _.clone(messages.isaac_to_denmark_stream);
+
+    assert.equal(in_message.alerted, undefined);
+    message_store.set_message_booleans(in_message);
+    assert.equal(in_message.alerted, true);
+
+    // Let's add a message into our message_store via
+    // add_message_metadata.
+    assert.equal(message_store.get(in_message.id), undefined);
+    message_store.add_message_metadata(in_message);
+    const message = message_store.get(in_message.id);
+    assert.equal(message, in_message);
+
+    // There are more side effects.
+    const topic_names = topic_data.get_recent_names(denmark_stream.stream_id);
+    assert.deepEqual(topic_names, ['copenhagen']);
+}());
+
+// Tracking unread messages is a very fundamental part of the Zulip
+// app, and we use the unread object to track unread messages.
+zrequire('unread');
+
+(function test_unread() {
+    const stream_id = denmark_stream.stream_id;
+    const topic_name = 'copenhagen';
+
+    assert.equal(unread.num_unread_for_topic(stream_id, topic_name), 0);
+
+    var in_message = _.clone(messages.isaac_to_denmark_stream);
+    message_store.set_message_booleans(in_message);
+
+    unread.process_loaded_messages([in_message]);
+    assert.equal(unread.num_unread_for_topic(stream_id, topic_name), 1);
+}());
+
+// In the Zulip app you can narrow your message stream by topic, by
+// sender, by PM recipient, by search keywords, etc.  We will discuss
+// narrows more broadly, but first let's test out a core piece of
+// code that makes things work.
+
+
+// Some quick housekeeping:  Let's clear page_params, which is a data
+// structure that the server sends down to us when the app starts.  We
+// prefer to test with a clean slate.
+
+set_global('page_params', {});
+
+// We use the second argument of zrequire to find the location of the
+// Filter class.
+zrequire('Filter', 'js/filter');
+
+(function test_filter() {
+    const filter_terms = [
+        {operator: 'stream', operand: 'Denmark'},
+        {operator: 'topic', operand: 'copenhagen'},
+    ];
+
+    const filter = new Filter(filter_terms);
+
+    const predicate = filter.predicate();
+
+    // We don't need full-fledged messages to test the gist of
+    // our filter.  If there are details that are distracting from
+    // your test, you should not feel guilty about removing them.
+    assert.equal(predicate({type: 'personal'}), false);
+
+    assert.equal(predicate({
+        type: 'stream',
+        stream_id: denmark_stream.stream_id,
+        subject: 'does not match filter',
+    }), false);
+
+    assert.equal(predicate({
+        type: 'stream',
+        stream_id: denmark_stream.stream_id,
+        subject: 'copenhagen',
+    }), true);
+}());
+
+// We have a "narrow" abstraction that sits roughly on top of the
+// "filter" abstraction.  If you are in a narrow, we track the
+// state with the narrow_state module.
+
+zrequire('narrow_state');
+
+(function test_narrow_state() {
+    // As we often do, first make assertions about the starting
+    // state:
+
+    assert.equal(narrow_state.stream(), undefined);
+
+    // Now set the state.
+    const filter_terms = [
+        {operator: 'stream', operand: 'Denmark'},
+        {operator: 'topic', operand: 'copenhagen'},
+    ];
+
+    const filter = new Filter(filter_terms);
+
+    narrow_state.set_current_filter(filter);
+
+    assert.equal(narrow_state.stream(), 'Denmark');
+    assert.equal(narrow_state.topic(), 'copenhagen');
+}());
+
+/*
+
+    Let's step back and review what we've done so far.
+
+    We've used fairly straightforward testing techniques
+    to explore the following modules:
+
+        filter
+        message_store
+        narrow_state
+        people
+        stream_data
+        util
+
+    We haven't gone deep on any of these objects, but if
+    you are interested, all of these objects have test
+    suites that have 100% line coverage on the modules
+    that implement those objects.  For example, you can look
+    at people.js in this directory for more tests on the
+    people object.
+
+    We can quickly review some testing concepts:
+
+        zrequire - bring in real code
+        set_global - create stubs
+        IIFE - enclose tests in their own scope
+        assert.equal - verify results
+
+    ------
+
+    It's time to elaborate a bit on set_global.
+
+    First, some context.  When we test certain objects,
+    we don't always want to test all the code they
+    depend on.  Often we want to completely ignore the
+    interactions with certain objects; other times, we
+    will want to simulate some behavior of the objects
+    we depend on without bringing in all the implementation
+    details.
+
+    Also, our test runner runs many tests back to back.
+    Between each test we need to essentially reset the global
+    object back to its original state, so that state doesn't
+    leak between tests.
+
+    That's where set_global comes in.  When you call
+    set_global, it updates the global namespace with an
+    object that you specify in the **test**, not real
+    code.  Using set_global explicitly tells your test
+    reader what your testing boundaries are between "real"
+    code and "simulated" code.  Finally, and perhaps most
+    importantly, the test runner will prevent this state
+    from leaking into the next test (and "zrequire" has
+    the same behavior attached to it as well).
+
+    ------
+
+    Let's talk about our next steps.
+
+    An app is pretty useless without an actual data source.
+    One of the primary ways that a Zulip client gets data
+    is through events.  (We also get data at page load, and
+    we can also ask the server for data, but that's not in
+    the scope of this conversation yet.)
+
+    Chat systems are dynamic.  If an admin adds a user, or
+    if a user sends a messages, the server immediately sends
+    events to all clients so that they can reflect appropriate
+    changes in their UI.  We're not going to discuss the entire
+    "full stack" mechanism here.  Instead, we'll focus on
+    the client code, starting at the boundary where we
+    process events.
+
+    Let's just get started...
+
+*/
+
+zrequire('server_events_dispatch');
+
+// We will use Bob in several tests.
+const bob = {
+    email: 'bob@example.com',
+    user_id: 33,
+    full_name: 'Bob Roberts',
+};
+
+(function test_add_user_event() {
+    const event = {
+        type: 'realm_user',
+        op: 'add',
+        person: bob,
+    };
+
+    assert(!people.is_known_user_id(bob.user_id));
+    server_events_dispatch.dispatch_normal_event(event);
+    assert(people.is_known_user_id(bob.user_id));
+}());
+
+/*
+
+   It's actually a little surprising that adding a user does
+   not have side effects beyond the people object.  I guess
+   we don't immediately update the buddy list, but that's
+   because the buddy list gets updated on the next server
+   fetch.
+
+   Let's try an update next.  To make this work, we will want
+   to put some stub objects into the global namespace (as
+   opposed to using the "real" code).
+
+*/
+
+set_global('activity', {});
+set_global('message_live_update', {});
+set_global('pm_list', {});
+set_global('settings_users', {});
+
+zrequire('user_events');
+
+(function test_update_user_event() {
+    const new_bob = {
+        email: 'bob@example.com',
+        user_id: bob.user_id,
+        full_name: 'The Artist Formerly Known as Bob',
+    };
+
+    const event = {
+        type: 'realm_user',
+        op: 'update',
+        person: new_bob,
+    };
+
+    // We have to stub a few things:
+    activity.redraw = noop;
+    message_live_update.update_user_full_name = noop;
+    pm_list.update_private_messages = noop;
+    settings_users.update_user_data = noop;
+
+    // Dispatch the realm_user/update event, which will update
+    // data structures and have other side effects that are
+    // stubbed out above.
+    server_events_dispatch.dispatch_normal_event(event);
+
+    const user = people.get_person_from_user_id(bob.user_id);
+
+    // Verify that the code actually did its main job:
+    assert.equal(user.full_name, 'The Artist Formerly Known as Bob');
+}());
+
+/*
+
+   Our test verifies that the update events leads to a name change
+   inside the people object, but it obviously kind of glosses over
+   the other interactions.
+
+   We can go a step further and verify the sequence of of operations
+   that happen during an event.  This concept is called "mocking",
+   and you can find libraries to help do mocking.  Here we will
+   just build our own lightweight mocking system, which is almost
+   trivially easy to do in a language like Javascript.
+
+*/
+
+function test_helper() {
+    var events = [];
+
+    return {
+        redirect: (module_name, func_name) => {
+            const full_name = module_name + '.' + func_name;
+            global[module_name][func_name] = () => {
+                events.push(full_name);
+            };
+        },
+        events: events,
+    };
+}
+
+/*
+
+   Our test_helper will allow us to redirect methods to an
+   events array, and we can then later verify that the sequence
+   of side effect is as predicted.
+
+   (Note that for now we don't simulate return values nor do we
+   inspect the arguments to these functions.  We could easily
+   extend our helper to do more.)
+
+   The forthcoming example is a pretty extreme example, where we
+   are calling a pretty high level method that dispatches
+   a lot of its work out to other objects.
+
+*/
+
+set_global('home_msg_list', {});
+set_global('message_list', {});
+set_global('message_util', {});
+set_global('notifications', {});
+set_global('resize', {});
+set_global('stream_list', {});
+set_global('unread_ops', {});
+set_global('unread_ui', {});
+
+zrequire('message_events');
+
+(function test_insert_message() {
+    const helper = test_helper();
+
+    const new_message = {
+        sender_id: isaac.user_id,
+        id: 1001,
+        content: 'example content',
+    };
+
+    assert.equal(message_store.get(new_message.id), undefined);
+
+    helper.redirect('activity', 'process_loaded_messages');
+    helper.redirect('message_util', 'add_messages');
+    helper.redirect('message_util', 'insert_new_messages');
+    helper.redirect('notifications', 'received_messages');
+    helper.redirect('resize', 'resize_page_components');
+    helper.redirect('stream_list', 'update_streams_sidebar');
+    helper.redirect('unread_ops', 'process_visible');
+    helper.redirect('unread_ui', 'update_unread_counts');
+
+    narrow_state.reset_current_filter();
+
+    message_events.insert_new_messages([new_message]);
+
+    // Even though we have stubbed a *lot* of code, our
+    // tests can still verify the main "narrative" of how
+    // the code invokes various objects when a new message
+    // comes in:
+    assert.deepEqual(helper.events, [
+        'message_util.add_messages',
+        'message_util.add_messages',
+        'activity.process_loaded_messages',
+        'unread_ui.update_unread_counts',
+        'resize.resize_page_components',
+        'unread_ops.process_visible',
+        'notifications.received_messages',
+        'stream_list.update_streams_sidebar',
+    ]);
+
+    // Despite all of our stubbing/mocking, the call to
+    // insert_new_messages will have created a very important
+    // side effect that we can verify:
+    const inserted_message = message_store.get(new_message.id);
+    assert.equal(inserted_message.id, new_message.id);
+    assert.equal(inserted_message.content, 'example content');
+}());
+
+
+/*
+
+   The previous example starts to get us out of the data layer of
+   the app and into more interesting interactions.
+
+   When a new message comes in, we update the three major
+   panes of the app:
+
+        * left sidebar - stream list
+        * middle pane - message view
+        * right sidebar - buddy list (aka "activity" list)
+
+    These are reflected by the following calls:
+
+        stream_list.update_streams_sidebar
+        message_util.add_messages
+        activity.process_loaded_messages
+
+    For now, though, let's focus on another side effect
+    of processing incoming messages:
+
+        unread_ops.process_visible
+
+    When new messages come in, they are often immediately
+    visible to users, so the app will communicate this
+    back to the server by calling unread_ops.process_visible.
+
+    In order to unit test this, we don't want to require
+    an actual server to be running.  Instead, this example
+    will stub many of the "boundaries" to focus on the
+    core behavior.
+
+*/
+
+set_global('channel', {});
+set_global('feature_flags', {});
+set_global('home_msg_list', {});
+set_global('message_list', {});
+set_global('message_viewport', {});
+zrequire('message_flags');
+
+zrequire('unread_ops');
+
+(function test_unread_ops() {
+    (function set_up() {
+        const test_messages = [
+            {
+                id: 50,
+                type: 'stream',
+                stream_id: denmark_stream.stream_id,
+                subject: 'copenhagen',
+                unread: true,
+            },
+        ];
+
+        // Make our test message appear to be unread, so that
+        // we then need to subsequently process them as read.
+        unread.process_loaded_messages(test_messages);
+
+        // Make our window appear visible.
+        notifications.window_has_focus = () => true;
+
+        // Make our "test" message appear visible.
+        message_viewport.visible_messages = () => test_messages;
+
+        // Make us not be in a narrow (somewhat hackily).
+        message_list.narrowed = undefined;
+        set_global('current_msg_list', 'not-narrowed-stub');
+
+        // Ignore these interactions for now:
+        home_msg_list.show_message_as_read = noop;
+        message_list.all = {};
+        message_list.all.show_message_as_read = noop;
+        notifications.close_notification = noop;
+    }());
+
+    // Set up a way to capture the options passed in to channel.post.
+    var channel_post_opts;
+    channel.post = (opts) => {
+        channel_post_opts = opts;
+    };
+
+    // Do the main thing we're testing!
+    unread_ops.process_visible();
+
+    // The most important side effect of the above call is that
+    // we post info to the server.  We can verify that the correct
+    // url and parameters are specified:
+    assert.deepEqual(channel_post_opts, {
+        url: '/json/messages/flags',
+        idempotent: true,
+        data: { messages: '[50]', op: 'add', flag: 'read' },
+        success: channel_post_opts.success,
+    });
+}());
+
+/*
+
+   Next we will explore this function:
+
+      stream_list.update_streams_sidebar
+
+    To make this test work, we will create a somewhat elaborate
+    function that fills in for jQuery (https://jquery.com/), so that
+    one boundary of our tests is how stream_list.js calls into
+    stream_list to manipulate DOM.
+
+*/
+
+set_global('topic_list', {});
+
+zrequire('stream_sort');
+zrequire('stream_list');
+
+const social_stream = {
+    color: 'red',
+    name: 'Social',
+    stream_id: 102,
+    subscribed: true,
+};
+
+(function set_up_filter() {
+    stream_data.add_sub('Social', social_stream);
+
+    const filter_terms = [
+        {operator: 'stream', operand: 'Social'},
+        {operator: 'topic', operand: 'lunch'},
+    ];
+
+    const filter = new Filter(filter_terms);
+
+    narrow_state.filter = () => filter;
+    narrow_state.active = () => true;
+}());
+
+function jquery_elem() {
+    // We create basic stubs for jQuery elements, so they
+    // just work.  We can extend these in cases where we want
+    // more detailed testing.
+    var elem = {};
+
+    elem.expectOne = () => elem;
+    elem.removeClass = () => elem;
+    elem.empty = () => elem;
+
+    return elem;
+}
+
+function make_jquery_helper() {
+    const stream_list_filter = jquery_elem();
+    stream_list_filter.is = () => true;
+    stream_list_filter.val = () => '';
+
+    const stream_filters = jquery_elem();
+
+    var appended_data;
+    stream_filters.append = function (data) {
+        appended_data = data;
+    };
+
+    function fake_jquery(selector) {
+        switch (selector) {
+        case '#stream_filters li.highlighted_stream':
+            return jquery_elem();
+        case '.stream-list-filter':
+            return stream_list_filter;
+        case '#stream_filters li.narrow-filter':
+            return jquery_elem();
+        case 'ul#stream_filters li':
+            return jquery_elem();
+        case '.stream-filters-label':
+            return jquery_elem();
+        case '#stream_filters':
+            return stream_filters;
+        default:
+            throw Error('unknown selector: ' + selector);
+        }
+    }
+
+    set_global('$', fake_jquery);
+
+    return {
+        verify_actions: () => {
+            const expected_data_to_append = [
+                [
+                    'stream stub',
+                ],
+            ];
+
+            assert.deepEqual(appended_data,
+                             expected_data_to_append);
+        },
+    };
+}
+
+function make_topic_list_helper() {
+    // We want to make sure that updating a stream_list
+    // closes the topic list and then rebuilds it.  We don't
+    // care about the implementation details of topic_list for
+    // now, just that it is invoked properly.
+    topic_list.active_stream_id = () => undefined;
+
+    var topic_list_closed;
+    topic_list.close = () => {
+        topic_list_closed = true;
+    };
+
+    var topic_list_rebuilt;
+    topic_list.rebuild = () => {
+        topic_list_rebuilt = true;
+    };
+
+    return {
+        verify_actions: () => {
+            assert(topic_list_closed);
+            assert(topic_list_rebuilt);
+        },
+    };
+}
+
+function make_sidebar_helper() {
+    var updated_whether_active;
+
+    function row_widget() {
+        return {
+            update_whether_active: () => {
+                updated_whether_active = true;
+            },
+            get_li: () => ['stream stub'],
+        };
+    }
+
+    stream_list.stream_sidebar.set_row(social_stream.stream_id, row_widget());
+    stream_list.stream_cursor = {
+        redraw: noop,
+    };
+
+    return {
+        verify_actions: () => {
+            assert(updated_whether_active);
+
+        },
+    };
+}
+
+(function test_stream_list() {
+    const jquery_helper = make_jquery_helper();
+    const sidebar_helper = make_sidebar_helper();
+    const topic_list_helper = make_topic_list_helper();
+
+    var streams_shown;
+    stream_list.show_all_streams = () => {
+        streams_shown = true;
+    };
+
+    // This is what we are testing!
+    stream_list.update_streams_sidebar();
+
+    assert(streams_shown);
+
+    jquery_helper.verify_actions();
+    sidebar_helper.verify_actions();
+    topic_list_helper.verify_actions();
+}());

frontend_tests/node_tests/hash_util.js

@@ -0,0 +1,54 @@
+zrequire('hash_util');
+zrequire('stream_data');
+zrequire('people');
+
+var hamlet = {
+    user_id: 1,
+    email: 'hamlet@example.com',
+    full_name: 'Hamlet',
+};
+
+people.add_in_realm(hamlet);
+
+var sub = {
+    stream_id: 99,
+    name: 'frontend',
+};
+
+stream_data.add_sub(sub.name, sub);
+
+(function test_hash_util() {
+    // Test encodeHashComponent
+    var str = 'https://www.zulipexample.com';
+    var result1 = hash_util.encodeHashComponent(str);
+    assert.equal(result1, 'https.3A.2F.2Fwww.2Ezulipexample.2Ecom');
+
+    // Test decodeHashComponent
+    var result2 = hash_util.decodeHashComponent(result1);
+    assert.equal(result2, str);
+
+    // Test encode_operand and decode_operand
+
+    function encode_decode_operand(operator, operand, expected_val) {
+        var encode_result = hash_util.encode_operand(operator, operand);
+        assert.equal(encode_result, expected_val);
+        var new_operand = encode_result;
+        var decode_result = hash_util.decode_operand(operator, new_operand);
+        assert.equal(decode_result, operand);
+    }
+
+    var operator = 'sender';
+    var operand = hamlet.email;
+
+    encode_decode_operand(operator, operand, '1-hamlet');
+
+    operator = 'stream';
+    operand = 'frontend';
+
+    encode_decode_operand(operator, operand, '99-frontend');
+
+    operator = 'topic';
+    operand = 'testing 123';
+
+    encode_decode_operand(operator, operand, 'testing.20123');
+}());

frontend_tests/node_tests/hashchange.js

@@ -3,6 +3,36 @@ zrequire('hash_util');
 zrequire('hashchange');
 zrequire('stream_data');
 
+set_global('document', 'document-stub');
+set_global('history', {});
+set_global('window', {
+    location: {
+        protocol: 'http:',
+        host: 'example.com',
+    },
+});
+
+set_global('admin', {});
+set_global('drafts', {});
+set_global('favicon', {});
+set_global('floating_recipient_bar', {});
+set_global('info_overlay', {});
+set_global('narrow', {});
+set_global('overlays', {});
+set_global('settings', {});
+set_global('subs', {});
+set_global('ui_util', {});
+
+function blueslip_wrap(f) {
+    return function (e) {
+        return f(e);
+    };
+}
+
+set_global('blueslip', {
+    wrap_function: blueslip_wrap,
+});
+
 (function test_operators_round_trip() {
     var operators;
     var hash;
@@ -79,3 +109,229 @@ zrequire('stream_data');
     hash = hashchange.operators_to_hash(operators);
     assert.equal(hash, '#narrow/pm-with/42-alice');
 }());
+
+function stub_trigger(f) {
+    set_global('$', () => {
+        return {
+            trigger: f,
+        };
+    });
+    $.Event = (name) => {
+        assert.equal(name, 'zuliphashchange.zulip');
+    };
+}
+
+function test_helper() {
+    var events = [];
+    var narrow_terms;
+
+    function stub(module_name, func_name) {
+        global[module_name][func_name] = () => {
+            events.push(module_name + '.' + func_name);
+        };
+    }
+
+    stub('admin', 'setup_page');
+    stub('drafts', 'launch');
+    stub('favicon', 'reset');
+    stub('floating_recipient_bar', 'update');
+    stub('narrow', 'deactivate');
+    stub('overlays', 'close_for_hash_change');
+    stub('settings', 'setup_page');
+    stub('subs', 'launch');
+    stub('ui_util', 'blur_active_element');
+
+    stub_trigger(() => { events.push('trigger event'); });
+
+    ui_util.change_tab_to = (hash) => {
+        events.push('change_tab_to ' + hash);
+    };
+
+    narrow.activate = (terms) => {
+        narrow_terms = terms;
+        events.push('narrow.activate');
+    };
+
+    info_overlay.show = (name) => {
+        events.push('info: ' + name);
+    };
+
+    return {
+        clear_events: () => {
+            events = [];
+        },
+        assert_events: (expected_events) => {
+            assert.deepEqual(expected_events, events);
+        },
+        get_narrow_terms: () => {
+            return narrow_terms;
+        },
+    };
+}
+
+(function test_hash_interactions() {
+    var helper = test_helper();
+
+    window.location.hash = '#';
+
+    helper.clear_events();
+    hashchange.initialize();
+    helper.assert_events([
+        'overlays.close_for_hash_change',
+        'trigger event',
+        'change_tab_to #home',
+        'narrow.deactivate',
+        'floating_recipient_bar.update',
+    ]);
+
+    helper.clear_events();
+    window.onhashchange();
+    helper.assert_events([
+        'overlays.close_for_hash_change',
+        'trigger event',
+        'change_tab_to #home',
+        'narrow.deactivate',
+        'floating_recipient_bar.update',
+    ]);
+
+    window.location.hash = '#narrow/stream/Denmark';
+
+    helper.clear_events();
+    window.onhashchange();
+    helper.assert_events([
+        'overlays.close_for_hash_change',
+        'trigger event',
+        'change_tab_to #home',
+        'narrow.activate',
+        'floating_recipient_bar.update',
+    ]);
+    var terms = helper.get_narrow_terms();
+    assert.equal(terms[0].operand, 'Denmark');
+
+    window.location.hash = '#narrow';
+
+    helper.clear_events();
+    window.onhashchange();
+    helper.assert_events([
+        'overlays.close_for_hash_change',
+        'trigger event',
+        'change_tab_to #home',
+        'narrow.activate',
+        'floating_recipient_bar.update',
+    ]);
+    terms = helper.get_narrow_terms();
+    assert.equal(terms.length, 0);
+
+    window.location.hash = '#streams/whatever';
+
+    helper.clear_events();
+    window.onhashchange();
+    helper.assert_events([
+        'overlays.close_for_hash_change',
+        'subs.launch',
+    ]);
+
+    window.location.hash = '#keyboard-shortcuts/whatever';
+
+    helper.clear_events();
+    window.onhashchange();
+    helper.assert_events([
+        'overlays.close_for_hash_change',
+        'trigger event',
+        'info: keyboard-shortcuts',
+    ]);
+
+    window.location.hash = '#markdown-help/whatever';
+
+    helper.clear_events();
+    window.onhashchange();
+    helper.assert_events([
+        'overlays.close_for_hash_change',
+        'trigger event',
+        'info: markdown-help',
+    ]);
+
+    window.location.hash = '#search-operators/whatever';
+
+    helper.clear_events();
+    window.onhashchange();
+    helper.assert_events([
+        'overlays.close_for_hash_change',
+        'trigger event',
+        'info: search-operators',
+    ]);
+
+    window.location.hash = '#drafts';
+
+    helper.clear_events();
+    window.onhashchange();
+    helper.assert_events([
+        'overlays.close_for_hash_change',
+        'drafts.launch',
+    ]);
+
+    window.location.hash = '#settings/alert-words';
+
+    helper.clear_events();
+    window.onhashchange();
+    helper.assert_events([
+        'overlays.close_for_hash_change',
+        'settings.setup_page',
+        'admin.setup_page',
+    ]);
+
+    window.location.hash = '#organization/user-list-admin';
+
+    helper.clear_events();
+    window.onhashchange();
+    helper.assert_events([
+        'settings.setup_page',
+        'admin.setup_page',
+    ]);
+
+    var called_back;
+
+    helper.clear_events();
+    hashchange.exit_overlay(() => {
+        called_back = true;
+    });
+
+    helper.assert_events([
+        'ui_util.blur_active_element',
+    ]);
+    assert(called_back);
+
+}());
+
+(function test_save_narrow() {
+    var helper = test_helper();
+
+    var operators = [
+        {operator: 'is', operand: 'private'},
+    ];
+
+    hashchange.save_narrow(operators);
+    helper.assert_events([
+        'trigger event',
+        'favicon.reset',
+    ]);
+    assert.equal(window.location.hash, '#narrow/is/private');
+
+    var url_pushed;
+    global.history.pushState = (state, title, url) => {
+        url_pushed = url;
+    };
+
+    operators = [
+        {operator: 'is', operand: 'starred'},
+    ];
+
+    helper.clear_events();
+    hashchange.save_narrow(operators);
+    helper.assert_events([
+        'trigger event',
+        'favicon.reset',
+    ]);
+    assert.equal(url_pushed, 'http://example.com/#narrow/is/starred');
+}());
+

frontend_tests/node_tests/hotkey.js

@@ -14,23 +14,23 @@
 set_global('activity', {
 });
 
+set_global('navigator', {
+    userAgent: '',
+});
+
 set_global('page_params', {
 });
 
 set_global('overlays', {
 });
 
-set_global('$', function () {
-    return {
-        // Hack: Used for reactions hotkeys; may want to restructure.
-        find: function () {return ['target'];},
-        keydown: function () {},
-        keypress: function () {},
-    };
-});
+var noop = () => {};
 
-set_global('document', {
-});
+// jQuery stuff should go away if we make an initialize() method.
+set_global('document', 'document-stub');
+set_global('$', global.make_zjquery());
+$.fn.keydown = noop;
+$.fn.keypress = noop;
 
 var hotkey = zrequire('hotkey');
 
@@ -70,11 +70,12 @@ function stubbing(func_name_to_stub, test_function) {
         });
     }
 
-    function map_down(which, shiftKey, ctrlKey) {
+    function map_down(which, shiftKey, ctrlKey, metaKey) {
         return hotkey.get_keydown_hotkey({
             which: which,
             shiftKey: shiftKey,
             ctrlKey: ctrlKey,
+            metaKey: metaKey,
         });
     }
 
@@ -97,7 +98,8 @@ function stubbing(func_name_to_stub, test_function) {
     assert.equal(map_press(47).name, 'search'); // slash
     assert.equal(map_press(106).name, 'vim_down'); // j
 
-    assert.equal(map_down(219, false, true).name, 'escape');
+    assert.equal(map_down(219, false, true).name, 'escape'); // ctrl + [
+    assert.equal(map_down(75, false, true).name, 'search_with_k'); // ctrl + k
 
     // More negative tests.
     assert.equal(map_down(47), undefined);
@@ -117,6 +119,17 @@ function stubbing(func_name_to_stub, test_function) {
     assert.equal(map_down(88, false, true), undefined); // ctrl + x
     assert.equal(map_down(78, false, true), undefined); // ctrl + n
     assert.equal(map_down(77, false, true), undefined); // ctrl + m
+    assert.equal(map_down(75, false, false, true), undefined); // cmd + k
+
+    // CMD tests for MacOS
+    global.navigator.userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.167 Safari/537.36";
+    assert.equal(map_down(219, false, false, true).name, 'escape'); // cmd + [
+    assert.equal(map_down(75, false, false, true).name, 'search_with_k'); // cmd + k
+
+    assert.equal(map_down(75, false, true, false), undefined); // ctrl + k
+
+    // Reset userAgent
+    global.navigator.userAgent = '';
 }());
 
 (function test_basic_chars() {

frontend_tests/node_tests/i18n.js

@@ -56,7 +56,6 @@ i18n.init({
             enable_offline_push_notifications: false,
             enable_online_push_notifications: false,
             enable_digest_emails: false,
-            default_desktop_notifications: false,
         },
     };
 

frontend_tests/node_tests/input_pill.js

@@ -5,8 +5,7 @@ zrequire('Handlebars', 'handlebars');
 zrequire('templates');
 global.compile_template('input_pill');
 
-set_global('blueslip', {
-});
+set_global('blueslip', global.make_zblueslip());
 
 var noop = function () {};
 
@@ -37,32 +36,28 @@ function pill_html(value, data_id) {
 }
 
 (function test_basics() {
-    var error;
-
     var config = {};
 
-    blueslip.error = function (err) {
-        error = err;
-    };
-
+    blueslip.set_test_data('error', 'Pill needs container.');
     input_pill.create(config);
-    assert.equal(error, 'Pill needs container.');
+    assert.equal(blueslip.get_test_logs('error').length, 1);
+    blueslip.clear_test_data();
 
     var pill_input = $.create('pill_input');
     var container = $.create('container');
     container.set_find_results('.input', pill_input);
 
+    blueslip.set_test_data('error', 'Pill needs create_item_from_text');
     config.container = container;
     input_pill.create(config);
-    assert.equal(error, 'Pill needs create_item_from_text');
+    assert.equal(blueslip.get_test_logs('error').length, 1);
+    blueslip.clear_test_data();
 
+    blueslip.set_test_data('error', 'Pill needs get_text_from_item');
     config.create_item_from_text = noop;
     input_pill.create(config);
-    assert.equal(error, 'Pill needs get_text_from_item');
-
-    blueslip.error = function () {
-        throw "unexpected error";
-    };
+    assert.equal(blueslip.get_test_logs('error').length, 1);
+    blueslip.clear_test_data();
 
     config.get_text_from_item = noop;
     var widget = input_pill.create(config);

frontend_tests/node_tests/list_render.js

@@ -0,0 +1,312 @@
+// TODO: make an initialize function for list_render.
+const initialize = (function () {
+    var initalize_function;
+
+    set_global('$', (f) => {
+        initalize_function = f;
+    });
+
+    // We can move this module scope when we have
+    // an explicit initialize function.
+    zrequire('list_render');
+
+    return initalize_function;
+}());
+
+// We need these stubs to get by instanceof checks.
+// The list_render library allows you to insert objects
+// that are either jQuery, Element, or just raw HTML
+// strings.  We initially test with raw strings.
+set_global('jQuery', 'stub');
+set_global('Element', function () {
+    return { };
+});
+
+// This function will be the anonymous click handler
+// for clicking on any element that matches the
+// CSS selector of "[data-sort]".
+var handle_sort_click;
+
+// We only need very simple jQuery wrappers for when the
+// "real" code wraps html or sets up click handlers.
+// We'll simulate most other objects ourselves.
+set_global('$', (arg) => {
+    if (arg.to_jquery) {
+        return arg.to_jquery();
+    }
+
+    if (arg === 'body') {
+        return {
+            on: (event_name, selector, f) => {
+                assert.equal(event_name, 'click');
+                assert.equal(selector, '[data-sort]');
+                handle_sort_click = f;
+            },
+        };
+    }
+
+    return {
+        html: () => arg,
+    };
+});
+
+function make_containers() {
+    // We build objects here that simulate jQuery containers.
+    // The main thing to do at first is simulate that our
+    // parent container is the nearest ancestor to our main
+    // container that has a max-height attribute, and then
+    // the parent container will have a scroll event attached to
+    // it.  This is a good time to read __set_events in the
+    // real code.
+    const parent_container = {};
+    const container = {};
+
+    container.parent = () => parent_container;
+    container.length = () => 1;
+    container.is = () => false;
+    container.css = (prop) => {
+        assert.equal(prop, 'max-height');
+        return 'none';
+    };
+
+    parent_container.is = () => false;
+    parent_container.length = () => 1;
+    parent_container.css = (prop) => {
+        assert.equal(prop, 'max-height');
+        return 100;
+    };
+
+    // Capture the scroll callback so we can call it in
+    // our tests.
+    parent_container.scroll = (f) => {
+        parent_container.call_scroll = () => {
+            f.call(parent_container);
+        };
+    };
+
+    // Make our append function just set a field we can
+    // check in our tests.
+    container.append = (data) => {
+        container.appended_data = data;
+    };
+
+    return {
+        container: container,
+        parent_container: parent_container,
+    };
+}
+
+function make_search_input() {
+    const $element = {};
+
+    // Allow ourselves to be wrapped by $(...) and
+    // return ourselves.
+    $element.to_jquery = () => $element;
+
+    $element.on = (event_name, f) => {
+        assert.equal(event_name, 'input');
+        $element.simulate_input_event = () => {
+            const elem = {
+                value: $element.val(),
+            };
+            f.call(elem);
+        };
+    };
+
+    return $element;
+}
+
+function div(item) {
+    return '<div>' + item + '</div>';
+}
+
+(function test_list_render() {
+    const {container, parent_container} = make_containers();
+
+    const search_input = make_search_input();
+
+    const list = [
+        'apple',
+        'banana',
+        'carrot',
+        'dog',
+        'egg',
+        'fence',
+        'grape',
+    ];
+    const opts = {
+        filter: {
+            element: search_input,
+        },
+        load_count: 2,
+        modifier: (item) => div(item),
+    };
+
+    const widget = list_render(container, list, opts);
+
+    widget.render();
+
+    var expected_html = '<div>apple</div><div>banana</div>';
+    assert.deepEqual(container.appended_data.html(), expected_html);
+
+    // Set up our fake geometry so it forces a scroll action.
+    parent_container.scrollTop = 180;
+    parent_container.clientHeight = 100;
+    parent_container.scrollHeight = 260;
+
+    // Scrolling gets the next two elements from the list into
+    // our widget.
+    parent_container.call_scroll();
+    expected_html = '<div>carrot</div><div>dog</div>';
+    assert.deepEqual(container.appended_data.html(), expected_html);
+
+    // Filtering will pick out dog/egg/grape when we put "g"
+    // into our search input.  (This uses the default filter, which
+    // is a glorified indexOf call.)
+    container.html = (html) => { assert.equal(html, ''); };
+    search_input.val = () => 'g';
+    search_input.simulate_input_event();
+    expected_html = '<div>dog</div><div>egg</div><div>grape</div>';
+    assert.deepEqual(container.appended_data.html(), expected_html);
+
+    // We can insert new data into the widget.
+    const new_data = [
+        'greta',
+        'faye',
+        'gary',
+        'frank',
+        'giraffe',
+        'fox',
+    ];
+
+    widget.data(new_data);
+    widget.render();
+    expected_html = '<div>greta</div><div>gary</div>';
+    assert.deepEqual(container.appended_data.html(), expected_html);
+}());
+
+function sort_button(opts) {
+    // The complications here are due to needing to find
+    // the list via complicated HTML assumptions. Also, we
+    // don't have any abstraction for the button and its
+    // siblings other than direct jQuery actions.
+
+    function data(sel) {
+        switch (sel) {
+        case "sort": return opts.sort_type;
+        case "sort-prop": return opts.prop_name;
+        default: throw Error('unknown selector: ' + sel);
+        }
+    }
+
+    function lookup(sel, value) {
+        return (selector) => {
+            assert.equal(sel, selector);
+            return value;
+        };
+    }
+
+    var button;
+
+    const $button = {
+        data: data,
+        parents: lookup('table', {
+            next: lookup('.progressive-table-wrapper', {
+                data: lookup('list-render', opts.list_name),
+            }),
+        }),
+        hasClass: lookup('active', opts.active),
+        siblings: lookup('.active', {
+            removeClass: (sel) => {
+                assert.equal(sel, 'active');
+                button.siblings_deactivated = true;
+            },
+        }),
+        addClass: (sel) => {
+            assert.equal(sel, 'active');
+            button.activated = true;
+        },
+    };
+
+    button = {
+        to_jquery: () => $button,
+        siblings_deactivated: false,
+        activated: false,
+    };
+
+    return button;
+}
+
+(function test_sorting() {
+    const {container} = make_containers();
+
+    var cleared;
+    container.html = (html) => {
+        assert.equal(html, '');
+        cleared = true;
+    };
+
+    const alice = { name: 'alice', salary: 50 };
+    const bob = { name: 'bob', salary: 40 };
+    const cal = { name: 'cal', salary: 30 };
+    const dave = { name: 'dave', salary: 25 };
+
+    const list = [bob, dave, alice, cal];
+
+    const opts = {
+        name: 'my-list',
+        load_count: 2,
+        modifier: (item) => {
+            return div(item.name) + div(item.salary);
+        },
+    };
+
+    function html_for(people) {
+        return _.map(people, opts.modifier).join('');
+    }
+
+    list_render(container, list, opts);
+    initialize();
+
+    var button_opts;
+    var button;
+    var expected_html;
+
+    button_opts = {
+        sort_type: 'alphabetic',
+        prop_name: 'name',
+        list_name: 'my-list',
+        active: false,
+    };
+
+    button = sort_button(button_opts);
+
+    handle_sort_click.call(button);
+
+    assert(cleared);
+    assert(button.siblings_deactivated);
+
+    expected_html = html_for([alice, bob, cal, dave]);
+    assert.deepEqual(container.appended_data.html(), expected_html);
+
+    // Now try a numeric sort.
+    button_opts = {
+        sort_type: 'numeric',
+        prop_name: 'salary',
+        list_name: 'my-list',
+        active: false,
+    };
+
+    button = sort_button(button_opts);
+
+    cleared = false;
+    button.siblings_deactivated = false;
+
+    handle_sort_click.call(button);
+
+    assert(cleared);
+    assert(button.siblings_deactivated);
+
+    expected_html = html_for([dave, cal, bob, alice]);
+    assert.deepEqual(container.appended_data.html(), expected_html);
+}());

frontend_tests/node_tests/markdown.js

@@ -1,7 +1,4 @@
 /*global Dict */
-var path = zrequire('path', 'path');
-var fs = zrequire('fs', 'fs');
-
 zrequire('hash_util');
 zrequire('katex', 'node_modules/katex/dist/katex.min.js');
 zrequire('marked', 'third/marked/lib/marked');
@@ -47,10 +44,10 @@ set_global('page_params', {
     translate_emoticons: false,
 });
 
-set_global('blueslip', {error: function () {}});
+set_global('blueslip', global.make_zblueslip());
 
 set_global('Image', function () {
-  return {};
+    return {};
 });
 emoji.initialize();
 
@@ -76,6 +73,12 @@ people.add({
     email: 'leo@zulip.com',
 });
 
+people.add({
+    full_name: 'Bobby <h1>Tables</h1>',
+    user_id: 103,
+    email: 'bobby@zulip.com',
+});
+
 people.initialize_current_user(cordelia.user_id);
 
 var hamletcharacters = {
@@ -92,8 +95,16 @@ var backend = {
     members: [],
 };
 
+var edgecase_group = {
+    name: "Bobby <h1>Tables</h1>",
+    id: 3,
+    description: "HTML Syntax to check for Markdown edge cases.",
+    members: [],
+};
+
 global.user_groups.add(hamletcharacters);
 global.user_groups.add(backend);
+global.user_groups.add(edgecase_group);
 
 var stream_data = global.stream_data;
 var denmark = {
@@ -111,8 +122,16 @@ var social = {
     in_home_view: true,
     invite_only: true,
 };
+var edgecase_stream = {
+    subscribed: true,
+    color: 'green',
+    name: 'Bobby <h1>Tables</h1>',
+    stream_id: 3,
+    in_home_view: true,
+};
 stream_data.add_sub('Denmark', denmark);
 stream_data.add_sub('social', social);
+stream_data.add_sub('Bobby <h1>Tables</h1>', edgecase_stream);
 
 // Check the default behavior of fenced code blocks
 // works properly before markdown is initialized.
@@ -125,47 +144,46 @@ stream_data.add_sub('social', social);
 
 markdown.initialize();
 
-var bugdown_data = JSON.parse(fs.readFileSync(path.join(__dirname, '../../zerver/fixtures/markdown_test_cases.json'), 'utf8', 'r'));
+var bugdown_data = global.read_fixture_data('markdown_test_cases.json');
 
 (function test_bugdown_detection() {
-
     var no_markup = [
-                     "This is a plaintext message",
-                     "This is a plaintext: message",
-                     "This is a :plaintext message",
-                     "This is a :plaintext message: message",
-                     "Contains a not an image.jpeg/ok file",
-                     "Contains a not an http://www.google.com/ok/image.png/stop file",
-                     "No png to be found here, a png",
-                     "No user mention **leo**",
-                     "No user mention @what there",
-                     "No group mention *hamletcharacters*",
-                     "We like to code\n~~~\ndef code():\n    we = \"like to do\"\n~~~",
-                     "This is a\nmultiline :emoji: here\n message",
-                     "This is an :emoji: message",
-                     "User Mention @**leo**",
-                     "User Mention @**leo f**",
-                     "User Mention @**leo with some name**",
-                     "Group Mention @*hamletcharacters*",
-                     "Stream #**Verona**",
-                     "This contains !gravatar(leo@zulip.com)",
-                     "And an avatar !avatar(leo@zulip.com) is here",
-                    ];
+        "This is a plaintext message",
+        "This is a plaintext: message",
+        "This is a :plaintext message",
+        "This is a :plaintext message: message",
+        "Contains a not an image.jpeg/ok file",
+        "Contains a not an http://www.google.com/ok/image.png/stop file",
+        "No png to be found here, a png",
+        "No user mention **leo**",
+        "No user mention @what there",
+        "No group mention *hamletcharacters*",
+        "We like to code\n~~~\ndef code():\n    we = \"like to do\"\n~~~",
+        "This is a\nmultiline :emoji: here\n message",
+        "This is an :emoji: message",
+        "User Mention @**leo**",
+        "User Mention @**leo f**",
+        "User Mention @**leo with some name**",
+        "Group Mention @*hamletcharacters*",
+        "Stream #**Verona**",
+        "This contains !gravatar(leo@zulip.com)",
+        "And an avatar !avatar(leo@zulip.com) is here",
+    ];
 
     var markup = [
-                   "Contains a https://zulip.com/image.png file",
-                   "Contains a https://zulip.com/image.jpg file",
-                   "https://zulip.com/image.jpg",
-                   "also https://zulip.com/image.jpg",
-                   "https://zulip.com/image.jpg too",
-                   "Contains a zulip.com/foo.jpeg file",
-                   "Contains a https://zulip.com/image.png file",
-                   "twitter url https://twitter.com/jacobian/status/407886996565016579",
-                   "https://twitter.com/jacobian/status/407886996565016579",
-                   "then https://twitter.com/jacobian/status/407886996565016579",
-                   "twitter url http://twitter.com/jacobian/status/407886996565016579",
-                   "youtube url https://www.youtube.com/watch?v=HHZ8iqswiCw&feature=youtu.be&a",
-                 ];
+        "Contains a https://zulip.com/image.png file",
+        "Contains a https://zulip.com/image.jpg file",
+        "https://zulip.com/image.jpg",
+        "also https://zulip.com/image.jpg",
+        "https://zulip.com/image.jpg too",
+        "Contains a zulip.com/foo.jpeg file",
+        "Contains a https://zulip.com/image.png file",
+        "twitter url https://twitter.com/jacobian/status/407886996565016579",
+        "https://twitter.com/jacobian/status/407886996565016579",
+        "then https://twitter.com/jacobian/status/407886996565016579",
+        "twitter url http://twitter.com/jacobian/status/407886996565016579",
+        "youtube url https://www.youtube.com/watch?v=HHZ8iqswiCw&feature=youtu.be&a",
+    ];
 
     no_markup.forEach(function (content) {
         assert.equal(markdown.contains_backend_only_syntax(content), false);
@@ -286,14 +304,14 @@ var bugdown_data = JSON.parse(fs.readFileSync(path.join(__dirname, '../../zerver
         {input: 'T\n#**Denmark**',
          expected: '<p>T<br>\n<a class="stream" data-stream-id="1" href="http://zulip.zulipdev.com/#narrow/stream/1-Denmark">#Denmark</a></p>'},
         {input: 'T\n@**Cordelia Lear**',
-          expected: '<p>T<br>\n<span class="user-mention" data-user-id="101">@Cordelia Lear</span></p>'},
+         expected: '<p>T<br>\n<span class="user-mention" data-user-id="101">@Cordelia Lear</span></p>'},
         {input: 'T\n@hamletcharacters',
          expected: '<p>T<br>\n@hamletcharacters</p>'},
         {input: 'T\n@*hamletcharacters*',
          expected: '<p>T<br>\n<span class="user-group-mention" data-user-group-id="1">@hamletcharacters</span></p>'},
         {input: 'T\n@*notagroup*',
          expected: '<p>T<br>\n@*notagroup*</p>'},
-       {input: 'T\n@*backend*',
+        {input: 'T\n@*backend*',
          expected: '<p>T<br>\n<span class="user-group-mention" data-user-group-id="2">@Backend</span></p>'},
         {input: '@*notagroup*',
          expected: '<p>@*notagroup*</p>'},
@@ -305,6 +323,23 @@ var bugdown_data = JSON.parse(fs.readFileSync(path.join(__dirname, '../../zerver
         {input: ':)',
          expected: '<p><span class="emoji emoji-1f603" title="smiley">:smiley:</span></p>',
          translate_emoticons: true},
+        // Test HTML Escape in Custom Zulip Rules
+        {input: '@**<h1>The Rogue One</h1>**',
+         expected: '<p>@**&lt;h1&gt;The Rogue One&lt;/h1&gt;**</p>'},
+        {input: '#**<h1>The Rogue One</h1>**',
+         expected: '<p>#**&lt;h1&gt;The Rogue One&lt;/h1&gt;**</p>'},
+        {input: '!avatar(<h1>The Rogue One</h1>)',
+         expected: '<p><img alt="&lt;h1&gt;The Rogue One&lt;/h1&gt;" class="message_body_gravatar" src="/avatar/&lt;h1&gt;The Rogue One&lt;/h1&gt;?s=30" title="&lt;h1&gt;The Rogue One&lt;/h1&gt;"></p>'},
+        {input: ':<h1>The Rogue One</h1>:',
+         expected: '<p>:&lt;h1&gt;The Rogue One&lt;/h1&gt;:</p>'},
+        {input: '@**O\'Connell**',
+         expected: '<p>@**O&#39;Connell**</p>'},
+        {input: '@*Bobby <h1>Tables</h1>*',
+         expected: '<p><span class="user-group-mention" data-user-group-id="3">@Bobby &lt;h1&gt;Tables&lt;/h1&gt;</span></p>'},
+        {input: '@**Bobby <h1>Tables</h1>**',
+         expected: '<p><span class="user-mention" data-user-id="103">@Bobby &lt;h1&gt;Tables&lt;/h1&gt;</span></p>'},
+        {input: '#**Bobby <h1>Tables</h1>**',
+         expected: '<p><a class="stream" data-stream-id="3" href="http://zulip.zulipdev.com/#narrow/stream/3-Bobby-.3Ch1.3ETables.3C.2Fh1.3E">#Bobby &lt;h1&gt;Tables&lt;/h1&gt;</a></p>'},
     ];
 
     // We remove one of the unicode emoji we put as input in one of the test
@@ -322,7 +357,6 @@ var bugdown_data = JSON.parse(fs.readFileSync(path.join(__dirname, '../../zerver
         var message = {raw_content: input};
         markdown.apply_markdown(message);
         var output = message.content;
-
         assert.equal(expected, output);
     });
 }());
@@ -386,6 +420,20 @@ var bugdown_data = JSON.parse(fs.readFileSync(path.join(__dirname, '../../zerver
     assert.equal(message.mentioned, true);
     assert.equal(message.mentioned_me_directly, true);
 
+    input = "test @**everyone**";
+    message = {subject: "No links here", raw_content: input};
+    markdown.apply_markdown(message);
+    assert.equal(message.is_me_message, false);
+    assert.equal(message.mentioned, true);
+    assert.equal(message.mentioned_me_directly, false);
+
+    input = "test @**stream**";
+    message = {subject: "No links here", raw_content: input};
+    markdown.apply_markdown(message);
+    assert.equal(message.is_me_message, false);
+    assert.equal(message.mentioned, true);
+    assert.equal(message.mentioned_me_directly, false);
+
     input = "test @all";
     message = {subject: "No links here", raw_content: input};
     markdown.apply_markdown(message);
@@ -443,12 +491,9 @@ var bugdown_data = JSON.parse(fs.readFileSync(path.join(__dirname, '../../zerver
 
 (function test_katex_throws_unexpected_exceptions() {
     katex.renderToString = function () { throw new Error('some-exception'); };
-    var blueslip_error_called = false;
-    blueslip.error = function (ex) {
-        assert.equal(ex.message, 'some-exception');
-        blueslip_error_called = true;
-    };
+    blueslip.set_test_data('error', 'Error: some-exception');
     var message = { raw_content: '$$a$$' };
     markdown.apply_markdown(message);
-    assert(blueslip_error_called);
+    assert(blueslip.get_test_logs('error').length, 1);
+    blueslip.clear_test_data();
 }());

frontend_tests/node_tests/message_edit.js

@@ -66,4 +66,27 @@ var editability_types = message_edit.editability_types;
     assert.equal(get_editability(message, 45), editability_types.NO_LONGER);
     // If we don't pass a second argument, treat it as 0
     assert.equal(get_editability(message), editability_types.NO_LONGER);
+
+    message = {
+        sent_by_me: false,
+        type: 'stream',
+    };
+    global.page_params = {
+        realm_allow_community_topic_editing: true,
+        realm_allow_message_editing: true,
+        realm_message_content_edit_limit_seconds: 0,
+    };
+    message.timestamp = current_timestamp - 60;
+    assert.equal(get_editability(message), editability_types.TOPIC_ONLY);
+
+    // Test `message_edit.is_topic_editable()`
+    assert.equal(message_edit.is_topic_editable(message), true);
+
+    message.sent_by_me = true;
+    global.page_params.realm_allow_community_topic_editing = false;
+    assert.equal(message_edit.is_topic_editable(message), true);
+
+    message.sent_by_me = false;
+    global.page_params.realm_allow_community_topic_editing = false;
+    assert.equal(message_edit.is_topic_editable(message), false);
 }());

frontend_tests/node_tests/message_fetch.js

@@ -9,6 +9,7 @@ set_global('MessageListView', function () { return {}; });
 
 zrequire('FetchStatus', 'js/fetch_status');
 zrequire('Filter', 'js/filter');
+zrequire('MessageListData', 'js/message_list_data');
 zrequire('message_list');
 zrequire('util');
 

frontend_tests/node_tests/message_list.js

@@ -11,6 +11,7 @@ set_global('document', null);
 zrequire('FetchStatus', 'js/fetch_status');
 zrequire('util');
 zrequire('muting');
+zrequire('MessageListData', 'js/message_list_data');
 zrequire('MessageListView', 'js/message_list_view');
 var MessageList = zrequire('message_list').MessageList;
 
@@ -198,18 +199,18 @@ var with_overrides = global.with_overrides; // make lint happy
 
     var list = new MessageList(table, filter);
     var items = [
-            {
-                id: 1,
-                sender_id: 3,
-            },
-            {
-                id: 2,
-                sender_id: 3,
-            },
-            {
-                id: 3,
-                sender_id: 6,
-            },
+        {
+            id: 1,
+            sender_id: 3,
+        },
+        {
+            id: 2,
+            sender_id: 3,
+        },
+        {
+            id: 3,
+            sender_id: 6,
+        },
     ];
 
     list.append(items);
@@ -245,8 +246,9 @@ var with_overrides = global.with_overrides; // make lint happy
 
 
     list = new MessageList(table, filter);
-    list.append([{id:10}, {id:20}, {id:30}, {id:20.02}, {id:20.03}, {id:40},
-                 {id:50}, {id: 50.01}, {id: 50.02}, {id:60}]);
+    list.append([
+        {id:10}, {id:20}, {id:30}, {id:20.02}, {id:20.03}, {id:40},
+        {id:50}, {id: 50.01}, {id: 50.02}, {id:60}]);
     list._local_only= {20.02: {id:20.02}, 20.03: {id:20.03},
                        50.01: {id: 50.01}, 50.02: {id: 50.02}};
 
@@ -395,7 +397,7 @@ var with_overrides = global.with_overrides; // make lint happy
 
     var messages = [{id: 1}, {id: 2}, {id: 3}];
 
-    list.unmuted_messages = function (m) { return m; };
+    list.data.unmuted_messages = function (msgs) { return msgs; };
     global.with_stub(function (stub) {
         list.view.rerender_the_whole_thing = stub.f;
         list.add_and_rerender(messages);

frontend_tests/node_tests/message_list_data.js

@@ -0,0 +1,113 @@
+zrequire('unread');
+zrequire('util');
+
+zrequire('Filter', 'js/filter');
+zrequire('MessageListData', 'js/message_list_data');
+
+set_global('page_params', {});
+set_global('blueslip', global.make_zblueslip());
+
+global.patch_builtin('setTimeout', (f, delay) => {
+    assert.equal(delay, 0);
+    return f();
+});
+
+function make_msg(msg_id) {
+    return {
+        id: msg_id,
+        unread: true,
+    };
+}
+
+function make_msgs(msg_ids) {
+    return _.map(msg_ids, make_msg);
+}
+
+(function test_basics() {
+    const mld = new MessageListData({
+        muting_enabled: false,
+        filter: undefined,
+    });
+
+    assert.equal(mld.is_search(), false);
+
+    mld.add(make_msgs([35, 25, 15, 45]));
+
+    function assert_contents(msg_ids) {
+        const msgs = mld.all_messages();
+        assert.deepEqual(msgs, make_msgs(msg_ids));
+    }
+    assert_contents([15, 25, 35, 45]);
+
+    const new_msgs = make_msgs([10, 20, 30, 40, 50, 60, 70]);
+    const info = mld.triage_messages(new_msgs);
+
+    assert.deepEqual(info, {
+        top_messages: make_msgs([10]),
+        interior_messages: make_msgs([20, 30, 40]),
+        bottom_messages: make_msgs([50, 60, 70]),
+    });
+
+    mld.prepend(info.top_messages);
+    mld.append(info.bottom_messages);
+
+    assert_contents([10, 15, 25, 35, 45, 50, 60, 70]);
+
+    mld.add(info.interior_messages);
+
+    assert_contents([10, 15, 20, 25, 30, 35, 40, 45, 50, 60, 70]);
+
+    assert.equal(mld.selected_id(), -1);
+    assert.equal(mld.closest_id(8), 10);
+    assert.equal(mld.closest_id(27), 25);
+    assert.equal(mld.closest_id(72), 70);
+
+    mld.set_selected_id(50);
+    assert.equal(mld.selected_id(), 50);
+    assert.equal(mld.selected_idx(), 8);
+
+    mld.remove([mld.get(50)]);
+    assert_contents([10, 15, 20, 25, 30, 35, 40, 45, 60, 70]);
+
+    mld.update_items_for_muting();
+    assert_contents([10, 15, 20, 25, 30, 35, 40, 45, 60, 70]);
+
+    mld.reset_select_to_closest();
+    assert.equal(mld.selected_id(), 45);
+    assert.equal(mld.selected_idx(), 7);
+
+    assert.equal(mld.first_unread_message_id(), 10);
+    mld.get(10).unread = false;
+    assert.equal(mld.first_unread_message_id(), 15);
+
+
+    mld.clear();
+    assert_contents([]);
+    assert.equal(mld.closest_id(99), -1);
+    assert.equal(mld.get_last_message_sent_by_me(), undefined);
+
+    mld.add(make_msgs([120, 125.01, 130, 140]));
+    assert_contents([120, 125.01, 130, 140]);
+    mld.set_selected_id(125.01);
+    assert.equal(mld.selected_id(), 125.01);
+
+    mld.get(125.01).id = 145;
+    mld.change_message_id(125.01, 145, {
+        re_render: () => {},
+    });
+    assert_contents([120, 130, 140, 145]);
+
+    _.each(mld.all_messages(), (msg) => {
+        msg.unread = false;
+    });
+
+    assert.equal(mld.first_unread_message_id(), 145);
+}());
+
+(function test_errors() {
+    const mld = new MessageListData({
+        muting_enabled: false,
+        filter: undefined,
+    });
+    assert.equal(mld.get('bogus-id'), undefined);
+}());

frontend_tests/node_tests/message_list_view.js

@@ -5,13 +5,14 @@ zrequire('util');
 zrequire('XDate', 'node_modules/xdate/src/xdate');
 zrequire('Filter', 'js/filter');
 zrequire('FetchStatus', 'js/fetch_status');
+zrequire('MessageListData', 'js/message_list_data');
 zrequire('MessageListView', 'js/message_list_view');
 zrequire('message_list');
 
 var noop = function () {};
 
 set_global('page_params', {
-  twenty_four_hour_time: false,
+    twenty_four_hour_time: false,
 });
 set_global('home_msg_list', null);
 set_global('feature_flags', {twenty_four_hour_time: false});
@@ -274,7 +275,7 @@ set_global('rows', {
         assert_message_groups_list_equal(result.append_groups, []);
         assert_message_groups_list_equal(result.prepend_groups, []);
         assert_message_groups_list_equal(result.rerender_groups,
-            [build_message_group([message2, message1])]);
+                                         [build_message_group([message2, message1])]);
         assert_message_list_equal(result.append_messages, []);
         assert_message_list_equal(result.rerender_messages, []);
     }());
@@ -376,7 +377,7 @@ set_global('rows', {
 
         // Stub out functionality that is not core to the rendering window
         // logic.
-        list.unmuted_messages = function (messages) {
+        list.data.unmuted_messages = function (messages) {
             return messages;
         };
 

frontend_tests/node_tests/message_store.js

@@ -27,7 +27,7 @@ set_global('page_params', {
     is_admin: true,
 });
 
-set_global('blueslip', {});
+set_global('blueslip', global.make_zblueslip());
 
 var me = {
     email: 'me@example.com',
@@ -137,21 +137,20 @@ global.people.initialize_current_user(me.user_id);
         display_recipient: [{user_id: 92714}],
     };
 
-    var blueslip_errors = 0;
-    blueslip.error = function () {
-        blueslip_errors += 1;
-    };
+    blueslip.set_test_data('error', 'Unknown user_id in get_person_from_user_id: 92714');
+    blueslip.set_test_data('error', 'Unknown user id 92714'); // From person.js
 
     // Expect each to throw two blueslip errors
     // One from message_store.js, one from person.js
     var emails = message_store.get_pm_emails(message);
     assert.equal(emails, '?');
-    assert.equal(blueslip_errors, 2);
+    assert.equal(blueslip.get_test_logs('error').length, 2);
 
-    blueslip_errors = 0;
     var names = message_store.get_pm_full_names(message);
     assert.equal(names, '?');
-    assert.equal(blueslip_errors, 2);
+    assert.equal(blueslip.get_test_logs('error').length, 4);
+
+    blueslip.clear_test_data();
 
     message = {
         type: 'stream',

frontend_tests/node_tests/narrow.js

@@ -77,9 +77,12 @@ function set_filter(operators) {
 
     var hide_id;
     var show_id;
-    global.$ = function (id) {
-      return {hide: function () {hide_id = id;}, show: function () {show_id = id;}};
-    };
+    set_global('$', (id) => {
+        return {
+            hide: () => {hide_id = id;},
+            show: () => {show_id = id;},
+        };
+    });
 
     narrow_state.reset_current_filter();
     narrow.show_empty_narrow_message();

frontend_tests/node_tests/narrow_activate.js

@@ -0,0 +1,215 @@
+set_global('$', global.make_zjquery());
+
+zrequire('narrow_state');
+zrequire('stream_data');
+zrequire('Filter', 'js/filter');
+zrequire('unread');
+zrequire('narrow');
+
+set_global('blueslip', {});
+set_global('channel', {});
+set_global('compose_actions', {});
+set_global('current_msg_list', {});
+set_global('hashchange', {});
+set_global('home_msg_list', {});
+set_global('message_fetch', {});
+set_global('message_list', {});
+set_global('message_scroll', {});
+set_global('message_util', {});
+set_global('notifications', {});
+set_global('page_params', {});
+set_global('search', {});
+set_global('stream_list', {});
+set_global('top_left_corner', {});
+set_global('ui_util', {});
+set_global('unread_ops', {});
+
+
+var noop = () => {};
+//
+// We have strange hacks in narrow.activate to sleep 0
+// seconds.
+global.patch_builtin('setTimeout', (f, t) => {
+    assert.equal(t, 0);
+    f();
+});
+
+function stub_trigger(f) {
+    set_global('document', 'document-stub');
+    $('document-stub').trigger = f;
+    $.Event = (name) => {
+        assert.equal(name, 'narrow_activated.zulip');
+    };
+}
+
+var denmark = {
+    subscribed: false,
+    color: 'blue',
+    name: 'Denmark',
+    stream_id: 1,
+    in_home_view: false,
+};
+stream_data.add_sub('Denmark', denmark);
+
+function test_helper() {
+    var events = [];
+
+    function stub(module_name, func_name) {
+        global[module_name][func_name] = () => {
+            events.push(module_name + '.' + func_name);
+        };
+    }
+
+    stub('compose_actions', 'on_narrow');
+    stub('hashchange', 'save_narrow');
+    stub('message_scroll', 'hide_indicators');
+    stub('notifications', 'clear_compose_notifications');
+    stub('notifications', 'redraw_title');
+    stub('search', 'update_button_visibility');
+    stub('stream_list', 'handle_narrow_activated');
+    stub('top_left_corner', 'handle_narrow_activated');
+    stub('ui_util', 'change_tab_to');
+    stub('unread_ops', 'process_visible');
+
+    stub_trigger(() => { events.push('trigger event'); });
+
+    blueslip.debug = noop;
+
+    message_util.add_messages = (messages, target_list, opts) => {
+        // The real function here doesn't do any more than this
+        // that we care about here.
+        target_list.add_messages(messages, opts);
+    };
+
+    return {
+        clear: () => {
+            events = [];
+        },
+        push_event: (event) => {
+            events.push(event);
+        },
+        assert_events: (expected_events) => {
+            assert.deepEqual(expected_events, events);
+        },
+    };
+}
+
+function stub_message_list() {
+    message_list.MessageList = function (table_name, filter) {
+        var list = this;
+        this.messages = [];
+        this.filter = filter;
+        this.view = {
+            set_message_offset: function (offset) {
+                list.view.offset = offset;
+            },
+        };
+
+        return this;
+    };
+
+    message_list.MessageList.prototype = {
+        add_messages: function (messages) {
+            var predicate = this.filter.predicate();
+            messages = _.filter(messages, predicate);
+            this.messages = this.messages.concat(messages);
+        },
+
+        get: function (msg_id) {
+            var msg = _.find(this.messages, (msg) => {
+                return msg.id === msg_id;
+            });
+            return msg;
+        },
+
+        empty: function () {
+            return this.messages.length === 0;
+        },
+
+        select_id: function (msg_id) {
+            this.selected_id = msg_id;
+        },
+    };
+}
+
+(function test_basics() {
+    stub_message_list();
+
+    var helper = test_helper();
+    var terms = [
+        { operator: 'stream', operand: 'Denmark' },
+    ];
+
+    var selected_id = 1000;
+
+    var selected_message = {
+        id: selected_id,
+        type: 'stream',
+        stream_id: denmark.stream_id,
+    };
+
+    var messages = [selected_message];
+
+    var row = {
+        length: 1,
+        offset: () => { return {top: 25}; },
+    };
+
+    current_msg_list.selected_id = () => { return -1; };
+    current_msg_list.get_row = () => { return row; };
+
+    message_list.all = {
+        all_messages: () => {
+            return messages;
+        },
+        get: (msg_id) => {
+            assert.equal(msg_id, selected_id);
+            return selected_message;
+        },
+    };
+
+    var cont;
+
+    message_fetch.load_messages_for_narrow = (opts) => {
+        cont = opts.cont;
+
+        assert.deepEqual(opts, {
+            cont: opts.cont,
+            then_select_id: 1000,
+            use_first_unread_anchor: false,
+        });
+    };
+
+    narrow.activate(terms, {
+        then_select_id: selected_id,
+    });
+
+    assert.equal(message_list.narrowed.selected_id, selected_id);
+    assert.equal(message_list.narrowed.view.offset, 25);
+
+    helper.assert_events([
+        'notifications.clear_compose_notifications',
+        'notifications.redraw_title',
+        'ui_util.change_tab_to',
+        'message_scroll.hide_indicators',
+        'unread_ops.process_visible',
+        'hashchange.save_narrow',
+        'search.update_button_visibility',
+        'compose_actions.on_narrow',
+        'top_left_corner.handle_narrow_activated',
+        'stream_list.handle_narrow_activated',
+        'trigger event',
+    ]);
+
+    channel.post = (opts) => {
+        assert.equal(opts.url, '/json/report/narrow_times');
+        helper.push_event('report narrow times');
+    };
+
+    helper.clear();
+    cont();
+    helper.assert_events([
+        'report narrow times',
+    ]);
+
+}());

frontend_tests/node_tests/narrow_state.js

@@ -3,9 +3,8 @@ zrequire('Filter', 'js/filter');
 zrequire('stream_data');
 zrequire('narrow_state');
 
-set_global('blueslip', {});
-set_global('page_params', {
-});
+set_global('blueslip', global.make_zblueslip());
+set_global('page_params', {});
 
 function set_filter(operators) {
     operators = _.map(operators, function (op) {
@@ -24,15 +23,6 @@ function set_filter(operators) {
 
     assert(!narrow_state.is_for_stream_id(test_stream.stream_id));
 
-    var bad_stream_id = 1000000;
-    var called = false;
-    global.blueslip.error = function (msg) {
-        assert.equal(msg, 'Bad stream id ' + bad_stream_id);
-        called = true;
-    };
-    assert(!narrow_state.is_for_stream_id(bad_stream_id));
-    assert(called);
-
     set_filter([
         ['stream', 'Test'],
         ['topic', 'Bar'],
@@ -41,6 +31,7 @@ function set_filter(operators) {
     assert(narrow_state.active());
 
     assert.equal(narrow_state.stream(), 'Test');
+    assert.equal(narrow_state.stream_id(), test_stream.stream_id);
     assert.equal(narrow_state.topic(), 'Bar');
     assert(narrow_state.is_for_stream_id(test_stream.stream_id));
 
@@ -65,6 +56,7 @@ function set_filter(operators) {
     assert(!narrow_state.narrowed_to_search());
     assert(!narrow_state.narrowed_to_topic());
     assert(!narrow_state.narrowed_by_stream_reply());
+    assert.equal(narrow_state.stream_id(), undefined);
 
     set_filter([['stream', 'Foo']]);
     assert(!narrow_state.narrowed_to_pms());
@@ -204,9 +196,17 @@ function set_filter(operators) {
 (function test_stream() {
     set_filter(undefined);
     assert.equal(narrow_state.stream(), undefined);
+    assert.equal(narrow_state.stream_id(), undefined);
 
     set_filter([['stream', 'Foo'], ['topic', 'Bar']]);
     assert.equal(narrow_state.stream(), 'Foo');
+    assert.equal(narrow_state.stream_sub(), undefined);
+    assert.equal(narrow_state.stream_id(), undefined);
+
+    const sub = {name: 'Foo', stream_id: 55};
+    stream_data.add_sub('Foo', sub);
+    assert.equal(narrow_state.stream_id(), 55);
+    assert.deepEqual(narrow_state.stream_sub(), sub);
 
     set_filter([['sender', 'someone'], ['topic', 'random']]);
     assert.equal(narrow_state.stream(), undefined);
@@ -225,15 +225,11 @@ function set_filter(operators) {
     set_filter([['pm-with', '']]);
     assert.equal(narrow_state.pm_string(), undefined);
 
-    var called = false;
-    blueslip.warn = function (error) {
-        assert.equal(error, 'Unknown emails: bogus@foo.com');
-        called = true;
-    };
-
+    blueslip.set_test_data('warn', 'Unknown emails: bogus@foo.com');
     set_filter([['pm-with', 'bogus@foo.com']]);
     assert.equal(narrow_state.pm_string(), undefined);
-    assert(called);
+    assert.equal(blueslip.get_test_logs('warn').length, 1);
+    blueslip.clear_test_data();
 
     var alice = {
         email: 'alice@foo.com',

frontend_tests/node_tests/narrow_unread.js

@@ -0,0 +1,197 @@
+zrequire('Filter', 'js/filter');
+zrequire('people');
+zrequire('stream_data');
+zrequire('unread');
+zrequire('util');
+set_global('blueslip', global.make_zblueslip());
+
+set_global('message_store', {});
+set_global('page_params', {});
+
+set_global('muting', {
+    is_topic_muted: () => false,
+});
+
+// The main code we are testing lives here.
+zrequire('narrow_state');
+
+const alice = {
+    email: 'alice@example.com',
+    user_id: 11,
+    full_name: 'Alice',
+};
+
+people.init();
+people.add(alice);
+people.is_my_user_id = () => false;
+
+function set_filter(terms) {
+    const filter = new Filter(terms);
+    narrow_state.set_current_filter(filter);
+}
+
+function assert_unread_info(expected) {
+    assert.deepEqual(narrow_state.get_first_unread_info(), expected);
+}
+
+function candidate_ids() {
+    return narrow_state._possible_unread_message_ids();
+}
+
+(function test_get_unread_ids() {
+    var unread_ids;
+    var terms;
+
+    const sub = {
+        name: 'My Stream',
+        stream_id: 55,
+    };
+
+    const stream_msg = {
+        id: 101,
+        type: 'stream',
+        stream_id: sub.stream_id,
+        subject: 'my topic',
+        unread: true,
+        mentioned: true,
+    };
+
+    const private_msg = {
+        id: 102,
+        type: 'private',
+        unread: true,
+        display_recipient: [
+            {user_id: alice.user_id},
+        ],
+    };
+
+    stream_data.add_sub(sub.name, sub);
+
+    unread_ids = candidate_ids();
+    assert.equal(unread_ids, undefined);
+
+    terms = [
+        {operator: 'bogus_operator', operand: 'me@example.com'},
+    ];
+    set_filter(terms);
+    unread_ids = candidate_ids();
+    assert.equal(unread_ids, undefined);
+    assert_unread_info({flavor: 'cannot_compute'});
+
+    terms = [
+        {operator: 'stream', operand: 'bogus'},
+    ];
+    set_filter(terms);
+    unread_ids = candidate_ids();
+    assert.deepEqual(unread_ids, []);
+
+    terms = [
+        {operator: 'stream', operand: sub.name},
+    ];
+    set_filter(terms);
+    unread_ids = candidate_ids();
+    assert.deepEqual(unread_ids, []);
+    assert_unread_info({flavor: 'not_found'});
+
+    unread.process_loaded_messages([stream_msg]);
+    message_store.get = (msg_id) => {
+        assert.equal(msg_id, stream_msg.id);
+        return stream_msg;
+    };
+
+    unread_ids = candidate_ids();
+    assert.deepEqual(unread_ids, [stream_msg.id]);
+    assert_unread_info({
+        flavor: 'found',
+        msg_id: stream_msg.id,
+    });
+
+    terms = [
+        {operator: 'stream', operand: 'bogus'},
+        {operator: 'topic', operand: 'my topic'},
+    ];
+    set_filter(terms);
+    unread_ids = candidate_ids();
+    assert.deepEqual(unread_ids, []);
+
+    terms = [
+        {operator: 'stream', operand: sub.name},
+        {operator: 'topic', operand: 'my topic'},
+    ];
+    set_filter(terms);
+    unread_ids = candidate_ids();
+    assert.deepEqual(unread_ids, [stream_msg.id]);
+
+    terms = [
+        {operator: 'is', operand: 'mentioned'},
+    ];
+    set_filter(terms);
+    unread_ids = candidate_ids();
+    assert.deepEqual(unread_ids, [stream_msg.id]);
+
+    terms = [
+        {operator: 'sender', operand: 'me@example.com'},
+    ];
+    set_filter(terms);
+    // note that our candidate ids are just "all" ids now
+    unread_ids = candidate_ids();
+    assert.deepEqual(unread_ids, [stream_msg.id]);
+
+    // this actually does filtering
+    assert_unread_info({flavor: 'not_found'});
+
+    terms = [
+        {operator: 'pm-with', operand: 'alice@example.com'},
+    ];
+    set_filter(terms);
+    unread_ids = candidate_ids();
+    assert.deepEqual(unread_ids, []);
+
+    unread.process_loaded_messages([private_msg]);
+
+    message_store.get = (msg_id) => {
+        assert.equal(msg_id, private_msg.id);
+        return private_msg;
+    };
+
+    unread_ids = candidate_ids();
+    assert.deepEqual(unread_ids, [private_msg.id]);
+
+    assert_unread_info({
+        flavor: 'found',
+        msg_id: private_msg.id,
+    });
+
+    terms = [
+        {operator: 'is', operand: 'private'},
+    ];
+    set_filter(terms);
+    unread_ids = candidate_ids();
+    assert.deepEqual(unread_ids, [private_msg.id]);
+
+    terms = [
+        {operator: 'pm-with', operand: 'bob@example.com'},
+    ];
+    set_filter(terms);
+
+    blueslip.set_test_data('warn', 'Unknown emails: bob@example.com');
+    unread_ids = candidate_ids();
+    assert.deepEqual(unread_ids, []);
+
+    terms = [
+        {operator: 'is', operand: 'starred'},
+    ];
+    set_filter(terms);
+    unread_ids = candidate_ids();
+    assert.deepEqual(unread_ids, []);
+
+    terms = [
+        {operator: 'search', operand: 'needle'},
+    ];
+    set_filter(terms);
+
+    blueslip.set_test_data('error', 'unexpected call to get_first_unread_info');
+    assert_unread_info({
+        flavor: 'cannot_compute',
+    });
+}());

frontend_tests/node_tests/notifications.js

@@ -1,7 +1,7 @@
 // Dependencies
-zrequire('muting');
-zrequire('stream_data');
-
+set_global('$', global.make_zjquery({
+    silent: true,
+}));
 set_global('document', {
     hasFocus: function () {
         return true;
@@ -12,6 +12,15 @@ set_global('page_params', {
     is_admin: false,
     realm_users: [],
 });
+// For people.js
+set_global('md5', function (s) {
+    return 'md5-' + s;
+});
+
+zrequire('muting');
+zrequire('stream_data');
+zrequire('ui');
+zrequire('people');
 
 zrequire('notifications');
 
@@ -55,94 +64,201 @@ stream_data.add_sub('stream_two', two);
 
     // Case 1: If the message was sent by this user,
     //  DO NOT notify the user
-        // In this test, all other circumstances should trigger notification
-        // EXCEPT sent_by_me, which should trump them
-        assert.equal(notifications.message_is_notifiable({
-            id: 10,
-            content: 'message number 1',
-            sent_by_me: true,
-            notification_sent: false,
-            mentioned_me_directly: true,
-            type: 'stream',
-            stream: 'stream_two',
-            stream_id: 20,
-            subject: 'topic_two',
-        }), false);
+    // In this test, all other circumstances should trigger notification
+    // EXCEPT sent_by_me, which should trump them
+    assert.equal(notifications.message_is_notifiable({
+        id: 10,
+        content: 'message number 1',
+        sent_by_me: true,
+        notification_sent: false,
+        mentioned_me_directly: true,
+        type: 'stream',
+        stream: 'stream_two',
+        stream_id: 20,
+        subject: 'topic_two',
+    }), false);
 
     // Case 2: If the user has already been sent a notificaton about this message,
     //  DO NOT notify the user
-        // In this test, all other circumstances should trigger notification
-        // EXCEPT notification_sent, which should trump them
-        // (ie: it mentions user, it's not muted, etc)
-        assert.equal(notifications.message_is_notifiable({
-            id: 20,
-            content: 'message number 2',
-            sent_by_me: false,
-            notification_sent: true,
-            mentioned_me_directly: true,
-            type: 'stream',
-            stream: 'stream_two',
-            stream_id: 20,
-            subject: 'topic_two',
-        }), false);
+    // In this test, all other circumstances should trigger notification
+    // EXCEPT notification_sent, which should trump them
+    // (ie: it mentions user, it's not muted, etc)
+    assert.equal(notifications.message_is_notifiable({
+        id: 20,
+        content: 'message number 2',
+        sent_by_me: false,
+        notification_sent: true,
+        mentioned_me_directly: true,
+        type: 'stream',
+        stream: 'stream_two',
+        stream_id: 20,
+        subject: 'topic_two',
+    }), false);
 
     // Case 3: If a message mentions the user directly,
     //  DO notify the user
-        // Mentioning trumps muting
-        assert.equal(notifications.message_is_notifiable({
-            id: 30,
-            content: 'message number three',
-            sent_by_me: false,
-            notification_sent: false,
-            mentioned_me_directly: true,
-            type: 'stream',
-            stream: 'stream_one',
-            stream_id: 10,
-            subject: 'topic_three',
-        }), true);
+    // Mentioning trumps muting
+    assert.equal(notifications.message_is_notifiable({
+        id: 30,
+        content: 'message number three',
+        sent_by_me: false,
+        notification_sent: false,
+        mentioned_me_directly: true,
+        type: 'stream',
+        stream: 'stream_one',
+        stream_id: 10,
+        subject: 'topic_three',
+    }), true);
 
-        // Mentioning should trigger notification in unmuted topic
-        assert.equal(notifications.message_is_notifiable({
-            id: 40,
-            content: 'message number 4',
-            sent_by_me: false,
-            notification_sent: false,
-            mentioned_me_directly: true,
-            type: 'stream',
-            stream: 'stream_two',
-            stream_id: 20,
-            subject: 'topic_two',
-        }), true);
+    // Mentioning should trigger notification in unmuted topic
+    assert.equal(notifications.message_is_notifiable({
+        id: 40,
+        content: 'message number 4',
+        sent_by_me: false,
+        notification_sent: false,
+        mentioned_me_directly: true,
+        type: 'stream',
+        stream: 'stream_two',
+        stream_id: 20,
+        subject: 'topic_two',
+    }), true);
 
     // Case 4: If a message is in a muted stream
     //  and does not mention the user DIRECTLY,
     //  DO NOT notify the user
-        assert.equal(notifications.message_is_notifiable({
-            id: 50,
-            content: 'message number 5',
-            sent_by_me: false,
-            notification_sent: false,
-            mentioned_me_directly: false,
-            type: 'stream',
-            stream: 'stream_one',
-            stream_id: 10,
-            subject: 'topic_one',
-        }), false);
+    assert.equal(notifications.message_is_notifiable({
+        id: 50,
+        content: 'message number 5',
+        sent_by_me: false,
+        notification_sent: false,
+        mentioned_me_directly: false,
+        type: 'stream',
+        stream: 'stream_one',
+        stream_id: 10,
+        subject: 'topic_one',
+    }), false);
 
     // Case 5
-        // If none of the above cases apply
-        // (ie: topic is not muted, message does not mention user,
-        //  no notification sent before, message not sent by user),
-        // return true to pass it to notifications settings
-        assert.equal(notifications.message_is_notifiable({
-            id: 60,
-            content: 'message number 6',
-            sent_by_me: false,
-            notification_sent: false,
-            mentioned_me_directly: false,
-            type: 'stream',
-            stream: 'stream_two',
-            stream_id: 20,
-            subject: 'topic_two',
-        }), true);
+    // If none of the above cases apply
+    // (ie: topic is not muted, message does not mention user,
+    //  no notification sent before, message not sent by user),
+    // return true to pass it to notifications settings
+    assert.equal(notifications.message_is_notifiable({
+        id: 60,
+        content: 'message number 6',
+        sent_by_me: false,
+        notification_sent: false,
+        mentioned_me_directly: false,
+        type: 'stream',
+        stream: 'stream_two',
+        stream_id: 20,
+        subject: 'topic_two',
+    }), true);
+}());
+
+
+(function test_basic_notifications() {
+
+    var n; // Object for storing all notification data for assertions.
+    var last_closed_message_id = null;
+    var last_shown_message_id = null;
+
+    // Notifications API stub
+    notifications.set_notification_api({
+        checkPermission: function checkPermission() {
+            if (window.Notification.permission === 'granted') {
+                return 0;
+            }
+            return 2;
+        },
+        requestPermission: function () {
+            return;
+        },
+        createNotification: function createNotification(icon, title, content, tag) {
+            var notification_object = {icon: icon, body: content, tag: tag};
+            // properties for testing.
+            notification_object.tests = {
+                shown: false,
+            };
+            notification_object.show = function () {
+                last_shown_message_id = this.tag;
+            };
+            notification_object.close = function () {
+                last_closed_message_id = this.tag;
+            };
+            notification_object.cancel = function () { notification_object.close(); };
+            return notification_object;
+        },
+    });
+
+    var message_1 = {
+        id: 1000,
+        content: '@-mentions the user',
+        avatar_url: 'url',
+        sent_by_me: false,
+        notification_sent: false,
+        mentioned_me_directly: true,
+        type: 'stream',
+        stream: 'stream_one',
+        stream_id: 10,
+        subject: 'topic_two',
+    };
+
+    var message_2 = {
+        id: 1500,
+        avatar_url: 'url',
+        content: '@-mentions the user',
+        sent_by_me: false,
+        notification_sent: false,
+        mentioned_me_directly: true,
+        type: 'stream',
+        stream: 'stream_one',
+        stream_id: 10,
+        subject: 'topic_four',
+    };
+
+    // Send notification.
+    notifications.process_notification({message: message_1, webkit_notify: true});
+    n = notifications.get_notifications();
+    assert.equal('undefined to stream_one > topic_two' in n, true);
+    assert.equal(Object.keys(n).length, 1);
+    assert.equal(last_shown_message_id, message_1.id);
+
+    // Remove notification.
+    notifications.close_notification(message_1);
+    n = notifications.get_notifications();
+    assert.equal('undefined to stream_one > topic_two' in n, false);
+    assert.equal(Object.keys(n).length, 0);
+    assert.equal(last_closed_message_id, message_1.id);
+
+    // Send notification.
+    message_1.id = 1001;
+    notifications.process_notification({message: message_1, webkit_notify: true});
+    n = notifications.get_notifications();
+    assert.equal('undefined to stream_one > topic_two' in n, true);
+    assert.equal(Object.keys(n).length, 1);
+    assert.equal(last_shown_message_id, message_1.id);
+
+    // Process same message again. Notification count shouldn't increase.
+    message_1.id = 1002;
+    notifications.process_notification({message: message_1, webkit_notify: true});
+    n = notifications.get_notifications();
+    assert.equal('undefined to stream_one > topic_two' in n, true);
+    assert.equal(Object.keys(n).length, 1);
+    assert.equal(last_shown_message_id, message_1.id);
+
+    // Send another message. Notification count should increase.
+    notifications.process_notification({message: message_2, webkit_notify: true});
+    n = notifications.get_notifications();
+    assert.equal('undefined to stream_one > topic_four' in n, true);
+    assert.equal(Object.keys(n).length, 2);
+    assert.equal(last_shown_message_id, message_2.id);
+
+    // Remove notifications.
+    notifications.close_notification(message_1);
+    notifications.close_notification(message_2);
+    n = notifications.get_notifications();
+    assert.equal('undefined to stream_one > topic_two' in n, false);
+    assert.equal(Object.keys(n).length, 0);
+    assert.equal(last_closed_message_id, message_2.id);
 }());

frontend_tests/node_tests/people.js

@@ -1,9 +1,9 @@
 zrequire('util');
 zrequire('people');
 
-set_global('blueslip', {
-    error: function () { return; },
-});
+set_global('blueslip', global.make_zblueslip({
+    error: false, // We check for errors in people_errors.js
+}));
 set_global('page_params', {});
 set_global('md5', function (s) {
     return 'md5-' + s;
@@ -86,13 +86,10 @@ initialize();
     assert.equal(people.is_active_user_for_popover(bot_botson.user_id), true);
 
     // Invalid user ID returns false and warns.
-    var message;
-    blueslip.warn = function (msg) {
-        message = msg;
-    };
-
+    blueslip.set_test_data('warn', 'Unexpectedly invalid user_id in user popover query: 123412');
     assert.equal(people.is_active_user_for_popover(123412), false);
-    assert.equal(message, 'Unexpectedly invalid user_id in user popover query: 123412');
+    assert.equal(blueslip.get_test_logs('warn').length, 1);
+    blueslip.clear_test_data();
 
     // We can still get their info for non-realm needs.
     person = people.get_by_email(email);
@@ -245,7 +242,7 @@ initialize();
 }());
 
 (function test_filtered_users() {
-     var charles = {
+    var charles = {
         email: 'charles@example.com',
         user_id: 301,
         full_name: 'Charles Dickens',
@@ -374,13 +371,15 @@ initialize();
         avatar_url: 'charles.com/foo.png',
     };
     var maria = {
-        email: 'athens@example.com',
+        email: 'Athens@example.com',
         user_id: 452,
         full_name: 'Maria Athens',
     };
     people.add(charles);
     people.add(maria);
 
+    assert.equal(people.small_avatar_url_for_person(maria),
+                 'https://secure.gravatar.com/avatar/md5-athens@example.com?d=identicon&s=50');
     var message = {
         type: 'private',
         display_recipient: [
@@ -392,9 +391,9 @@ initialize();
     };
     assert.equal(people.pm_with_url(message), '#narrow/pm-with/451,452-group');
     assert.equal(people.pm_reply_to(message),
-        'athens@example.com,charles@example.com');
+                 'Athens@example.com,charles@example.com');
     assert.equal(people.small_avatar_url(message),
-        'charles.com/foo.png&s=50');
+                 'charles.com/foo.png&s=50');
 
     message = {
         type: 'private',
@@ -406,16 +405,16 @@ initialize();
     };
     assert.equal(people.pm_with_url(message), '#narrow/pm-with/452-athens');
     assert.equal(people.pm_reply_to(message),
-        'athens@example.com');
+                 'Athens@example.com');
     assert.equal(people.small_avatar_url(message),
-        'legacy.png&s=50');
+                 'legacy.png&s=50');
 
     message = {
         avatar_url: undefined,
         sender_id: maria.user_id,
     };
     assert.equal(people.small_avatar_url(message),
-        'https://secure.gravatar.com/avatar/md5-athens@example.com?d=identicon&s=50'
+                 'https://secure.gravatar.com/avatar/md5-athens@example.com?d=identicon&s=50'
     );
 
     message = {
@@ -424,7 +423,7 @@ initialize();
         sender_id: 9999999,
     };
     assert.equal(people.small_avatar_url(message),
-        'https://secure.gravatar.com/avatar/md5-foo@example.com?d=identicon&s=50'
+                 'https://secure.gravatar.com/avatar/md5-foo@example.com?d=identicon&s=50'
     );
 
     message = {
@@ -616,14 +615,13 @@ initialize();
 
     // Test shim where we can still retrieve user info using the
     // old email.
-    var warning;
-    global.blueslip.warn = function (w) {
-        warning = w;
-    };
-
+    blueslip.set_test_data('warn',
+                           'Obsolete email passed to get_by_email: ' +
+                           'FOO@example.com new email = bar@example.com');
     person = people.get_by_email(old_email);
-    assert(/Obsolete email.*FOO.*bar/.test(warning));
     assert.equal(person.user_id, user_id);
+    assert.equal(blueslip.get_test_logs('warn').length, 1);
+    blueslip.clear_test_data();
 }());
 
 initialize();

frontend_tests/node_tests/people_errors.js

@@ -1,6 +1,11 @@
 zrequire('people');
+set_global('reload', {
+    is_in_progress: false,
+});
 
-set_global('blueslip', {});
+set_global('blueslip', global.make_zblueslip({
+    debug: true, // testing for debug is disabled by default.
+}));
 
 var me = {
     email: 'me@example.com',
@@ -14,68 +19,68 @@ people.add(me);
 people.initialize_current_user(me.user_id);
 
 (function test_report_late_add() {
-    var message;
-    global.blueslip.error = function (msg) {
-        message = msg;
-    };
-
+    blueslip.set_test_data('error', 'Added user late: user_id=55 email=foo@example.com');
     people.report_late_add(55, 'foo@example.com');
-    assert.equal(message, 'Added user late: user_id=55 email=foo@example.com');
+    assert.equal(blueslip.get_test_logs('error').length, 1);
+    blueslip.clear_test_data();
+
+    reload.is_in_progress = true;
+    people.report_late_add(55, 'foo@example.com');
+    assert.equal(blueslip.get_test_logs('log').length, 1);
+    assert.equal(blueslip.get_test_logs('log')[0], 'Added user late: user_id=55 email=foo@example.com');
+    assert.equal(blueslip.get_test_logs('error').length, 0);
+    blueslip.clear_test_data();
 }());
 
 (function test_blueslip() {
     var unknown_email = "alicebobfred@example.com";
 
-    global.blueslip.debug = function (msg) {
-        assert.equal(msg, 'User email operand unknown: ' + unknown_email);
-    };
-
-    var warning;
-    global.blueslip.warn = function (w) {
-        warning = w;
-    };
-
+    blueslip.set_test_data('debug', 'User email operand unknown: ' + unknown_email);
     people.id_matches_email_operand(42, unknown_email);
+    assert.equal(blueslip.get_test_logs('debug').length, 1);
+    blueslip.clear_test_data();
 
-    global.blueslip.error = function (msg) {
-        assert.equal(msg, 'Unknown email for get_user_id: ' + unknown_email);
-    };
+    blueslip.set_test_data('error', 'Unknown email for get_user_id: ' + unknown_email);
     people.get_user_id(unknown_email);
+    assert.equal(blueslip.get_test_logs('error').length, 1);
+    blueslip.clear_test_data();
 
+    blueslip.set_test_data('warn', 'No user_id provided for person@example.com');
     var person = {
         email: 'person@example.com',
         user_id: undefined,
         full_name: 'Person Person',
     };
     people.add(person);
-    assert.equal(warning, 'No user_id provided for person@example.com');
+    assert.equal(blueslip.get_test_logs('warn').length, 1);
+    blueslip.clear_test_data();
 
-    global.blueslip.error = function (msg) {
-        assert.equal(msg, 'No user_id found for person@example.com');
-    };
+    blueslip.set_test_data('error', 'No user_id found for person@example.com');
     var user_id = people.get_user_id('person@example.com');
     assert.equal(user_id, undefined);
+    assert.equal(blueslip.get_test_logs('error').length, 1);
+    blueslip.clear_test_data();
 
-    global.blueslip.error = function (msg) {
-        assert.equal(msg, 'Unknown user ids: 1,2');
-    };
+    blueslip.set_test_data('error', 'Unknown user ids: 1,2');
     people.user_ids_string_to_emails_string('1,2');
+    assert.equal(blueslip.get_test_logs('error').length, 1);
+    blueslip.clear_test_data();
 
-    global.blueslip.warn = function (msg) {
-        assert.equal(msg, 'Unknown emails: ' + unknown_email);
-    };
+    blueslip.set_test_data('warn', 'Unknown emails: ' + unknown_email);
     people.email_list_to_user_ids_string(unknown_email);
+    assert.equal(blueslip.get_test_logs('warn').length, 1);
+    blueslip.clear_test_data();
 
     var message = {
         type: 'private',
         display_recipient: [],
         sender_id: me.user_id,
     };
-    global.blueslip.error = function (msg) {
-        assert.equal(msg, 'Empty recipient list in message');
-    };
+    blueslip.set_test_data('error', 'Empty recipient list in message');
     people.pm_with_user_ids(message);
     people.group_pm_with_user_ids(message);
+    assert.equal(blueslip.get_test_logs('error').length, 2);
+    blueslip.clear_test_data();
 
     var charles = {
         email: 'charles@example.com',
@@ -100,18 +105,17 @@ people.initialize_current_user(me.user_id);
         ],
         sender_id: charles.user_id,
     };
-    global.blueslip.error = function (msg) {
-        assert.equal(msg, 'Unknown user id in message: 42');
-    };
+    blueslip.set_test_data('error', 'Unknown user id in message: 42');
     var reply_to = people.pm_reply_to(message);
     assert(reply_to.indexOf('?') > -1);
+    assert.equal(blueslip.get_test_logs('error').length, 1);
+    blueslip.clear_test_data();
 
     people.pm_with_user_ids = function () { return [42]; };
     people.get_person_from_user_id = function () { return; };
-    global.blueslip.error = function (msg) {
-        assert.equal(msg, 'Unknown people in message');
-    };
+    blueslip.set_test_data('error', 'Unknown people in message');
     var uri = people.pm_with_url({});
     assert.equal(uri.indexOf('unk'), uri.length - 3);
+    assert.equal(blueslip.get_test_logs('error').length, 1);
+    blueslip.clear_test_data();
 }());
-

frontend_tests/node_tests/pm_list.js

@@ -9,7 +9,7 @@ set_global('stream_popover', {
 });
 set_global('unread', {});
 set_global('unread_ui', {});
-set_global('blueslip', {});
+set_global('blueslip', global.make_zblueslip());
 set_global('popovers', {
     hide_all: function () {},
 });
@@ -45,12 +45,11 @@ global.people.initialize_current_user(me.user_id);
 
 (function test_get_conversation_li() {
     var test_conversation = 'foo@example.com,bar@example.com';
-    var error_msg;
-    global.blueslip.warn = function (error) {
-        error_msg = error;
-    };
+    blueslip.set_test_data('warn', 'Unknown conversation: ' + test_conversation);
+    blueslip.set_test_data('warn', 'Unknown emails: ' + test_conversation); // people.js
     pm_list.get_conversation_li(test_conversation);
-    assert.equal(error_msg, 'Unknown conversation: ' + test_conversation);
+    assert.equal(blueslip.get_test_logs('warn').length, 2);
+    blueslip.clear_test_data();
 }());
 
 (function test_close() {

frontend_tests/node_tests/popovers.js

@@ -0,0 +1,177 @@
+set_global('$', global.make_zjquery());
+set_global('i18n', global.stub_i18n);
+
+zrequire('hashchange');
+zrequire('hash_util');
+zrequire('narrow');
+zrequire('narrow_state');
+zrequire('people');
+zrequire('presence');
+
+var noop =  function () {};
+$.fn.popover = noop; // this will get wrapped by our code
+
+zrequire('popovers');
+
+set_global('current_msg_list', {});
+set_global('page_params', {
+    custom_profile_fields: [],
+});
+set_global('rows', {});
+set_global('templates', {});
+
+
+set_global('message_viewport', {
+    height: () => 500,
+});
+
+set_global('emoji_picker', {
+    hide_emoji_popover: noop,
+});
+
+set_global('stream_popover', {
+    hide_stream_popover: noop,
+    hide_topic_popover: noop,
+    hide_all_messages_popover: noop,
+    restore_stream_list_size: noop,
+});
+
+set_global('ClipboardJS', function (sel) {
+    assert.equal(sel, '.copy_link');
+});
+
+var alice = {
+    email: 'alice@example.com',
+    full_name: 'Alice Smith',
+    user_id: 42,
+};
+
+var me = {
+    email: 'me@example.com',
+    user_id: 30,
+    full_name: 'Me Myself',
+    timezone: 'US/Pacific',
+};
+
+function initialize_people() {
+    people.init();
+    people.add_in_realm(me);
+    people.add_in_realm(alice);
+    people.initialize_current_user(me.user_id);
+}
+
+initialize_people();
+
+function make_image_stubber() {
+    var images = [];
+
+    function stub_image() {
+        var image = {};
+        image.to_$ = () => {
+            return {
+                on: (name, f) => {
+                    assert.equal(name, "load");
+                    image.load_f = f;
+                },
+            };
+        };
+        images.push(image);
+        return image;
+    }
+
+    set_global('Image', function () {
+        return stub_image();
+    });
+
+    return {
+        get: (i) => images[i],
+    };
+}
+
+(function test_sender_hover() {
+    popovers.register_click_handlers();
+
+    var handler = $('#main_div').get_on_handler('click', '.sender_info_hover');
+    var e = {
+        stopPropagation: noop,
+    };
+
+    var message = {
+        id: 999,
+        sender_id: alice.user_id,
+    };
+
+    var target = $.create('click target');
+
+    target.offset = () => {
+        return {
+            top: 10,
+        };
+    };
+
+    rows.id = () => message.id;
+
+    current_msg_list.get = (msg_id) => {
+        assert.equal(msg_id, message.id);
+        return message;
+    };
+
+    current_msg_list.select_id = (msg_id) => {
+        assert.equal(msg_id, message.id);
+    };
+
+    target.closest = (sel) => {
+        assert.equal(sel, '.message_row');
+        return {};
+    };
+
+    templates.render = function (fn, opts) {
+        switch (fn) {
+        case 'user_info_popover':
+            assert.deepEqual(opts, {
+                class: 'message-info-popover',
+            });
+            return 'popover-html';
+
+        case 'user_info_popover_title':
+            assert.deepEqual(opts, {
+                user_avatar: 'avatar/alice@example.com',
+            });
+            return 'title-html';
+
+        case 'user_info_popover_content':
+            assert.deepEqual(opts, {
+                user_full_name: 'Alice Smith',
+                user_email: 'alice@example.com',
+                user_id: 42,
+                user_time: undefined,
+                presence_status: 'offline',
+                user_last_seen_time_status: 'translated: Unknown',
+                pm_with_uri: '#narrow/pm-with/42-alice',
+                sent_by_uri: '#narrow/sender/42-alice',
+                narrowed: false,
+                private_message_class: 'respond_personal_button',
+                show_user_profile: false,
+                is_me: false,
+                is_active: true,
+                is_bot: undefined,
+                is_sender_popover: true,
+            });
+            return 'content-html';
+
+        default:
+            throw Error('unrecognized template: ' + fn);
+        }
+    };
+
+    $('.user_popover_email').each = noop;
+
+    var image_stubber = make_image_stubber();
+
+    handler.call(target, e);
+
+    var avatar_img = image_stubber.get(0);
+    assert.equal(avatar_img.src, 'avatar/42/medium');
+
+    // todo: load image
+}());

frontend_tests/node_tests/presence.js

@@ -159,15 +159,15 @@ people.initialize_current_user(me.user_id);
     presence.set_info(presences, base_time);
 
     assert.deepEqual(presence.presence_info[alice.user_id],
-        { status: 'active', mobile: false, last_active: 500}
+                     { status: 'active', mobile: false, last_active: 500}
     );
 
     assert.deepEqual(presence.presence_info[fred.user_id],
-        { status: 'idle', mobile: false, last_active: 500}
+                     { status: 'idle', mobile: false, last_active: 500}
     );
 
     assert.deepEqual(presence.presence_info[zoe.user_id],
-        { status: 'offline', mobile: false, last_active: undefined}
+                     { status: 'offline', mobile: false, last_active: undefined}
     );
 
     assert(!presence.presence_info[bot.user_id]);

frontend_tests/node_tests/reactions.js

@@ -158,43 +158,43 @@ set_global('current_msg_list', {
     result.sort(function (a, b) { return a.count - b.count; });
 
     var expected_result = [
-      {
-         emoji_name: 'frown',
-         reaction_type: 'unicode_emoji',
-         emoji_code: '1f626',
-         local_id: 'unicode_emoji,frown,1f626',
-         count: 1,
-         user_ids: [7],
-         title: 'Cali reacted with :frown:',
-         emoji_alt_code: false,
-         class: 'message_reaction',
-      },
-      {
-         emoji_name: 'inactive_realm_emoji',
-         reaction_type: 'realm_emoji',
-         emoji_code: '992',
-         local_id: 'realm_emoji,inactive_realm_emoji,992',
-         count: 1,
-         user_ids: [5],
-         title: 'You (click to remove) reacted with :inactive_realm_emoji:',
-         emoji_alt_code: false,
-         is_realm_emoji: true,
-         url: 'TBD',
-         class: 'message_reaction reacted',
-      },
-      {
-         emoji_name: 'smile',
-         reaction_type: 'unicode_emoji',
-         emoji_code: '1f604',
-         local_id: 'unicode_emoji,smile,1f604',
-         count: 2,
-         user_ids: [5, 6],
-         title: 'You (click to remove) and Bob van Roberts reacted with :smile:',
-         emoji_alt_code: false,
-         class: 'message_reaction reacted',
-      },
-   ];
-   assert.deepEqual(result, expected_result);
+        {
+            emoji_name: 'frown',
+            reaction_type: 'unicode_emoji',
+            emoji_code: '1f626',
+            local_id: 'unicode_emoji,frown,1f626',
+            count: 1,
+            user_ids: [7],
+            title: 'Cali reacted with :frown:',
+            emoji_alt_code: false,
+            class: 'message_reaction',
+        },
+        {
+            emoji_name: 'inactive_realm_emoji',
+            reaction_type: 'realm_emoji',
+            emoji_code: '992',
+            local_id: 'realm_emoji,inactive_realm_emoji,992',
+            count: 1,
+            user_ids: [5],
+            title: 'You (click to remove) reacted with :inactive_realm_emoji:',
+            emoji_alt_code: false,
+            is_realm_emoji: true,
+            url: 'TBD',
+            class: 'message_reaction reacted',
+        },
+        {
+            emoji_name: 'smile',
+            reaction_type: 'unicode_emoji',
+            emoji_code: '1f604',
+            local_id: 'unicode_emoji,smile,1f604',
+            count: 2,
+            user_ids: [5, 6],
+            title: 'You (click to remove) and Bob van Roberts reacted with :smile:',
+            emoji_alt_code: false,
+            class: 'message_reaction reacted',
+        },
+    ];
+    assert.deepEqual(result, expected_result);
 }());
 
 (function test_sending() {
@@ -288,7 +288,7 @@ set_global('current_msg_list', {
 
     reactions.set_reaction_count(reaction_element, 5);
 
-    assert.equal(count_element.html(), '5');
+    assert.equal(count_element.text(), '5');
 }());
 
 (function test_get_reaction_section() {
@@ -388,7 +388,7 @@ set_global('current_msg_list', {
 
     reactions.add_reaction(bob_event);
     assert(title_set);
-    assert.equal(count_element.html(), '2');
+    assert.equal(count_element.text(), '2');
 
     // Now, remove Bob's 8ball emoji.  The event has the same exact
     // structure as the add event.
@@ -402,7 +402,7 @@ set_global('current_msg_list', {
 
     reactions.remove_reaction(bob_event);
     assert(title_set);
-    assert.equal(count_element.html(), '1');
+    assert.equal(count_element.text(), '1');
 
     var current_emojis = reactions.get_emojis_used_by_user_for_message_id(1001);
     assert.deepEqual(current_emojis, ['smile', 'inactive_realm_emoji', '8ball']);

frontend_tests/node_tests/scroll_util.js

@@ -0,0 +1,93 @@
+zrequire('scroll_util');
+
+(function test_scroll_delta() {
+    // If we are entirely on-screen, don't scroll
+    assert.equal(0, scroll_util.scroll_delta({
+        elem_top: 1,
+        elem_bottom: 9,
+        container_height: 10,
+    }));
+
+    assert.equal(0, scroll_util.scroll_delta({
+        elem_top: -5,
+        elem_bottom: 15,
+        container_height: 10,
+    }));
+
+    // The top is offscreen.
+    assert.equal(-3, scroll_util.scroll_delta({
+        elem_top: -3,
+        elem_bottom: 5,
+        container_height: 10,
+    }));
+
+    assert.equal(-3, scroll_util.scroll_delta({
+        elem_top: -3,
+        elem_bottom: -1,
+        container_height: 10,
+    }));
+
+    assert.equal(-11, scroll_util.scroll_delta({
+        elem_top: -150,
+        elem_bottom: -1,
+        container_height: 10,
+    }));
+
+    // The bottom is offscreen.
+    assert.equal(3, scroll_util.scroll_delta({
+        elem_top: 7,
+        elem_bottom: 13,
+        container_height: 10,
+    }));
+
+    assert.equal(3, scroll_util.scroll_delta({
+        elem_top: 11,
+        elem_bottom: 13,
+        container_height: 10,
+    }));
+
+    assert.equal(11, scroll_util.scroll_delta({
+        elem_top: 11,
+        elem_bottom: 99,
+        container_height: 10,
+    }));
+
+}());
+
+(function test_scroll_element_into_container() {
+    const container = (function () {
+        var top = 3;
+        return {
+            height: () => 100,
+            scrollTop: (arg) => {
+                if (arg === undefined) {
+                    return top;
+                }
+                top = arg;
+            },
+        };
+    }());
+
+    const elem1 = {
+        height: () => 25,
+        position: () => {
+            return {
+                top: 0,
+            };
+        },
+    };
+    scroll_util.scroll_element_into_container(elem1, container);
+    assert.equal(container.scrollTop(), 3);
+
+    const elem2 = {
+        height: () => 15,
+        position: () => {
+            return {
+                top: 250,
+            };
+        },
+    };
+    scroll_util.scroll_element_into_container(elem2, container);
+    assert.equal(container.scrollTop(), 250 - 100 + 3 + 15);
+}());
+

frontend_tests/node_tests/search_suggestion.js

@@ -248,9 +248,9 @@ topic_data.reset();
     };
 
     set_global('activity', {
-            get_huddles: function () {
-                return [];
-            },
+        get_huddles: function () {
+            return [];
+        },
     });
 
     var ted =
@@ -382,9 +382,9 @@ topic_data.reset();
     assert.deepEqual(suggestions.strings, expected);
 
     set_global('activity', {
-            get_huddles: function () {
-                return ['101,42', '101,103,42'];
-            },
+        get_huddles: function () {
+            return ['101,42', '101,103,42'];
+        },
     });
 
     // Simulate a past huddle which should now prioritize ted over alice
@@ -560,8 +560,8 @@ init();
 
     global.stream_data.get_stream_id = function (stream_name) {
         switch (stream_name) {
-            case 'office': return office_id;
-            case 'devel': return devel_id;
+        case 'office': return office_id;
+        case 'devel': return devel_id;
         }
     };
 
@@ -596,7 +596,7 @@ init();
         return suggestions.lookup_table[q].description;
     }
     assert.equal(describe('te'), "Search for te");
-    assert.equal(describe('stream:office topic:team'), "Stream office > team");
+    assert.equal(describe('stream:office topic:team'), "Stream office > team");
 
     suggestions = search.get_suggestions('topic:staplers stream:office');
     expected = [
@@ -758,9 +758,9 @@ init();
         return suggestions.lookup_table[q].description;
     }
     assert.equal(describe('pm-with:ted@zulip.com'),
-        "Private messages with <strong>Te</strong>d Smith &lt;<strong>te</strong>d@zulip.com&gt;");
+                 "Private messages with <strong>Te</strong>d Smith &lt;<strong>te</strong>d@zulip.com&gt;");
     assert.equal(describe('sender:ted@zulip.com'),
-        "Sent by <strong>Te</strong>d Smith &lt;<strong>te</strong>d@zulip.com&gt;");
+                 "Sent by <strong>Te</strong>d Smith &lt;<strong>te</strong>d@zulip.com&gt;");
 
     suggestions = search.get_suggestions('Ted '); // note space