Release Zulip server 1.8.0.

This logging was apparently broken when sorting imports; it's a fairly
unique thing in our codebase that this would be a problem.  Prevent
future regressions by adding this exception explicitly to the isort
configuration.

.circleci/config.yml

@@ -19,12 +19,13 @@ jobs:
               dirs=(/srv/zulip-{npm,venv}-cache)
               sudo mkdir -p "${dirs[@]}"
               sudo chown -R circleci "${dirs[@]}"
+
       - restore_cache:
           keys:
-          - v1-npm-base.trusty.1
+          - v1-npm-base.trusty-{{ checksum "package.json" }}-{{ checksum "yarn.lock" }}
       - restore_cache:
           keys:
-          - v1-venv-base.trusty.1
+          - v1-venv-base.trusty-{{ checksum "requirements/thumbor.txt" }}-{{ checksum "requirements/dev.txt" }}
 
       - run:
           name: install dependencies
@@ -50,11 +51,11 @@ jobs:
       - save_cache:
           paths:
             - /srv/zulip-npm-cache
-          key: v1-npm-base.trusty.1
+          key: v1-npm-base.trusty-{{ checksum "package.json" }}-{{ checksum "yarn.lock" }}
       - save_cache:
           paths:
             - /srv/zulip-venv-cache
-          key: v1-venv-base.trusty.1
+          key: v1-venv-base.trusty-{{ checksum "requirements/thumbor.txt" }}-{{ checksum "requirements/dev.txt" }}
       # TODO: in Travis we also cache ~/zulip-emoji-cache, ~/node, ~/misc
 
       # The moment of truth!  Run the tests.
@@ -99,12 +100,13 @@ jobs:
               dirs=(/srv/zulip-{npm,venv}-cache)
               sudo mkdir -p "${dirs[@]}"
               sudo chown -R circleci "${dirs[@]}"
+
       - restore_cache:
           keys:
-          - v1-npm-base.xenial.1
+          - v1-npm-base.xenial-{{ checksum "package.json" }}-{{ checksum "yarn.lock" }}
       - restore_cache:
           keys:
-          - v1-venv-base.xenial.1
+          - v1-venv-base.xenial-{{ checksum "requirements/thumbor.txt" }}-{{ checksum "requirements/dev.txt" }}
 
       - run:
           name: install dependencies
@@ -116,11 +118,11 @@ jobs:
       - save_cache:
           paths:
             - /srv/zulip-npm-cache
-          key: v1-npm-base.xenial.1
+          key: v1-npm-base.xenial-{{ checksum "package.json" }}-{{ checksum "yarn.lock" }}
       - save_cache:
           paths:
             - /srv/zulip-venv-cache
-          key: v1-venv-base.xenial.1
+          key: v1-venv-base.xenial-{{ checksum "requirements/thumbor.txt" }}-{{ checksum "requirements/dev.txt" }}
 
       - run:
           name: run backend tests

.eslintrc.json

@@ -15,7 +15,6 @@
         "XDate": false,
         "zxcvbn": false,
         "LazyLoad": false,
-        "Dropbox": false,
         "SockJS": false,
         "marked": false,
         "md5": false,
@@ -33,6 +32,7 @@
         "server_events": false,
         "server_events_dispatch": false,
         "message_scroll": false,
+        "keydown_util": false,
         "info_overlay": false,
         "ui": false,
         "ui_report": false,
@@ -167,7 +167,7 @@
         "emoji_codes": false,
         "drafts": false,
         "katex": false,
-        "Clipboard": false,
+        "ClipboardJS": false,
         "emoji_picker": false,
         "hotspots": false,
         "compose_ui": false,

.isort.cfg

@@ -6,3 +6,5 @@ known_third_party = django, ujson, sqlalchemy
 known_first_party = zerver, zproject, version, confirmation, zilencer, analytics, frontend_tests, scripts, corporate
 sections = FUTURE, STDLIB, THIRDPARTY, FIRSTPARTY, LOCALFOLDER
 lines_after_imports = 1
+# See the comment related to ioloop_logging for why this is skipped.
+skip = zerver/management/commands/runtornado.py

Vagrantfile

@@ -167,7 +167,7 @@ Welcome to the Zulip development environment!  Popular commands:
 * tools/lint - Run the linter (quick and catches many problmes)
 * tools/test-* - Run tests (use --help to learn about options)
 
-Read https://zulip.readthedocs.io/en/latest/testing.html to learn
+Read https://zulip.readthedocs.io/en/latest/testing/testing.html to learn
 how to run individual test suites so that you can get a fast debug cycle.
 
 EndOfMessage'

docs/conf.py

@@ -44,7 +44,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = 'Zulip'
-copyright = '2015-2017, The Zulip Team'
+copyright = '2015-2018, The Zulip Team'
 author = 'The Zulip Team'
 
 # The version info for the project you're documenting, acts as replacement for
@@ -52,9 +52,9 @@ author = 'The Zulip Team'
 # built documents.
 #
 # The short X.Y version.
-version = '1.7+git'
+version = '1.8'
 # The full version, including alpha/beta/rc tags.
-release = '1.7.1+git'
+release = '1.8.0'
 
 # This allows us to insert a warning that appears only on an unreleased
 # version, e.g. to say that something is likely to have changed.

docs/overview/changelog.md

@@ -7,111 +7,80 @@ All notable changes to the Zulip server are documented in this file.
 This section lists notable unreleased changes; it is generally updated
 in bursts.
 
-**Highlights:**
+### 1.8.0 -- 2018-04-17
 
-- Added a user setting to choose the emoji set used in Zulip: Google,
-  Twitter, Apple, or Emoji One.
-- Added a video call integration powered by Jitsi.
+**Highlights:**
 - Dramatically simplified the server installation process; it's now possible
   to install Zulip without first setting up outgoing email.
-- Added certbot support to the installer for getting certificates.
-- Added support for mentioning groups of users.
+- Added experimental support for importing an organization's history
+  from Slack.
 - Added a new "night mode" theme for dark environments.
-- Added experimental support for importing an organization from Slack.
-- Overhauled our settings system to eliminate the ugly "save changes"
-  button system.
-- Rewrote our API documentation to be much more friendly and
-  expansive; it now covers most important endpoints, with nice examples.
+- Added a video call integration powered by Jitsi.
+- Lots of visual polish improvements.
+- Countless small bugfixes both in the backend and the UI.
+
+
+**Security and privacy:**
+- Several important security fixes since 1.7.0, which were released
+  already in 1.7.1 and 1.7.2.
 - The security model for private streams has changed.  Now
   organization administrators can remove users, edit descriptions, and
   rename private streams they are not subscribed to.  See Zulip's
   security model documentation for details.
-- Lots of visual polish improvements.
-
-**Full feature changelog:**
-
-- New integrations: ErrBot, GoCD, Google Code-In, Opbeat, Groove, Raygun,
-  Insping, Dropbox, Front, Intercom, Statuspage.io, Flock and Beeminder.
-- The local uploads backend now does the same security checks that the
-  S3 backend did before serving files to users.
-- Added support for users in multiple realms having the same email.
-- Added support for embedded interactive bots.
-- Added inline preview + player for Vimeo videos.
-- Added a setting to allow users to delete their messages.
+- On Xenial, the local uploads backend now does the same security
+  checks that the S3 backend did before serving files to users.
+  Ubuntu Trusty's version of nginx is too old to support this and so
+  the legacy model is the default; we recommend upgrading.
 - Added an organization setting to limit creation of bots.
+- Refactored the authentication backends codebase to be much easier to
+  verify.
+- Added a user setting to control whether email notifications include
+  message content (or just the fact that there are new messages).
+
+
+**Visual and UI:**
+- Added a user setting to translate emoticons/smileys to emoji.
+- Added a user setting to choose the emoji set used in Zulip: Google,
+  Twitter, Apple, or Emoji One.
+- Expanded setting for displaying emoji as text to cover all display
+  settings (previously only affected reactions).
+- Overhauled our settings system to eliminate the old "save changes"
+  button system.
+- Redesigned the "uploaded files" UI.
+- Redesigned the "account settings" UI.
+- Redesigned error pages for the various email confirmation flows.
+- Our emoji now display at full resolution on retina displays.
+- Improved placement of text when inserting emoji via picker.
+- Improved the descriptions and UI for many settings.
+- Improved visual design of the help center (/help/).
+
+
+**Core chat experience:**
+- Added support for mentioning groups of users.
+- Added a setting to allow users to delete their messages.
 - Added support for uploading files in the message-edit UI.
-- Added new event types to several webhook integrations.
-- Added a display for whether the user is logged-in in logged-out
-  pages.
-- Added support for hosting multiple domains, not all as subdomains of
-  the same base domain.
-- Added a new /team/ page explaining the team, with a nice
-  visualization of our contributors.
-- Added support for default bots to receive messages when they're
-mentioned, even if they are not subscribed.
-- Added support for inviting a new user as an administrator.
-- Added a new organization settings page for managing invites.
-- Added a user setting to control whether the organization's name is
-  included in email subject lines.
-- Added support for clicking on a mention to see a user's profile.
-- Added new compose features for pasting HTML.
 - Redesigned the compose are for private messages to use pretty pills
   rather than raw email addresses to display recipients.
 - Added new ctrl+B, ctrl+I, ctrl+L compose shortcuts for inserting
   common syntax.
 - Added warning when linking to a private stream via typeahead.
-- Added rate-limiting on inviting users to join a realm (prevents spam).
 - Added support for automatically-numbered markdown lists.
 - Added a big warning when posting to #announce.
-- Added a user setting to control whether email notifications include
-  message content (or just the fact that there are new messages).
 - Added a notification when drafts are saved, to make them more
-discoverable.
+  discoverable.
 - Added a fast local echo to emoji reactions.
-- Added new "basics" section to keyboard shortcuts documentation.
-- Added a new ">" keyboard shortcut for quote-and-reply.
-- Added a new "p" keyboard shortcut to just to next unread PM thread.
-- Added support for overriding the topic is all incoming webhook integrations.
-- Added a new nagios check for the Zulip analytics state.
-- Added a menu item to mark all messages as read.
-- Added support for logging into the mobile apps with RemoteUserBackend.
-- Added an organization setting to disable welcome emails to new users.
-- Added traffic statistics (messages/week) to the "Manage streams" UI.
-- Added a display setting to translate emoticons/smileys to emoji.
-- Added an organization setting to ban disposable email addresses
-  (I.e.. those from sites like mailinator.com).
-- Added a server setting to control whether digest emails are sent.
-- Links to logged-in content in Zulip now take the user to the
-  appropriate upload or view after a user logs in.
-- Incoming webhooks now send a private message to the bot owner for
-  more convenient testing.
-- Rewrote documentation for many integrations to use a cleaner
-  numbered-list format.
-- Renamed "Home" to "All messages", to avoid users clicking on it too
-  early in using Zulip.
 - Messages containing just a link to an image (or an uploaded image)
   now don't clutter the feed with the URL: we just display the image.
-- Refactored the authentication backends codebase to be much easier to
-  verify.
-- Expanded setting for displaying emoji as text to cover all display
-  settings (previously only affected reactions).
 - Redesigned the API for emoji reactions to support the full range of
   how emoji reactions are used.
-- Migrated the codebase to use the nice Python 3 typing syntax.
-- Optimized how user avatar URLs are transmitted over the wire.
-- Optimized message sending performance a bit more.
-- Split the Notifications Stream setting in two settings, one for new
-  users, the other for new streams.
-- Fixed numerous issues in the "stream settings" UI.
 - Fixed most of the known (mostly obscure) bugs in how messages are
   formatted in Zulip.
 - Fixed "more topics" to correctly display all historical topics for
   public streams, even though from before a user subscribed.
-- Fixed several bugs around interacting with deactivated users.
+- Added a menu item to mark all messages as read.
 - Fixed image upload file pickers offering non-image files.
 - Fixed some subtle bugs with full-text search and unicode.
 - Fixed bugs in the "edit history" HTML rendering process.
-- Fixed several hotkeys scope bugs.
 - Fixed popovers being closed when new messages come in.
 - Fixed unexpected code blocks when using the email mirror.
 - Fixed clicking on links to a narrow opening a new window.
@@ -119,47 +88,132 @@ discoverable.
 - Fixed layering issues with mobile Safari.
 - Fixed several obscure real-time synchronization bugs.
 - Fixed handling of messages with a very large HTML rendering.
-- Fixed buggy APNs logic that could cause extra exception emails.
+- Fixed several bugs around interacting with deactivated users.
 - Fixed interaction bugs with unread counts and deleting messages.
 - Fixed support for replacing deactivated custom emoji.
-- Fixed a missing dependency for the localhost_sso auth backend.
-- Fixed uploading user avatars encoded using the CMYK mode.
 - Fixed scrolling downwards in narrows.
-- Fixed numerous subtle bugs with the stream creation UI.
-- Dramatically improved organization of developer docs.
-- Statistics on translation percentages now include mobile apps.
+- Optimized how user avatar URLs are transmitted over the wire.
+- Optimized message sending performance a bit more.
+- Fixed a subtle and hard-to-reproduce bug that resulted in every
+  message being condensed ([More] appearing on every message).
 - Improved typeahead's handling of editing an already-completed mention.
 - Improved syntax for inline LaTeX to be more convenient.
-- Improve keyboard navigation of left and right sidebars with arrow keys.
-- Changes the URL scheme for stream narrows to encode the stream ID,
-  so that they can be robust to streams being renamed.  The change is
-  backwards-compatible; existing narrow URLs still work.
-- APIs for fetching messages now provide more metadata to help clients.
-- Clarified instructions for server settings (especially LDAP auth).
-- Redesigned the "uploaded files" UI.
-- Redesigned the "account settings" UI.
-- Redesigned error pages for the various email confirmation flows.
-- Added missing information on requesting user in many exception emails.
-- Our emoji now display at full resolution on retina displays.
-- Improved placement of text when inserting emoji via picker.
-- Improved the password reset flow to be less confusing if you don't
-  have an account.
+- Improved syntax for permanent links to streams in Zulip.
 - Improved behavior of copy-pasting a large number of messages.
-- Improved Tornado retry logic for connecting to RabbitMQ.
-- Improved the descriptions and UI for many settings.
 - Improved handling of browser undo in compose.
-- Improved mobile notifications to support narrowing when one click a
-  mobile push notification.
-- Improved visual design of the help center (/help/).
 - Improved saved drafts system to garbage-collect old drafts and sort
   by last modification, not creation.
 - Removed the legacy "Zulip labs" autoscroll_forever setting.  It was
   enabled mostly by accident.
 - Removed some long-deprecated markdown syntax for mentions.
+- Added support for clicking on a mention to see a user's profile.
+- Links to logged-in content in Zulip now take the user to the
+  appropriate upload or view after a user logs in.
+- Renamed "Home" to "All messages", to avoid users clicking on it too
+  early in using Zulip.
+- Added a user setting to control whether the organization's name is
+  included in email subject lines.
+- Fixed uploading user avatars encoded using the CMYK mode.
+
+
+**User accounts and invites:**
+- Added support for users in multiple realms having the same email.
+- Added a display for whether the user is logged-in in logged-out
+  pages.
+- Added support for inviting a new user as an administrator.
+- Added a new organization settings page for managing invites.
+- Added rate-limiting on inviting users to join a realm (prevents spam).
+- Added an organization setting to disable welcome emails to new users.
+- Added an organization setting to ban disposable email addresses
+  (I.e.. those from sites like mailinator.com).
+- Improved the password reset flow to be less confusing if you don't
+  have an account.
+- Split the Notifications Stream setting in two settings, one for new
+  users, the other for new streams.
+
+
+**Stream subscriptions and settings:**
+- Added traffic statistics (messages/week) to the "Manage streams" UI.
+- Fixed numerous issues in the "stream settings" UI.
+- Fixed numerous subtle bugs with the stream creation UI.
+- Changes the URL scheme for stream narrows to encode the stream ID,
+  so that they can be robust to streams being renamed.  The change is
+  backwards-compatible; existing narrow URLs still work.
+
+
+**API, bots, and integrations:**
+- Rewrote our API documentation to be much more friendly and
+  expansive; it now covers most important endpoints, with nice examples.
+- New integrations: ErrBot, GoCD, Google Code-In, Opbeat, Groove,
+  Raygun, Insping, Dialogflow, Dropbox, Front, Intercom,
+  Statuspage.io, Flock and Beeminder.
+- Added support for embedded interactive bots.
+- Added inline preview + player for Vimeo videos.
+- Added new event types and fixed bugs in several webhook integrations.
+- Added support for default bots to receive messages when they're
+  mentioned, even if they are not subscribed.
+- Added support for overriding the topic is all incoming webhook integrations.
+- Incoming webhooks now send a private message to the bot owner for
+  more convenient testing if a stream is not specified.
+- Rewrote documentation for many integrations to use a cleaner
+  numbered-list format.
+- APIs for fetching messages now provide more metadata to help clients.
+
+
+**Keyboard shortcuts:**
+- Added new "basics" section to keyboard shortcuts documentation.
+- Added a new ">" keyboard shortcut for quote-and-reply.
+- Added a new "p" keyboard shortcut to just to next unread PM thread.
+- Fixed several hotkeys scope bugs.
+- Changed the hotkey for compose-private-message from "C" to "x".
+- Improve keyboard navigation of left and right sidebars with arrow keys.
+
+
+**Mobile apps backend:**
+- Added support for logging into the mobile apps with RemoteUserBackend.
+- Improved mobile notifications to support narrowing when one clicks a
+  mobile push notification.
 - Statistics on the fraction of strings that are translated now
   include strings in the mobile apps as well.
+
+
+**For server admins:**
+- Added certbot support to the installer for getting certificates.
+- Added support for hosting multiple domains, not all as subdomains of
+  the same base domain.
+- Added a new nagios check for the Zulip analytics state.
+- Fixed buggy APNs logic that could cause extra exception emails.
+- Fixed a missing dependency for the localhost_sso auth backend.
+- Fixed subtle bugs in garbage-collection of old node_modules versions.
+- Clarified instructions for server settings (especially LDAP auth).
+- Added missing information on requesting user in many exception emails.
+- Improved Tornado retry logic for connecting to RabbitMQ.
+- Added a server setting to control whether digest emails are sent.
+
+
+**For Zulip developers:**
+- Migrated the codebase to use the nice Python 3 typing syntax.
+- Added a new /team/ page explaining the team, with a nice
+  visualization of our contributors.
+- Dramatically improved organization of developer docs.
 - Backend test coverage is now 95%.
-- Countless other little bug fixes both in the backend and the UI.
+
+
+### 1.7.2 -- 2018-04-12
+
+This is a security release, with a handful of cherry-picked changes
+since 1.7.1.  All Zulip server admins are encouraged to upgrade
+promptly.
+
+- CVE-2018-9986: Fix XSS issues with frontend markdown processor.
+- CVE-2018-9987: Fix XSS issue with muting notifications.
+- CVE-2018-9990: Fix XSS issue with stream names in topic typeahead.
+- CVE-2018-9999: Fix XSS issue with user uploads.  The fix for this
+  adds a Content-Security-Policy for the `LOCAL_UPLOADS_DIR` storage
+  backend for user-uploaded files.
+
+Thanks to Suhas Sunil Gaikwad for reporting CVE-2018-9987 and w2w for
+reporting CVE-2018-9986 and CVE-2018-9990.
 
 ### 1.7.1 -- 2017-11-21
 

docs/overview/gsoc-ideas.md

@@ -61,7 +61,7 @@ contributors to the project today).
 
 ### Expectations for GSoC students
 
-[Our guide for having a great summer with Zulip](../contributing/summer-with-zulip)
+[Our guide for having a great summer with Zulip](../contributing/summer-with-zulip.html)
 is focused on what one should know once doing a summer project with
 Zulip.  But it has a lot of useful advice on how we expect students to
 interact, above and beyond what is discussed in Google's materials.

docs/production/email.md

@@ -1,30 +1,58 @@
 # Outgoing email
 
-This page documents everything you need to know about setting up
-outgoing email in a Zulip production environment.  It's pretty simple
-if you already have an outgoing SMTP provider; just start reading from
-[the configuration section](#configuration).
+Zulip needs to be able to send email so it can confirm new users'
+email addresses and send notifications.
+
+## How to configure
+
+1. Identify an outgoing email (SMTP) account where you can have Zulip
+   send mail.  If you don't already have one you want to use, see
+   [Email services](#email-services) below.
+
+2. Fill out the section of `/etc/zulip/settings.py` headed "Outgoing
+   email (SMTP) settings".  This includes the hostname and typically
+   the port to reach your SMTP provider, and the username to log into
+   it as.
+
+3. Put the password for the SMTP user account in
+   `/etc/zulip/zulip-secrets.conf` by setting `email_password`. For
+   example: `email_password = abcd1234`.
+
+   Like any other change to the Zulip configuration, be sure to
+   [restart the server](settings.html) to make your changes take
+   effect.
+
+4. Test that your configuration is working.  See the test command in
+   the [Troubleshooting](#troubleshooting) section below.  If it's not
+   working, see the suggestions in that section.
+
+## Email services
 
 ### Free outgoing email services
 
 For sending outgoing email from your Zulip server, we highly recommend
 using a "transactional email" service like
-[Mailgun](https://documentation.mailgun.com/en/latest/quickstart-sending.html#send-via-smtp)
-or for AWS users,
+[SendGrid](https://sendgrid.com/docs/API_Reference/SMTP_API/integrating_with_the_smtp_api.html),
+[Mailgun](https://documentation.mailgun.com/en/latest/quickstart-sending.html#send-via-smtp),
+or, for AWS users,
 [Amazon SES](http://docs.aws.amazon.com/ses/latest/DeveloperGuide/send-email-smtp.html).
 These services are designed to send email from servers, and are by far
 the easiest way to get outgoing email working reliably.
 
 If you don't have an existing outgoing SMTP provider, don't worry!
-Both of the options we recommend above (as well as dozens of other
-services) have free options; we recommend Mailgun as the easiest to
-get setup with.  Once you've signed up, you'll want to find the
-service's provided "SMTP credentials", and configure Zulip as follows:
+Each of the options we recommend above (as well as dozens of other
+services) have free options.  Once you've signed up, you'll want to
+find the service's provided "SMTP credentials", and configure Zulip as
+follows:
 
-* The hostname as `EMAIL_HOST = 'smtp.mailgun.org'` in `/etc/zulip/settings.py`
-* The username as `EMAIL_HOST_USER = 'username@example.com` in
+* The hostname like `EMAIL_HOST = 'smtp.mailgun.org'` in `/etc/zulip/settings.py`
+* The username like `EMAIL_HOST_USER = 'username@example.com` in
   `/etc/zulip/settings.py`.
-* The password as `email_password = abcd1234` in `/etc/zulip/zulip-secrets.conf`.
+* The TLS setting as `EMAIL_USE_TLS = True` in
+  `/etc/zulip/settings.py`, for most providers
+* The port as `EMAIL_PORT = 587` in `/etc/zulip/settings.py`, for most
+  providers
+* The password like `email_password = abcd1234` in `/etc/zulip/zulip-secrets.conf`.
 
 ### Using Gmail for outgoing email
 
@@ -44,38 +72,28 @@ how to make it work:
 * Note also that the rate limits for Gmail are also quite low
   (e.g. 100 / day), so it's easy to get rate-limited if your server
   has significant traffic.  For more active servers, we recommend
-  moving to a free account from a transaction email service.
+  moving to a free account on a transactional email service.
 
 ### Logging outgoing email to a file for prototyping
 
-If for prototyping, you don't want to bother setting up an email
-provider, you can add to `/etc/zulip/settings.py` the following:
+For prototyping, you might want to proceed without setting up an email
+provider.  If you want to see the emails Zulip would have sent, you
+can log them to a file instead.
+
+To do so, add these lines to `/etc/zulip/settings.py`:
 
 ```
 EMAIL_BACKEND = 'django.core.mail.backends.filebased.EmailBackend'
 EMAIL_FILE_PATH = '/var/log/zulip/emails'
 ```
 
-Outgoing emails that Zulip would have sent will just be written to
-files in `/var/log/zulip/emails/`.  This is enough to get you through
-initial user registration without an SMTP provider.
+Then outgoing emails that Zulip would have sent will just be written
+to files in `/var/log/zulip/emails/`.
 
-Remember to delete this configuration and restart the server if you
-later setup a real SMTP provider!
+Remember to delete this configuration (and restart the server) if you
+later set up a real SMTP provider!
 
-### Configuration
-
-To configure outgoing SMTP, you will need to complete the following steps:
-
-1. Fill out the outgoing email sending configuration block in
-`/etc/zulip/settings.py`, including `EMAIL_HOST`, and
-`EMAIL_HOST_USER`.  You may also need to set `EMAIL_PORT` if your
-provider doesn't use the standard SMTP submission port (587).
-
-2. Put the SMTP password for `EMAIL_HOST_USER` in
-`/etc/zulip/zulip-secrets.conf` as `email_password = yourPassword`.
-
-#### Testing and troubleshooting
+## Troubleshooting
 
 You can quickly test your outgoing email configuration using:
 
@@ -87,42 +105,50 @@ su zulip
 If it doesn't throw an error, it probably worked; you can confirm by
 checking your email.
 
-It's important to test, because outgoing email often doesn't work the
-first time.  Common causes of failures are:
+If it doesn't work, check these common failure causes:
 
-* Your hosting provider blocking outgoing SMTP traffic in its
-default firewall rules.  Check whether `EMAIL_PORT` is blocked in your
-hosting provider's firewall.
-* Forgetting to put the password in `/etc/zulip/zulip-secrets.conf`.
-* Typos in transcribing the username or password.
+* Your hosting provider may block outgoing SMTP traffic in its default
+  firewall rules.  Check whether the port `EMAIL_PORT` is blocked in
+  your hosting provider's firewall.
 
-Once you have it working from the management command, remember to
-restart your Zulip server using
-`/home/zulip/deployments/current/scripts/restart-server` so that the running
-server is using the latest configuration.
+* Make sure you set the password in `/etc/zulip/zulip-secrets.conf`.
 
-#### Advanced troubleshooting
+* Check the username and password for typos.
+
+* Be sure to restart your Zulip server after editing either
+  `settings.py` or `zulip-secrets.conf`, using
+  `/home/zulip/deployments/current/scripts/restart-server` .
+  Note that the `manage.py` command above will read the latest
+  configuration from the config files, even if the server is still
+  running with an old configuration.
+
+### Advanced troubleshooting
 
 Here are a few final notes on what to look at when debugging why you
 aren't receiving emails from Zulip:
 
 * Most transactional email services have an "outgoing email" log where
-  you can inspect the emails that reached the service, whether it was
-  flagged as spam, etc.
+  you can inspect the emails that reached the service, whether an
+  email was flagged as spam, etc.
+
 * Starting with Zulip 1.7, Zulip logs an entry in
   `/var/log/zulip/send_email.log` whenever it attempts to send an
-  email, including whether the request succeeded or failed.
+  email.  The log entry includes whether the request succeeded or failed.
+
 * If attempting to send an email throws an exception, a traceback
   should be in `/var/log/zulip/errors.log`, along with any other
   exceptions Zulip encounters.
+
 * Zulip's email sending configuration is based on the standard Django
-  [SMTP backend](https://docs.djangoproject.com/en/1.10/topics/email/#smtp-backend)
+  [SMTP backend](https://docs.djangoproject.com/en/2.0/topics/email/#smtp-backend)
   configuration.  So if you're having trouble getting your email
   provider working, you may want to search for documentation related
-  to using your email provider with Django.  The one thing we've
-  changed from the defaults is reading the email password from the
-  `email_password` entry in the Zulip secrets file, as part of our
-  policy of not having any secret information in the
-  `/etc/zulip/settings.py` file.  In other words, if Django
-  documentation references setting `EMAIL_HOST_PASSWORD`, you should
-  instead set `email_password` in `/etc/zulip/zulip-secrets.conf`.
+  to using your email provider with Django.
+
+  The one thing we've changed from the Django defaults is that we read
+  the email password from the `email_password` entry in the Zulip
+  secrets file, as part of our policy of not having any secret
+  information in the `/etc/zulip/settings.py` file.  In other words,
+  if Django documentation references setting `EMAIL_HOST_PASSWORD`,
+  you should instead set `email_password` in
+  `/etc/zulip/zulip-secrets.conf`.

docs/production/install-existing-server.md

@@ -16,7 +16,7 @@ recommended, and you may break your server. Make sure you have backups
 and a provisioning script ready to go to wipe and restore your
 existing services if (when) your server goes down.
 
-These instructions are only for experts.  If you're not an experiecned
+These instructions are only for experts.  If you're not an experienced
 Linux sysadmin, you will have a much better experience if you get a
 dedicated VM to install Zulip on instead (or [use zulipchat.com](https://zulipchat.com).
 

docs/production/ssl-certificates.md

@@ -13,10 +13,14 @@ If you already have an SSL certificate, just install (or symlink) its
 files into place at the following paths:
 * `/etc/ssl/private/zulip.key` for the private key
 * `/etc/ssl/certs/zulip.combined-chain.crt` for the certificate.
-  Because Zulip uses nginx as its web server, this should be in the
-  format of a [chained certificate bundle][nginx-https].
 
-[nginx-https]: http://nginx.org/en/docs/http/configuring_https_servers.html
+Your certificate file should contain not only your own certificate but
+its full chain, including any intermediate certificates used by your
+CA.  See the [nginx documentation][nginx-chains] for details on what
+this means and how to do it and test it.  If you're missing part of
+the chain, your server may work with some browsers but not others.
+
+[nginx-chains]: http://nginx.org/en/docs/http/configuring_https_servers.html#chains
 
 ## Certbot (recommended)
 

docs/subsystems/index.rst

@@ -36,5 +36,6 @@ Subsystems Documentation
    documentation
    conversion
    input-pills
+   presence
    unread_messages
    user-docs

docs/subsystems/presence.md

@@ -0,0 +1,55 @@
+# Presence
+
+This document explains the model for Zulip's presence.
+
+In a chat tool like Zulip, users expect to see the “presence” status
+of other users: is the person I want to talk to currently online? If
+not, were they last online 5 minutes ago, or more like an hour ago, or
+a week?  Presence helps set expectations for whether someone is likely
+to respond soon.  To a user, this feature can seem like a simple thing
+that should be easy.  But presence is actually one of the hardest
+scalability problems for a team chat tool like Zulip.
+
+There's a lot of performance-related details in the backend and
+network protocol design that we won't get into here.  The focus of
+this is what one needs to know to correctly implement a Zulip client's
+presence implementation (e.g. webapp, mobile app, terminal client, or
+other tool that's intended to represent whether a user is online and
+using Zulip).
+
+A client should report to the server every minute a `POST` request to
+`/users/me/presence`, containing the current user's status.  The
+requests contains a few parameters.  The most important is "status",
+which had 2 valid values:
+
+* "active" -- this means the user has interacted with the client
+  recently.  We use this for the "green" state in the webapp.
+* "idle" -- the user has not interacted with the client recently.
+  This is important for the case where a user left a Zulip tab open on
+  their desktop at work and went home for the weekend.  We use this
+  for the "orange" state in the webapp.
+
+The client receives in the response to that request a data set that,
+for each user, contains their status and timestamp that we last heard
+from that client.  There are a few important details to understand
+about that data structure:
+
+* It's really important that the timestamp is the last time we heard
+  from the client.  A client can only interpret the status to display
+  about another user by doing a simple computation using the (status,
+  timestamp) pair.  E.g. a user who last used Zulip 1 week ago will
+  have a timestamp of 1 week ago and a status of "active".  Why?
+  Because this correctly handles the race conditions.  For example, if
+  the threshhold for displaying a user as "offline" was 5 minutes
+  since the user was last online, the client can at any time
+  accurately compute whether that user is offline (even if the last
+  data from the server was 45 seconds ago, and the user was last
+  online 4:30 before the client received that server data).
+* The `status_from_timestamp` function in `static/js/presence.js` is
+  useful sample code; the `OFFLINE_THRESHOLD_SECS` check is critical
+  to correct output.
+* We provide the data for e.g. whether the user was online on their
+  desktop or the mobile app, but for a basic client, you will likely
+  only want to parse the "aggregated" key, which shows the summary
+  answer for "is this user online".
+

docs/subsystems/user-docs.md

@@ -249,9 +249,9 @@ always create a new macro by adding a new file to that folder.
 
 ### **Organization settings** `{!admin.md!}` macro
 
-* **About:** Links to the **Organization settings** documentation.
-Usually preceded by the [**Go to the** macro](#go-to-the-go-to-the-md-macro)
-and a link to a particular section on the **Organization settings** page.
+* **About:** Links to the **Organization settings** documentation. Usually
+preceded by a link to a particular section on the **Organization settings**
+page.
 
 * **Contents:**
     ```md
@@ -260,7 +260,7 @@ and a link to a particular section on the **Organization settings** page.
 
 * **Example usage and rendering:**
     ```md
-    {!go-to-the.md!} [Organization settings](/#organization/organization-settings)
+    1. Go to the [Organization settings](/#organization/organization-settings)
     {!admin.md!}
     ```
     ```md
@@ -284,7 +284,7 @@ immediately after the title.
     ```md
     {!admin-only.md!}
 
-    {!follow-steps.md!} change who can join your stream by changing the stream's
+    Follow the following steps to change who can join your stream by changing the stream's
     accessibility.
     ```
     ```md
@@ -348,27 +348,6 @@ macro](#message-actions-message-actions-md-macro).
     down chevron (<i class="fa fa-chevron-down"></i>) icon to reveal an actions dropdown.
     ```
 
-### **Go to the** `{!go-to-the.md}` macro
-
-* **About:** Usually precedes the [**Settings** macro](#settings-settings-md-macro)
-or the [**Organization settings** macro](#organization-settings-admin-md-macro). Transforms
-following content into a step.
-
-* **Contents:**
-    ```md
-    1. Go to the
-    ```
-
-* **Example usage and rendering:**
-    ```md
-    {!go-to-the.md!} [Notifications](/#settings/notifications)
-    {!settings.md!}
-    ```
-    ```md
-    1. Go to the [Notifications](/#settings/notifications) tab on the
-    [Settings](/help/edit-settings) page.
-    ```
-
 ### **Filter streams** `{!filter-streams.md!}` macro
 
 * **About:** Explains how to search for specific streams in the
@@ -392,24 +371,6 @@ following content into a step.
     name of the stream in the **Filter streams** input.
     ```
 
-### **Follow steps** `{!follow-steps.md!}` macro
-
-* **About:** Prepends phrases with instructions to follow the following steps.
-
-* **Contents:**
-    ```md
-    Follow the following steps to
-    ```
-
-* **Example usage and rendering:**
-    ```md
-    {!follow-steps.md!}  change your mobile notification settings.
-    ```
-    ```md
-    Follow the following steps to change your mobile notification
-    settings.
-    ```
-
 ### **Message actions** `{!message-actions.md!}` macro
 
 * **About:** Explains how to view the actions of message. Usually followed by an instruction
@@ -456,8 +417,7 @@ describing the settings they modified.
 ### **Settings** `{!settings.md!}` macro
 
 * **About:** Links to the **Edit Settings** documentation. Usually preceded by
-the [**Go to the** macro](#go-to-the-go-to-the-md-macro) and a link to a
-particular section on the **Settings** page.
+a link to a particular section on the **Settings** page.
 
 * **Contents:**
     ```md
@@ -466,7 +426,7 @@ particular section on the **Settings** page.
 
 * **Example usage and rendering:**
     ```md
-    {!go-to-the.md!} [Notifications](/#settings/notifications)
+    1. Go to the [Notifications](/#settings/notifications)
     {!settings.md!}
     ```
     ```md

docs/tutorials/new-feature-tutorial.md

@@ -483,8 +483,9 @@ to be added to the admin page (and its value added to the data sent back
 to server when a realm is updated) and the change event needs to be
 handled on the client.
 
-To add the checkbox to the admin page, modify the relevant template,
-`static/templates/settings/organization-permissions-admin.handlebars`
+To add the checkbox to the admin page, modify the relevant template in
+`static/templates/settings/`, which can be
+`organization-permissions-admin.handlebars` or `organization-settings-admin.handlebars`
 (omitted here since it is relatively straightforward).
 
 Then add the new form control in `static/js/admin.js`.
@@ -507,66 +508,81 @@ function _setup_page() {
 The JavaScript code for organization settings and permissions can be found in
 `static/js/settings_org.js`.
 
-There is a front-end version of `property_types`, which reduces the code
-needed on the front end for a new feature.
+In frontend, we have split the `property_types` into three objects:
 
-Add the new feature to the `property_types` object in `settings_org.js`.
-The key should be the setting name and the value should be an object with
-the following keys:
+- `org_profile`: This contains properties for the "organization
+    profile" settings page.
 
-* type
-* checked_msg (what message the user sees when they enable the setting)
-* unchecked_msg (what message the user sees when they disable the setting)
+- `org_settings`: This contains properties for the "organization
+    settings" page. Settings belonging to this section generally
+    decide what features should be available to a user like deleting a
+    message, message edit history etc.  Our `mandatory_topics` feature
+    belongs in this section.
+
+- `org_permissions`: This contains properties for the "organization
+    permissions" section. These properties control security controls
+    like who can join the organization and whether normal users can
+    create streams or upload custom emoji.
+
+Once you've determined wheter the new setting belongs, the next step
+is to find the right subsection of that page to put the setting
+in. For example in this case of `mandatory_topics` it will lie in
+"Message feed" (`msg_feed`) subsection.
+
+*If you're not sure in which section your feature belongs, it's is
+better to discuss it in the [community](https://chat.zulip.org/)
+before implementing it.*
+
+When defining the property, you'll also need to specify the property
+field type (i.e. whether it's a `bool`, `integer` or `text`).
 
 ``` diff
 
 // static/js/settings_org.js
-
-var property_types = {
-    settings: {
+var org_settings = {
+    msg_editing: {
         // ...
     },
-    permissions: { // ...
+    msg_feed: {
+        // ...
 +       mandatory_topics: {
 +           type: 'bool',
-+           checked_msg: i18n.t("Topics are required in messages to streams"),
-+           unchecked_msg: i18n.t("Topics are not required in messages to streams"),
-        },
++       },
     },
 };
+
 ```
 
-Additionally, any code needed to update the UI when the setting is changed
-should be written in a function inside `settings_org.js`.
-For example, when a realm description is updated, that value change should
-occur in other windows where the description field is visible:
+Note that some settings, like `realm_create_stream_permission`,
+reuqire special treatment, because they don't match the common
+pattern.  We can't extract the property name and compare the value of
+such input elements with those in `page_params`, so we have to
+manually handle such situations in a couple key functions:
 
-    # static/js/settings_org.js
+- `settings_org.get_property_value`: This processes the property name
+    when it doesn't match a corresponding key in `page_params`, and
+    returns the current value of that property, which we can use to
+    compare and set the values of corresponding DOM element.
 
-    exports.update_realm_description = function () {
-        if (!meta.loaded) {
-            return;
-        }
-
-        $('#id_realm_description').val(page_params.realm_description);
-    };
-
-
-This ensures the appropriate code will run even if the
-changes are made in another browser window.
-
-In the example of updating a `mandatory_topics` setting, most of the changes
-are on the backend, so no UI updates are required.
+- `settings_org.update_dependent_subsettings`: This handles settings
+    whose value and state depend on other elements.  For example,
+    `realm_waiting_period_threshold` is only shown for with the right
+    state of `realm_create_stream_permission`.
 
 Finally, update `server_events_dispatch.js` to handle related events coming from
 the server. There is an object, `realm_settings`, in the function
 `dispatch_normal_event`. The keys in this object are setting names and the
 values are the UI updating functions to run when an event has occurred.
 
-If there is no relevant UI change to make, the value should be `noop`
-(this is the case for `mandatory_topics`). However, if you had written
-a function in `settings_org.js` to update UI, that function should
-be the value in the `realm_settings` object.
+If there is no relevant UI change to make other than in settings page
+itself, the value should be `noop` (this is the case for
+`mandatory_topics`, since this setting only has an effect on the
+backend, so no UI updates are required.).
+
+However, if you had written a function to update the UI after a given
+setting has changed, your function should be referenced in the
+`realm_settings` of `server_events_dispatch.js`.  See for example
+`settings_emoji.update_custom_emoji_ui`.
 
 ``` diff
 
@@ -585,10 +601,35 @@ function dispatch_normal_event(event) {
       };
 ```
 
+Checkboxes and other common input elements handle the UI updates
+automatically through the logic in `settings_org.sync_realm_settings`.
+
 The rest of the `dispatch_normal_events` function updates the state of the
 application if an update event has occurred on a realm property and runs
 the associated function to update the application's UI, if necessary.
 
+Here are few important cases you should consider when testing your changes:
+
+- For organization settings where we have a "save/discard" model, make
+  sure both the "Save" and "Discard changes" buttons are working
+  properly.
+
+- If your setting is dependent on another setting, carefully check
+  that both are properly synchronized.  For example, the input element
+  for `realm_waiting_period_threshold` is shown only when we have
+  selected the custom time limit option in the
+  `realm_create_stream_permission` dropdown.
+
+- Do some manual testing for the real-time synchronization of input
+  elements across the browsers and just like "Discard changes" button,
+  check whether dependent settings are synchronized properly (this is
+  easy to do by opening two browser windows to the settings page, and
+  making changes in one while watching the other).
+
+- Each subsection has independent "Save" and "Discard changes"
+  buttons, so changes and saving in one subsection shouldn't affect
+  the others.
+
 ### Front End Tests
 
 A great next step is to write front end tests. There are two types of

frontend_tests/.eslintrc.json

@@ -11,6 +11,7 @@
         "zrequire": false
     },
     "rules": {
-        "no-sync": 0
+        "no-sync": 0,
+        "prefer-const": "error"
     }
 }

frontend_tests/casper_tests/06-settings.js

@@ -302,7 +302,7 @@ casper.thenClick('a[data-code="en"]');
  * Changing the language back to English so that subsequent tests pass.
  */
 casper.waitUntilVisible('#language-settings-status a', function () {
-    casper.test.assertSelectorHasText('#language-settings-status', 'Saved. Please reload for the change to take effect.');
+    casper.test.assertSelectorHasText('#language-settings-status', 'Gespeichert. Bitte lade die Seite neu um die Änderungen zu aktivieren.');
 });
 
 casper.thenOpen("http://zulip.zulipdev.com:9981/");

frontend_tests/casper_tests/10-admin.js

@@ -198,9 +198,8 @@ casper.then(function () {
 });
 
 casper.then(function () {
-    casper.waitUntilVisible('div#admin-profile-field-status', function () {
-        casper.test.assertSelectorHasText('div#admin-profile-field-status',
-                                          'Custom profile field added!');
+    casper.waitUntilVisible('#admin-profile-field-status img', function () {
+        casper.test.assertSelectorHasText('div#admin-profile-field-status', 'Saved');
         casper.test.assertSelectorHasText('.profile-field-row span.profile_field_name', 'Teams');
         casper.test.assertSelectorHasText('.profile-field-row span.profile_field_type', 'Short Text');
         casper.click('.profile-field-row button.open-edit-form');
@@ -217,9 +216,8 @@ casper.then(function () {
 });
 
 casper.then(function () {
-    casper.waitUntilVisible('div#admin-profile-field-status', function () {
-        casper.test.assertSelectorHasText('div#admin-profile-field-status',
-                                          'Custom profile field updated!');
+    casper.waitUntilVisible('#admin-profile-field-status img', function () {
+        casper.test.assertSelectorHasText('div#admin-profile-field-status', 'Saved');
         casper.test.assertSelectorHasText('.profile-field-row span.profile_field_name', 'team');
         casper.test.assertSelectorHasText('.profile-field-row span.profile_field_type', 'Short Text');
         casper.click('.profile-field-row button.delete');
@@ -227,9 +225,8 @@ casper.then(function () {
 });
 
 casper.then(function () {
-    casper.waitUntilVisible('div#admin-profile-field-status', function () {
-        casper.test.assertSelectorHasText('div#admin-profile-field-status',
-                                          'Custom profile field deleted!');
+    casper.waitUntilVisible('#admin-profile-field-status img', function () {
+        casper.test.assertSelectorHasText('div#admin-profile-field-status', 'Saved');
     });
 });
 

frontend_tests/node_tests/activity.js

@@ -40,7 +40,7 @@ zrequire('activity');
 zrequire('stream_list');
 
 set_global('blueslip', {
-    log: function () {},
+    log: () => {},
 });
 
 set_global('popovers', {
@@ -58,52 +58,52 @@ set_global('stream_popover', {
 
 
 set_global('reload', {
-    is_in_progress: function () {return false;},
+    is_in_progress: () => false,
 });
 set_global('resize', {
-    resize_page_components: function () {},
+    resize_page_components: () => {},
 });
 set_global('window', 'window-stub');
 
-var me = {
+const me = {
     email: 'me@zulip.com',
     user_id: 999,
     full_name: 'Me Myself',
 };
 
-var alice = {
+const alice = {
     email: 'alice@zulip.com',
     user_id: 1,
     full_name: 'Alice Smith',
 };
-var fred = {
+const fred = {
     email: 'fred@zulip.com',
     user_id: 2,
     full_name: "Fred Flintstone",
 };
-var jill = {
+const jill = {
     email: 'jill@zulip.com',
     user_id: 3,
     full_name: 'Jill Hill',
 };
-var mark = {
+const mark = {
     email: 'mark@zulip.com',
     user_id: 4,
     full_name: 'Marky Mark',
 };
-var norbert = {
+const norbert = {
     email: 'norbert@zulip.com',
     user_id: 5,
     full_name: 'Norbert Oswald',
 };
 
-var zoe = {
+const zoe = {
     email: 'zoe@example.com',
     user_id: 6,
     full_name: 'Zoe Yang',
 };
 
-var people = global.people;
+const people = global.people;
 
 people.add_in_realm(alice);
 people.add_in_realm(fred);
@@ -114,16 +114,16 @@ people.add_in_realm(zoe);
 people.add_in_realm(me);
 people.initialize_current_user(me.user_id);
 
-compose_fade.update_faded_users = function () {};
+compose_fade.update_faded_users = () => {};
 
-var real_update_huddles = activity.update_huddles;
-activity.update_huddles = function () {};
+const real_update_huddles = activity.update_huddles;
+activity.update_huddles = () => {};
 
 global.compile_template('user_presence_row');
 global.compile_template('user_presence_rows');
 global.compile_template('group_pms');
 
-var presence_info = {};
+const presence_info = {};
 presence_info[alice.user_id] = { status: 'inactive' };
 presence_info[fred.user_id] = { status: 'active' };
 presence_info[jill.user_id] = { status: 'active' };
@@ -149,7 +149,7 @@ presence.presence_info = presence_info;
 }());
 
 (function test_sort_users() {
-    var user_ids = [alice.user_id, fred.user_id, jill.user_id];
+    const user_ids = [alice.user_id, fred.user_id, jill.user_id];
 
     activity._sort_users(user_ids);
 
@@ -162,15 +162,15 @@ presence.presence_info = presence_info;
 
 (function test_process_loaded_messages() {
 
-    var huddle1 = 'jill@zulip.com,norbert@zulip.com';
-    var timestamp1 = 1382479029; // older
+    const huddle1 = 'jill@zulip.com,norbert@zulip.com';
+    const timestamp1 = 1382479029; // older
 
-    var huddle2 = 'alice@zulip.com,fred@zulip.com';
-    var timestamp2 = 1382479033; // newer
+    const huddle2 = 'alice@zulip.com,fred@zulip.com';
+    const timestamp2 = 1382479033; // newer
 
-    var old_timestamp = 1382479000;
+    const old_timestamp = 1382479000;
 
-    var messages = [
+    const messages = [
         {
             type: 'private',
             display_recipient: [{id: jill.user_id}, {id: norbert.user_id}],
@@ -197,8 +197,8 @@ presence.presence_info = presence_info;
 
     activity.process_loaded_messages(messages);
 
-    var user_ids_string1 = people.emails_strings_to_user_ids_string(huddle1);
-    var user_ids_string2 = people.emails_strings_to_user_ids_string(huddle2);
+    const user_ids_string1 = people.emails_strings_to_user_ids_string(huddle1);
+    const user_ids_string2 = people.emails_strings_to_user_ids_string(huddle2);
     assert.deepEqual(activity.get_huddles(), [user_ids_string2, user_ids_string1]);
 }());
 
@@ -249,7 +249,7 @@ presence.presence_info = presence_info;
     var huddle = 'alice@zulip.com,fred@zulip.com,jill@zulip.com,mark@zulip.com';
     huddle = people.emails_strings_to_user_ids_string(huddle);
 
-    var presence_info = {};
+    const presence_info = {};
     presence_info[alice.user_id] = { status: 'active' };
     presence_info[fred.user_id] = { status: 'idle' }; // counts as present
     // jill not in list
@@ -272,9 +272,9 @@ presence.presence_info[me.user_id] = { status: activity.ACTIVE };
 
 activity.set_user_list_filter();
 
-var user_order = [fred.user_id, jill.user_id, norbert.user_id,
+const user_order = [fred.user_id, jill.user_id, norbert.user_id,
                      zoe.user_id, alice.user_id, mark.user_id];
-var user_count = 6;
+const user_count = 6;
 
 // Mock the jquery is func
 $('.user-list-filter').is = function (sel) {
@@ -297,7 +297,7 @@ $('#user_presences li.user_sidebar_entry.narrow-filter').last = function () {
     $('.user-list-filter').focus();
 
     $('#user_presences li.user_sidebar_entry.narrow-filter');
-    var users = activity.build_user_sidebar();
+    const users = activity.build_user_sidebar();
     assert.deepEqual(users, [{
             name: 'Fred Flintstone',
             href: '#narrow/pm-with/2-fred',
@@ -350,15 +350,15 @@ $('#user_presences li.user_sidebar_entry.narrow-filter').last = function () {
 }());
 
 (function test_PM_update_dom_counts() {
-    var value = $.create('alice-value');
-    var count = $.create('alice-count');
-    var pm_key = alice.user_id.toString();
-    var li = $("li.user_sidebar_entry[data-user-id='" + pm_key + "']");
+    const value = $.create('alice-value');
+    const count = $.create('alice-count');
+    const pm_key = alice.user_id.toString();
+    const li = $("li.user_sidebar_entry[data-user-id='" + pm_key + "']");
     count.set_find_results('.value', value);
     li.set_find_results('.count', count);
     count.set_parent(li);
 
-    var counts = new Dict();
+    const counts = new Dict();
     counts.set(pm_key, 5);
     li.addClass('user_sidebar_entry');
 
@@ -374,16 +374,16 @@ $('#user_presences li.user_sidebar_entry.narrow-filter').last = function () {
 }());
 
 (function test_group_update_dom_counts() {
-    var value = $.create('alice-fred-value');
-    var count = $.create('alice-fred-count');
-    var pm_key = alice.user_id.toString() + "," + fred.user_id.toString();
-    var li_selector = "li.group-pms-sidebar-entry[data-user-ids='" + pm_key + "']";
-    var li = $(li_selector);
+    const value = $.create('alice-fred-value');
+    const count = $.create('alice-fred-count');
+    const pm_key = alice.user_id.toString() + "," + fred.user_id.toString();
+    const li_selector = "li.group-pms-sidebar-entry[data-user-ids='" + pm_key + "']";
+    const li = $(li_selector);
     count.set_find_results('.value', value);
     li.set_find_results('.count', count);
     count.set_parent(li);
 
-    var counts = new Dict();
+    const counts = new Dict();
     counts.set(pm_key, 5);
     li.addClass('group-pms-sidebar-entry');
 
@@ -446,12 +446,12 @@ $('#user_presences li.user_sidebar_entry.narrow-filter').last = function () {
     // Disable scrolling into place
     stream_list.scroll_element_into_container = function () {};
     // up
-    var e = {
+    const e = {
         keyCode: 38,
         stopPropagation: function () {},
         preventDefault: function () {},
     };
-    var keydown_handler = $('.user-list-filter').get_on_handler('keydown');
+    const keydown_handler = $('.user-list-filter').get_on_handler('keydown');
     keydown_handler(e);
     // Now the last element is selected
     sel_index = user_count - 1;
@@ -459,20 +459,12 @@ $('#user_presences li.user_sidebar_entry.narrow-filter').last = function () {
     sel_index = sel_index - 1;
 
     // down
-    e = {
-        keyCode: 40,
-        stopPropagation: function () {},
-        preventDefault: function () {},
-    };
+    e.keyCode = 40;
     keydown_handler(e);
     sel_index = sel_index + 1;
     keydown_handler(e);
 
-    e = {
-        keyCode: 13,
-        stopPropagation: function () {},
-        preventDefault: function () {},
-    };
+    e.keyCode = 13;
 
     // Enter text and narrow users
     $(".user-list-filter").expectOne().val('ali');
@@ -486,16 +478,16 @@ $('#user_presences li.user_sidebar_entry.narrow-filter').last = function () {
 }());
 
 (function test_focus_user_filter() {
-    var e = {
-        stopPropagation: function () {},
+    const e = {
+        stopPropagation: () => {},
     };
     var click_handler = $('.user-list-filter').get_on_handler('click');
     click_handler(e);
 }());
 
 (function test_focusout_user_filter() {
-    var e = { };
-    var click_handler = $('.user-list-filter').get_on_handler('blur');
+    const e = { };
+    const click_handler = $('.user-list-filter').get_on_handler('blur');
     click_handler(e);
 }());
 
@@ -508,7 +500,7 @@ presence.presence_info[norbert.user_id] = { status: activity.ACTIVE };
 presence.presence_info[zoe.user_id] = { status: activity.ACTIVE };
 
 (function test_filter_user_ids() {
-    var user_filter = $('.user-list-filter');
+    const user_filter = $('.user-list-filter');
     user_filter.val(''); // no search filter
 
     activity.set_user_list_filter();
@@ -551,7 +543,7 @@ presence.presence_info[zoe.user_id] = { status: activity.ACTIVE };
 }());
 
 (function test_insert_one_user_into_empty_list() {
-    var alice_li = $.create('alice list item');
+    const alice_li = $.create('alice list item');
 
     // These selectors are here to avoid some short-circuit logic.
     $('#user_presences').set_find_results('[data-user-id="1"]', alice_li);
@@ -562,9 +554,7 @@ presence.presence_info[zoe.user_id] = { status: activity.ACTIVE };
     };
 
     $.stub_selector('#user_presences li', {
-        toArray: function () {
-            return [];
-        },
+        toArray: () => [],
     });
     activity.insert_user_into_list(alice.user_id);
     assert(appended_html.indexOf('data-user-id="1"') > 0);
@@ -572,7 +562,7 @@ presence.presence_info[zoe.user_id] = { status: activity.ACTIVE };
 }());
 
 (function test_insert_fred_after_alice() {
-    var fred_li = $.create('fred list item');
+    const fred_li = $.create('fred list item');
 
     // These selectors are here to avoid some short-circuit logic.
     $('#user_presences').set_find_results('[data-user-id="2"]', fred_li);
@@ -601,7 +591,7 @@ presence.presence_info[zoe.user_id] = { status: activity.ACTIVE };
 }());
 
 (function test_insert_fred_before_jill() {
-    var fred_li = $.create('fred-li');
+    const fred_li = $.create('fred-li');
 
     // These selectors are here to avoid some short-circuit logic.
     $('#user_presences').set_find_results('[data-user-id="2"]', fred_li);
@@ -637,7 +627,7 @@ activity.set_user_list_filter();
     // This test only tests that we do not explode when
     // try to insert Fred into a list where he does not
     // match the search filter.
-    var user_filter = $('.user-list-filter');
+    const user_filter = $('.user-list-filter');
     user_filter.val('do-not-match-filter');
     activity.insert_user_into_list(fred.user_id);
 }());
@@ -716,28 +706,24 @@ $('.user-list-filter').parent = function () {
 }());
 
 (function test_update_huddles_and_redraw() {
-    var value = $.create('alice-fred-value');
-    var count = $.create('alice-fred-count');
-    var pm_key = alice.user_id.toString() + "," + fred.user_id.toString();
-    var li_selector = "li.group-pms-sidebar-entry[data-user-ids='" + pm_key + "']";
-    var li = $(li_selector);
+    const value = $.create('alice-fred-value');
+    const count = $.create('alice-fred-count');
+    const pm_key = alice.user_id.toString() + "," + fred.user_id.toString();
+    const li_selector = "li.group-pms-sidebar-entry[data-user-ids='" + pm_key + "']";
+    const li = $(li_selector);
     count.set_find_results('.value', value);
     li.set_find_results('.count', count);
     count.set_parent(li);
 
-    var real_get_huddles = activity.get_huddles;
-    activity.get_huddles = function () {
-        return ['1,2'];
-    };
+    const real_get_huddles = activity.get_huddles;
+    activity.get_huddles = () => ['1,2'];
     activity.update_huddles = real_update_huddles;
     activity.redraw();
     assert.equal($('#group-pm-list').hasClass('show'), false);
     page_params.realm_presence_disabled = false;
     activity.redraw();
     assert.equal($('#group-pm-list').hasClass('show'), true);
-    activity.get_huddles = function () {
-        return [];
-    };
+    activity.get_huddles = () => [];
     activity.redraw();
     assert.equal($('#group-pm-list').hasClass('show'), false);
     activity.get_huddles = real_get_huddles;
@@ -745,35 +731,33 @@ $('.user-list-filter').parent = function () {
 }());
 
 (function test_set_user_status() {
-    var server_time = 500;
-    var info = {
+    const server_time = 500;
+    const info = {
         website: {
             status: "active",
             timestamp: server_time,
         },
     };
-    var alice_li = $.create('alice-li');
+    const alice_li = $.create('alice-li');
 
     $('#user_presences').set_find_results('[data-user-id="1"]', alice_li);
 
     $('#user_presences').append = function () {};
 
     $.stub_selector('#user_presences li', {
-        toArray: function () {
-            return [];
-        },
+        toArray: () => [],
     });
     presence.presence_info[alice.user_id] = undefined;
     activity.set_user_status(me.email, info, server_time);
     assert.equal(presence.presence_info[alice.user_id], undefined);
     activity.set_user_status(alice.email, info, server_time);
-    var expected = { status: 'active', mobile: false, last_active: 500 };
+    const expected = { status: 'active', mobile: false, last_active: 500 };
     assert.deepEqual(presence.presence_info[alice.user_id], expected);
     activity.set_user_status(alice.email, info, server_time);
     blueslip.warn = function (msg) {
         assert.equal(msg, 'unknown email: foo@bar.com');
     };
-    blueslip.error = function () {};
+    blueslip.error = () => {};
     activity.set_user_status('foo@bar.com', info, server_time);
 }());
 
@@ -783,10 +767,8 @@ $('.user-list-filter').parent = function () {
           func();
       },
   });
-  $(window).focus = function (func) {
-      func();
-  };
-  $(window).idle = function () {};
+  $(window).focus = func => func();
+  $(window).idle = () => {};
 
   channel.post = function (payload) {
       payload.success({});
@@ -808,9 +790,8 @@ $('.user-list-filter').parent = function () {
           zephyr_mirror_active: false,
       });
   };
-  global.setInterval = function (func) {
-      func();
-  };
+  global.setInterval = (func) => func();
+
   activity.initialize();
   assert($('#zephyr-mirror-error').hasClass('show'));
   assert(!activity.new_user_input);

frontend_tests/node_tests/components.js

@@ -1,9 +1,12 @@
 set_global('i18n', global.stub_i18n);
 
+zrequire('keydown_util');
 zrequire('components');
 
-var LEFT_KEY = { which: 37 };
-var RIGHT_KEY = { which: 39 };
+var noop = function () {};
+
+var LEFT_KEY = { which: 37, preventDefault: noop };
+var RIGHT_KEY = { which: 39, preventDefault: noop };
 
 (function test_basics() {
     var keydown_f;
@@ -118,7 +121,10 @@ var RIGHT_KEY = { which: 39 };
         }
     });
 
-    var widget = components.toggle({
+    var callback_value;
+
+    var widget;
+    widget = components.toggle({
         name: "info-overlay-toggle",
         selected: 0,
         values: [
@@ -129,6 +135,12 @@ var RIGHT_KEY = { which: 39 };
         callback: function (name, key) {
             assert.equal(callback_args, undefined);
             callback_args = [name, key];
+
+            // The subs code tries to get a widget value in the middle of a
+            // callback, which can lead to obscure bugs.
+            if (widget) {
+                callback_value = widget.value();
+            }
         },
     });
 
@@ -162,6 +174,7 @@ var RIGHT_KEY = { which: 39 };
     assert.equal(tabs[2].class, 'last selected');
     assert.deepEqual(callback_args, ['translated: Search operators', 'search-operators']);
     assert.equal(widget.value(), 'translated: Search operators');
+    assert.equal(widget.value(), callback_value);
 
     // try to crash the key handler
     keydown_f.call(tabs[focused_tab], RIGHT_KEY);

frontend_tests/node_tests/compose.js

@@ -43,6 +43,7 @@ set_global('notifications', {
     notify_above_composebox: noop,
     clear_compose_notifications: noop,
 });
+set_global('subs', {});
 
 // Setting these up so that we can test that links to uploads within messages are
 // automatically converted to server relative links.
@@ -101,8 +102,12 @@ people.add(bob);
 
     sub.subscribed = false;
     stream_data.add_sub('social', sub);
+    templates.render = function (template_name) {
+        assert.equal(template_name, 'compose_not_subscribed');
+        return 'compose_not_subscribed_stub';
+    };
     assert(!compose.validate_stream_message_address_info('social'));
-    assert.equal($('#compose-error-msg').html(), "translated: <p>You're not subscribed to the stream <b>social</b>.</p><p>Manage your subscriptions <a href='#streams/all'>on your Streams page</a>.</p>");
+    assert.equal($('#compose-error-msg').html(), 'compose_not_subscribed_stub');
 
     global.page_params.narrow_stream = false;
     channel.post = function (payload) {
@@ -121,7 +126,7 @@ people.add(bob);
         payload.success(payload.data);
     };
     assert(!compose.validate_stream_message_address_info('Frontend'));
-    assert.equal($('#compose-error-msg').html(), "translated: <p>You're not subscribed to the stream <b>Frontend</b>.</p><p>Manage your subscriptions <a href='#streams/all'>on your Streams page</a>.</p>");
+    assert.equal($('#compose-error-msg').html(), 'compose_not_subscribed_stub');
 
     channel.post = function (payload) {
         assert.equal(payload.data.stream, 'Frontend');
@@ -631,6 +636,8 @@ people.add(bob);
     }());
 }());
 
+set_global('document', 'document-stub');
+
 (function test_enter_with_preview_open() {
     // Test sending a message with content.
     compose_state.set_message_type('stream');
@@ -697,12 +704,10 @@ people.add(bob);
         };
 
         var compose_finished_event_checked = false;
-        $.stub_selector(document, {
-            trigger: function (e) {
-                assert.equal(e.name, 'compose_finished.zulip');
-                compose_finished_event_checked = true;
-            },
-        });
+        $(document).trigger = function (e) {
+            assert.equal(e.name, 'compose_finished.zulip');
+            compose_finished_event_checked = true;
+        };
         var send_message_called = false;
         compose.send_message = function () {
             send_message_called = true;
@@ -1142,6 +1147,51 @@ function test_raw_file_drop(raw_drop_func) {
         assert(!$("#compose_invite_users").visible());
     }());
 
+    (function test_compose_not_subscribed_clicked() {
+        var handler = $("#compose-send-status")
+                      .get_on_handler('click', '.sub_unsub_button');
+        var subscription = {
+            stream_id: 102,
+            name: 'test',
+            subscribed: false,
+        };
+        var compose_not_subscribed_called = false;
+        subs.sub_or_unsub = function () {
+            compose_not_subscribed_called = true;
+        };
+
+        setup_parents_and_mock_remove('compose-send-status',
+                                      'sub_unsub_button',
+                                      '.compose_not_subscribed');
+
+        handler(event);
+
+        assert(compose_not_subscribed_called);
+
+        stream_data.add_sub('test', subscription);
+        $('#stream').val('test');
+        $("#compose-send-status").show();
+
+        handler(event);
+
+        assert(!$("#compose-send-status").visible());
+    }());
+
+    (function test_compose_not_subscribed_close_clicked() {
+        var handler = $("#compose-send-status")
+                      .get_on_handler('click', '#compose_not_subscribed_close');
+
+        setup_parents_and_mock_remove('compose_user_not_subscribed_close',
+                                      'compose_not_subscribed_close',
+                                      '.compose_not_subscribed');
+
+        $("#compose-send-status").show();
+
+        handler(event);
+
+        assert(!$("#compose-send-status").visible());
+    }());
+
     event = {
         preventDefault: noop,
     };

frontend_tests/node_tests/compose_actions.js

@@ -3,8 +3,7 @@ var return_false = function () { return false; };
 var return_true = function () { return true; };
 
 set_global('document', {
-    location: {
-    },
+    location: {}, // we need this to load compose.js
 });
 
 set_global('page_params', {
@@ -26,6 +25,8 @@ zrequire('util');
 zrequire('compose_state');
 zrequire('compose_actions');
 
+set_global('document', 'document-stub');
+
 var start = compose_actions.start;
 var cancel = compose_actions.cancel;
 var get_focus_area = compose_actions._get_focus_area;
@@ -71,7 +72,7 @@ set_global('narrow_state', {
 });
 
 set_global('unread_ops', {
-    mark_message_as_read: noop,
+    notify_server_message_read: noop,
 });
 
 set_global('common', {

frontend_tests/node_tests/compose_fade.js

@@ -26,11 +26,11 @@ var bob = {
     full_name: 'Bob',
 };
 
-people.add(me);
+people.add_in_realm(me);
 people.initialize_current_user(me.user_id);
 
-people.add(alice);
-people.add(bob);
+people.add_in_realm(alice);
+people.add_in_realm(bob);
 
 
 (function test_set_focused_recipient() {
@@ -65,6 +65,7 @@ people.add(bob);
     assert(compose_fade.would_receive_message('me@example.com'));
     assert(compose_fade.would_receive_message('alice@example.com'));
     assert(!compose_fade.would_receive_message('bob@example.com'));
+    assert.equal(compose_fade.would_receive_message('nonrealmuser@example.com'), undefined);
 
     var good_msg = {
         type: 'stream',

frontend_tests/node_tests/compose_pm_pill.js

@@ -0,0 +1,144 @@
+zrequire('compose_pm_pill');
+zrequire('input_pill');
+zrequire('user_pill');
+
+set_global('$', global.make_zjquery());
+set_global('people', {});
+var pills = {
+    pill: {},
+};
+
+(function test_pills() {
+    var othello = {
+        user_id: 1,
+        email: 'othello@example.com',
+        full_name: 'Othello',
+    };
+
+    var iago = {
+        email: 'iago@zulip.com',
+        user_id: 2,
+        full_name: 'Iago',
+    };
+
+    var hamlet = {
+        email: 'hamlet@example.com',
+        user_id: 3,
+        full_name: 'Hamlet',
+    };
+
+    people.get_realm_persons = function () {
+        return [iago, othello, hamlet];
+    };
+
+    var recipient_stub = $("#private_message_recipient");
+    var pill_container_stub = $('.pill-container[data-before="You and"]');
+    recipient_stub.set_parent(pill_container_stub);
+    var create_item_handler;
+
+    var all_pills = {};
+
+    pills.appendValidatedData = function (item) {
+        var id = item.user_id;
+        assert.equal(all_pills[id], undefined);
+        all_pills[id] = item;
+    };
+    pills.items = function () {
+        return _.values(all_pills);
+    };
+
+    var text_cleared;
+    pills.clear_text = function () {
+        text_cleared = true;
+    };
+
+    var pills_cleared;
+    pills.clear = function () {
+        pills_cleared = true;
+        pills = {
+            pill: {},
+        };
+        all_pills= {};
+    };
+
+    var appendValue_called;
+    pills.appendValue = function (value) {
+        appendValue_called = true;
+        assert.equal(value, 'othello@example.com');
+        this.appendValidatedData(othello);
+    };
+
+    var get_by_email_called = false;
+    people.get_by_email = function (user_email) {
+        get_by_email_called = true;
+        if (user_email === iago.email) {
+            return iago;
+        }
+        if (user_email === othello.email) {
+            return othello;
+        }
+    };
+
+    var get_person_from_user_id_called = false;
+    people.get_person_from_user_id = function (id) {
+        get_person_from_user_id_called = true;
+        if (id === othello.user_id) {
+            return othello;
+        }
+        assert.equal(id, 3);
+        return hamlet;
+    };
+
+    function test_create_item(handler) {
+        (function test_rejection_path() {
+            var item = handler(othello.email, pills.items());
+            assert(get_by_email_called);
+            assert.equal(item, undefined);
+        }());
+
+        (function test_success_path() {
+            get_by_email_called = false;
+            var res = handler(iago.email, pills.items());
+            assert(get_by_email_called);
+            assert.equal(typeof(res), 'object');
+            assert.equal(res.user_id, iago.user_id);
+            assert.equal(res.display_value, iago.full_name);
+        }());
+    }
+
+    function input_pill_stub(opts) {
+        assert.equal(opts.container, pill_container_stub);
+        create_item_handler = opts.create_item_from_text;
+        assert(create_item_handler);
+        return pills;
+    }
+
+    set_global('input_pill', {
+        create: input_pill_stub,
+    });
+
+    compose_pm_pill.initialize();
+    assert(compose_pm_pill.my_pill);
+
+    compose_pm_pill.set_from_typeahead(othello);
+    compose_pm_pill.set_from_typeahead(hamlet);
+
+    var user_ids = compose_pm_pill.get_user_ids();
+    assert.deepEqual(user_ids, [othello.user_id, hamlet.user_id]);
+
+    var emails = compose_pm_pill.get_emails();
+    assert.equal(emails, 'othello@example.com,hamlet@example.com');
+
+    var items = compose_pm_pill.get_typeahead_items();
+    assert.deepEqual(items, [{email: 'iago@zulip.com', user_id: 2, full_name: 'Iago'}]);
+
+    test_create_item(create_item_handler);
+
+    compose_pm_pill.set_from_emails('othello@example.com');
+    assert(compose_pm_pill.my_pill);
+
+    assert(get_person_from_user_id_called);
+    assert(pills_cleared);
+    assert(appendValue_called);
+    assert(text_cleared);
+}());

frontend_tests/node_tests/composebox_typeahead.js

@@ -1,3 +1,4 @@
+set_global('i18n', global.stub_i18n);
 zrequire('compose_state');
 zrequire('ui_util');
 zrequire('pm_conversations');
@@ -11,6 +12,9 @@ zrequire('stream_data');
 zrequire('user_pill');
 zrequire('compose_pm_pill');
 zrequire('composebox_typeahead');
+set_global('md5', function (s) {
+    return 'md5-' + s;
+});
 
 var ct = composebox_typeahead;
 var noop = function () {};
@@ -446,17 +450,17 @@ user_pill.get_user_ids = function () {
         // corresponding parts in bold.
         options.query = 'oth';
         actual_value = options.highlighter(othello);
-        expected_value = '<strong>Othello, the Moor of Venice</strong>&nbsp;&nbsp;\n<small class="autocomplete_secondary">othello@zulip.com</small>\n';
+        expected_value = '        <img class="typeahead-image" src="https://secure.gravatar.com/avatar/md5-othello@zulip.com?d&#x3D;identicon&amp;s&#x3D;50" />\n<strong>Othello, the Moor of Venice</strong>&nbsp;&nbsp;\n<small class="autocomplete_secondary">othello@zulip.com</small>\n';
         assert.equal(actual_value, expected_value);
 
         options.query = 'Lear';
         actual_value = options.highlighter(cordelia);
-        expected_value = '<strong>Cordelia Lear</strong>&nbsp;&nbsp;\n<small class="autocomplete_secondary">cordelia@zulip.com</small>\n';
+        expected_value = '        <img class="typeahead-image" src="https://secure.gravatar.com/avatar/md5-cordelia@zulip.com?d&#x3D;identicon&amp;s&#x3D;50" />\n<strong>Cordelia Lear</strong>&nbsp;&nbsp;\n<small class="autocomplete_secondary">cordelia@zulip.com</small>\n';
         assert.equal(actual_value, expected_value);
 
         options.query = 'othello@zulip.com, co';
         actual_value = options.highlighter(cordelia);
-        expected_value = '<strong>Cordelia Lear</strong>&nbsp;&nbsp;\n<small class="autocomplete_secondary">cordelia@zulip.com</small>\n';
+        expected_value = '        <img class="typeahead-image" src="https://secure.gravatar.com/avatar/md5-cordelia@zulip.com?d&#x3D;identicon&amp;s&#x3D;50" />\n<strong>Cordelia Lear</strong>&nbsp;&nbsp;\n<small class="autocomplete_secondary">cordelia@zulip.com</small>\n';
         assert.equal(actual_value, expected_value);
 
         // options.matcher()
@@ -561,12 +565,12 @@ user_pill.get_user_ids = function () {
         // content_highlighter.
         fake_this = { completing: 'mention', token: 'othello' };
         actual_value = options.highlighter.call(fake_this, othello);
-        expected_value = '<strong>Othello, the Moor of Venice</strong>&nbsp;&nbsp;\n<small class="autocomplete_secondary">othello@zulip.com</small>\n';
+        expected_value = '        <img class="typeahead-image" src="https://secure.gravatar.com/avatar/md5-othello@zulip.com?d&#x3D;identicon&amp;s&#x3D;50" />\n<strong>Othello, the Moor of Venice</strong>&nbsp;&nbsp;\n<small class="autocomplete_secondary">othello@zulip.com</small>\n';
         assert.equal(actual_value, expected_value);
 
         fake_this = { completing: 'mention', token: 'hamletcharacters' };
         actual_value = options.highlighter.call(fake_this, hamletcharacters);
-        expected_value = '<strong>hamletcharacters</strong>&nbsp;&nbsp;\n<small class="autocomplete_secondary">Characters of Hamlet</small>\n';
+        expected_value = '        <i class="typeahead-image icon icon-vector-group"></i>\n<strong>hamletcharacters</strong>&nbsp;&nbsp;\n<small class="autocomplete_secondary">Characters of Hamlet</small>\n';
         assert.equal(actual_value, expected_value);
 
         // options.matcher()
@@ -892,20 +896,14 @@ user_pill.get_user_ids = function () {
         assert.deepEqual(returned, reference);
     }
 
-    var all_items = [
-        {
-            special_item_text: 'all (Notify everyone)',
-            email: 'all',
+    var all_items = _.map(['all', 'everyone', 'stream'], function (mention) {
+        return {
+            special_item_text: 'translated: ' + mention +" (Notify stream)",
+            email: mention,
             pm_recipient_count: Infinity,
-            full_name: 'all',
-        },
-        {
-            special_item_text: 'everyone (Notify everyone)',
-            email: 'everyone',
-            pm_recipient_count: Infinity,
-            full_name: 'everyone',
-        },
-    ];
+            full_name: mention,
+        };
+    });
 
     var people_with_all = global.people.get_realm_persons().concat(all_items);
     var all_mentions = people_with_all.concat(global.user_groups.get_realm_user_groups());
@@ -1090,20 +1088,14 @@ user_pill.get_user_ids = function () {
 }());
 
 (function test_typeahead_results() {
-    var all_items = [
-        {
-            special_item_text: 'all (Notify everyone)',
-            email: 'all',
+    var all_items = _.map(['all', 'everyone', 'stream'], function (mention) {
+        return {
+            special_item_text: 'translated: ' + mention +" (Notify stream)",
+            email: mention,
             pm_recipient_count: Infinity,
-            full_name: 'all',
-        },
-        {
-            special_item_text: 'everyone (Notify everyone)',
-            email: 'everyone',
-            pm_recipient_count: Infinity,
-            full_name: 'everyone',
-        },
-    ];
+            full_name: mention,
+        };
+    });
     var people_with_all = global.people.get_realm_persons().concat(all_items);
     var all_mentions = people_with_all.concat(global.user_groups.get_realm_user_groups());
     var stream_list = [denmark_stream, sweden_stream, netherland_stream];

frontend_tests/node_tests/copy_and_paste.js

@@ -1,5 +1,9 @@
 global.stub_out_jquery();
 
+set_global('page_params', {
+    development: true,
+});
+
 var jsdom = require("jsdom");
 global.document = jsdom.jsdom('<!DOCTYPE html><p>Hello world</p>');
 var window = jsdom.jsdom().defaultView;

frontend_tests/node_tests/dispatch.js

@@ -360,6 +360,7 @@ var event_fixtures = {
                 name: 'devel',
                 stream_id: 42,
                 subscribers: ['alice@example.com', 'bob@example.com'],
+                email_address: 'devel+0138515295f4@zulipdev.com:9991',
                 // etc.
             },
         ],
@@ -749,6 +750,7 @@ with_overrides(function (override) {
     event = event_fixtures.realm_user__remove;
     global.with_stub(function (stub) {
         override('people.deactivate', stub.f);
+        override('stream_data.remove_deactivated_user_from_all_streams', noop);
         dispatch(event);
         var args = stub.get_args('person');
         assert_same(args.person, event.person);
@@ -801,15 +803,21 @@ with_overrides(function (override) {
     });
 
     var event = event_fixtures.subscription__add;
-    global.with_stub(function (stub) {
-        override('stream_data.get_sub_by_id', function (stream_id) {
-            return {stream_id: stream_id};
+    global.with_stub(function (subscription_stub) {
+        global.with_stub(function (stream_email_stub) {
+            override('stream_data.get_sub_by_id', function (stream_id) {
+                return {stream_id: stream_id};
+            });
+            override('stream_events.mark_subscribed', subscription_stub.f);
+            override('stream_data.update_stream_email_address', stream_email_stub.f);
+            dispatch(event);
+            var args = subscription_stub.get_args('sub', 'subscribers');
+            assert_same(args.sub.stream_id, event.subscriptions[0].stream_id);
+            assert_same(args.subscribers, event.subscriptions[0].subscribers);
+            args = stream_email_stub.get_args('sub', 'email_address');
+            assert_same(args.email_address, event.subscriptions[0].email_address);
+            assert_same(args.sub.stream_id, event.subscriptions[0].stream_id);
         });
-        override('stream_events.mark_subscribed', stub.f);
-        dispatch(event);
-        var args = stub.get_args('sub', 'subscribers');
-        assert_same(args.sub.stream_id, event.subscriptions[0].stream_id);
-        assert_same(args.subscribers, event.subscriptions[0].subscribers);
     });
 
     event = event_fixtures.subscription__peer_add;
@@ -935,7 +943,7 @@ with_overrides(function (override) {
     });
 });
 
-// mark_message_as_read requires message_store and these dependencies.
+// notify_server_message_read requires message_store and these dependencies.
 zrequire('unread_ops');
 zrequire('unread');
 zrequire('topic_data');

frontend_tests/node_tests/filter.js

@@ -2,6 +2,7 @@ zrequire('util');
 zrequire('unread');
 zrequire('stream_data');
 zrequire('people');
+zrequire('Handlebars', 'handlebars');
 zrequire('Filter', 'js/filter');
 
 set_global('page_params', {});
@@ -585,7 +586,7 @@ function make_sub(name, stream_id) {
         {operator: 'stream', operand: 'devel'},
         {operator: 'topic', operand: 'JS'},
     ];
-    string = 'stream devel > JS';
+    string = 'stream devel > JS';
     assert.equal(Filter.describe(narrow), string);
 
     narrow = [

frontend_tests/node_tests/markdown.js

@@ -76,6 +76,12 @@ people.add({
     email: 'leo@zulip.com',
 });
 
+people.add({
+    full_name: 'Bobby <h1>Tables</h1>',
+    user_id: 103,
+    email: 'bobby@zulip.com',
+});
+
 people.initialize_current_user(cordelia.user_id);
 
 var hamletcharacters = {
@@ -92,8 +98,16 @@ var backend = {
     members: [],
 };
 
+var edgecase_group = {
+    name: "Bobby <h1>Tables</h1>",
+    id: 3,
+    description: "HTML Syntax to check for Markdown edge cases.",
+    members: [],
+};
+
 global.user_groups.add(hamletcharacters);
 global.user_groups.add(backend);
+global.user_groups.add(edgecase_group);
 
 var stream_data = global.stream_data;
 var denmark = {
@@ -111,8 +125,16 @@ var social = {
     in_home_view: true,
     invite_only: true,
 };
+var edgecase_stream = {
+    subscribed: true,
+    color: 'green',
+    name: 'Bobby <h1>Tables</h1>',
+    stream_id: 3,
+    in_home_view: true,
+};
 stream_data.add_sub('Denmark', denmark);
 stream_data.add_sub('social', social);
+stream_data.add_sub('Bobby <h1>Tables</h1>', edgecase_stream);
 
 // Check the default behavior of fenced code blocks
 // works properly before markdown is initialized.
@@ -305,6 +327,23 @@ var bugdown_data = JSON.parse(fs.readFileSync(path.join(__dirname, '../../zerver
         {input: ':)',
          expected: '<p><span class="emoji emoji-1f603" title="smiley">:smiley:</span></p>',
          translate_emoticons: true},
+         // Test HTML Escape in Custom Zulip Rules
+        {input: '@**<h1>The Rogue One</h1>**',
+         expected: '<p>@**&lt;h1&gt;The Rogue One&lt;/h1&gt;**</p>'},
+        {input: '#**<h1>The Rogue One</h1>**',
+         expected: '<p>#**&lt;h1&gt;The Rogue One&lt;/h1&gt;**</p>'},
+        {input: '!avatar(<h1>The Rogue One</h1>)',
+         expected: '<p><img alt="&lt;h1&gt;The Rogue One&lt;/h1&gt;" class="message_body_gravatar" src="/avatar/&lt;h1&gt;The Rogue One&lt;/h1&gt;?s=30" title="&lt;h1&gt;The Rogue One&lt;/h1&gt;"></p>'},
+        {input: ':<h1>The Rogue One</h1>:',
+         expected: '<p>:&lt;h1&gt;The Rogue One&lt;/h1&gt;:</p>'},
+        {input: '@**O\'Connell**',
+         expected: '<p>@**O&#39;Connell**</p>'},
+        {input: '@*Bobby <h1>Tables</h1>*',
+         expected: '<p><span class="user-group-mention" data-user-group-id="3">@Bobby &lt;h1&gt;Tables&lt;/h1&gt;</span></p>'},
+        {input: '@**Bobby <h1>Tables</h1>**',
+         expected: '<p><span class="user-mention" data-user-id="103">@Bobby &lt;h1&gt;Tables&lt;/h1&gt;</span></p>'},
+        {input: '#**Bobby <h1>Tables</h1>**',
+         expected: '<p><a class="stream" data-stream-id="3" href="http://zulip.zulipdev.com/#narrow/stream/3-Bobby-.3Ch1.3ETables.3C.2Fh1.3E">#Bobby &lt;h1&gt;Tables&lt;/h1&gt;</a></p>'},
     ];
 
     // We remove one of the unicode emoji we put as input in one of the test
@@ -322,7 +361,6 @@ var bugdown_data = JSON.parse(fs.readFileSync(path.join(__dirname, '../../zerver
         var message = {raw_content: input};
         markdown.apply_markdown(message);
         var output = message.content;
-
         assert.equal(expected, output);
     });
 }());
@@ -386,6 +424,20 @@ var bugdown_data = JSON.parse(fs.readFileSync(path.join(__dirname, '../../zerver
     assert.equal(message.mentioned, true);
     assert.equal(message.mentioned_me_directly, true);
 
+    input = "test @**everyone**";
+    message = {subject: "No links here", raw_content: input};
+    markdown.apply_markdown(message);
+    assert.equal(message.is_me_message, false);
+    assert.equal(message.mentioned, true);
+    assert.equal(message.mentioned_me_directly, false);
+
+    input = "test @**stream**";
+    message = {subject: "No links here", raw_content: input};
+    markdown.apply_markdown(message);
+    assert.equal(message.is_me_message, false);
+    assert.equal(message.mentioned, true);
+    assert.equal(message.mentioned_me_directly, false);
+
     input = "test @all";
     message = {subject: "No links here", raw_content: input};
     markdown.apply_markdown(message);

frontend_tests/node_tests/narrow.js

@@ -77,9 +77,12 @@ function set_filter(operators) {
 
     var hide_id;
     var show_id;
-    global.$ = function (id) {
-      return {hide: function () {hide_id = id;}, show: function () {show_id = id;}};
-    };
+    set_global('$', (id) => {
+        return {
+            hide: () => {hide_id = id;},
+            show: () => {show_id = id;},
+        };
+    });
 
     narrow_state.reset_current_filter();
     narrow.show_empty_narrow_message();

frontend_tests/node_tests/notifications.js

@@ -1,7 +1,7 @@
 // Dependencies
-zrequire('muting');
-zrequire('stream_data');
-
+set_global('$', global.make_zjquery({
+    silent: true,
+}));
 set_global('document', {
     hasFocus: function () {
         return true;
@@ -12,6 +12,15 @@ set_global('page_params', {
     is_admin: false,
     realm_users: [],
 });
+// For people.js
+set_global('md5', function (s) {
+    return 'md5-' + s;
+});
+
+zrequire('muting');
+zrequire('stream_data');
+zrequire('ui');
+zrequire('people');
 
 zrequire('notifications');
 
@@ -146,3 +155,108 @@ stream_data.add_sub('stream_two', two);
             subject: 'topic_two',
         }), true);
 }());
+
+
+(function test_basic_notifications() {
+
+    var n; // Object for storing all notification data for assertions.
+    var last_closed_message_id = null;
+    var last_shown_message_id = null;
+
+    // Notifications API stub
+    notifications.set_notification_api({
+        checkPermission: function checkPermission() {
+            if (window.Notification.permission === 'granted') {
+                return 0;
+            }
+            return 2;
+        },
+        requestPermission: function () {
+            return;
+        },
+        createNotification: function createNotification(icon, title, content, tag) {
+            var notification_object = {icon: icon, body: content, tag: tag};
+            // properties for testing.
+            notification_object.tests = {
+                shown: false,
+            };
+            notification_object.show = function () {
+                last_shown_message_id = this.tag;
+            };
+            notification_object.close = function () {
+                last_closed_message_id = this.tag;
+            };
+            notification_object.cancel = function () { notification_object.close(); };
+            return notification_object;
+        },
+    });
+
+   var message_1 = {
+        id: 1000,
+        content: '@-mentions the user',
+        sent_by_me: false,
+        notification_sent: false,
+        mentioned_me_directly: true,
+        type: 'stream',
+        stream: 'stream_one',
+        stream_id: 10,
+        subject: 'topic_two',
+    };
+
+    var message_2 = {
+        id: 1500,
+        content: '@-mentions the user',
+        sent_by_me: false,
+        notification_sent: false,
+        mentioned_me_directly: true,
+        type: 'stream',
+        stream: 'stream_one',
+        stream_id: 10,
+        subject: 'topic_four',
+    };
+
+    // Send notification.
+    notifications.process_notification({message: message_1, webkit_notify: true});
+    n = notifications.get_notifications();
+    assert.equal('undefined to stream_one > topic_two' in n, true);
+    assert.equal(Object.keys(n).length, 1);
+    assert.equal(last_shown_message_id, message_1.id);
+
+    // Remove notification.
+    notifications.close_notification(message_1);
+    n = notifications.get_notifications();
+    assert.equal('undefined to stream_one > topic_two' in n, false);
+    assert.equal(Object.keys(n).length, 0);
+    assert.equal(last_closed_message_id, message_1.id);
+
+    // Send notification.
+    message_1.id = 1001;
+    notifications.process_notification({message: message_1, webkit_notify: true});
+    n = notifications.get_notifications();
+    assert.equal('undefined to stream_one > topic_two' in n, true);
+    assert.equal(Object.keys(n).length, 1);
+    assert.equal(last_shown_message_id, message_1.id);
+
+    // Process same message again. Notification count shouldn't increase.
+    message_1.id = 1002;
+    notifications.process_notification({message: message_1, webkit_notify: true});
+    n = notifications.get_notifications();
+    assert.equal('undefined to stream_one > topic_two' in n, true);
+    assert.equal(Object.keys(n).length, 1);
+    assert.equal(last_shown_message_id, message_1.id);
+
+    // Send another message. Notification count should increase.
+    notifications.process_notification({message: message_2, webkit_notify: true});
+    n = notifications.get_notifications();
+    assert.equal('undefined to stream_one > topic_four' in n, true);
+    assert.equal(Object.keys(n).length, 2);
+    assert.equal(last_shown_message_id, message_2.id);
+
+    // Remove notifications.
+    notifications.close_notification(message_1);
+    notifications.close_notification(message_2);
+    n = notifications.get_notifications();
+    assert.equal('undefined to stream_one > topic_two' in n, false);
+    assert.equal(Object.keys(n).length, 0);
+    assert.equal(last_closed_message_id, message_2.id);
+}());

frontend_tests/node_tests/people.js

@@ -381,6 +381,8 @@ initialize();
     people.add(charles);
     people.add(maria);
 
+    assert.equal(people.small_avatar_url_for_person(maria),
+                 'https://secure.gravatar.com/avatar/md5-athens@example.com?d=identicon&s=50');
     var message = {
         type: 'private',
         display_recipient: [

frontend_tests/node_tests/reactions.js

@@ -288,7 +288,7 @@ set_global('current_msg_list', {
 
     reactions.set_reaction_count(reaction_element, 5);
 
-    assert.equal(count_element.html(), '5');
+    assert.equal(count_element.text(), '5');
 }());
 
 (function test_get_reaction_section() {
@@ -388,7 +388,7 @@ set_global('current_msg_list', {
 
     reactions.add_reaction(bob_event);
     assert(title_set);
-    assert.equal(count_element.html(), '2');
+    assert.equal(count_element.text(), '2');
 
     // Now, remove Bob's 8ball emoji.  The event has the same exact
     // structure as the add event.
@@ -402,7 +402,7 @@ set_global('current_msg_list', {
 
     reactions.remove_reaction(bob_event);
     assert(title_set);
-    assert.equal(count_element.html(), '1');
+    assert.equal(count_element.text(), '1');
 
     var current_emojis = reactions.get_emojis_used_by_user_for_message_id(1001);
     assert.deepEqual(current_emojis, ['smile', 'inactive_realm_emoji', '8ball']);

frontend_tests/node_tests/search_suggestion.js

@@ -596,7 +596,7 @@ init();
         return suggestions.lookup_table[q].description;
     }
     assert.equal(describe('te'), "Search for te");
-    assert.equal(describe('stream:office topic:team'), "Stream office > team");
+    assert.equal(describe('stream:office topic:team'), "Stream office > team");
 
     suggestions = search.get_suggestions('topic:staplers stream:office');
     expected = [

frontend_tests/node_tests/settings_muting.js

@@ -0,0 +1,56 @@
+set_global('$', global.make_zjquery());
+
+zrequire('settings_muting');
+zrequire('muting');
+set_global('muting_ui', {});
+
+var noop = function () {};
+
+(function test_settings() {
+
+    muting.add_muted_topic('frontend', 'js');
+    var set_up_ui_called = false;
+    muting_ui.set_up_muted_topics_ui = function (opts) {
+        assert.deepEqual(opts, [['frontend', 'js']]);
+        set_up_ui_called = true;
+    };
+
+    settings_muting.set_up();
+
+    var click_handler = $('body').get_on_handler('click', '.settings-unmute-topic');
+    assert.equal(typeof(click_handler), 'function');
+
+    var event = {
+        stopImmediatePropagation: noop,
+    };
+
+    var fake_this = $.create('fake.settings-unmute-topic');
+    var tr_html = $('tr[data-topic="js"]');
+    fake_this.closest = function (opts) {
+        assert.equal(opts, 'tr');
+        return tr_html;
+    };
+
+    var data_called = 0;
+    tr_html.data = function (opts) {
+        if (opts === 'stream') {
+            data_called += 1;
+            return 'frontend';
+        }
+        if (opts === 'topic') {
+            data_called += 1;
+            return 'js';
+        }
+    };
+
+    var unmute_called = false;
+    muting_ui.unmute = function (stream, topic) {
+        assert.equal(stream, 'frontend');
+        assert.equal(topic, 'js');
+        unmute_called = true;
+    };
+    click_handler.call(fake_this, event);
+    assert(unmute_called);
+    assert(set_up_ui_called);
+    assert.equal(data_called, 2);
+}());

frontend_tests/node_tests/settings_org.js

@@ -5,6 +5,7 @@ zrequire('stream_data');
 zrequire('settings_account');
 zrequire('settings_org');
 zrequire('settings_ui');
+zrequire('settings_ui');
 
 var noop = function () {};
 
@@ -136,12 +137,63 @@ function test_realms_domain_modal(add_realm_domain) {
     error_callback({});
     assert.equal(info.val(), 'translated: Failed');
 }
-
+function createSaveButtons() {
+    var stub_save_button_header = $('.subsection-header');
+    var save_btn_controls = $.create('.save-btn-controls');
+    var stub_save_button = $('#org-submit-msg-editing');
+    var stub_save_button_text = $.create('.icon-button-text');
+    stub_save_button_header.prevAll = function () {
+        return $.create('<stub failed alert status element>');
+    };
+    stub_save_button.closest = function () {
+        return stub_save_button_header;
+    };
+    save_btn_controls.set_find_results(
+        '.save-button', stub_save_button
+    );
+    stub_save_button.set_find_results(
+        '.icon-button-text', stub_save_button_text
+    );
+    stub_save_button_header.set_find_results(
+        '.save-button-controls', save_btn_controls
+    );
+    stub_save_button_header.set_find_results(
+        '.subsection-changes-discard .button', $.create('#org-discard-msg-editing')
+    );
+    var props  = {};
+    props.hidden = false;
+    props.status = "";
+    stub_save_button.attr = function (name, val) {
+        if (name === "data-status") {
+            if (val !== null) {
+                props.status = val;
+                return;
+            }
+            return props.status;
+        } else if (name === "id") {
+            return 'org-submit-msg-editing';
+        }
+    };
+    save_btn_controls.animate = function (obj) {
+        if (obj.opacity === 0) {
+            props.hidden = true;
+        } else {
+            props.hidden = false;
+        }
+    };
+    return {
+        props: props,
+        save_button: stub_save_button,
+        save_button_header: stub_save_button_header,
+        save_button_controls: save_btn_controls,
+        save_button_text: stub_save_button_text,
+    };
+}
 function test_submit_settings_form(submit_form) {
     var ev = {
         preventDefault: noop,
         stopPropagation: noop,
-        target: '#org-submit-msg-editing',
+        currentTarget: '#org-submit-msg-editing',
     };
 
     $('#id_realm_default_language').val('fr');
@@ -155,20 +207,7 @@ function test_submit_settings_form(submit_form) {
         success_callback = req.success;
     };
 
-    var stub_save_button = $('#org-submit-msg-editing');
-    stub_save_button.attr = function () {
-        return 'org-submit-msg-editing';
-    };
-    var stub_save_button_header = $('.subsection-header');
-    stub_save_button_header.prevAll = function () {
-        return $.create('<stub failed alert status element>');
-    };
-    stub_save_button.closest = function () {
-        return stub_save_button_header;
-    };
-    stub_save_button_header.set_find_results(
-        '.subsection-changes-discard button', $.create('#org-discard-msg-editing')
-    );
+    createSaveButtons();
 
     submit_form(ev);
     assert(patched);
@@ -177,7 +216,6 @@ function test_submit_settings_form(submit_form) {
         allow_message_editing: true,
         message_content_edit_limit_seconds: 210,
     };
-
     success_callback(response_data);
 
     var updated_value_from_response = $('#id_realm_message_content_edit_limit_minutes').val();
@@ -195,6 +233,38 @@ function test_submit_settings_form(submit_form) {
     assert(updated_value_from_response, 0);
 }
 
+function test_change_save_button_state() {
+    set_global('$', global.make_zjquery());
+    var stubs = createSaveButtons();
+    var $save_btn_controls = stubs.save_button_controls;
+    var $save_btn_text = stubs.save_button_text;
+    var $save_btn = stubs.save_button;
+    var props = stubs.props;
+    settings_org.change_save_button_state($save_btn_controls, "unsaved");
+    assert.equal($save_btn_text.text(), 'translated: Save changes');
+    assert.equal(props.hidden, false);
+    assert.equal(props.status, "unsaved");
+    settings_org.change_save_button_state($save_btn_controls, "saved");
+    assert.equal($save_btn_text.text(), 'translated: Save changes');
+    assert.equal(props.hidden, true);
+    assert.equal(props.status, "");
+    settings_org.change_save_button_state($save_btn_controls, "saving");
+    assert.equal($save_btn_text.text(), 'translated: Saving');
+    assert.equal(props.status, "saving");
+    assert.equal($save_btn.hasClass('saving'), true);
+    settings_org.change_save_button_state($save_btn_controls, "discarded");
+    assert.equal(props.hidden, true);
+    assert.equal($save_btn.hasClass('saving'), false);
+    settings_org.change_save_button_state($save_btn_controls, "succeeded");
+    assert.equal(props.hidden, true);
+    assert.equal(props.status, "saved");
+    assert.equal($save_btn_text.text(), 'translated: Saved');
+    settings_org.change_save_button_state($save_btn_controls, "failed");
+    assert.equal(props.hidden, false);
+    assert.equal(props.status, "failed");
+    assert.equal($save_btn_text.text(), 'translated: Save changes');
+}
+
 function test_upload_realm_icon(upload_realm_icon) {
     var form_data = {
         append: function (field, val) {
@@ -332,14 +402,14 @@ function test_change_allow_subdomains(change_allow_subdomains) {
     domain_obj.text(domain);
 
 
-    var elem_obj = $('<elem html>');
+    var elem_obj = $.create('<elem html>');
     var parents_obj = $.create('parents object');
 
     elem_obj.set_parents_result('tr', parents_obj);
     parents_obj.set_find_results('.domain', domain_obj);
     elem_obj.prop('checked', allow);
 
-    change_allow_subdomains.apply('<elem html>', [ev]);
+    change_allow_subdomains.apply(elem_obj, [ev]);
 
     success_callback();
     assert.equal(info.val(),
@@ -350,7 +420,7 @@ function test_change_allow_subdomains(change_allow_subdomains) {
 
     allow = false;
     elem_obj.prop('checked', allow);
-    change_allow_subdomains.apply('<elem html>', [ev]);
+    change_allow_subdomains.apply(elem_obj, [ev]);
     success_callback();
     assert.equal(info.val(),
                  'translated: Update successful: Subdomains no longer allowed for example.com');
@@ -400,7 +470,7 @@ function test_extract_property_name() {
 
     var submit_settings_form;
     $('.organization').on = function (action, selector, f) {
-        if (selector === '.subsection-header .subsection-changes-save button') {
+        if (selector === '.subsection-header .subsection-changes-save .button') {
             assert.equal(action, 'click');
             submit_settings_form = f;
         }
@@ -422,6 +492,8 @@ function test_extract_property_name() {
         upload_realm_icon = f;
     };
 
+    var stub_render_notifications_stream_ui = settings_org.render_notifications_stream_ui;
+    settings_org.render_notifications_stream_ui = noop;
     var parent_elem = $.create('waiting-period-parent-stub');
     $('#id_realm_waiting_period_threshold').set_parent(parent_elem);
     // TEST set_up() here, but this mostly just allows us to
@@ -439,11 +511,17 @@ function test_extract_property_name() {
     test_disable_signup_notifications_stream(callbacks.disable_signup_notifications_stream);
     test_change_allow_subdomains(change_allow_subdomains);
     test_extract_property_name();
+    settings_org.render_notifications_stream_ui = stub_render_notifications_stream_ui;
+    test_change_save_button_state();
 }());
 
 (function test_misc() {
     page_params.is_admin = false;
 
+    var stub_notification_disable_parent = $.create('<stub notification_disable parent');
+    stub_notification_disable_parent.set_find_results('.notification-disable',
+        $.create('<disable link>'));
+
     page_params.realm_name_changes_disabled = false;
     settings_account.update_name_change_display();
     assert.equal($('#full_name').attr('disabled'), false);
@@ -469,6 +547,9 @@ function test_extract_property_name() {
     assert.equal($("#change_email .button").attr('disabled'), false);
 
     var elem = $('#realm_notifications_stream_name');
+    elem.closest = function () {
+        return stub_notification_disable_parent;
+    };
     stream_data.get_sub_by_id = function (stream_id) {
         assert.equal(stream_id, 42);
         return { name: 'some_stream' };
@@ -483,6 +564,9 @@ function test_extract_property_name() {
     assert(elem.hasClass('text-warning'));
 
     elem = $('#realm_signup_notifications_stream_name');
+    elem.closest = function () {
+        return stub_notification_disable_parent;
+    };
     stream_data.get_sub_by_id = function (stream_id) {
         assert.equal(stream_id, 75);
         return { name: 'some_stream' };

frontend_tests/node_tests/stream_data.js

@@ -169,33 +169,34 @@ zrequire('marked', 'third/marked/lib/marked');
 
     stream_data.set_subscribers(sub, [fred.user_id, george.user_id]);
     stream_data.update_calculated_fields(sub);
-    assert(stream_data.user_is_subscribed('Rome', 'FRED@zulip.com'));
-    assert(stream_data.user_is_subscribed('Rome', 'fred@zulip.com'));
-    assert(stream_data.user_is_subscribed('Rome', 'george@zulip.com'));
-    assert(!stream_data.user_is_subscribed('Rome', 'not_fred@zulip.com'));
+    assert(stream_data.is_user_subscribed('Rome', fred.user_id));
+    assert(stream_data.is_user_subscribed('Rome', george.user_id));
+    assert(!stream_data.is_user_subscribed('Rome', not_fred.user_id));
 
     stream_data.set_subscribers(sub, []);
 
-    var email = 'brutus@zulip.com';
     var brutus = {
-        email: email,
+        email: 'brutus@zulip.com',
         full_name: 'Brutus',
         user_id: 104,
     };
     people.add(brutus);
-    assert(!stream_data.user_is_subscribed('Rome', email));
+    assert(!stream_data.is_user_subscribed('Rome', brutus.user_id));
 
     // add
     var ok = stream_data.add_subscriber('Rome', brutus.user_id);
     assert(ok);
-    assert(stream_data.user_is_subscribed('Rome', email));
+    assert(stream_data.is_user_subscribed('Rome', brutus.user_id));
     sub = stream_data.get_sub('Rome');
     stream_data.update_subscribers_count(sub);
     assert.equal(sub.subscriber_count, 1);
+    var sub_email = "Rome:214125235@zulipdev.com:9991";
+    stream_data.update_stream_email_address(sub, sub_email);
+    assert.equal(sub.email_address, sub_email);
 
     // verify that adding an already-added subscriber is a noop
     stream_data.add_subscriber('Rome', brutus.user_id);
-    assert(stream_data.user_is_subscribed('Rome', email));
+    assert(stream_data.is_user_subscribed('Rome', brutus.user_id));
     sub = stream_data.get_sub('Rome');
     stream_data.update_subscribers_count(sub);
     assert.equal(sub.subscriber_count, 1);
@@ -203,20 +204,22 @@ zrequire('marked', 'third/marked/lib/marked');
     // remove
     ok = stream_data.remove_subscriber('Rome', brutus.user_id);
     assert(ok);
-    assert(!stream_data.user_is_subscribed('Rome', email));
+    assert(!stream_data.is_user_subscribed('Rome', brutus.user_id));
     sub = stream_data.get_sub('Rome');
     stream_data.update_subscribers_count(sub);
     assert.equal(sub.subscriber_count, 0);
 
-    // verify that checking subscription with bad email is a noop
-    var bad_email = 'notbrutus@zulip.org';
-    global.blueslip.error = function (msg) {
-        assert.equal(msg, "Unknown email for get_user_id: " + bad_email);
-    };
+    // verify that deactivating user should unsubscribe user from all streams
+    assert(stream_data.add_subscriber('Rome', george.user_id));
+    set_global('subs', { rerender_subscriptions_settings: function () {} });
+    stream_data.remove_deactivated_user_from_all_streams(george.user_id);
+    assert(!stream_data.is_user_subscribed('Rome', george.user_id));
+
+    // verify that checking subscription with undefined user id
     global.blueslip.warn = function (msg) {
-        assert.equal(msg, "Bad email passed to user_is_subscribed: " + bad_email);
+        assert.equal(msg, "Undefined user_id passed to function is_user_subscribed");
     };
-    assert(!stream_data.user_is_subscribed('Rome', bad_email));
+    assert.equal(stream_data.is_user_subscribed('Rome', undefined), undefined);
 
     // Verify noop for bad stream when removing subscriber
     var bad_stream = 'UNKNOWN';
@@ -233,7 +236,7 @@ zrequire('marked', 'third/marked/lib/marked');
     // verify that removing an already-removed subscriber is a noop
     ok = stream_data.remove_subscriber('Rome', brutus.user_id);
     assert(!ok);
-    assert(!stream_data.user_is_subscribed('Rome', email));
+    assert(!stream_data.is_user_subscribed('Rome', brutus.user_id));
     sub = stream_data.get_sub('Rome');
     stream_data.update_subscribers_count(sub);
     assert.equal(sub.subscriber_count, 0);
@@ -244,24 +247,24 @@ zrequire('marked', 'third/marked/lib/marked');
     stream_data.add_sub('Rome', sub);
     stream_data.add_subscriber('Rome', brutus.user_id);
     sub.subscribed = true;
-    assert(stream_data.user_is_subscribed('Rome', email));
+    assert(stream_data.is_user_subscribed('Rome', brutus.user_id));
 
     // Verify that we noop and don't crash when unsubscribed.
     sub.subscribed = false;
     stream_data.update_calculated_fields(sub);
     ok = stream_data.add_subscriber('Rome', brutus.user_id);
     assert(ok);
-    assert.equal(stream_data.user_is_subscribed('Rome', email), true);
+    assert.equal(stream_data.is_user_subscribed('Rome', brutus.user_id), true);
     stream_data.remove_subscriber('Rome', brutus.user_id);
-    assert.equal(stream_data.user_is_subscribed('Rome', email), false);
+    assert.equal(stream_data.is_user_subscribed('Rome', brutus.user_id), false);
     stream_data.add_subscriber('Rome', brutus.user_id);
-    assert.equal(stream_data.user_is_subscribed('Rome', email), true);
+    assert.equal(stream_data.is_user_subscribed('Rome', brutus.user_id), true);
 
     sub.invite_only = true;
     stream_data.update_calculated_fields(sub);
-    assert.equal(stream_data.user_is_subscribed('Rome', email), undefined);
+    assert.equal(stream_data.is_user_subscribed('Rome', brutus.user_id), undefined);
     stream_data.remove_subscriber('Rome', brutus.user_id);
-    assert.equal(stream_data.user_is_subscribed('Rome', email), undefined);
+    assert.equal(stream_data.is_user_subscribed('Rome', brutus.user_id), undefined);
 
     // Verify that we don't crash and return false for a bad stream.
     ok = stream_data.add_subscriber('UNKNOWN', brutus.user_id);

frontend_tests/node_tests/templates.js

@@ -622,6 +622,13 @@ function render(template_name, args) {
     assert.equal(error_msg, "translated:         This stream is reserved for announcements.\n        \n        Are you sure you want to message all 101 people in this stream?");
 }());
 
+(function compose_not_subscribed() {
+    var html = render('compose_not_subscribed');
+    global.write_handlebars_output("compose_not_subscribed", html);
+    var button = $(html).find("button:first");
+    assert.equal(button.text(), "translated: Subscribe");
+}());
+
 (function compose_notification() {
     var args = {
         note: "You sent a message to a muted topic.",
@@ -1064,6 +1071,33 @@ function render(template_name, args) {
     assert.equal(label.text().trim(), 'King Lear (lear@zulip.com)');
 }());
 
+(function non_editable_user_group() {
+    var args = {
+        user_group: {
+            id: "9",
+            name: "uranohoshi",
+            description: "Students at Uranohoshi Academy",
+        },
+    };
+
+    var html = '';
+    html += '<div id="user-groups">';
+    html += render('non_editable_user_group', args);
+    html += '</div>';
+
+    global.write_handlebars_output('non_editable_user_group', html);
+
+    var group_id = $(html).find('.user-group:first').prop('id');
+    var group_name_pills = $(html).find('.user-group:first .pill-container').attr('data-group-pills');
+    var group_name_display = $(html).find('.user-group:first .name').text().trim().replace(/\s+/g, ' ');
+    var group_description = $(html).find('.user-group:first .description').text().trim().replace(/\s+/g, ' ');
+
+    assert.equal(group_id, '9');
+    assert.equal(group_name_pills, 'uranohoshi');
+    assert.equal(group_name_display, 'uranohoshi');
+    assert.equal(group_description, 'Students at Uranohoshi Academy');
+}());
+
 (function notification() {
     var args = {
         content: "Hello",

frontend_tests/node_tests/timerender.js

@@ -197,7 +197,7 @@ zrequire('timerender');
 
 (function test_set_full_datetime() {
     var message = {
-        timestamp: 1495091573, // 5/18/2017 7:12:53 AM (UTC+0)
+        timestamp: 1495091573, // 2017-5-18 07:12:53 AM (UTC+0)
     };
     var time_element = $('<span/>');
     var attrs = new Dict();
@@ -207,7 +207,14 @@ zrequire('timerender');
         return time_element;
     };
 
-    var expected = '5/18/2017 7:12:53 AM (UTC+0)';
+    // since node >= 8 date.toLocaleDateString and
+    // date.toLocaleTimeString have been changed, instead of
+    // returning 5/18/2017 7:12:53 AM (UTC+0) they now return
+    // 2017-5-18 07:12:53 (UTC+0) - This change does not affect browsers
+    // since browsers have their own way of returning string that is
+    // or maybe inconsistenc with node's way.
+    var time = new Date(message.timestamp * 1000);
+    var expected = `${time.toLocaleDateString()} 07:12:53 (UTC+0)`;
     timerender.set_full_datetime(message, time_element);
     var actual = attrs.get('title');
     assert.equal(expected, actual);

frontend_tests/node_tests/typeahead_helper.js

@@ -1,5 +1,8 @@
 set_global('page_params', {realm_is_zephyr_mirror_realm: false});
 set_global('templates', {});
+set_global('md5', function (s) {
+    return 'md5-' + s;
+});
 
 zrequire('Handlebars', 'handlebars');
 zrequire('recent_senders');

frontend_tests/node_tests/upload.js

@@ -32,12 +32,11 @@ var upload_opts = upload.options({ mode: "compose" });
         assert(handler);
     };
     $("#compose-error-msg").html('');
-    var test_html = '<div class="progress progress-striped active">' +
-                    '<div class="bar" id="compose-upload-bar" style="width: 00%;">' +
-                    '</div></div>';
-    $("<p>").after = function (html) {
+    var test_html = '<div class="progress active">' +
+                    '<div class="bar" id="compose-upload-bar" style="width: 0"></div>' +
+                    '</div>';
+    $("#compose-send-status").append = function (html) {
         assert.equal(html, test_html);
-        return 'fake-html';
     };
 
     upload_opts.drop();
@@ -46,7 +45,6 @@ var upload_opts = upload.options({ mode: "compose" });
     assert($("#compose-send-status").hasClass("alert-info"));
     assert($("#compose-send-status").visible());
     assert.equal($("<p>").text(), 'translated: Uploading…');
-    assert.equal($("#compose-error-msg").html(), 'fake-html');
 }());
 
 (function test_progress_updated() {
@@ -65,6 +63,10 @@ var upload_opts = upload.options({ mode: "compose" });
         $("#compose-send-status").addClass("alert-info");
         $("#compose-send-button").attr("disabled", 'disabled');
         $("#compose-error-msg").text('');
+
+        $("#compose-upload-bar").parent = function () {
+            return { remove: function () {} };
+        };
     }
 
     function assert_side_effects(msg) {
@@ -144,8 +146,17 @@ var upload_opts = upload.options({ mode: "compose" });
             }
         }
 
+        global.patch_builtin('setTimeout', function (func) {
+            func();
+        });
+
+        $("#compose-upload-bar").width = function (width_percent) {
+            assert.equal(width_percent, '100%');
+        };
+
         setup();
         upload_opts.uploadFinished(i, {}, response);
+        upload_opts.progressUpdated(1, '', 100);
         assert_side_effects();
     }
 

frontend_tests/node_tests/util.js

@@ -166,6 +166,13 @@ zrequire('util');
         'some text before only @**everyone**',
     ];
 
+    var messages_with_stream_mentions = [
+        '@**stream**',
+        'some text before @**stream** some text after',
+        '@**stream** some text after only',
+        'some text before only @**stream**',
+    ];
+
     var messages_without_all_mentions = [
         '@all',
         'some text before @all some text after',
@@ -183,6 +190,16 @@ zrequire('util');
         '`@**everyone**`',
         'some_email@**everyone**.com',
     ];
+
+    var messages_without_stream_mentions = [
+        'some text before @stream some text after',
+        '@stream',
+        '`@stream`',
+        'some_email@stream.com',
+        '`@**stream**`',
+        'some_email@**stream**.com',
+    ];
+
     var i;
     for (i=0; i<messages_with_all_mentions.length; i += 1) {
         assert(util.is_all_or_everyone_mentioned(messages_with_all_mentions[i]));
@@ -192,6 +209,10 @@ zrequire('util');
         assert(util.is_all_or_everyone_mentioned(messages_with_everyone_mentions[i]));
     }
 
+    for (i=0; i<messages_with_stream_mentions.length; i += 1) {
+        assert(util.is_all_or_everyone_mentioned(messages_with_stream_mentions[i]));
+    }
+
     for (i=0; i<messages_without_all_mentions.length; i += 1) {
         assert(!util.is_all_or_everyone_mentioned(messages_without_everyone_mentions[i]));
     }
@@ -199,6 +220,10 @@ zrequire('util');
     for (i=0; i<messages_without_everyone_mentions.length; i += 1) {
         assert(!util.is_all_or_everyone_mentioned(messages_without_everyone_mentions[i]));
     }
+
+    for (i=0; i<messages_without_stream_mentions.length; i += 1) {
+        assert(!util.is_all_or_everyone_mentioned(messages_without_stream_mentions[i]));
+    }
 }());
 
 (function test_move_array_elements_to_front() {

frontend_tests/node_tests/zjquery.js

@@ -174,3 +174,26 @@ set_global('$', global.make_zjquery());
     obj2.addClass('.striped');
     assert(obj2.hasClass('.striped'));
 }());
+
+(function test_extensions() {
+    // You can extend $.fn so that all subsequent objects
+    // we create get a new function.
+
+    $.fn.area = function () {
+        return this.width() * this.height();
+    };
+
+    // Before we use area, though, let's illustrate that
+    // the predominant Zulip testing style is to stub objects
+    // using direct syntax:
+
+    var rect = $.create('rectangle');
+    rect.width = () => { return 5; };
+    rect.height = () => { return 7; };
+
+    assert.equal(rect.width(), 5);
+    assert.equal(rect.height(), 7);
+
+    // But we also have area available from general extension.
+    assert.equal(rect.area(), 35);
+}());

frontend_tests/zjsunit/zjquery.js

@@ -2,10 +2,19 @@ var noop = function () {};
 
 var exports = {};
 
-exports.make_zjquery = function () {
+exports.make_zjquery = function (opts) {
 
     var elems = {};
 
+    // Our fn structure helps us simulate extending jQuery.
+    var fn = {};
+
+    function add_extensions(obj) {
+        _.each(fn, (v, k) => {
+            obj[k] = v;
+        });
+    }
+
     function new_elem(selector) {
         var html = 'never-been-set';
         var text = 'never-been-set';
@@ -82,7 +91,9 @@ exports.make_zjquery = function () {
                 if (child) {
                     return child;
                 }
-
+                if (opts.silent) {
+                    return self;
+                }
                 throw Error("Cannot find " + child_selector + " in " + selector);
             },
             focus: function () {
@@ -219,6 +230,9 @@ exports.make_zjquery = function () {
                 return self;
             },
             removeData: noop,
+            replaceWith: function () {
+                return self;
+            },
             select: function (arg) {
                 generic_event('select', arg);
                 return self;
@@ -253,7 +267,7 @@ exports.make_zjquery = function () {
                 // legitimate for code to trigger multiple handlers.
                 // But up until now, we haven't needed this, and if
                 // you come across this assertion, it's possible that
-                // you can sselfify your tests by just doing your own
+                // you can simplify your tests by just doing your own
                 // mocking of trigger().  If you really know what you
                 // are doing, you can remove this limitation.
                 assert(funcs.length <= 1, 'multiple functions set up');
@@ -281,10 +295,13 @@ exports.make_zjquery = function () {
 
         self[0] = 'you-must-set-the-child-yourself';
 
+        add_extensions(self);
+
+        self.selector = selector;
         return self;
     }
 
-    var zjquery = function (arg) {
+    var zjquery = function (arg, arg2) {
         if (typeof arg === "function") {
             // If somebody is passing us a function, we emulate
             // jQuery's behavior of running this function after
@@ -292,25 +309,44 @@ exports.make_zjquery = function () {
             // so we just call it right away.
             arg();
             return;
-        } else if (typeof arg === "object") {
-            // If somebody is passing us an element, we return
-            // the element itself if it's been created with
-            // zjquery.
-            // This may happen in cases like $(this).
-            if (arg.debug) {
-                var this_selector = arg.debug().selector;
-                if (elems[this_selector]) {
-                    return arg;
-                }
+        }
+
+
+        // If somebody is passing us an element, we return
+        // the element itself if it's been created with
+        // zjquery.
+        // This may happen in cases like $(this).
+        if (arg.selector) {
+            if (elems[arg.selector]) {
+                return arg;
             }
         }
 
+        // We occasionally create stub objects that know
+        // they want to be wrapped by jQuery (so they can
+        // in turn return stubs).  The convention is that
+        // they provide a to_$ attribute.
+        if (arg.to_$) {
+            assert(typeof arg.to_$ === "function");
+            return arg.to_$();
+        }
+
+        if (arg2 !== undefined) {
+            throw Error("We only use one-argument variations of $(...) in Zulip code.");
+        }
+
         var selector = arg;
 
+        if (typeof selector !== "string") {
+            console.info(arg);
+            throw Error("zjquery does not know how to wrap this object yet");
+        }
+
         var valid_selector =
             ('<#.'.indexOf(selector[0]) >= 0) ||
             (selector === 'window-stub') ||
             (selector === 'document-stub') ||
+            (selector === 'body') ||
             (selector === 'html') ||
             (selector.location) ||
             (selector.indexOf('#') >= 0) ||
@@ -369,6 +405,8 @@ exports.make_zjquery = function () {
         return _.extend(content, container);
     };
 
+    zjquery.fn = fn;
+
     return zjquery;
 };
 

package.json

@@ -8,7 +8,7 @@
     "@types/node": "8.0.34",
     "@types/webpack": "3.0.13",
     "blueimp-md5": "2.10.0",
-    "clipboard": "1.5.16",
+    "clipboard": "2.0.0",
     "emoji-datasource-apple": "3.0.0",
     "emoji-datasource-emojione": "3.0.0",
     "emoji-datasource-google": "3.0.0",
@@ -32,6 +32,8 @@
     "script-loader": "0.7.2",
     "source-map-loader": "0.2.3",
     "string.prototype.codepointat": "0.2.0",
+    "string.prototype.endswith": "0.2.0",
+    "string.prototype.startswith": "0.2.0",
     "to-markdown": "3.1.0",
     "ts-loader": "2.1.0",
     "ts-node": "3.3.0",
@@ -48,7 +50,7 @@
     "casperjs": "casperjs/casperjs",
     "cssstyle": "0.2.29",
     "difflib": "0.2.4",
-    "eslint": "3.9.1",
+    "eslint": "4.19.1",
     "eslint-plugin-empty-returns": "1.0.2",
     "htmlparser2": "3.8.3",
     "istanbul": "0.4.5",

puppet/zulip/files/nginx/sites-available/zulip-enterprise

@@ -14,6 +14,7 @@ server {
 
     location /user_avatars {
         add_header X-Content-Type-Options nosniff;
+        add_header Content-Security-Policy "default-src 'none' img-src 'self'";
         include /etc/nginx/zulip-include/uploads.types;
         alias /home/zulip/uploads/avatars;
     }

puppet/zulip/files/nginx/zulip-include-frontend/uploads.types

@@ -2,6 +2,8 @@
 types {
     text/plain                            txt;
 
+    application/pdf                       pdf;
+
     image/gif                             gif;
     image/jpeg                            jpeg jpg;
     image/png                             png;

puppet/zulip/files/nginx/zulip-include-maybe/uploads-route.direct

@@ -1,5 +1,6 @@
 location /user_uploads {
     add_header X-Content-Type-Options nosniff;
+    add_header Content-Security-Policy "default-src 'none'; style-src 'self' 'unsafe-inline'; img-src 'self'; object-src 'self'; plugin-types application/pdf;";
     include /etc/nginx/zulip-include/uploads.types;
     alias /home/zulip/uploads/files;
 }

puppet/zulip/files/nginx/zulip-include-maybe/uploads-route.internal

@@ -1,6 +1,7 @@
 location /serve_uploads {
     internal;
     add_header X-Content-Type-Options nosniff;
+    add_header Content-Security-Policy "default-src 'none'; style-src 'self' 'unsafe-inline'; img-src 'self'; object-src 'self'; plugin-types application/pdf;";
     include /etc/nginx/zulip-include/uploads.types;
     alias /home/zulip/uploads/files;
 }

requirements/common.in

@@ -95,7 +95,8 @@ oauthlib==2.0.7
 ndg-httpsclient==0.4.4
 
 # Needed to access rabbitmq
-pika==0.11.2
+# See #8466 for why we're not using the latest version.
+pika==0.11.0
 
 # Needed to access our database
 psycopg2==2.7.4 --no-binary psycopg2
@@ -164,11 +165,12 @@ ijson==2.3
 beautifulsoup4==4.6.0
 pyoembed==0.1.2
 
-# The Zulip API bindings, from its own repository.
-# We integrate with these tightly, so often it makes sense to pin a
-# version from Git rather than a release.
--e "git+https://github.com/zulip/python-zulip-api.git@0.4.2#egg=zulip==0.4.2_git&subdirectory=zulip"
--e "git+https://github.com/zulip/python-zulip-api.git@0.4.2#egg=zulip_bots==0.4.2+git&subdirectory=zulip_bots"
+# The Zulip API bindings, from its own repository.  We integrate with
+# these tightly, including fetching content not included in the normal
+# release tarballs (which is a bug).  So we need to pin it makes sense
+# to pin a version from Git rather than a release.
+-e "git+https://github.com/zulip/python-zulip-api.git@0.4.4#egg=zulip==0.4.4_git&subdirectory=zulip"
+-e "git+https://github.com/zulip/python-zulip-api.git@0.4.4#egg=zulip_bots==0.4.4+git&subdirectory=zulip_bots"
 
 # Used for Hesiod lookups, etc.
 py3dns==3.1.0

requirements/dev.txt

@@ -11,8 +11,8 @@
 
 git+https://github.com/zulip/talon.git@7d8bdc4dbcfcc5a73298747293b99fe53da55315#egg=talon==1.2.10.zulip1
 git+https://github.com/zulip/ultrajson@70ac02bec#egg=ujson==1.35+git
-git+https://github.com/zulip/python-zulip-api.git@0.4.2#egg=zulip==0.4.2_git&subdirectory=zulip
-git+https://github.com/zulip/python-zulip-api.git@0.4.2#egg=zulip_bots==0.4.2+git&subdirectory=zulip_bots
+git+https://github.com/zulip/python-zulip-api.git@0.4.4#egg=zulip==0.4.4_git&subdirectory=zulip
+git+https://github.com/zulip/python-zulip-api.git@0.4.4#egg=zulip_bots==0.4.4+git&subdirectory=zulip_bots
 alabaster==0.7.10
 apns2==0.3.0
 argon2-cffi==18.1.0
@@ -101,7 +101,7 @@ pbr==3.1.1                # via mock
 pexpect==4.3.0            # via ipython
 phonenumberslite==8.8.6   # via django-phonenumber-field
 pickleshare==0.7.4        # via ipython
-pika==0.11.2
+pika==0.11.0
 pillow==5.0.0
 pip-tools==1.11.0
 polib==1.1.0

requirements/prod.txt

@@ -11,8 +11,8 @@
 
 git+https://github.com/zulip/talon.git@7d8bdc4dbcfcc5a73298747293b99fe53da55315#egg=talon==1.2.10.zulip1
 git+https://github.com/zulip/ultrajson@70ac02bec#egg=ujson==1.35+git
-git+https://github.com/zulip/python-zulip-api.git@0.4.2#egg=zulip==0.4.2_git&subdirectory=zulip
-git+https://github.com/zulip/python-zulip-api.git@0.4.2#egg=zulip_bots==0.4.2+git&subdirectory=zulip_bots
+git+https://github.com/zulip/python-zulip-api.git@0.4.4#egg=zulip==0.4.4_git&subdirectory=zulip
+git+https://github.com/zulip/python-zulip-api.git@0.4.4#egg=zulip_bots==0.4.4+git&subdirectory=zulip_bots
 apns2==0.3.0
 argon2-cffi==18.1.0
 asn1crypto==0.23.0        # via cryptography
@@ -71,7 +71,7 @@ pbr==3.1.1                # via mock
 pexpect==4.3.0            # via ipython
 phonenumberslite==8.8.6   # via django-phonenumber-field
 pickleshare==0.7.4        # via ipython
-pika==0.11.2
+pika==0.11.0
 pillow==5.0.0
 polib==1.1.0
 premailer==3.1.1

scripts/lib/install-node

@@ -7,8 +7,13 @@ if [ "$TRAVIS" ] ; then
     ZULIP_SRV="/home/travis"
 fi
 YARN_BIN="$ZULIP_SRV/zulip-yarn/bin/yarn"
-node_version=6.6.0
-yarn_version=0.27.5
+node_version=8.11.1
+yarn_version=1.5.1
+
+# This is a fix for the fact that nvm uses $HOME to determine which
+# user account's home directory to ~/.config to.  Ideally, we'd have a
+# more systematic fix, like using `sudo -H` everywhere.
+export HOME=/root
 
 current_node_version="none"
 if hash node 2>/dev/null; then
@@ -23,7 +28,7 @@ fi
 if [ "$current_node_version" != "v$node_version" ]; then
     export NVM_DIR=/usr/local/nvm
     if ! [ -e "$NVM_DIR/nvm.sh" ]; then
-        wget -qO- https://raw.githubusercontent.com/creationix/nvm/v0.32.0/install.sh | bash
+        wget -qO- https://raw.githubusercontent.com/creationix/nvm/v0.33.8/install.sh | bash
     fi
 
     source "$NVM_DIR/nvm.sh"
@@ -41,5 +46,11 @@ if [ "$current_node_version" != "v$node_version" ]; then
     sed -i "s|NODE_PATH|$NODE_BIN|" /usr/local/bin/node
 fi
 
+# Work around the fact that apparently sudo doesn't clear the HOME
+# environment variable in some cases; we don't want root
+# accessing/storing yarn configuration in the non-root user's home
+# directory.
+export HOME=/root
+
 # Install yarn if not installed
 bash "$ZULIP_PATH/scripts/lib/third/install-yarn.sh" "$ZULIP_SRV" --version "$yarn_version"

scripts/lib/third/install-yarn.sh

@@ -68,7 +68,8 @@ yarn_verify_integrity() {
 
   printf "Verifying integrity...\n"
   # Grab the public key if it doesn't already exist
-  gpg --list-keys $gpg_key >/dev/null 2>&1 || (curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | gpg --import)
+  # Zulip patch: Fix the fact that Yarn has extended this keyring and we should always redownload.
+  curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | gpg --import
 
   if [ ! -f "$1.asc" ]; then
     printf "$red> Could not download GPG signature for this Yarn release. This means the release cannot be verified!$reset\n"

scripts/setup/postgres-reset-sequences

@@ -1,5 +0,0 @@
-#!/bin/bash
-set -e
-
-./manage.py sqlsequencereset zerver | ./manage.py dbshell
-echo "Sequence has been reset successfully!"

static/images/checkbox-green.svg

@@ -1,5 +1,5 @@
 <svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" id="Layer_1" x="0px" y="0px" width="96px" height="96px" viewBox="21 0 75 75" enable-background="new 0 0 96 96" xml:space="preserve">
 <g>
-	<path fill="#00cbc3" d="M49.844,68.325c-1.416,0-2.748-0.554-3.75-1.557L27.523,48.191c-1.003-1.002-1.555-2.334-1.555-3.75   s0.552-2.749,1.555-3.75c1.001-1.001,2.333-1.552,3.75-1.552s2.75,0.551,3.753,1.553l14.019,14.017L82.14,5.504   c0.989-1.468,2.639-2.345,4.412-2.345c1.054,0,2.075,0.312,2.956,0.902c2.424,1.631,3.07,4.934,1.439,7.361L54.25,65.98   c-0.892,1.316-2.312,2.162-3.895,2.314C50.17,68.315,50.01,68.325,49.844,68.325z"/>
+	<path fill="#5bbd95" d="M49.844,68.325c-1.416,0-2.748-0.554-3.75-1.557L27.523,48.191c-1.003-1.002-1.555-2.334-1.555-3.75   s0.552-2.749,1.555-3.75c1.001-1.001,2.333-1.552,3.75-1.552s2.75,0.551,3.753,1.553l14.019,14.017L82.14,5.504   c0.989-1.468,2.639-2.345,4.412-2.345c1.054,0,2.075,0.312,2.956,0.902c2.424,1.631,3.07,4.934,1.439,7.361L54.25,65.98   c-0.892,1.316-2.312,2.162-3.895,2.314C50.17,68.315,50.01,68.325,49.844,68.325z"/>
 </g>
 </svg>

static/images/integrations/logos/facebook.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="266.893" height="266.895"><path fill="#3C5A99" d="M248.082 262.307c7.854 0 14.223-6.369 14.223-14.225V18.812c0-7.857-6.368-14.224-14.223-14.224H18.812c-7.857 0-14.224 6.367-14.224 14.224v229.27c0 7.855 6.366 14.225 14.224 14.225h229.27z"/><path fill="#FFF" d="M182.409 262.307v-99.803h33.499l5.016-38.895h-38.515V98.777c0-11.261 3.127-18.935 19.275-18.935l20.596-.009V45.045c-3.562-.474-15.788-1.533-30.012-1.533-29.695 0-50.025 18.126-50.025 51.413v28.684h-33.585v38.895h33.585v99.803h40.166z"/></svg>

static/js/admin.js

@@ -82,8 +82,8 @@ function _setup_page() {
     };
 
     options.bot_creation_policy_values = settings_bots.bot_creation_policy_values;
-    var admin_tab = templates.render('admin_tab', options);
-    $("#settings_content .organization-box").html(admin_tab);
+    var rendered_admin_tab = templates.render('admin_tab', options);
+    $("#settings_content .organization-box").html(rendered_admin_tab);
     $("#settings_content .alert").removeClass("show");
 
     settings_bots.update_bot_settings_tip();

static/js/components.js

@@ -52,6 +52,7 @@ exports.toggle = (function () {
             elem.addClass("selected");
 
             if (idx !== meta.idx) {
+                meta.idx = idx;
                 if (opts.callback) {
                     opts.callback(
                         opts.values[idx].label,
@@ -61,19 +62,22 @@ exports.toggle = (function () {
                 }
             }
 
-            meta.idx = idx;
-            elem.focus();
+            if (!opts.child_wants_focus) {
+                elem.focus();
+            }
         }
 
         function maybe_go_left() {
             if (meta.idx > 0) {
                 select_tab(meta.idx - 1);
+                return true;
             }
         }
 
         function maybe_go_right() {
             if (meta.idx < opts.values.length - 1) {
                 select_tab(meta.idx + 1);
+                return true;
             }
         }
 
@@ -83,22 +87,24 @@ exports.toggle = (function () {
                 select_tab(idx);
             });
 
-            meta.$ind_tab.keydown(function (e) {
-                var key = e.which || e.keyCode;
-
-                if (key === 37) {
-                    maybe_go_left();
-                } else if (key === 39) {
-                    maybe_go_right();
-                }
+            keydown_util.handle({
+                elem: meta.$ind_tab,
+                handlers: {
+                    left_arrow: maybe_go_left,
+                    right_arrow: maybe_go_right,
+                },
             });
 
+            // We should arguably default opts.selected to 0.
             if (typeof opts.selected === "number") {
                 select_tab(opts.selected);
             }
         }());
 
         var prototype = {
+            maybe_go_left: maybe_go_left,
+            maybe_go_right: maybe_go_right,
+
             value: function () {
                 if (meta.idx >= 0) {
                     return opts.values[meta.idx].label;

static/js/compose.js

@@ -185,6 +185,19 @@ function nonexistent_stream_reply_error() {
     }, 5000);
 }
 
+function compose_not_subscribed_error(error_text, bad_input) {
+    $('#compose-send-status').removeClass(common.status_classes)
+               .addClass('home-error-bar')
+               .stop(true).fadeTo(0, 1);
+    $('#compose-error-msg').html(error_text);
+    $("#compose-send-button").prop('disabled', false);
+    $("#sending-indicator").hide();
+    $(".compose-send-status-close").hide();
+    if (bad_input !== undefined) {
+        bad_input.focus().select();
+    }
+}
+
 exports.nonexistent_stream_reply_error = nonexistent_stream_reply_error;
 
 function clear_compose_box() {
@@ -513,8 +526,8 @@ exports.validation_error = function (error_type, stream_name) {
         compose_error(i18n.t("Error checking subscription"), $("#stream"));
         return false;
     case "not-subscribed":
-        response = i18n.t("<p>You're not subscribed to the stream <b>__stream_name__</b>.</p><p>Manage your subscriptions <a href='#streams/all'>on your Streams page</a>.</p>", context);
-        compose_error(response, $('#stream'));
+        var new_row = templates.render("compose_not_subscribed");
+        compose_not_subscribed_error(new_row, $('#stream'));
         return false;
     }
     return true;
@@ -692,22 +705,6 @@ exports.initialize = function () {
 
     upload.feature_check($("#compose #attach_files"));
 
-    // Lazy load the Dropbox script, since it can slow our page load
-    // otherwise, and isn't enabled for all users. Also, this Dropbox
-    // script isn't under an open source license, so we can't (for legal
-    // reasons) minify it with our own code.
-    if (feature_flags.dropbox_integration) {
-        LazyLoad.js('https://www.dropbox.com/static/api/1/dropins.js', function () {
-            // Successful load. We should now have window.Dropbox.
-            if (! _.has(window, 'Dropbox')) {
-                blueslip.error('Dropbox script reports loading but window.Dropbox undefined');
-            } else if (Dropbox.isBrowserSupported()) {
-                Dropbox.init({appKey: window.dropboxAppKey});
-                $("#compose #attach_dropbox_files").removeClass("notdisplayed");
-            }
-        });
-    }
-
     // Show a warning if a user @-mentions someone who will not receive this message
     $(document).on('usermention_completed.zulip', function (event, data) {
         if (compose_state.get_message_type() !== 'stream') {
@@ -722,8 +719,8 @@ exports.initialize = function () {
         if (data !== undefined && data.mentioned !== undefined) {
             var email = data.mentioned.email;
 
-            // warn if @all or @everyone is mentioned
-            if (data.mentioned.full_name  === 'all' || data.mentioned.full_name === 'everyone') {
+            // warn if @all, @everyone or @stream is mentioned
+            if (data.mentioned.full_name  === 'all' || data.mentioned.full_name === 'everyone' || data.mentioned.full_name === 'stream') {
                 return; // don't check if @all or @everyone is subscribed to a stream
             }
 
@@ -769,6 +766,24 @@ exports.initialize = function () {
         compose.finish();
     });
 
+    $("#compose-send-status").on('click', '.sub_unsub_button', function (event) {
+        event.preventDefault();
+
+        var stream_name = $('#stream').val();
+        if (stream_name === undefined) {
+            return;
+        }
+        var sub = stream_data.get_sub(stream_name);
+        subs.sub_or_unsub(sub);
+        $("#compose-send-status").hide();
+    });
+
+    $("#compose-send-status").on('click', '#compose_not_subscribed_close', function (event) {
+        event.preventDefault();
+
+        $("#compose-send-status").hide();
+    });
+
     $("#compose_invite_users").on('click', '.compose_invite_link', function (event) {
         event.preventDefault();
 
@@ -885,15 +900,15 @@ exports.initialize = function () {
     // content is passed to check for status messages ("/me ...")
     // and will be undefined in case of errors
     function show_preview(rendered_content, content) {
-        var preview_html;
+        var rendered_preview_html;
         if (content !== undefined && markdown.is_status_message(content, rendered_content)) {
             // Handle previews of /me messages
-            preview_html = "<strong>" + page_params.full_name + "</strong> " + rendered_content.slice(4 + 3, -4);
+            rendered_preview_html = "<strong>" + page_params.full_name + "</strong> " + rendered_content.slice(4 + 3, -4);
         } else {
-            preview_html = rendered_content;
+            rendered_preview_html = rendered_content;
         }
 
-        $("#preview_content").html(preview_html);
+        $("#preview_content").html(rendered_preview_html);
         if (page_params.emojiset === "text") {
             $("#preview_content").find(".emoji").replaceWith(function () {
                 var text = $(this).attr("title");
@@ -967,24 +982,6 @@ exports.initialize = function () {
         exports.clear_preview_area();
     });
 
-    $("#compose").on("click", "#attach_dropbox_files", function (e) {
-        e.preventDefault();
-        var options = {
-            // Required. Called when a user selects an item in the Chooser.
-            success: function (files) {
-                var textbox = $("#compose-textarea");
-                var links = _.map(files, function (file) { return '[' + file.name + '](' + file.link +')'; })
-                             .join(' ') + ' ';
-                textbox.val(textbox.val() + links);
-            },
-            // Optional. A value of false (default) limits selection to a single file, while
-            // true enables multiple file selection.
-            multiselect: true,
-            iframe: true,
-        };
-        Dropbox.choose(options);
-    });
-
     $("#compose").filedrop(
         upload.options({
             mode: 'compose',

static/js/compose_actions.js

@@ -288,7 +288,7 @@ exports.respond_to_message = function (opts) {
         return;
     }
 
-    unread_ops.mark_message_as_read(message);
+    unread_ops.notify_server_message_read(message);
 
     var stream = '';
     var subject = '';
@@ -349,28 +349,26 @@ exports.on_topic_narrow = function () {
         return;
     }
 
-    if (compose_state.subject()) {
-        // If the user has filled in a subject, we have
-        // a risk of a mix, and we can't reliably guess
-        // whether the old topic is appropriate (otherwise,
-        // why did they narrow?) or the new one is
-        // appropriate (after all, they were starting to
-        // compose on the old topic and may now be looking
-        // for info), so we punt and cancel.
-
-        // If subject is not same as topic narrowed to then
-        // stop composing
-        if (compose_state.subject().toLowerCase() !== narrow_state.topic().toLowerCase()) {
-            exports.cancel();
-        }
+    if (compose_state.subject() && compose_state.has_message_content()) {
+        // If the user has written something to a different topic,
+        // they probably want that content, so leave compose open.
+        //
+        // This effectively uses the heuristic of whether there is
+        // content in compose to determine whether the user had firmly
+        // decided to compose to the old topic or is just looking to
+        // reply to what they see.
+        compose_fade.update_message_list();
         return;
     }
 
-    // If we got this far, then the compose box has the correct
-    // stream filled in, and we just need to update the topic.
-    // See #3300 for context--a couple users specifically asked
-    // for this convenience.
+    // If we got this far, then the compose box has the correct stream
+    // filled in, and either compose is empty or no topic was set, so
+    // we should update the compose topic to match the new narrow.
+    // See #3300 for context--a couple users specifically asked for
+    // this convenience.
     compose_state.subject(narrow_state.topic());
+    compose_fade.set_focused_recipient("stream");
+    compose_fade.update_message_list();
     $('#compose-textarea').focus().select();
 };
 

static/js/compose_fade.js

@@ -118,8 +118,8 @@ exports.would_receive_message = function (email) {
     if (focused_recipient.type === 'stream') {
         var user = people.get_active_user_for_email(email);
         var sub = stream_data.get_sub(focused_recipient.stream);
-        if (!sub) {
-            // If the stream isn't valid, there is no risk of a mix
+        if (!sub || !user) {
+            // If the stream or user isn't valid, there is no risk of a mix
             // yet, so don't fade.
             return;
         }
@@ -129,7 +129,7 @@ exports.would_receive_message = function (email) {
             // not subscribed.
             return;
         }
-        return stream_data.user_is_subscribed(focused_recipient.stream, email);
+        return stream_data.is_user_subscribed(focused_recipient.stream, user.user_id);
     }
 
     // PM, so check if the given email is in the recipients list.

static/js/composebox_typeahead.js

@@ -374,23 +374,20 @@ exports.compose_content_begins_typeahead = function (query) {
 
         this.completing = 'mention';
         this.token = current_token;
-        var all_item = {
-            special_item_text: "all (Notify everyone)",
-            email: "all",
-            // Always sort above, under the assumption that names will
-            // be longer and only contain "all" as a substring.
-            pm_recipient_count: Infinity,
-            full_name: "all",
-        };
-        var everyone_item = {
-            special_item_text: "everyone (Notify everyone)",
-            email: "everyone",
-            pm_recipient_count: Infinity,
-            full_name: "everyone",
-        };
+        var all_items = _.map(['all', 'everyone', 'stream'], function (mention) {
+            return {
+                special_item_text: i18n.t("__wildcard_mention_token__ (Notify stream)",
+                {wildcard_mention_token: mention}),
+                email: mention,
+                // Always sort above, under the assumption that names will
+                // be longer and only contain "all" as a substring.
+                pm_recipient_count: Infinity,
+                full_name: mention,
+            };
+        });
         var persons = people.get_realm_persons();
         var groups = user_groups.get_realm_user_groups();
-        return [].concat(persons, [all_item, everyone_item], groups);
+        return [].concat(persons, all_items, groups);
     }
 
     if (this.options.completions.stream && current_token[0] === '#') {

static/js/copy_and_paste.js

@@ -224,7 +224,7 @@ exports.paste_handler = function (event) {
 
     if (clipboardData.getData) {
         var paste_html = clipboardData.getData('text/html');
-        if (paste_html) {
+        if (paste_html && page_params.development_environment) {
             event.preventDefault();
             var text = exports.paste_handler_converter(paste_html);
             compose_ui.insert_syntax_and_focus(text);

static/js/emoji_picker.js

@@ -247,11 +247,11 @@ function filter_emojis() {
                 });
             });
         });
-        var search_results_rendered = templates.render('emoji_popover_search_results', {
+        var rendered_search_results = templates.render('emoji_popover_search_results', {
             search_results: search_results,
             message_id: message_id,
         });
-        $('.emoji-search-results').html(search_results_rendered);
+        $('.emoji-search-results').html(rendered_search_results);
         ui.update_scrollbar($(".emoji-search-results-container"));
         if (!search_results_visible) {
             show_search_results();

static/js/feature_flags.js

@@ -13,7 +13,6 @@ exports.mark_read_at_bottom = true;
 exports.propagate_topic_edits = true;
 exports.clicking_notification_causes_narrow = true;
 exports.collapsible = false;
-exports.dropbox_integration = false;
 exports.reminders_in_message_action_menu = false;
 
 return exports;

static/js/filter.js

@@ -483,7 +483,7 @@ Filter.operator_to_prefix = function (operator, negated) {
 };
 
 // Convert a list of operators to a human-readable description.
-Filter.describe = function (operators) {
+function describe_unescaped(operators) {
     if (operators.length === 0) {
         return 'all messages';
     }
@@ -530,8 +530,11 @@ Filter.describe = function (operators) {
         return "unknown operator";
     });
     return parts.concat(more_parts).join(', ');
-};
+}
 
+Filter.describe = function (operators) {
+    return Handlebars.Utils.escapeExpression(describe_unescaped(operators));
+};
 
 return Filter;
 

static/js/info_overlay.js

@@ -21,10 +21,15 @@ function adjust_mac_shortcuts() {
     });
 }
 
+// Make it explicit that our toggler is undefined until
+// _setup_info_overlay is called via ensure_i18n.
+exports.toggler = undefined;
+
 function _setup_info_overlay() {
-    var info_overlay_toggle = components.toggle({
+    var opts = {
         name: "info-overlay-toggle",
         selected: 0,
+        child_wants_focus: true,
         values: [
             { label: i18n.t("Keyboard shortcuts"), key: "keyboard-shortcuts" },
             { label: i18n.t("Message formatting"), key: "markdown-help" },
@@ -35,14 +40,35 @@ function _setup_info_overlay() {
             $("#" + key).show();
             $("#" + key).find(".modal-body").focus();
         },
-    }).get();
+    };
 
-    $(".informational-overlays .overlay-tabs")
-        .append($(info_overlay_toggle).addClass("large"));
+    var toggler = components.toggle(opts);
+    var elem = toggler.get();
+    elem.addClass('large');
+
+    var modals = _.map(opts.values, function (item) {
+        var key = item.key; // e.g. markdown-help
+        var modal = $('#' + key).find('.modal-body');
+        return modal;
+    });
+
+    _.each(modals, function (modal) {
+        keydown_util.handle({
+            elem: modal,
+            handlers: {
+                left_arrow: toggler.maybe_go_left,
+                right_arrow: toggler.maybe_go_right,
+            },
+        });
+    });
+
+    $(".informational-overlays .overlay-tabs").append(elem);
 
     if (/Mac/i.test(navigator.userAgent)) {
         adjust_mac_shortcuts();
     }
+
+    exports.toggler = toggler;
 }
 
 exports.show = function (target) {
@@ -59,7 +85,9 @@ exports.show = function (target) {
     }
 
     if (target) {
-        components.toggle.lookup("info-overlay-toggle").goto(target);
+        if (exports.toggler) {
+            exports.toggler.goto(target);
+        }
     }
 };
 

static/js/input_pill.js

@@ -367,6 +367,61 @@ exports.create = function (opts) {
     return prototype;
 };
 
+// Following function is used for creating non-editable pills.
+exports.create_non_editable_pills = function (opts) {
+
+    if (!opts.container) {
+        blueslip.error('Pill needs container.');
+        return;
+    }
+
+    var store = {
+        pills: [],
+        $parent: opts.container,
+    };
+
+    var funcs = {
+        // This is generally called by typeahead logic, where we have all
+        // the data we need (as opposed to, say, just a user-typed email).
+        appendValidatedData: function (item) {
+            var id = exports.random_id();
+
+            if (!item.display_value) {
+                blueslip.error('no display_value returned');
+                return;
+            }
+
+            var payload = {
+                id: id,
+                item: item,
+            };
+
+            store.pills.push(payload);
+
+            var opts = {
+                id: payload.id,
+                display_value: item.display_value,
+                cannot_edit: true,
+            };
+
+            var pill_html = templates.render('input_pill', opts);
+            payload.$element = $(pill_html);
+            store.$parent.append(payload.$element);
+        },
+
+        items: function () {
+            return _.pluck(store.pills, 'item');
+        },
+    };
+
+    var prototype = {
+        appendValidatedData: funcs.appendValidatedData.bind(funcs),
+        items: funcs.items,
+    };
+
+    return prototype;
+
+};
 return exports;
 
 }());

static/js/invite.js

@@ -76,8 +76,8 @@ exports.initialize = function () {
             invitee_emails.val('');
 
             if (page_params.development_environment) {
-                var email_msg = templates.render('dev_env_email_access');
-                $('#dev_env_msg').html(email_msg).addClass('alert-info').show();
+                var rendered_email_msg = templates.render('dev_env_email_access');
+                $('#dev_env_msg').html(rendered_email_msg).addClass('alert-info').show();
             }
 
         },