Add changelog for Zulip 1.3.11 release.

The default of 200 was shorter than the Camo URLs use by Zulip.

.gitattributes

@@ -14,7 +14,7 @@
 /zproject/local_settings.py export-ignore
 /zproject/test_settings.py export-ignore
 /zerver/fixtures export-ignore
-/zerver/tests.py export-ignore
+/zerver/tests export-ignore
 /frontend_tests export-ignore
 /node_modules export-ignore
 /humbug export-ignore

.gitignore

@@ -13,6 +13,7 @@
 /update-prod-static.log
 frontend_tests/casper_tests/server.log
 frontend_tests/casper_lib/test_credentials.js
+memcached_prefix
 /prod-static
 /errors/*
 *.sw[po]
@@ -24,7 +25,7 @@ zerver/fixtures/migration-status
 zerver/fixtures/test_data1.json
 .kdev4
 zulip.kdev4
-memcached_prefix
+remote_cache_prefix
 coverage/
 /queue_error
 .test-js-with-node.html
@@ -41,3 +42,5 @@ tools/emoji_dump/bitmaps/
 tools/emoji_dump/*.ttx
 tools/phantomjs
 node_modules
+uploads/
+test_uploads/

.travis.yml

@@ -14,6 +14,10 @@ env:
 language: python
 python:
   - "2.7"
+matrix:
+  include:
+    - python: "3.4"
+      env: TEST_SUITE=mypy
 # command to run tests
 script:
   - ./tools/travis/$TEST_SUITE

README.dev.md

@@ -25,23 +25,23 @@ such as Mac via Virtualbox (but everything will be 2-3x slower).
   sudo apt-get install vagrant lxc lxc-templates cgroup-lite redir
   vagrant plugin install vagrant-lxc
   ```
+  You may want to [configure sudo to be passwordless when using Vagrant LXC][avoiding-sudo].
 
 * If your host is Ubuntu 14.04, you will need to [download a newer
-  version of Vagrant](https://www.vagrantup.com/downloads.html), and
-  then do the following:
+  version of Vagrant][vagrant-dl], and then do the following:
   ```
   sudo apt-get install lxc lxc-templates cgroup-lite redir
   sudo dpkg -i vagrant*.deb # in directory where you downloaded vagrant
   vagrant plugin install vagrant-lxc
   ```
+  You may want to [configure sudo to be passwordless when using Vagrant LXC][avoiding-sudo].
 
 * For other Linux hosts with a kernel above 3.12, [follow the Vagrant
-  LXC installation
-  instructions](https://github.com/fgrehm/vagrant-lxc) to get Vagrant
-  with LXC for your platform.
+  LXC installation instructions][vagrant-lxc] to get Vagrant with LXC
+  for your platform.
 
-* If your host is OS X or older Linux, [download VirtualBox](https://www.virtualbox.org/wiki/Downloads),
-  [download Vagrant](https://www.vagrantup.com/downloads.html), and install them both.
+* If your host is OS X or older Linux, [download VirtualBox][vbox-dl],
+  [download Vagrant][vagrant-dl], and install them both.
 
 * If you're on OS X and have VMWare, it should be possible to patch
   Vagrantfile to use the VMWare vagrant provider which should perform
@@ -54,6 +54,11 @@ such as Mac via Virtualbox (but everything will be 2-3x slower).
   core.autocrlf=false` to avoid Windows line endings being added to
   files (this causes weird errors).
 
+[vagrant-dl]: https://www.vagrantup.com/downloads.html
+[vagrant-lxc]: https://github.com/fgrehm/vagrant-lxc
+[vbox-dl]: https://www.virtualbox.org/wiki/Downloads
+[avoiding-sudo]: https://github.com/fgrehm/vagrant-lxc#avoiding-sudo-passwords
+
 Once that's done, simply change to your zulip directory and run
 `vagrant up` in your terminal to install the development server.  This
 will take a long time on the first run because Vagrant needs to
@@ -74,13 +79,40 @@ source /srv/zulip-venv/bin/activate
 To get shell access to the virtual machine running the server to run
 lint, management commands, etc., use `vagrant ssh`.
 
-(A small note on tools/run-dev.py: the `--interface=''` option will make
-the development server listen on all network interfaces.  While this
-is correct for the Vagrant guest sitting behind a NAT, you probably
-don't want to use that option when using run-dev.py in other environments).
+(A small note on tools/run-dev.py: the `--interface=''` option will
+make the development server listen on all network interfaces.  While
+this is correct for the Vagrant guest sitting behind a NAT, you
+probably don't want to use that option when using run-dev.py in other
+environments).
 
-At this point you should [read about using the development environment](https://github.com/zulip/zulip/blob/master/README.dev.md#using-the-development-environment).
+At this point you should [read about using the development
+environment][using-dev].
 
+[using-dev]: #using-the-development-environment
+
+## Specifying a proxy
+
+If you need to use a proxy server to access the Internet, you will
+need to specify the proxy settings before running `Vagrant up`.
+First, install the Vagrant plugin `vagrant-proxyconf`:
+
+```
+vagrant plugin install vagrant-proxyconf.
+```
+
+Then create `~/.zulip-vagrant-config` and add the following lines to
+it (with the appropriate values in it for your proxy):
+
+```
+HTTP_PROXY http://proxy_host:port
+HTTPS_PROXY http://proxy_host:port
+NO_PROXY localhost,127.0.0.1,.example.com
+
+```
+
+Now run `vagrant up` in your terminal to install the development
+server. If you ran `vagrant up` before and failed, you'll need to run
+`vagrant destroy` first to clean up the failed installation.
 
 Using provision.py without Vagrant
 ----------------------------------
@@ -91,7 +123,6 @@ running:
 
 ```
 sudo apt-get update
-sudo apt-get install -y python-pbs
 python /srv/zulip/provision.py
 
 cd /srv/zulip
@@ -110,7 +141,7 @@ instructions should work.
 
 Install the following non-Python dependencies:
  * libffi-dev — needed for some Python extensions
- * postgresql 9.1 or later — our database (also install development headers)
+ * postgresql 9.1 or later — our database (client, server, headers)
  * nodejs 0.10 (and npm)
  * memcached (and headers)
  * rabbitmq-server
@@ -118,12 +149,16 @@ Install the following non-Python dependencies:
  * python-dev
  * redis-server — rate limiting
  * tsearch-extras — better text search
- * libfreetype6-dev - needed before you pip install Pillow to properly generate emoji PNGs
+ * libfreetype6-dev — needed before you pip install Pillow to properly generate emoji PNGs
 
 ### On Debian or Ubuntu systems:
 
 ```
-sudo apt-get install closure-compiler libfreetype6-dev libffi-dev memcached rabbitmq-server libldap2-dev redis-server postgresql-server-dev-all libmemcached-dev python-dev hunspell-en-us nodejs nodejs-legacy npm git yui-compressor puppet gettext
+sudo apt-get install closure-compiler libfreetype6-dev libffi-dev \
+    memcached rabbitmq-server libldap2-dev redis-server \
+    postgresql-server-dev-all libmemcached-dev python-dev \
+    hunspell-en-us nodejs nodejs-legacy npm git yui-compressor \
+    puppet gettext
 
 # If on 12.04 or wheezy:
 sudo apt-get install postgresql-9.1
@@ -145,21 +180,26 @@ Now continue with the "All systems" instructions below.
 
 ### On Fedora 22 (experimental):
 
-These instructions are experimental and may have bugs; patches welcome!
+These instructions are experimental and may have bugs; patches
+welcome!
 
 ```
-sudo dnf install libffi-devel memcached rabbitmq-server openldap-devel python-devel redis postgresql-server postgresql-devel postgresql libmemcached-devel freetype-devel nodejs npm yuicompressor closure-compiler gettext
+sudo dnf install libffi-devel memcached rabbitmq-server \
+    openldap-devel python-devel redis postgresql-server \
+    postgresql-devel postgresql libmemcached-devel freetype-devel \
+    nodejs npm yuicompressor closure-compiler gettext
 ```
 
 Now continue with the Common to Fedora/CentOS instructions below.
 
 ### On CentOS 7 Core (experimental):
 
-These instructions are experimental and may have bugs; patches welcome!
+These instructions are experimental and may have bugs; patches
+welcome!
 
 ```
-# Add user zulip to the system (not necessary if you configured zulip as the administrator
-# user during the install process of CentOS 7).
+# Add user zulip to the system (not necessary if you configured zulip
+# as the administrator user during the install process of CentOS 7).
 useradd zulip
 
 # Create a password for zulip user
@@ -173,13 +213,16 @@ zulip   ALL=(ALL)       ALL
 # Switch to zulip user
 su zulip
 
-# Enable EPEL 7 repo so we can install rabbitmq-server, redis and other dependencies
+# Enable EPEL 7 repo so we can install rabbitmq-server, redis and
+# other dependencies
 sudo yum install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
 
 # Install dependencies
-sudo yum install libffi-devel memcached rabbitmq-server openldap-devel python-devel redis postgresql-server \
-postgresql-devel postgresql libmemcached-devel wget python-pip openssl-devel freetype-devel libjpeg-turbo-devel \
-zlib-devel nodejs yuicompressor closure-compiler gettext
+sudo yum install libffi-devel memcached rabbitmq-server openldap-devel
+    python-devel redis postgresql-server postgresql-devel postgresql \
+    libmemcached-devel wget python-pip openssl-devel freetype-devel \
+    libjpeg-turbo-devel zlib-devel nodejs yuicompressor \
+    closure-compiler gettext
 
 # We need these packages to compile tsearch-extras
 sudo yum groupinstall "Development Tools"
@@ -204,6 +247,40 @@ host    all             all             ::1/128                 md5
 
 Now continue with the Common to Fedora/CentOS instructions below.
 
+### On OpenBSD 5.8 (experimental):
+
+These instructions are experimental and may have bugs; patches
+welcome!
+
+```
+doas pkg_add sudo bash gcc postgresql-server redis rabbitmq \
+    memcached node libmemcached py-Pillow py-cryptography py-cffi
+
+# Get tsearch_extras and build it (using a modified version which
+# aliases int4 on OpenBSD):
+git clone https://github.com/blablacio/tsearch_extras
+cd tsearch_extras
+gmake && sudo gmake install
+
+# Point environment to custom include locations and use newer GCC
+# (needed for Node modules):
+export CFLAGS="-I/usr/local/include -I/usr/local/include/sasl"
+export CXX=eg++
+
+# Create tsearch_data directory:
+sudo mkdir /usr/local/share/postgresql/tsearch_data
+
+
+# Hack around missing dictionary files -- need to fix this to get the
+# proper dictionaries from what in debian is the hunspell-en-us
+# package.
+sudo touch /usr/local/share/postgresql/tsearch_data/english.stop
+sudo touch /usr/local/share/postgresql/tsearch_data/en_us.dict
+sudo touch /usr/local/share/postgresql/tsearch_data/en_us.affix
+```
+
+Now continue with the All Systems instructions below.
+
 ### Common to Fedora/CentOS instructions
 
 ```
@@ -214,8 +291,9 @@ cd ts2
 make
 sudo make install
 
-# Hack around missing dictionary files -- need to fix this to get
-# the proper dictionaries from what in debian is the hunspell-en-us package.
+# Hack around missing dictionary files -- need to fix this to get the
+# proper dictionaries from what in debian is the hunspell-en-us
+# package.
 sudo touch /usr/share/pgsql/tsearch_data/english.stop
 sudo touch /usr/share/pgsql/tsearch_data/en_us.dict
 sudo touch /usr/share/pgsql/tsearch_data/en_us.affix
@@ -223,7 +301,8 @@ sudo touch /usr/share/pgsql/tsearch_data/en_us.affix
 # Edit the postgres settings:
 sudo vi /var/lib/pgsql/data/pg_hba.conf
 
-# Add this line before the first uncommented line to enable password auth:
+# Add this line before the first uncommented line to enable password
+# auth:
 host    all             all             127.0.0.1/32            md5
 
 # Start the services
@@ -238,19 +317,25 @@ Finally continue with the All Systems instructions below.
 ### All Systems:
 
 ```
-pip install -r requirements.txt
-npm install
+pip install --no-deps -r requirements.txt
+./tools/install-phantomjs
+./tools/install-mypy
 ./tools/download-zxcvbn
 ./tools/emoji_dump/build_emoji
 ./scripts/setup/generate_secrets.py -d
-sudo cp ./puppet/zulip/files/postgresql/zulip_english.stop /usr/share/postgresql/9.3/tsearch_data/
+if [ $(uname) = "OpenBSD" ]; then sudo cp ./puppet/zulip/files/postgresql/zulip_english.stop /var/postgresql/tsearch_data/; else sudo cp ./puppet/zulip/files/postgresql/zulip_english.stop /usr/share/postgresql/9.3/tsearch_data/; fi
 ./scripts/setup/configure-rabbitmq
 ./tools/postgres-init-dev-db
 ./tools/do-destroy-rebuild-database
 ./tools/postgres-init-test-db
 ./tools/do-destroy-rebuild-test-database
+npm install
 ```
 
+If `npm install` fails, the issue may be that you need a newer version
+of `npm`.  You can use `npm install -g npm` to update your version of
+`npm` and try again.
+
 To start the development server:
 
 ```
@@ -259,17 +344,45 @@ To start the development server:
 
 … and visit [http://localhost:9991/](http://localhost:9991/).
 
-Using Docker
--------------
+#### Proxy setup for by-hand installation
 
-You can also use Docker to develop, first you need to install Docker in your development machine following the [instructions](https://docs.docker.com/engine/installation/). Some other interesting links for somebody new in Docker are:
+If you are building the development environment on a network where a
+proxy is required to access the Internet, you will need to set the
+proxy in the environment as follows:
+
+- On Ubuntu, set the proxy environment variables using:
+ ```
+ export https_proxy=http://proxy_host:port
+ export http_proxy=http://proxy_host:port
+ ```
+
+- And set the npm proxy and https-proxy using:
+ ```
+ npm config set proxy http://proxy_host:port
+ npm config set https-proxy http://proxy_host:port
+ ```
+
+Using Docker (experimental)
+---------------------------
+
+The docker instructions for development are experimental, so they may
+have bugs.  If you try them and run into any issues, please report
+them!
+
+You can also use Docker to run a Zulip development environment.
+First, you need to install Docker in your development machine
+following the [instructions][docker-install].  Some other interesting
+links for somebody new in Docker are:
 
 * [Get Started](https://docs.docker.com/linux/started/)
 * [Understand the architecture](https://docs.docker.com/engine/introduction/understanding-docker/)
-* [Docker run reference]https://docs.docker.com/engine/reference/run/()
+* [Docker run reference](https://docs.docker.com/engine/reference/run/)
 * [Dockerfile reference](https://docs.docker.com/engine/reference/builder/)
 
-Then you should create the Docker image based on Ubuntu Linux, first go to the directory with the Zulip source code:
+[docker-install]: https://docs.docker.com/engine/installation/
+
+Then you should create the Docker image based on Ubuntu Linux, first
+go to the directory with the Zulip source code:
 
 ```
 docker build -t user/zulipdev .
@@ -279,7 +392,7 @@ Now you're going to install Zulip dependencies in the image:
 
 ```
 docker run -itv $(pwd):/srv/zulip -p 80:9991 user/zulipdev /bin/bash
-$ /usr/bin/python /srv/zulip/provision.py --docker 
+$ /usr/bin/python /srv/zulip/provision.py --docker
 docker ps -af ancestor=user/zulipdev
 docker commit -m "Zulip installed" <container id> user/zulipdev:v2
 ```
@@ -287,10 +400,12 @@ docker commit -m "Zulip installed" <container id> user/zulipdev:v2
 Finally you can run the docker server with:
 
 ```
-docker run -itv $(pwd):/srv/zulip -p 80:9991 user/zulipdev:v2 /srv/zulip/scripts/start-dockers
+docker run -itv $(pwd):/srv/zulip -p 80:9991 user/zulipdev:v2 \
+    /srv/zulip/scripts/start-dockers
 ```
 
-If you want to connect to the Docker instance to build a release tarball you can use:
+If you want to connect to the Docker instance to build a release
+tarball you can use:
 
 ```
 docker ps
@@ -306,14 +421,16 @@ docker ps
 docker kill <container id>
 ```
 
-If you want to run all the tests you need to start the servers first, you can do it with:
+If you want to run all the tests you need to start the servers first,
+you can do it with:
 
 ```
 docker run -itv $(pwd):/srv/zulip user/zulipdev:v2 /bin/bash
 $ scripts/test-all-docker
 ```
 
-You can modify the source code in your development machine and review the results in your browser.
+You can modify the source code in your development machine and review
+the results in your browser.
 
 
 Using the Development Environment
@@ -325,7 +442,7 @@ server homepage just shows a list of the users that exist on the
 server and you can login as any of them by just clicking on a user.
 This setup saves time for the common case where you want to test
 something other than the login process; to test the login process
-you'll want to change AUTHENTICATION_BACKENDS in the not-PRODUCTION
+you'll want to change `AUTHENTICATION_BACKENDS` in the not-PRODUCTION
 case of `zproject/settings.py` from zproject.backends.DevAuthBackend
 to use the auth method(s) you'd like to test.
 
@@ -341,10 +458,10 @@ browser window to see changes take effect.
 
 * If you change Python code used by the the main Django/Tornado server
 processes, these services are run on top of Django's [manage.py
-runserver](https://docs.djangoproject.com/en/1.8/ref/django-admin/#runserver-port-or-address-port),
-which will automatically restart the Zulip Django and Tornado servers
-whenever you save changes to Python code.  You can watch this happen
-in the `run-dev.py` console to make sure the backend has reloaded.
+runserver][django-runserver] which will automatically restart the
+Zulip Django and Tornado servers whenever you save changes to Python
+code.  You can watch this happen in the `run-dev.py` console to make
+sure the backend has reloaded.
 
 * The Python queue workers don't automatically restart when you save
 changes (or when they stop running), so you will want to ctrl-C and
@@ -353,28 +470,34 @@ queue workers or if a queue worker has crashed.
 
 * If you change the database schema, you'll need to use the standard
 Django migrations process to create and then run your migrations; see
-the [new feature
-tutorial](http://zulip.readthedocs.org/en/latest/new-feature-tutorial.html)
-for an example.  Additionally you should check out the [detailed
-testing docs](http://zulip.readthedocs.org/en/latest/testing.html) for
-how to run the tests properly after doing a migration.
+the [new feature tutorial][new-feature-tutorial] for an example.
+Additionally you should check out the [detailed testing
+docs][testing-docs] for how to run the tests properly after doing a
+migration.
 
 (In production, everything runs under supervisord and thus will
 restart if it crashes, and `upgrade-zulip` will take care of running
 migrations and then cleanly restaring the server for you).
 
+[django-runserver]: https://docs.djangoproject.com/en/1.8/ref/django-admin/#runserver-port-or-address-port
+[new-feature-tutorial]: http://zulip.readthedocs.io/en/latest/new-feature-tutorial.html
+[testing-docs]: http://zulip.readthedocs.io/en/latest/testing.html
+
 Running the test suite
 ======================
 
-For more details, check out the [detailed testing
-docs](http://zulip.readthedocs.org/en/latest/testing.html).
+For more details, especially on how to write tests, check out the
+[detailed testing docs][tdocs].
+
+[tdocs]: http://zulip.readthedocs.io/en/latest/testing.html
 
 To run all the tests, do this:
 ```
 ./tools/test-all
 ```
 
-For the Vagrant environment, you'll want to first enter the environment:
+For the Vagrant environment, you'll want to first enter the
+environment:
 ```
 vagrant ssh
 source /srv/zulip-venv/bin/activate
@@ -388,7 +511,7 @@ time debugging a test failure, e.g.:
 
 ```
 ./tools/lint-all # Runs all the linters in parallel
-./tools/test-backend zerver.test_bugdown.BugdownTest.test_inline_youtube
+./tools/test-backend zerver.tests.test_bugdown.BugdownTest.test_inline_youtube
 ./tools/test-js-with-casper 10-navigation.js
 ./tools/test-js-with-node # Runs all node tests but is very fast
 ```
@@ -417,5 +540,15 @@ Possible testing issues
   above. Afterwards, re-run the `init*-db` and the
   `do-destroy-rebuild*-database` scripts.
 
-- When building the development environment using Vagrant and the LXC provider, if you encounter permissions errors, you may need to `chown -R 1000:$(whoami) /path/to/zulip` on the host before running `vagrant up` in order to ensure that the synced directory has the correct owner during provision. This issue will arise if you run `id username` on the host where `username` is the user running Vagrant and the output is anything but 1000.
-  This seems to be caused by Vagrant behavior; more information can be found here https://github.com/fgrehm/vagrant-lxc/wiki/FAQ#help-my-shared-folders-have-the-wrong-owner
+- When building the development environment using Vagrant and the LXC
+  provider, if you encounter permissions errors, you may need to
+  `chown -R 1000:$(whoami) /path/to/zulip` on the host before running
+  `vagrant up` in order to ensure that the synced directory has the
+  correct owner during provision. This issue will arise if you run `id
+  username` on the host where `username` is the user running Vagrant
+  and the output is anything but 1000.
+  This seems to be caused by Vagrant behavior; for more information,
+  see [the vagrant-lxc FAQ entry about shared folder permissions
+  ][lxc-sf].
+
+[lxc-sf]: https://github.com/fgrehm/vagrant-lxc/wiki/FAQ#help-my-shared-folders-have-the-wrong-owner)

README.md

@@ -12,6 +12,11 @@ missed-message emails, desktop apps, and much more.
 Further information on the Zulip project and its features can be found
 at https://www.zulip.org.
 
+[![Build Status][1]][2]
+
+[1]: https://travis-ci.org/zulip/zulip.svg?branch=master
+[2]: https://travis-ci.org/zulip/zulip
+
 Installing the Zulip Development environment
 ============================================
 
@@ -33,64 +38,92 @@ Contributing to Zulip
 Zulip welcomes all forms of contributions!  The page documents the
 Zulip development process.
 
-* **Pull requests**. Before a pull request can be merged, you need to to sign the [Dropbox
-Contributor License Agreement](https://opensource.dropbox.com/cla/).
-Also, please skim our [commit message style
-guidelines](http://zulip.readthedocs.org/en/latest/code-style.html#commit-messages).
+* **Pull requests**. Before a pull request can be merged, you need to
+to sign the [Dropbox Contributor License Agreement][cla].  Also,
+please skim our [commit message style guidelines][doc-commit-style].
 
 * **Testing**. The Zulip automated tests all run automatically when
 you submit a pull request, but you can also run them all in your
 development environment following the instructions in the [testing
-section](https://github.com/zulip/zulip#running-the-test-suite) below.
+docs][doc-test].
 
 * **Developer Documentation**.  Zulip has a growing collection of
-developer documentation on [Read The Docs](https://zulip.readthedocs.org/).
-Recommended reading for new contributors includes the
-[directory structure](http://zulip.readthedocs.org/en/latest/directory-structure.html) and
-[new feature tutorial](http://zulip.readthedocs.org/en/latest/new-feature-tutorial.html).
+developer documentation on [Read The Docs][doc].  Recommended reading
+for new contributors includes the [directory structure][doc-dirstruct]
+and [new feature tutorial][doc-newfeat].
 
-* **Mailing list and bug tracker** Zulip has a [development discussion
-mailing list](https://groups.google.com/forum/#!forum/zulip-devel) and
-uses [GitHub issues](https://github.com/zulip/zulip/issues).  Feel
-free to send any questions or suggestions of areas where you'd love to
-see more documentation to the list!  Please report any security issues
-you discover to support@zulip.com.
+* **Mailing lists and bug tracker**. Zulip has a [development
+discussion mailing list][gg-devel] and uses [GitHub issues
+][gh-issues].  There are also lists for the [Android][email-android]
+and [iOS][email-ios] apps.  Feel free to send any questions or
+suggestions of areas where you'd love to see more documentation to the
+relevant list!  Please report any security issues you discover to
+zulip-security@googlegroups.com.
 
-* **App codebases** This repository is for the Zulip server and web app; the
-[desktop](https://github.com/zulip/zulip-desktop),
-[Android](https://github.com/zulip/zulip-android), and
-[iOS](https://github.com/zulip/zulip-ios) apps are separate
+* **App codebases**. This repository is for the Zulip server and web
+app; the [desktop][], [Android][], and [iOS][] apps are separate
 repositories.
 
+* **Translations**.  Zulip is in the process of being translated into
+10+ languages, and we love contributions to our translations.  See our
+[translating documentation](transifex) if you're interested in
+contributing!
+
+[cla]: https://opensource.dropbox.com/cla/
+[doc]: https://zulip.readthedocs.io/
+[doc-commit-style]: http://zulip.readthedocs.io/en/latest/code-style.html#commit-messages
+[doc-dirstruct]: http://zulip.readthedocs.io/en/latest/directory-structure.html
+[doc-newfeat]: http://zulip.readthedocs.io/en/latest/new-feature-tutorial.html
+[doc-test]: https://github.com/zulip/zulip/blob/master/README.dev.md#running-the-test-suite
+[gg-devel]: https://groups.google.com/forum/#!forum/zulip-devel
+[gh-issues]: https://github.com/zulip/zulip/issues
+[desktop]: https://github.com/zulip/zulip-desktop
+[android]: https://github.com/zulip/zulip-android
+[ios]: https://github.com/zulip/zulip-ios
+[email-android]: https://groups.google.com/forum/#!forum/zulip-android
+[email-ios]: https://groups.google.com/forum/#!forum/zulip-ios
+[transifex]: https://www.transifex.com/zulip/zulip/
+
 How to get involved with contributing to Zulip
 ==============================================
 
-First, subscribe to the Zulip [development discussion mailing list](https://groups.google.com/forum/#!forum/zulip-devel).
+First, subscribe to the Zulip [development discussion mailing
+list][gg-devel].
 
 The Zulip project uses a system of labels in our [issue
-tracker](https://github.com/zulip/zulip/issues) to make it easy to
-find a project if you don't have your own project idea in mind or want
-to get some experience with working on Zulip before embarking on a
-larger project you have in mind:
+tracker][gh-issues] to make it easy to find a project if you don't
+have your own project idea in mind or want to get some experience with
+working on Zulip before embarking on a larger project you have in
+mind:
 
-* [Bite Size](https://github.com/zulip/zulip/labels/bite%20size):
-  Smaller projects that could be a great first contribution.
 * [Integrations](https://github.com/zulip/zulip/labels/integrations).
   Integrate Zulip with another piece of software and contribute it
-  back to the community!  Writing an integration can be a great
-  started project.  There's some brief documentation on the best way
-  to write integrations at https://github.com/zulip/zulip/issues/70.
-* [Documentation](https://github.com/zulip/zulip/labels/documentation).
+  back to the community!  Writing an integration can be a great first
+  contribution.  There's detailed documentation on how to write
+  integrations in [the Zulip integration writing
+  guide](https://zulip.readthedocs.io/en/latest/integration-guide.html).
+
+* [Bite Size](https://github.com/zulip/zulip/labels/bite%20size):
+  Smaller projects that might be a great first contribution.
+
+* [Documentation](https://github.com/zulip/zulip/labels/documentation):
   The Zulip project loves contributions of new documentation.
+
 * [Help Wanted](https://github.com/zulip/zulip/labels/help%20wanted):
   A broader list of projects that nobody is currently working on.
-* [Platform support](https://github.com/zulip/zulip/labels/Platform%20support).
-  These are open issues about making it possible to install Zulip on a wider
-  range of platforms.
-* [Bugs](https://github.com/zulip/zulip/labels/bug). Open bugs.
-* [Feature requests](https://github.com/zulip/zulip/labels/enhancement).
-  Browsing this list can be a great way to find feature ideas to implement that
-  other Zulip users are excited about.
+
+* [Platform support](https://github.com/zulip/zulip/labels/Platform%20support):
+  These are open issues about making it possible to install Zulip on a
+  wider range of platforms.
+
+* [Bugs](https://github.com/zulip/zulip/labels/bug): Open bugs.
+
+* [Feature requests](https://github.com/zulip/zulip/labels/enhancement):
+  Browsing this list can be a great way to find feature ideas to
+  implement that other Zulip users are excited about.
+
+* [2016 roadmap milestone](http://zulip.readthedocs.io/en/latest/roadmap.html): The
+  projects that are [priorities for the Zulip project](https://zulip.readthedocs.io/en/latest/roadmap.html).  These are great projects if you're looking to make an impact.
 
 If you're excited about helping with an open issue, just post on the
 conversation thread that you're working on it.  You're encouraged to
@@ -123,7 +156,7 @@ looking at the new feature tutorial and coding style guidelines on
 ReadTheDocs.
 
 Feedback on how to make this development process more efficient, fun,
-and friendly to new contributors is very welcome!  Just shoot an email
+and friendly to new contributors is very welcome!  Just send an email
 to the Zulip Developers list with your thoughts.
 
 License

README.prod.md

@@ -56,14 +56,24 @@ These instructions should be followed as root.
   You will eventually want to get a properly signed certificate (and
   note that at present the Zulip desktop app doesn't support
   self-signed certificates), but this will let you finish the
-  installation process.
+  installation process.  You can get a free properly signed
+  certificate from the new [Letsencrypt](https://letsencrypt.org/)
+  service, by following their [nginx
+  instructions](https://letsencrypt.readthedocs.io/en/latest/using.html#nginx).
+
+  When you do get an actual certificate, you will need to install as
+  /etc/ssl/certs/zulip.combined-chain.crt the full certificate
+  authority chain, not just the certificate; see the section on "SSL
+  certificate chains" [in the nginx
+  docs](http://nginx.org/en/docs/http/configuring_https_servers.html)
+  for how to do this:
 
 (2) Download [the latest built server tarball](https://www.zulip.com/dist/releases/zulip-server-latest.tar.gz)
   and unpack it to `/root/zulip`, e.g.
   ```
   wget https://www.zulip.com/dist/releases/zulip-server-latest.tar.gz
   tar -xf zulip-server-latest.tar.gz
-  mv zulip-server-1.3.6 /root/zulip
+  mkdir -p /root/zulip && tar -xf zulip-server-latest.tar.gz --directory=/root/zulip --strip-components=1
   ```
 
 (3) Run
@@ -138,8 +148,11 @@ need to do some additional setup documented in the `settings.py` template:
 
 * For Google authentication, you need to follow the configuration
   instructions around `GOOGLE_OAUTH2_CLIENT_ID` and `GOOGLE_CLIENT_ID`.
+
 * For Email authentication, you will need to follow the configuration
-  instructions around outgoing SMTP from Django.
+  instructions for outgoing SMTP from Django.  You can use `manage.py
+  send_test_email username@example.com` to test whether you've
+  successfully configured outgoing SMTP.
 
 You should be able to login now.  If you get an error, check
 `/var/log/zulip/errors.log` for a traceback, and consult the next
@@ -313,15 +326,17 @@ only supports talking to servers with a properly signed SSL
 certificate, so you may find that you get a blank screen when you
 connect to a Zulip server using a self-signed certificate.
 
-The Zulip iOS and Android apps in their respective stores don't yet
-support talking to non-zulip.com servers; the iOS app is waiting on
-Apple's app store review, while the Android app is waiting on someone
-to do the small project of adding a field to specify what Zulip server
-to talk to.
+The Zulip Android app in the Google Play store doesn't yet support
+talking to non-zulip.com servers (and the iOS one doesn't support
+Google auth SSO against non-zulip.com servers; there's a design for
+how to fix that which wouldn't be a ton of work to implement).  If you
+are interested in helping out with the Zulip mobile apps, shoot an
+email to zulip-devel@googlegroups.com and the maintainers can guide
+you on how to help.
 
-These issues will likely all be addressed in the coming weeks; make
-sure to join the zulip-announce@googlegroups.com list so that you can
-receive the announcements when these become available.
+For announcements about improvements to the apps, make sure to join
+the zulip-announce@googlegroups.com list so that you can receive the
+announcements when these become available.
 
 (5) All the other features: Hotkeys, emoji, search filters,
 @-mentions, etc.  Zulip has lots of great features, make sure your
@@ -377,7 +392,7 @@ upgrade.
 * The Zulip upgrade process works by creating a new deployment under
   /home/zulip/deployments/ containing a complete copy of the Zulip
   server code, and then moving the symlinks at
-  `/home/zulip/deployments/current` and /root/zulip` as part of the
+  `/home/zulip/deployments/current` and `/root/zulip` as part of the
   upgrade process.  This means that if the new version isn't working,
   you can quickly downgrade to the old version by using
   `/home/zulip/deployments/<date>/scripts/restart-server` to return to
@@ -443,7 +458,7 @@ computed using a hash of avatar_salt and user's email), etc.
 they do get large on a busy server, and it's definitely
 lower-priority.
 
-### Restoration
+#### Restoration
 
 To restore from backups, the process is basically the reverse of the above:
 
@@ -475,10 +490,11 @@ that they are up to date using the Nagios plugin at:
 Contributions to more fully automate this process or make this section
 of the guide much more explicit and detailed are very welcome!
 
-### Postgres streaming replication
 
-Zulip has database configuration for doing with Postgres streaming
-replication ; you can see the configuration in these files:
+#### Postgres streaming replication
+
+Zulip has database configuration for using Postgres streaming
+replication; you can see the configuration in these files:
 
 * puppet/zulip_internal/manifests/postgres_slave.pp
 * puppet/zulip_internal/manifests/postgres_master.pp
@@ -488,65 +504,64 @@ Contribution of a step-by-step guide for setting this up (and moving
 this configuration to be available in the main `puppet/zulip/` tree)
 would be very welcome!
 
-### Using a remote postgres host
-
-This is a bit annoying to setup, but you can configure Zulip to use a
-dedicated postgres server by setting the `REMOTE_POSTGRES_HOST`
-variable in /etc/zulip/settings.py, and configuring Postgres
-certificate authentication (see
-http://www.postgresql.org/docs/9.1/static/ssl-tcp.html and
-http://www.postgresql.org/docs/9.1/static/libpq-ssl.html for
-documentation on how to set this up and deploy the certificates) to
-make the DATABASES configuration in `zproject/settings.py` work (or
-override that configuration).
 
 ### Monitoring Zulip
 
-The complete Nagios configuration (sans secret keys) we used to
+The complete Nagios configuration (sans secret keys) used to
 monitor zulip.com is available under `puppet/zulip_internal` in the
 Zulip Git repository (those files are not installed in the release
-tarballs); there are a number of useful Nagios plugins available
-there, including:
+tarballs).
 
-Frontend server monitoring:
+The Nagios plugins used by that configuration are installed
+automatically by the Zulip installation process in subdirectories
+under `/usr/lib/nagios/plugins/`.  The following is a summary of the
+various Nagios plugins included with Zulip and what they check:
+
+Application server and queue worker monitoring:
 
 * check_send_receive_time (sends a test message through the system
   between two bot users to check that end-to-end message sending works)
-* check_website_response.sh (standard HTTP check)
-
-Queue worker monitoring:
 
 * check_rabbitmq_consumers and check_rabbitmq_queues (checks for
   rabbitmq being down or the queue workers being behind)
+
 * check_queue_worker_errors (checks for errors reported by the queue workers)
+
 * check_worker_memory (monitors for memory leaks in queue workers)
+
 * check_email_deliverer_backlog and check_email_deliverer_process
   (monitors for whether outgoing emails are being sent)
 
 Database monitoring:
 
-* check_pg_replication_lag
+* check_postgres_replication_lag (checks streaming replication is up
+  to date).
+
 * check_postgres (checks the health of the postgres database)
+
 * check_postgres_backup (checks backups are up to date; see above)
+
 * check_fts_update_log (monitors for whether full-text search updates
   are being processed)
 
 Standard server monitoring:
 
-* check_debian_packages
+* check_website_response.sh (standard HTTP check)
 
-Contributions on making it easier to monitor Zulip and maintain it in
-production, e.g.  https://github.com/zulip/zulip/issues/371, are very
-welcome!
+* check_debian_packages (checks apt repository is up to date)
+
+If you're using these plugins, bug reports and pull requests to make
+it easier to monitor Zulip and maintain it in production are
+encouraged!
 
 ### Scalability of Zulip
 
 This section attempts to address the considerations involved with
 running Zulip with a large team (>1000 users).
 
-* We recommend using a remote postgres database (see
-  REMOTE_POSTGRES_HOST docs above) for isolation, though it is not
-  required.  In the following, we discuss a relatively simple
+* We recommend using a [remote postgres
+  database](#postgres-database-details) for isolation, though it is
+  not required.  In the following, we discuss a relatively simple
   configuration with two types of servers: application servers
   (running Django, Tornado, RabbitMQ, Redis, Memcached, etc.) and
   database servers.
@@ -593,12 +608,6 @@ running Zulip with a large team (>1000 users).
   likely the first part of any project to support exchanging events
   amongst multiple servers.
 
-* The first scalability issue encountered by a very large realm (more
-  than a few thousand users), will be with the [frontend buddy list
-  perf and UI](https://github.com/zulip/zulip/issues/262).  Fixing
-  this should be a small project; the code for that part of the UI
-  layer doesn't do proper incremental updates.
-
 Questions, concerns, and bug reports about this area of Zulip are very
 welcome!  This is an area we are hoping to improve.
 
@@ -683,7 +692,7 @@ we can do a responsible security announcement).
 #### Users and Bots
 
 * There are three types of users in a Zulip realm: Administrators,
-  normal users, and botsq.  Administrators have the ability to
+  normal users, and bots.  Administrators have the ability to
   deactivate and reactivate other human and bot users, delete streams,
   add/remove administrator privileges, as well as change configuration
   for the overall realm (e.g. whether an invitation is required to
@@ -857,10 +866,25 @@ hostname/DNS side of the configuration.  Suggestions for how to
 improve this SSO setup documentation are very welcome!
 
 
-Remote Postgresql database
-==========================
+Postgres database details
+=========================
 
-If you want to use a remote Postgresql database, you should configure the information about the connection with the server. You need a user called "zulip" in your database server. You can configure these options in /etc/zulip/settings.py
+#### Remote Postgres database
+
+This is a bit annoying to setup, but you can configure Zulip to use a
+dedicated postgres server by setting the `REMOTE_POSTGRES_HOST`
+variable in /etc/zulip/settings.py, and configuring Postgres
+certificate authentication (see
+http://www.postgresql.org/docs/9.1/static/ssl-tcp.html and
+http://www.postgresql.org/docs/9.1/static/libpq-ssl.html for
+documentation on how to set this up and deploy the certificates) to
+make the DATABASES configuration in `zproject/settings.py` work (or
+override that configuration).
+
+If you want to use a remote Postgresql database, you should configure
+the information about the connection with the server. You need a user
+called "zulip" in your database server. You can configure these
+options in /etc/zulip/settings.py:
 
 * REMOTE_POSTGRES_HOST: Name or IP address of the remote host
 * REMOTE_POSTGRES_SSLMODE: SSL Mode used to connect to the server, different options you can use are:
@@ -877,10 +901,101 @@ Then you should specify the password of the user zulip for the database in /etc/
 postgres_password = xxxx
 ```
 
-Finally you can stop your database in the zulip server to save some memory, you can do it with:
+Finally, you can stop your database on the Zulip server via:
 
 ```
 sudo service postgresql stop
 sudo update-rc.d postgresql disable
 ```
 
+In future versions of this feature, we'd like to implement and
+document how to the remote postgres database server itself
+automatically by using the Zulip install script with a different set
+of puppet manifests than the all-in-one feature; if you're interested
+in working on this, post to the Zulip development mailing list and we
+can give you some tips.
+
+#### Debugging postgres database issues
+
+When debugging postgres issues, in addition to the standard `pg_top`
+tool, often it can be useful to use this query:
+
+```
+SELECT procpid,waiting,query_start,current_query FROM pg_stat_activity ORDER BY procpid;
+```
+
+which shows the currently running backends and their activity. This is
+similar to the pg_top output, with the added advantage of showing the
+complete query, which can be valuable in debugging.
+
+To stop a runaway query, you can run `SELECT pg_cancel_backend(pid
+int)` or `SELECT pg_terminate_backend(pid int)` as the 'postgres'
+user. The former cancels the backend's current query and the latter
+terminates the backend process. They are implemented by sending SIGINT
+and SIGTERM to the processes, respectively.  We recommend against
+sending a Postgres process SIGKILL. Doing so will cause the database
+to kill all current connections, roll back any pending transactions,
+and enter recovery mode.
+
+#### Stopping the Zulip postgres database
+
+To start or stop postgres manually, use the pg_ctlcluster command:
+
+```
+pg_ctlcluster 9.1 [--force] main {start|stop|restart|reload}
+```
+
+By default, using stop uses "smart" mode, which waits for all clients
+to disconnect before shutting down the database. This can take
+prohibitively long. If you use the --force option with stop,
+pg_ctlcluster will try to use the "fast" mode for shutting
+down. "Fast" mode is described by the manpage thusly:
+
+  With the --force option the "fast" mode is used which rolls back all
+  active transactions, disconnects clients immediately and thus shuts
+  down cleanly. If that does not work, shutdown is attempted again in
+  "immediate" mode, which can leave the cluster in an inconsistent state
+  and thus will lead to a recovery run at the next start. If this still
+  does not help, the postmaster process is killed. Exits with 0 on
+  success, with 2 if the server is not running, and with 1 on other
+  failure conditions. This mode should only be used when the machine is
+  about to be shut down.
+
+Many database parameters can be adjusted while the database is
+running. Just modify /etc/postgresql/9.1/main/postgresql.conf and
+issue a reload. The logs will note the change.
+
+#### Debugging issues starting postgres
+
+pg_ctlcluster often doesn't give you any information on why the
+database failed to start. It may tell you to check the logs, but you
+won't find any information there. pg_ctlcluster runs the following
+command underneath when it actually goes to start Postgres:
+
+```
+/usr/lib/postgresql/9.1/bin/pg_ctl start -D /var/lib/postgresql/9.1/main -s -o  '-c config_file="/etc/postgresql/9.1/main/postgresql.conf"'
+```
+
+Since pg_ctl doesn't redirect stdout or stderr, running the above can
+give you better diagnostic information. However, you might want to
+stop Postgres and restart it using pg_ctlcluster after you've debugged
+with this approach, since it does bypass some of the work that
+pg_ctlcluster does.
+
+
+#### Postgres Vacuuming alerts
+
+The `autovac_freeze` postgres alert from `check_postgres` is
+particularly important.  This alert indicates that the age (in terms
+of number of transactions) of the oldest transaction id (XID) is
+getting close to the `autovacuum_freeze_max_age` setting.  When the
+oldest XID hits that age, Postgres will force a VACUUM operation,
+which can often lead to sudden downtime until the operation finishes.
+If it did not do this and the age of the oldest XID reached 2 billion,
+transaction id wraparound would occur and there would be data loss.
+To clear the nagios alert, perform a `VACUUM` in each indicated
+database as a database superuser (`postgres`).
+
+See
+http://www.postgresql.org/docs/9.1/static/routine-vacuuming.html#VACUUM-FOR-WRAPAROUND
+for more details on postgres vacuuming.

THIRDPARTY

@@ -63,16 +63,16 @@ Copyright: 2006 Otheus Shelling
 License: GPL-2.0
 Comment: Not linked.
 
-Files: puppet/zulip_internal/files/nagios_plugins/check_debian_packages
+Files: puppet/zulip/files/nagios_plugins/zulip_base/check_debian_packages
 Copyright: 2005 Francesc Guasch
 License: GPL-2.0
 Comment: Not linked.
 
-Files: puppet/zulip_internal/files/nagios_plugins/check_postgres.pl
+Files: puppet/zulip/files/nagios_plugins/zulip_postgres_appdb/check_postgres.pl
 Copyright: 2007-2015 Greg Sabino Mullane
 License: BSD-2-Clause
 
-Files: puppet/zulip_internal/files/nagios_plugins/check_website_response.sh
+Files: puppet/zulip/files/nagios_plugins/zulip_nagios_server/check_website_response.sh
 Copyright: 2011 Chris Freeman
 License: GPL-2.0
 
@@ -229,10 +229,6 @@ Files: tools/jslint/jslint.js
 Copyright: 2002 Douglas Crockford
 License: XXX-good-not-evil
 
-Files: tools/python-proxy
-Copyright: 2009 F.bio Domingues
-License: Expat
-
 Files: tools/review
 Copyright: 2010 Ksplice, Inc.
 License: Apache-2.0
@@ -246,6 +242,10 @@ Files: zerver/lib/ccache.py
 Copyright: 2013 David Benjamin and Alan Huang
 License: Expat
 
+Files: zerver/lib/decorator.py zerver/management/commands/runtornado.py scripts/setup/generate_secrets.py
+Copyright: Django Software Foundation and individual contributors
+License: BSD-3-Clause
+
 Files: frontend_tests/casperjs/*
 Copyright: 2011-2012 Nicolas Perriault
  Joyent, Inc. and other Node contributors

Vagrantfile

@@ -2,6 +2,11 @@
 
 VAGRANTFILE_API_VERSION = "2"
 
+def command?(name)
+  `which #{name}`
+  $?.success?
+end
+
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
   # For LXC. VirtualBox hosts use a different box, described below.
@@ -13,6 +18,49 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.synced_folder ".", "/vagrant", disabled: true
   config.vm.synced_folder ".", "/srv/zulip"
 
+  proxy_config_file = ENV['HOME'] + "/.zulip-vagrant-config"
+  if File.file?(proxy_config_file)
+    http_proxy = https_proxy = no_proxy = ""
+
+    IO.foreach(proxy_config_file) do |line|
+      line.chomp!
+      key, value = line.split(nil, 2)
+      case key
+      when /^([#;]|$)/; # ignore comments
+      when "HTTP_PROXY"; http_proxy = value
+      when "HTTPS_PROXY"; https_proxy = value
+      when "NO_PROXY"; no_proxy = value
+      end
+    end
+
+    if Vagrant.has_plugin?("vagrant-proxyconf")
+      if http_proxy != ""
+        config.proxy.http = http_proxy
+      end
+      if https_proxy != ""
+        config.proxy.https = https_proxy
+      end
+      if https_proxy != ""
+        config.proxy.no_proxy = no_proxy
+      end
+    end
+  end
+
+  # Specify LXC provider before VirtualBox provider so it's preferred.
+  config.vm.provider "lxc" do |lxc|
+    if command? "lxc-ls"
+      LXC_VERSION = `lxc-ls --version`.strip unless defined? LXC_VERSION
+      if LXC_VERSION >= "1.1.0"
+        # Allow start without AppArmor, otherwise Box will not Start on Ubuntu 14.10
+        # see https://github.com/fgrehm/vagrant-lxc/issues/333
+        lxc.customize 'aa_allow_incomplete', 1
+      end
+      if LXC_VERSION >= "2.0.0"
+        lxc.backingstore = 'dir'
+      end
+    end
+  end
+
   config.vm.provider "virtualbox" do |vb, override|
     override.vm.box = "ubuntu/trusty64"
     # 2GiB seemed reasonable here. The VM OOMs with only 1024MiB.
@@ -22,8 +70,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 $provision_script = <<SCRIPT
 set -x
 set -e
-sudo apt-get update
-sudo apt-get install -y python-pbs
 /usr/bin/python /srv/zulip/provision.py
 SCRIPT
 

analytics/management/commands/active_user_stats.py

@@ -22,7 +22,7 @@ class Command(BaseCommand):
 
         # Calculate 10min, 2hrs, 12hrs, 1day, 2 business days (TODO business days), 1 week bucket of stats
         hour_buckets = [0.16, 2, 12, 24, 48, 168]
-        user_info = defaultdict(dict)
+        user_info = defaultdict(dict) # type: Dict[str, Dict[float, List[str]]]
 
         for last_presence in users:
             if last_presence.status == UserPresence.IDLE:
@@ -43,7 +43,7 @@ class Command(BaseCommand):
                 statsd.gauge("users.active.%s.%shr" %  (statsd_key(realm, True), statsd_key(hr, True)), len(users))
 
         # Also do stats for how many users have been reading the app.
-        users_reading = UserActivity.objects.select_related().filter(query="/json/update_message_flags")
+        users_reading = UserActivity.objects.select_related().filter(query="/json/messages/flags")
         user_info = defaultdict(dict)
         for activity in users_reading:
             for bucket in hour_buckets:

analytics/management/commands/analyze_mit.py

@@ -27,15 +27,15 @@ def compute_stats(log_level):
                            "bitcoin@mit.edu", "lp@mit.edu", "clocks@mit.edu",
                            "root@mit.edu", "nagios@mit.edu",
                            "www-data|local-realm@mit.edu"])
-    user_counts = {}
+    user_counts = {} # type: Dict[str, Dict[str, int]]
     for m in mit_query.select_related("sending_client", "sender"):
         email = m.sender.email
         user_counts.setdefault(email, {})
         user_counts[email].setdefault(m.sending_client.name, 0)
         user_counts[email][m.sending_client.name] += 1
 
-    total_counts = {}
-    total_user_counts = {}
+    total_counts = {} # type: Dict[str, int]
+    total_user_counts = {} # type: Dict[str, int]
     for email, counts in user_counts.items():
         total_user_counts.setdefault(email, 0)
         for client_name, count in counts.items():
@@ -44,9 +44,9 @@ def compute_stats(log_level):
             total_user_counts[email] += count
 
     logging.debug("%40s | %10s | %s" % ("User", "Messages", "Percentage Zulip"))
-    top_percents = {}
+    top_percents = {} # type: Dict[int, float]
     for size in [10, 25, 50, 100, 200, len(total_user_counts.keys())]:
-        top_percents[size] = 0
+        top_percents[size] = 0.0
     for i, email in enumerate(sorted(total_user_counts.keys(),
                                      key=lambda x: -total_user_counts[x])):
         percent_zulip = round(100 - (user_counts[email].get("zephyr_mirror", 0)) * 100. /

analytics/management/commands/analyze_user_activity.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+from __future__ import division
 from __future__ import print_function
 
 from zerver.lib.statistics import seconds_usage_between

analytics/management/commands/realm_stats.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+from __future__ import division
 from __future__ import print_function
 
 import datetime
@@ -28,7 +29,7 @@ class Command(BaseCommand):
                     UserActivity.objects.filter(user_profile__realm=realm,
                                                 user_profile__is_active=True,
                                                 last_visit__gt=activity_cutoff,
-                                                query="/json/update_pointer",
+                                                query="/json/users/me/pointer",
                                                 client__name="website")]
 
     def messages_sent_by(self, user, days_ago):

analytics/views.py

@@ -1,4 +1,7 @@
 from __future__ import absolute_import
+from __future__ import division
+from typing import Any, Dict, List, Tuple
+
 from django.db import connection
 from django.template import RequestContext, loader
 from django.utils.html import mark_safe
@@ -75,7 +78,7 @@ def get_realm_day_counts():
     rows = dictfetchall(cursor)
     cursor.close()
 
-    counts = defaultdict(dict)
+    counts = defaultdict(dict) # type: Dict[str, Dict[int, int]]
     for row in rows:
         counts[row['domain']][row['age']] = row['cnt']
 
@@ -137,7 +140,8 @@ def realm_summary_table(realm_minutes):
                         '/json/send_message',
                         'send_message_backend',
                         '/api/v1/send_message',
-                        '/json/update_pointer'
+                        '/json/update_pointer',
+                        '/json/users/me/pointer'
                     )
                 AND
                     last_visit > now() - interval '1 day'
@@ -166,8 +170,9 @@ def realm_summary_table(realm_minutes):
                         ua.query in (
                             '/json/send_message',
                             'send_message_backend',
-                           '/api/v1/send_message',
-                            '/json/update_pointer'
+                            '/api/v1/send_message',
+                            '/json/update_pointer',
+                            '/json/users/me/pointer'
                         )
                     GROUP by realm.id, up.email
                     HAVING max(last_visit) between
@@ -187,7 +192,8 @@ def realm_summary_table(realm_minutes):
                         '/json/send_message',
                         '/api/v1/send_message',
                         'send_message_backend',
-                        '/json/update_pointer'
+                        '/json/update_pointer',
+                        '/json/users/me/pointer'
                     )
                 AND
                     up.realm_id = realm.id
@@ -619,7 +625,8 @@ def raw_user_activity_table(records):
     return make_table(title, cols, rows)
 
 def get_user_activity_summary(records):
-    summary = {}
+    # type: (Any) -> Any
+    summary = {} # type: Dict[str, Dict[str, Any]]
     def update(action, record):
         if action not in summary:
             summary[action] = dict(
@@ -654,7 +661,7 @@ def get_user_activity_summary(records):
             update('website', record)
         if ('send_message' in query) or re.search('/api/.*/external/.*', query):
             update('send', record)
-        if query in ['/json/update_pointer', '/api/v1/update_pointer']:
+        if query in ['/json/update_pointer', '/json/users/me/pointer', '/api/v1/update_pointer']:
             update('pointer', record)
         update(client, record)
 
@@ -816,9 +823,9 @@ def realm_user_summary_table(all_records, admin_emails):
 
 @zulip_internal
 def get_realm_activity(request, realm):
-    data = []
-    all_records = {}
-    all_user_records = {}
+    # type: (Any, Any) -> Any
+    data = [] # type: List[Tuple[str, str]]
+    all_user_records = {} # type: Dict[str, Any]
 
     try:
         admins = get_realm(realm).get_admin_users()
@@ -828,8 +835,7 @@ def get_realm_activity(request, realm):
     admin_emails = {admin.email for admin in admins}
 
     for is_bot, page_title in [(False,  'Humans'), (True, 'Bots')]:
-        all_records = get_user_activity_records_for_realm(realm, is_bot)
-        all_records = list(all_records)
+        all_records = list(get_user_activity_records_for_realm(realm, is_bot))
 
         user_records, content = realm_user_summary_table(all_records, admin_emails)
         all_user_records.update(user_records)
@@ -861,7 +867,7 @@ def get_realm_activity(request, realm):
 def get_user_activity(request, email):
     records = get_user_activity_records_for_email(email)
 
-    data = []
+    data = [] # type: List[Tuple[str, str]]
     user_summary = get_user_activity_summary(records)
     content = user_activity_summary_table(user_summary)
 

api/MANIFEST.in

@@ -8,3 +8,4 @@ include examples/unsubscribe
 include examples/list-members
 include examples/list-subscriptions
 include examples/print-messages
+include examples/recent-messages

api/examples/create-user

@@ -21,6 +21,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+from __future__ import print_function
 import sys
 from os import path
 import optparse
@@ -46,9 +47,9 @@ parser.add_option('--new-short-name')
 
 client = zulip.init_from_options(options)
 
-print client.create_user({
+print(client.create_user({
         'email': options.new_email,
         'password': options.new_password,
         'full_name': options.new_full_name,
         'short_name': options.new_short_name
-        })
+        }))

api/examples/edit-message

@@ -21,6 +21,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+from __future__ import print_function
 import sys
 import os
 import optparse
@@ -53,4 +54,4 @@ if options.subject != "":
     message_data["subject"] = options.subject
 if options.content != "":
     message_data["content"] = options.content
-print client.update_message(message_data)
+print(client.update_message(message_data))

api/examples/get-public-streams

@@ -21,6 +21,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+from __future__ import print_function
 import sys
 import os
 import optparse
@@ -43,4 +44,4 @@ parser.add_option_group(zulip.generate_option_group(parser))
 
 client = zulip.init_from_options(options)
 
-print client.get_streams(include_public=True, include_subscribed=False)
+print(client.get_streams(include_public=True, include_subscribed=False))

api/examples/list-members

@@ -21,6 +21,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+from __future__ import print_function
 import sys
 import os
 import optparse
@@ -42,4 +43,4 @@ parser.add_option_group(zulip.generate_option_group(parser))
 client = zulip.init_from_options(options)
 
 for user in client.get_members()["members"]:
-    print user["full_name"], user["email"]
+    print(user["full_name"], user["email"])

api/examples/list-subscriptions

@@ -21,6 +21,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+from __future__ import print_function
 import sys
 import os
 import optparse
@@ -42,4 +43,4 @@ parser.add_option_group(zulip.generate_option_group(parser))
 
 client = zulip.init_from_options(options)
 
-print client.list_subscriptions()
+print(client.list_subscriptions())

api/examples/print-events

@@ -21,6 +21,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+from __future__ import print_function
 import sys
 import os
 import optparse
@@ -43,7 +44,7 @@ parser.add_option_group(zulip.generate_option_group(parser))
 client = zulip.init_from_options(options)
 
 def print_event(event):
-    print event
+    print(event)
 
 # This is a blocking call, and will continuously poll for new events
 # Note also the filter here is messages to the stream Denmark; if you

api/examples/print-messages

@@ -21,6 +21,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+from __future__ import print_function
 import sys
 import os
 import optparse
@@ -43,7 +44,7 @@ parser.add_option_group(zulip.generate_option_group(parser))
 client = zulip.init_from_options(options)
 
 def print_message(message):
-    print message
+    print(message)
 
 # This is a blocking call, and will continuously poll for new messages
 client.call_on_each_message(print_message)

api/examples/print-next-message

@@ -21,6 +21,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+from __future__ import print_function
 import sys
 import os
 import optparse
@@ -42,4 +43,4 @@ parser.add_option_group(zulip.generate_option_group(parser))
 
 client = zulip.init_from_options(options)
 
-print client.get_messages({})
+print(client.get_messages({}))

api/examples/recent-messages

@@ -0,0 +1,61 @@
+#!/usr/bin/env python2.7
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import json
+import optparse
+
+usage = """recent-messages [options] --count=<no. of previous messages> --user=<sender's email address> --api-key=<sender's api key>
+
+Prints out last count messages recieved by the indicated bot or user
+
+Example: recent-messages --count=101 --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option('--count', default=100)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+req = {
+    'narrow': [["stream", "Denmark"]],
+    'num_before': options.count,
+    'num_after': 0,
+    'anchor': 1000000000,
+    'apply_markdown': False
+}
+
+old_messages = client.do_api_query(req, zulip.API_VERSTRING + 'messages', method='GET')
+if 'messages' in old_messages:
+    for message in old_messages['messages']:
+        print(json.dumps(message, indent=4))
+else:
+    print([])

api/examples/send-message

@@ -21,6 +21,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+from __future__ import print_function
 import sys
 import os
 import optparse
@@ -54,4 +55,4 @@ message_data = {
     "subject": options.subject,
     "to": args,
 }
-print client.send_message(message_data)
+print(client.send_message(message_data))

api/examples/subscribe

@@ -21,6 +21,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+from __future__ import print_function
 import sys
 import os
 import optparse
@@ -45,8 +46,8 @@ parser.add_option('--streams', default='')
 client = zulip.init_from_options(options)
 
 if options.streams == "":
-    print >>sys.stderr, "Usage:", parser.usage
+    print("Usage:", parser.usage, file=sys.stderr)
     sys.exit(1)
 
-print client.add_subscriptions([{"name": stream_name} for stream_name in
-                                options.streams.split()])
+print(client.add_subscriptions([{"name": stream_name} for stream_name in
+                                options.streams.split()]))

api/examples/unsubscribe

@@ -21,6 +21,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+from __future__ import print_function
 import sys
 import os
 import optparse
@@ -45,7 +46,7 @@ parser.add_option('--streams', default='')
 client = zulip.init_from_options(options)
 
 if options.streams == "":
-    print >>sys.stderr, "Usage:", parser.usage
+    print("Usage:", parser.usage, file=sys.stderr)
     sys.exit(1)
 
-print client.remove_subscriptions(options.streams.split())
+print(client.remove_subscriptions(options.streams.split()))

api/integrations/asana/zulip_asana_mirror

@@ -30,6 +30,7 @@
 #
 # python-dateutil is a dependency for this script.
 
+from __future__ import print_function
 import base64
 from datetime import datetime, timedelta
 
@@ -37,16 +38,16 @@ import json
 import logging
 import os
 import time
-import urllib2
+from six.moves import urllib
 
 import sys
 
 try:
     import dateutil.parser
     import dateutil.tz
-except ImportError, e:
-    print >>sys.stderr, e
-    print >>sys.stderr, "Please install the python-dateutil package."
+except ImportError as e:
+    print(e, file=sys.stderr)
+    print("Please install the python-dateutil package.", file=sys.stderr)
     exit(1)
 
 sys.path.insert(0, os.path.dirname(__file__))
@@ -74,8 +75,8 @@ def fetch_from_asana(path):
     headers = {"Authorization": "Basic %s" % auth}
 
     url = "https://app.asana.com/api/1.0" + path
-    request = urllib2.Request(url, None, headers)
-    result = urllib2.urlopen(request)
+    request = urllib.request.Request(url, None, headers)
+    result = urllib.request.urlopen(request)
 
     return json.load(result)
 
@@ -189,7 +190,7 @@ def since():
                 timestamp = float(datestring)
                 max_timestamp_processed = datetime.fromtimestamp(timestamp)
                 logging.info("Reading from resume file: " + datestring)
-        except (ValueError,IOError) as e:
+        except (ValueError, IOError) as e:
             logging.warn("Could not open resume file: %s" % (
                     e.message or e.strerror,))
             max_timestamp_processed = default_since()

api/integrations/basecamp/zulip_basecamp_mirror

@@ -26,6 +26,7 @@
 # or preferably on a server.
 # You may need to install the python-requests library.
 
+from __future__ import absolute_import
 import requests
 import logging
 import time
@@ -33,7 +34,8 @@ import re
 import sys
 import os
 from datetime import datetime, timedelta
-from HTMLParser import HTMLParser
+from six.moves.html_parser import HTMLParser
+import six
 
 sys.path.insert(0, os.path.dirname(__file__))
 import zulip_basecamp_config as config
@@ -80,7 +82,7 @@ def check_permissions():
 
 # builds the message dict for sending a message with the Zulip API
 def build_message(event):
-    if not (event.has_key('bucket') and event.has_key('creator') and event.has_key('html_url')):
+    if not ('bucket' in event and 'creator' in event and 'html_url' in event):
         logging.error("Perhaps the Basecamp API changed behavior? "
                       "This event doesn't have the expected format:\n%s" %(event,))
         return None
@@ -92,7 +94,7 @@ def build_message(event):
     action = htmlParser.unescape(re.sub(r"<[^<>]+>", "", event.get('action', '')))
     target = htmlParser.unescape(event.get('target', ''))
     # Some events have "excerpts", which we blockquote
-    excerpt = htmlParser.unescape(event.get('excerpt',''))
+    excerpt = htmlParser.unescape(event.get('excerpt', ''))
     if excerpt.strip() == "":
         message = '**%s** %s [%s](%s).' % (event['creator']['name'], action, target, event['html_url'])
     else:
@@ -116,13 +118,13 @@ def run_mirror():
         since = re.search(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}-\d{2}:\d{2}", since)
         assert since, "resume file does not meet expected format"
         since = since.string
-    except (AssertionError,IOError) as e:
+    except (AssertionError, IOError) as e:
         logging.warn("Could not open resume file: %s" % (e.message or e.strerror,))
         since = (datetime.utcnow() - timedelta(hours=config.BASECAMP_INITIAL_HISTORY_HOURS)).isoformat() + "-00:00"
     try:
         # we use an exponential backoff approach when we get 429 (Too Many Requests).
         sleepInterval = 1
-        while 1:
+        while True:
             time.sleep(sleepInterval)
             response = requests.get("https://basecamp.com/%s/api/v1/events.json" % (config.BASECAMP_ACCOUNT_ID),
                                     params={'since': since},
@@ -170,7 +172,7 @@ def run_mirror():
 
 
 if __name__ == "__main__":
-    if not isinstance(config.RESUME_FILE, basestring):
+    if not isinstance(config.RESUME_FILE, six.string_types):
         sys.stderr("RESUME_FILE path not given; refusing to continue")
     check_permissions()
     if config.LOG_FILE:

api/integrations/codebase/zulip_codebase_mirror

@@ -29,6 +29,8 @@
 #
 # python-dateutil is a dependency for this script.
 
+from __future__ import print_function
+from __future__ import absolute_import
 import requests
 import logging
 import time
@@ -36,13 +38,14 @@ import sys
 import os
 
 from datetime import datetime, timedelta
+import six
 
 
 try:
     import dateutil.parser
-except ImportError, e:
-    print >>sys.stderr, e
-    print >>sys.stderr, "Please install the python-dateutil package."
+except ImportError as e:
+    print(e, file=sys.stderr)
+    print("Please install the python-dateutil package.", file=sys.stderr)
     exit(1)
 
 sys.path.insert(0, os.path.dirname(__file__))
@@ -271,13 +274,13 @@ def run_mirror():
         else:
             timestamp = int(timestamp, 10)
             since = datetime.fromtimestamp(timestamp)
-    except (ValueError,IOError) as e:
+    except (ValueError, IOError) as e:
         logging.warn("Could not open resume file: %s" % (e.message or e.strerror,))
         since = default_since()
 
     try:
         sleepInterval = 1
-        while 1:
+        while True:
             events = make_api_call("activity")[::-1]
             if events is not None:
                 sleepInterval = 1
@@ -314,7 +317,7 @@ def check_permissions():
         sys.stderr(e)
 
 if __name__ == "__main__":
-    if not isinstance(config.RESUME_FILE, basestring):
+    if not isinstance(config.RESUME_FILE, six.string_types):
         sys.stderr("RESUME_FILE path not given; refusing to continue")
     check_permissions()
     if config.LOG_FILE:

api/integrations/git/post-receive

@@ -29,13 +29,14 @@
 # For example:
 #  aa453216d1b3e49e7f6f98441fa56946ddcd6a20 68f7abf4e6f922807889f52bc043ecd31b79f814 refs/heads/master
 
+from __future__ import absolute_import
 import os
 import sys
 import subprocess
 import os.path
 
 sys.path.insert(0, os.path.dirname(__file__))
-import zulip_git_config as config
+from . import zulip_git_config as config
 VERSION = "0.9"
 
 if config.ZULIP_API_PATH is not None:

api/integrations/jira/org/humbug/jira/ZulipListener.groovy

@@ -67,7 +67,7 @@ class ZulipListener extends AbstractIssueEventListener {
                                   author, issueUrlMd, comment)
           break
         case ISSUE_CREATED_ID:
-          content = String.format("%s **created** %s priority %s, assigned to **%s**: \n\n> %s",
+          content = String.format("%s **created** %s priority %s, assigned to @**%s**: \n\n> %s",
                                   author, issueUrlMd, event.issue.priorityObject.name,
                                   assignee, title)
           break

api/integrations/perforce/git_p4.py

@@ -9,6 +9,7 @@
 #
 from __future__ import print_function
 from __future__ import absolute_import
+from __future__ import division
 import sys
 import six
 from six.moves import input
@@ -2346,7 +2347,7 @@ class P4Sync(Command, P4UserMap):
             self.labels[newestChange] = [output, revisions]
 
         if self.verbose:
-            print("Label changes: %s" % self.labels.keys())
+            print("Label changes: %s" % (list(self.labels.keys()),))
 
     # Import p4 labels as git tags. A direct mapping does not
     # exist, so assume that if all the files are at the same revision
@@ -2779,7 +2780,7 @@ class P4Sync(Command, P4UserMap):
                 if short in branches:
                     self.p4BranchesInGit = [ short ]
             else:
-                self.p4BranchesInGit = branches.keys()
+                self.p4BranchesInGit = list(branches.keys())
 
             if len(self.p4BranchesInGit) > 1:
                 if not self.silent:
@@ -2921,7 +2922,7 @@ class P4Sync(Command, P4UserMap):
                     b = b[len(self.projectName):]
                 self.createdBranches.add(b)
 
-        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
+        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) // 60))
 
         self.importProcess = subprocess.Popen(["git", "fast-import"],
                                               stdin=subprocess.PIPE,
@@ -3214,7 +3215,7 @@ commands = {
 
 def main():
     if len(sys.argv[1:]) == 0:
-        printUsage(commands.keys())
+        printUsage(list(commands.keys()))
         sys.exit(2)
 
     cmdName = sys.argv[1]
@@ -3224,7 +3225,7 @@ def main():
     except KeyError:
         print("unknown command %s" % cmdName)
         print("")
-        printUsage(commands.keys())
+        printUsage(list(commands.keys()))
         sys.exit(2)
 
     options = cmd.options

api/integrations/rss/rss-bot

@@ -23,16 +23,17 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+from __future__ import print_function
 import calendar
 import errno
 import hashlib
-from HTMLParser import HTMLParser
+from six.moves.html_parser import HTMLParser
 import logging
 import optparse
 import os
 import sys
 import time
-import urlparse
+from six.moves import urllib
 
 import feedparser
 import zulip
@@ -87,7 +88,7 @@ def mkdir_p(path):
     # Python doesn't have an analog to `mkdir -p` < Python 3.2.
     try:
         os.makedirs(path)
-    except OSError, e:
+    except OSError as e:
         if e.errno == errno.EEXIST and os.path.isdir(path):
             pass
         else:
@@ -97,7 +98,7 @@ try:
     mkdir_p(opts.data_dir)
 except OSError:
     # We can't write to the logfile, so just print and give up.
-    print >>sys.stderr, "Unable to store RSS data at %s." % (opts.data_dir,)
+    print("Unable to store RSS data at %s." % (opts.data_dir,), file=sys.stderr)
     exit(1)
 
 log_file = os.path.join(opts.data_dir, "rss-bot.log")
@@ -169,7 +170,7 @@ client = zulip.Client(email=opts.email, api_key=opts.api_key,
 first_message = True
 
 for feed_url in feed_urls:
-    feed_file = os.path.join(opts.data_dir, urlparse.urlparse(feed_url).netloc)
+    feed_file = os.path.join(opts.data_dir, urllib.parse.urlparse(feed_url).netloc)
 
     try:
         with open(feed_file, "r") as f:

api/integrations/twitter/twitter-bot

@@ -23,10 +23,11 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+from __future__ import print_function
 import os
 import sys
 import optparse
-import ConfigParser
+import six.moves.configparser
 
 import zulip
 VERSION = "0.9"
@@ -85,14 +86,14 @@ if not options.twitter_id:
     parser.error('You must specify --twitter-id')
 
 try:
-    config = ConfigParser.ConfigParser()
+    config = six.moves.configparser.ConfigParser()
     config.read(CONFIGFILE)
 
     consumer_key = config.get('twitter', 'consumer_key')
     consumer_secret = config.get('twitter', 'consumer_secret')
     access_token_key = config.get('twitter', 'access_token_key')
     access_token_secret = config.get('twitter', 'access_token_secret')
-except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+except (six.moves.configparser.NoSectionError, six.moves.configparser.NoOptionError):
    parser.error("Please provide a ~/.zulip_twitterrc")
 
 if not consumer_key or not consumer_secret or not access_token_key or not access_token_secret:
@@ -112,17 +113,17 @@ api = twitter.Api(consumer_key=consumer_key,
 user = api.VerifyCredentials()
 
 if not user.GetId():
-    print "Unable to log in to twitter with supplied credentials. Please double-check and try again"
+    print("Unable to log in to twitter with supplied credentials. Please double-check and try again")
     sys.exit()
 
 try:
     since_id = config.getint('twitter', 'since_id')
-except ConfigParser.NoOptionError:
+except six.moves.configparser.NoOptionError:
     since_id = -1
 
 try:
     user_id = config.get('twitter', 'user_id')
-except ConfigParser.NoOptionError:
+except six.moves.configparser.NoOptionError:
     user_id = options.twitter_id
 
 client = zulip.Client(
@@ -154,7 +155,7 @@ for status in statuses[::-1][:options.limit_tweets]:
 
     if ret['result'] == 'error':
         # If sending failed (e.g. no such stream), abort and retry next time
-        print "Error sending message to zulip: %s" % ret['msg']
+        print("Error sending message to zulip: %s" % ret['msg'])
         break
     else:
         since_id = status.GetId()

api/integrations/twitter/twitter-search-bot

@@ -23,10 +23,11 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+from __future__ import print_function
 import os
 import sys
 import optparse
-import ConfigParser
+import six.moves.configparser
 
 import zulip
 VERSION = "0.9"
@@ -107,14 +108,14 @@ if not opts.search_terms:
     parser.error('You must specify a search term.')
 
 try:
-    config = ConfigParser.ConfigParser()
+    config = six.moves.configparser.ConfigParser()
     config.read(CONFIGFILE)
 
     consumer_key = config.get('twitter', 'consumer_key')
     consumer_secret = config.get('twitter', 'consumer_secret')
     access_token_key = config.get('twitter', 'access_token_key')
     access_token_secret = config.get('twitter', 'access_token_secret')
-except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+except (six.moves.configparser.NoSectionError, six.moves.configparser.NoOptionError):
    parser.error("Please provide a ~/.zulip_twitterrc")
 
 if not (consumer_key and consumer_secret and access_token_key and access_token_secret):
@@ -122,7 +123,7 @@ if not (consumer_key and consumer_secret and access_token_key and access_token_s
 
 try:
     since_id = config.getint('search', 'since_id')
-except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
+except (six.moves.configparser.NoOptionError, six.moves.configparser.NoSectionError):
     since_id = 0
 
 try:
@@ -138,8 +139,8 @@ api = twitter.Api(consumer_key=consumer_key,
 user = api.VerifyCredentials()
 
 if not user.GetId():
-    print "Unable to log in to Twitter with supplied credentials.\
-Please double-check and try again."
+    print("Unable to log in to Twitter with supplied credentials.\
+Please double-check and try again.")
     sys.exit()
 
 client = zulip.Client(
@@ -182,7 +183,7 @@ for status in statuses[::-1][:opts.limit_tweets]:
 
     if ret['result'] == 'error':
         # If sending failed (e.g. no such stream), abort and retry next time
-        print "Error sending message to zulip: %s" % ret['msg']
+        print("Error sending message to zulip: %s" % ret['msg'])
         break
     else:
         since_id = status.GetId()

api/setup.py

@@ -2,6 +2,8 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import print_function
+from typing import Any, Generator, List, Tuple
+
 import os
 import sys
 
@@ -16,6 +18,7 @@ def version():
     return version
 
 def recur_expand(target_root, dir):
+    # type: (Any, Any) -> Generator[Tuple[str, List[str]], None, None]
     for root, _, files in os.walk(dir):
         paths = [os.path.join(root, f) for f in files]
         if len(paths):
@@ -40,7 +43,7 @@ package_info = dict(
     data_files=[('share/zulip/examples', ["examples/zuliprc", "examples/send-message", "examples/subscribe",
                                            "examples/get-public-streams", "examples/unsubscribe",
                                            "examples/list-members", "examples/list-subscriptions",
-                                           "examples/print-messages"])] + \
+                                           "examples/print-messages", "examples/recent-messages"])] + \
         list(recur_expand('share/zulip', 'integrations/')),
     scripts=["bin/zulip-send"],
 )
@@ -48,6 +51,8 @@ package_info = dict(
 setuptools_info = dict(
     install_requires=['requests>=0.12.1',
                       'simplejson',
+                      'six',
+                      'typing',
     ],
 )
 
@@ -65,7 +70,7 @@ except ImportError:
         sys.exit(1)
     try:
         import requests
-        assert(LooseVersion(requests.__version__) >= LooseVersion('0.12.1'))
+        assert(LooseVersion(requests.__version__) >= LooseVersion('0.12.1')) # type: ignore # https://github.com/JukkaL/mypy/issues/1165
     except (ImportError, AssertionError):
         print("requests >=0.12.1 is not installed", file=sys.stderr)
         sys.exit(1)

api/zulip/__init__.py

@@ -22,25 +22,25 @@
 
 from __future__ import print_function
 from __future__ import absolute_import
+from __future__ import division
 import simplejson
 import requests
 import time
 import traceback
-import urlparse
 import sys
 import os
 import optparse
 import platform
-import urllib
 import random
 from distutils.version import LooseVersion
 
 from six.moves.configparser import SafeConfigParser
+from six.moves import urllib
 import logging
 import six
 
 
-__version__ = "0.2.4"
+__version__ = "0.2.5"
 
 logger = logging.getLogger(__name__)
 
@@ -164,7 +164,7 @@ class Client(object):
             config_file = get_default_config_filename()
         if os.path.exists(config_file):
             config = SafeConfigParser()
-            with file(config_file, 'r') as f:
+            with open(config_file, 'r') as f:
                 config.readfp(f, config_file)
             if api_key is None:
                 api_key = config.get("api", "key")
@@ -245,7 +245,7 @@ class Client(object):
     def do_api_query(self, orig_request, url, method="POST", longpolling = False):
         request = {}
 
-        for (key, val) in orig_request.iteritems():
+        for (key, val) in six.iteritems(orig_request):
             if not (isinstance(val, str) or isinstance(val, six.text_type)):
                 request[key] = simplejson.dumps(val)
             else:
@@ -289,7 +289,7 @@ class Client(object):
                 kwargs = {kwarg: query_state["request"]}
                 res = requests.request(
                         method,
-                        urlparse.urljoin(self.base_url, url),
+                        urllib.parse.urljoin(self.base_url, url),
                         auth=requests.auth.HTTPBasicAuth(self.email,
                                                          self.api_key),
                         verify=self.tls_verification, timeout=90,
@@ -468,7 +468,7 @@ Client._register('list_subscriptions', method='GET', url='users/me/subscriptions
 Client._register('add_subscriptions', url='users/me/subscriptions', make_request=_mk_subs)
 Client._register('remove_subscriptions', method='PATCH', url='users/me/subscriptions', make_request=_mk_rm_subs)
 Client._register('get_subscribers', method='GET',
-                 computed_url=lambda request: 'streams/%s/members' % (urllib.quote(request['stream'], safe=''),),
+                 computed_url=lambda request: 'streams/%s/members' % (urllib.parse.quote(request['stream'], safe=''),),
                  make_request=_kwargs_to_dict)
 Client._register('render_message', method='GET', url='messages/render')
 Client._register('create_user', method='POST', url='users')

assets/favicon/generate

@@ -1,6 +1,8 @@
 #!/usr/bin/env python2.7
+from __future__ import absolute_import
 import xml.etree.ElementTree as ET
 import subprocess
+from six.moves import range
 
 # Generates the favicon images containing unread message counts.
 
@@ -10,7 +12,7 @@ elems = [tree.getroot().findall(
     ".//*[@id='%s']/{http://www.w3.org/2000/svg}tspan" % (name,))[0]
      for name in ('number_back', 'number_front')]
 
-for i in xrange(1,100):
+for i in range(1, 100):
     # Prepare a modified SVG
     s = '%2d' % (i,)
     for e in elems:

bin/get-django-setting

@@ -1,5 +1,6 @@
 #!/usr/bin/env python2.7
 from __future__ import absolute_import
+from __future__ import print_function
 
 import os
 import sys
@@ -8,4 +9,4 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
 os.environ['DJANGO_SETTINGS_MODULE'] = 'zproject.settings'
 from django.conf import settings
 
-print getattr(settings, sys.argv[1])
+print(getattr(settings, sys.argv[1]))

bin/write-rabbitmq-consumers-state-file

@@ -1,4 +1,5 @@
-#!/bin/bash -e
+#!/usr/bin/env bash
+set -e
 
 queue=$1
 

bots/check-mirroring

@@ -1,4 +1,6 @@
 #!/usr/bin/env python2.7
+from __future__ import print_function
+from __future__ import absolute_import
 import sys
 import time
 import optparse
@@ -7,6 +9,7 @@ import random
 import logging
 import subprocess
 import hashlib
+from six.moves import range
 
 parser = optparse.OptionParser()
 parser.add_option('--verbose',
@@ -100,7 +103,7 @@ def print_status_and_exit(status):
     # e.g. true success and punting due to a SERVNAK, result in a
     # non-alert case, so to give us something unambiguous to check in
     # Nagios, print the exit status.
-    print status
+    print(status)
     sys.exit(status)
 
 def send_zulip(message):
@@ -149,7 +152,7 @@ for (stream, test) in test_streams:
         zephyr_subs_to_add.append((stream, '*', '*'))
 
 actually_subscribed = False
-for tries in xrange(10):
+for tries in range(10):
     try:
         zephyr.init()
         zephyr._z.subAll(zephyr_subs_to_add)
@@ -163,7 +166,7 @@ for tries in xrange(10):
         if missing == 0:
             actually_subscribed = True
             break
-    except IOError, e:
+    except IOError as e:
         if "SERVNAK received" in e:
             logger.error("SERVNAK repeatedly received, punting rest of test")
         else:
@@ -276,7 +279,7 @@ logger.info("Finished receiving Zulip messages!")
 receive_zephyrs()
 logger.info("Finished receiving Zephyr messages!")
 
-all_keys = set(zhkeys.keys() + hzkeys.keys())
+all_keys = set(list(zhkeys.keys()) + list(hzkeys.keys()))
 def process_keys(content_list):
     # Start by filtering out any keys that might have come from
     # concurrent check-mirroring processes

bots/check-rabbitmq-consumers

@@ -1,5 +1,6 @@
 #!/usr/bin/env python2.7
 
+from __future__ import print_function
 import sys
 import time
 import optparse
@@ -15,7 +16,7 @@ states = {
 }
 
 if 'USER' in os.environ and not os.environ['USER'] in ['root', 'rabbitmq']:
-    print "This script must be run as the root or rabbitmq user"
+    print("This script must be run as the root or rabbitmq user")
 
 
 usage = """Usage: check-rabbitmq-consumers --queue=[queue-name] --min-threshold=[min-threshold]"""

bots/check-rabbitmq-queue

@@ -1,5 +1,6 @@
 #!/usr/bin/env python2.7
 
+from __future__ import print_function
 import sys
 import re
 import time
@@ -30,7 +31,7 @@ max_count = 0
 warn_queues = []
 
 if 'USER' in os.environ and not os.environ['USER'] in ['root', 'rabbitmq']:
-    print "This script must be run as the root or rabbitmq user"
+    print("This script must be run as the root or rabbitmq user")
 
 for line in output.split("\n"):
     line = line.strip()

bots/cron_file_helper.py

@@ -10,7 +10,7 @@ def nagios_from_file(results_file):
     This file is created by various nagios checking cron jobs such as
     check-rabbitmq-queues and check-rabbitmq-consumers"""
 
-    data = file(results_file).read().strip()
+    data = open(results_file).read().strip()
     pieces = data.split('|')
 
     if not len(pieces) == 4:

bots/gcal-bot

@@ -1,9 +1,10 @@
 #!/usr/bin/env python2.7
+from __future__ import print_function
 import sys
 import time
 import datetime
 import optparse
-import urlparse
+from six.moves import urllib
 import itertools
 import traceback
 import os
@@ -56,13 +57,13 @@ except ImportError:
     parser.error('Install python-gdata')
 
 def get_calendar_url():
-    parts = urlparse.urlparse(options.calendar)
+    parts = urllib.parse.urlparse(options.calendar)
     pat = os.path.split(parts.path)
     if pat[1] != 'basic':
         parser.error('The --calendar URL should be the XML "Private Address" ' +
                      'from your calendar settings')
-    return urlparse.urlunparse((parts.scheme, parts.netloc, pat[0] + '/full',
-                                '', 'futureevents=true&orderby=startdate', ''))
+    return urllib.parse.urlunparse((parts.scheme, parts.netloc, pat[0] + '/full',
+                                   '', 'futureevents=true&orderby=startdate', ''))
 
 calendar_url = get_calendar_url()
 
@@ -99,7 +100,7 @@ def send_reminders():
             key = (uid, start)
             if key not in sent:
                 line = '%s starts at %s' % (title, start.strftime('%H:%M'))
-                print 'Sending reminder:', line
+                print('Sending reminder:', line)
                 messages.append(line)
                 keys.add(key)
 

bots/jabber_mirror_backend.py

@@ -37,6 +37,7 @@
 #               | other sender|  x  |    |        |
 # public mode   +-------------+-----+----+--------+----
 #               | self sender |     |    |        |
+from typing import Set
 
 import logging
 import threading
@@ -78,11 +79,11 @@ class JabberToZulipBot(ClientXMPP):
             self.nick = jid.username
             jid.resource = "zulip"
         ClientXMPP.__init__(self, jid, password)
-        self.rooms = set()
+        self.rooms = set() # type: Set[str]
         self.rooms_to_join = rooms
         self.add_event_handler("session_start", self.session_start)
         self.add_event_handler("message", self.message)
-        self.zulip = None
+        self.zulip = None # type: zulip.Client
         self.use_ipv6 = False
 
         self.register_plugin('xep_0045') # Jabber chatrooms
@@ -195,7 +196,7 @@ class JabberToZulipBot(ClientXMPP):
 class ZulipToJabberBot(object):
     def __init__(self, zulip_client):
         self.client = zulip_client
-        self.jabber = None
+        self.jabber = None # type: JabberToZulipBot
 
     def set_jabber_client(self, client):
         self.jabber = client
@@ -376,7 +377,7 @@ option does not affect login credentials.'''.replace("\n", " "))
 
     config = SafeConfigParser()
     try:
-        with file(config_file, 'r') as f:
+        with open(config_file, 'r') as f:
             config.readfp(f, config_file)
     except IOError:
         pass

bots/log2zulip

@@ -1,4 +1,5 @@
 #!/usr/bin/env python2.7
+from __future__ import print_function
 import subprocess
 import os
 import sys
@@ -20,7 +21,7 @@ def mkdir_p(path):
     # Python doesn't have an analog to `mkdir -p` < Python 3.2.
     try:
         os.makedirs(path)
-    except OSError, e:
+    except OSError as e:
         if e.errno == errno.EEXIST and os.path.isdir(path):
             pass
         else:
@@ -55,14 +56,14 @@ def process_logs():
         data_file_path = "/var/tmp/log2zulip.state"
     mkdir_p(os.path.dirname(data_file_path))
     if not os.path.exists(data_file_path):
-        file(data_file_path, "w").write("{}")
-    last_data = ujson.loads(file(data_file_path).read())
+        open(data_file_path, "w").write("{}")
+    last_data = ujson.loads(open(data_file_path).read())
     new_data = {}
     for log_file in log_files:
         file_data = last_data.get(log_file, {})
         if not os.path.exists(log_file):
             # If the file doesn't exist, log an error and then move on to the next file
-            print "Log file %s does not exist!" % (log_file,)
+            print("Log file does not exist or could not stat log file: %s" % (log_file,))
             continue
         length = int(subprocess.check_output(["wc", "-l", log_file]).split()[0])
         if file_data.get("last") is None:
@@ -78,26 +79,26 @@ def process_logs():
             process_lines(new_lines, filename)
             file_data["last"] += len(new_lines)
         new_data[log_file] = file_data
-    file(data_file_path, "w").write(ujson.dumps(new_data))
+    open(data_file_path, "w").write(ujson.dumps(new_data))
 
 if __name__ == "__main__":
     if os.path.exists(lock_path):
-        print "Log2zulip lock held; not doing anything"
+        print("Log2zulip lock held; not doing anything")
         sys.exit(0)
 
     try:
-        file(lock_path, "w").write("1")
+        open(lock_path, "w").write("1")
         zulip_client = zulip.Client(config_file="/etc/log2zulip.zuliprc")
         try:
-            log_files = ujson.loads(file(control_path, "r").read())
+            log_files = ujson.loads(open(control_path, "r").read())
         except Exception:
-            print "Could not load control data from %s" % (control_path,)
+            print("Could not load control data from %s" % (control_path,))
             traceback.print_exc()
             sys.exit(1)
         process_logs()
     finally:
         try:
             os.remove(lock_path)
-        except OSError, IOError:
+        except OSError as IOError:
             pass
 

bots/process_ccache

@@ -9,27 +9,27 @@ ccache_data_encoded = sys.argv[3]
 
 # Update the Kerberos ticket cache file
 program_name = "zmirror-%s" % (short_user,)
-with file("/home/zulip/ccache/%s" % (program_name,), "w") as f:
+with open("/home/zulip/ccache/%s" % (program_name,), "w") as f:
     f.write(base64.b64decode(ccache_data_encoded))
 
 # Setup API key
 api_key_path = "/home/zulip/api-keys/%s" % (program_name,)
-file(api_key_path, "w").write(api_key + "\n")
+open(api_key_path, "w").write(api_key + "\n")
 
 # Setup supervisord configuration
 supervisor_path = "/etc/supervisor/conf.d/%s.conf" % (program_name,)
 template = "/home/zulip/zulip/bots/zmirror_private.conf.template"
-template_data = file(template).read()
+template_data = open(template).read()
 session_path = "/home/zulip/zephyr_sessions/%s" % (program_name,)
 
 # Preserve mail zephyrs forwarding setting across rewriting the config file
 
 try:
-    if "--forward-mail-zephyrs" in file(supervisor_path, "r").read():
+    if "--forward-mail-zephyrs" in open(supervisor_path, "r").read():
         template_data = template_data.replace("--use-sessions", "--use-sessions --forward-mail-zephyrs")
 except Exception:
     pass
-file(supervisor_path, "w").write(template_data.replace("USERNAME", short_user))
+open(supervisor_path, "w").write(template_data.replace("USERNAME", short_user))
 
 # Delete your session
 subprocess.check_call(["rm", "-f", session_path])

bots/summarize_stream.py

@@ -1,4 +1,5 @@
 from __future__ import print_function
+from typing import Any, Dict, List
 # This is hacky code to analyze data on our support stream.  The main
 # reusable bits are get_recent_messages and get_words.
 
@@ -50,13 +51,13 @@ def generate_support_stats():
     narrow = 'stream:support'
     count = 2000
     msgs = get_recent_messages(client, narrow, count)
-    msgs_by_topic = collections.defaultdict(list)
+    msgs_by_topic = collections.defaultdict(list) # type: Dict[str, List[Dict[str, Any]]]
     for msg in msgs:
         topic = msg['subject']
         msgs_by_topic[topic].append(msg)
 
-    word_count = collections.defaultdict(int)
-    email_count = collections.defaultdict(int)
+    word_count = collections.defaultdict(int) # type: Dict[str, int]
+    email_count = collections.defaultdict(int) # type: Dict[str, int]
 
     if False:
         for topic in msgs_by_topic:
@@ -64,16 +65,14 @@ def generate_support_stats():
     analyze_messages(msgs, word_count, email_count)
 
     if True:
-        words = word_count.keys()
-        words = [w for w in words if word_count[w] >= 10]
-        words = [w for w in words if len(w) >= 5]
+        words = [w for w in word_count.keys() if word_count[w] >= 10 and len(w) >= 5]
         words = sorted(words, key=lambda w: word_count[w], reverse=True)
         for word in words:
             print(word, word_count[word])
 
     if False:
-        emails = email_count.keys()
-        emails = sorted(emails, key=lambda w: email_count[w], reverse=True)
+        emails = sorted(list(email_count.keys()),
+                        key=lambda w: email_count[w], reverse=True)
         for email in emails:
             print(email, email_count[email])
 

bots/sync-public-streams

@@ -59,7 +59,7 @@ if __name__ == "__main__":
         if public_streams is None:
             continue
 
-        f = file("/home/zulip/public_streams.tmp", "w")
+        f = open("/home/zulip/public_streams.tmp", "w")
         f.write(simplejson.dumps(list(public_streams)) + "\n")
         f.close()
 

bots/zephyr_mirror_backend.py

@@ -21,6 +21,7 @@
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 from __future__ import absolute_import
+from typing import Any, List
 
 import sys
 from six.moves import map
@@ -28,7 +29,7 @@ from six.moves import range
 try:
     import simplejson
 except ImportError:
-    import json as simplejson
+    import json as simplejson # type: ignore
 import re
 import time
 import subprocess
@@ -48,6 +49,8 @@ class States(object):
     Startup, ZulipToZephyr, ZephyrToZulip, ChildSending = list(range(4))
 CURRENT_STATE = States.Startup
 
+logger = None # type: logging.Logger
+
 def to_zulip_username(zephyr_username):
     if "@" in zephyr_username:
         (user, realm) = zephyr_username.split("@")
@@ -191,7 +194,7 @@ def zephyr_bulk_subscribe(subs):
 
 def update_subscriptions():
     try:
-        f = file(options.stream_file_path, "r")
+        f = open(options.stream_file_path, "r")
         public_streams = simplejson.loads(f.read())
         f.close()
     except:
@@ -287,7 +290,7 @@ def parse_zephyr_body(zephyr_data):
 
 def parse_crypt_table(zephyr_class, instance):
     try:
-        crypt_table = file(os.path.join(os.environ["HOME"], ".crypt-table"))
+        crypt_table = open(os.path.join(os.environ["HOME"], ".crypt-table"))
     except IOError:
         return None
 
@@ -349,7 +352,7 @@ def process_notice(notice, log):
 
     if zephyr_class == options.nagios_class:
         # Mark that we got the message and proceed
-        with file(options.nagios_path, "w") as f:
+        with open(options.nagios_path, "w") as f:
             f.write("0\n")
         return
 
@@ -468,7 +471,7 @@ def zephyr_load_session_autoretry(session_path):
     backoff = zulip.RandomExponentialBackoff()
     while backoff.keep_going():
         try:
-            session = file(session_path, "r").read()
+            session = open(session_path, "r").read()
             zephyr._z.initialize()
             zephyr._z.load_session(session)
             zephyr.__inited = True
@@ -510,7 +513,7 @@ def zephyr_to_zulip(options):
         if options.nagios_class:
             zephyr_subscribe_autoretry((options.nagios_class, "*", "*"))
         if options.use_sessions:
-            file(options.session_path, "w").write(zephyr._z.dump_session())
+            open(options.session_path, "w").write(zephyr._z.dump_session())
 
     if options.logs_to_resend is not None:
         with open(options.logs_to_resend, 'r') as log:
@@ -804,9 +807,9 @@ def add_zulip_subscriptions(verbose):
         unauthorized = res.get("unauthorized")
         if verbose:
             if already is not None and len(already) > 0:
-                logger.info("\nAlready subscribed to: %s" % (", ".join(already.values()[0]),))
+                logger.info("\nAlready subscribed to: %s" % (", ".join(list(already.values())[0]),))
             if new is not None and len(new) > 0:
-                logger.info("\nSuccessfully subscribed to: %s" % (", ".join(new.values()[0]),))
+                logger.info("\nSuccessfully subscribed to: %s" % (", ".join(list(new.values())[0]),))
             if unauthorized is not None and len(unauthorized) > 0:
                 logger.info("\n" + "\n".join(textwrap.wrap("""\
 The following streams you have NOT been subscribed to,
@@ -857,7 +860,7 @@ def parse_zephyr_subs(verbose=False):
             logger.error("Couldn't find ~/.zephyr.subs!")
         return []
 
-    for line in file(subs_file, "r").readlines():
+    for line in open(subs_file, "r").readlines():
         line = line.strip()
         if len(line) == 0:
             continue
@@ -878,6 +881,7 @@ def parse_zephyr_subs(verbose=False):
     return zephyr_subscriptions
 
 def open_logger():
+    # type: () -> logging.Logger
     if options.log_path is not None:
         log_file = options.log_path
     elif options.forward_class_messages:
@@ -1025,7 +1029,7 @@ if __name__ == "__main__":
 
     signal.signal(signal.SIGINT, die_gracefully)
 
-    (options, args) = parse_args()
+    (options, args) = parse_args() # type: Any, List[str]
 
     logger = open_logger()
     configure_logger(logger, "parent")
@@ -1050,7 +1054,7 @@ Could not find API key file.
 You need to either place your api key file at %s,
 or specify the --api-key-file option.""" % (options.api_key_file,))))
             sys.exit(1)
-        api_key = file(options.api_key_file).read().strip()
+        api_key = open(options.api_key_file).read().strip()
         # Store the API key in the environment so that our children
         # don't need to read it in
         os.environ["HUMBUG_API_KEY"] = api_key
@@ -1113,7 +1117,7 @@ or specify the --api-key-file option.""" % (options.api_key_file,))))
         options.session_path = "/var/tmp/%s" % (options.user,)
 
     if options.forward_from_zulip:
-        child_pid = os.fork()
+        child_pid = os.fork() # type: int
         if child_pid == 0:
             CURRENT_STATE = States.ZulipToZephyr
             # Run the zulip => zephyr mirror in the child

changelog.md

@@ -4,6 +4,34 @@ All notable changes to this project will be documented in this file.
 
 [Unreleased]
 
+[1.3.11]
+- Moved email digest support into the default Zulip production configuration.
+- Added options for configuring Postgres, RabbitMQ, Redis, and memcached
+  in settings.py.
+- Added documentation on using Hubot to integrate with useful services
+  not yet integrated with Zulip directly (e.g. Google Hangouts).
+- Added new management command to test sending email from Zulip.
+- Added Codeship, Pingdom, Taiga, Teamcity, and Yo integrations.
+- Added Nagios plugins to the main distribution.
+- Added ability for realm administrators to manage custom emoji.
+- Added guide to writing new integrations.
+- Enabled camo image proxy to fix mixed-content warnings for http images.
+- Refactored the Zulip puppet modules to be more modular.
+- Refactored the Tornado event system, fixing old memory leaks.
+- Removed many old-style /json API endpoints
+- Implemented running queue processors multithreaded in development,
+  decreasing RAM requirements for a Zulip development environment from
+  ~1GB to ~300MB.
+- Fixed rerendering the complete buddy list whenever a user came back from
+  idle, which was a significant performance issue in larger realms.
+- Fixed the disabling of desktop notifications from 1.3.7 for new users.
+- Fixed the (admin) create_user API enforcing restricted_to_domain, even
+  if that setting was disabled for the realm.
+- Fixed bugs changing certain settings in administration pages.
+- Fixed collapsing messages in narrowed views.
+- Fixed 500 errors when uploading a non-image file as an avatar.
+- Fixed Jira integration incorrectly not @-mentioning assignee.
+
 [1.3.10]
 - Added new integration for Travis CI.
 - Added settings option to control maximum file upload size.

confirmation/settings.py

@@ -2,9 +2,10 @@
 
 # Copyright: (c) 2008, Jarek Zgoda <jarek.zgoda@gmail.com>
 
+from typing import Any, Dict
+
 __revision__ = '$Id: settings.py 12 2008-11-23 19:38:52Z jarek.zgoda $'
 
 STATUS_ACTIVE = 1
 
-STATUS_FIELDS = {
-}
+STATUS_FIELDS = {} # type: Dict[Any, Any]

corporate/urls.py

@@ -6,4 +6,10 @@ urlpatterns = patterns('',
     url(r'^zephyr/$', TemplateView.as_view(template_name='corporate/zephyr.html')),
     url(r'^mit/$', TemplateView.as_view(template_name='corporate/mit.html')),
     url(r'^zephyr-mirror/$', TemplateView.as_view(template_name='corporate/zephyr-mirror.html')),
+
+    # Terms of service and privacy policy
+    url(r'^terms/$',   TemplateView.as_view(template_name='corporate/terms.html')),
+    url(r'^terms-enterprise/$',  TemplateView.as_view(template_name='corporate/terms-enterprise.html')),
+    url(r'^privacy/$', TemplateView.as_view(template_name='corporate/privacy.html')),
+
 )

docs/code-style.rst

@@ -79,13 +79,13 @@ In our Django code, never do direct
 use ``get_user_profile_by_{email,id}``. There are 3 reasons for this:
 
 #. It's guaranteed to correctly do a case-inexact lookup
-#. It fetches the user object from memcached, which is faster
+#. It fetches the user object from remote cache, which is faster
 #. It always fetches a UserProfile object which has been queried using
    .selected\_related(), and thus will perform well when one later
    accesses related models like the Realm.
 
 Similarly we have ``get_client`` and ``get_stream`` functions to fetch
-those commonly accessed objects via memcached.
+those commonly accessed objects via remote cache.
 
 Using Django model objects as keys in sets/dicts
 ------------------------------------------------
@@ -391,7 +391,11 @@ Commit Discipline
 -----------------
 
 We follow the Git project's own commit discipline practice of "Each
-commit is a minimal coherent idea".
+commit is a minimal coherent idea".  This discipline takes a bit of
+work, but it makes it much easier for code reviewers to spot bugs, and
+makesthe commit history a much more useful resource for developers
+trying to understand why the code works the way it does, which also
+helps a lot in preventing bugs.
 
 Coherency requirements for any commit:
 
@@ -436,12 +440,28 @@ Other considerations:
 
 -  Overly fine commits are easily squashed, but not vice versa, so err
    toward small commits, and the code reviewer can advise on squashing.
+-  If a commit you write doesn't pass tests, you should usually fix
+   that by amending the commit to fix the bug, not writing a new "fix
+   tests" commit on top of it.
+-  When you fix a GitHub issue, `mark that you've fixed the issue in
+   your commit message
+   <https://help.github.com/articles/closing-issues-via-commit-messages/>`__
+   so that the issue is automatically closed when your code is merged.
+   Zulip's preferred style for this is to have the final paragraph
+   of the commit message read e.g. "Fixes: #123."
 
-It can take some practice to get used to writing your commits this
-way.  For example, often you'll start adding a feature, and discover
-you need to a refactoring partway through writing the feature.  When
-that happens, we recommend stashing your partial feature, do the
-refactoring, commit it, and then finish implementing your feature.
+Zulip expects you to structure the commits in your pull requests to
+form a clean history before we will merge them; it's best to write
+your commits following these guidelines in the first place, but if you
+don't, you can always fix your history using `git rebase -i`.
+
+It can take some practice to get used to writing your commits with a
+clean history so that you don't spend much time doing interactive
+rebases.  For example, often you'll start adding a feature, and
+discover you need to a refactoring partway through writing the
+feature.  When that happens, we recommend stashing your partial
+feature, do the refactoring, commit it, and then finish implementing
+your feature.
 
 Commit Messages
 ---------------
@@ -454,18 +474,35 @@ Bad::
 
    bugfix
    gather_subscriptions was broken
+   fix bug #234.
 
 Good::
 
-   Prevent gather_subscriptions from throwing an exception when given bad input.
+   Fix gather_subscriptions throwing an exception when given bad input.
 
--  Please use a complete sentence, ending with a period.
+-  Use present-tense action verbs in your commit messages.
+
+Bad::
+
+   Fixing gather_subscriptions throwing an exception when given bad input.
+   Fixed gather_subscriptions throwing an exception when given bad input.
+
+Good::
+
+   Fix gather_subscriptions throwing an exception when given bad input.
+
+-  Please use a complete sentence in the summary, ending with a
+   period.
 
 -  The rest of the commit message should be written in full prose and
    explain why and how the change was made. If the commit makes
    performance improvements, you should generally include some rough
    benchmarks showing that it actually improves the performance.
 
+-  Any paragraph content in the commit message should be line-wrapped
+   to less than 76 characters per line, so that your commit message
+   will be reasonably readable in `git log` in a normal terminal.
+
 -  In your commit message, you should describe any manual testing you
    did in addition to running the automated tests, and any aspects of
    the commit that you think are questionable and you'd like special

docs/conf.py

@@ -293,3 +293,11 @@ texinfo_documents = [
 
 # If true, do not generate a @detailmenu in the "Top" node's menu.
 #texinfo_no_detailmenu = False
+
+from recommonmark.parser import CommonMarkParser
+
+source_parsers = {
+        '.md': CommonMarkParser,
+        }
+
+source_suffix = ['.rst', '.md']

docs/directory-structure.rst

@@ -75,13 +75,13 @@ Templates
 Tests
 =====
 
-+------------------------+-----------------------------------+
-| ``zerver/test*.py``             | Backend tests            |
-+------------------------+-----------------------------------+
-| ``frontend_tests/node``         | Node Frontend unit tests |
-+------------------------+-----------------------------------+
-| ``frontend_tests/tests``        | Casper frontend tests    |
-+------------------------+-----------------------------------+
++-------------------------+-----------------------------------+
+| ``zerver/tests/``       |          Backend tests            |
++-------------------------+-----------------------------------+
+| ``frontend_tests/node`` |          Node Frontend unit tests |
++-------------------------+-----------------------------------+
+| ``frontend_tests/tests``|          Casper frontend tests    |
++-------------------------+-----------------------------------+
 
 Documentation
 =============

docs/front-end-build-process.rst

@@ -7,20 +7,65 @@ This page documents additional information that may be useful when developing ne
 Primary build process
 =====================
 
-Most of the exisiting JS in Zulip is written in IIFE-wrapped modules, one per file in the `static/js` directory. When running Zulip in development mode each file is loaded seperately. In production mode (and when creating a release tarball) JavaScript files are concatenated and minified.
+Most of the existing JS in Zulip is written in IIFE-wrapped modules,
+one per file in the `static/js` directory. When running Zulip in
+development mode, each file is loaded seperately.  In production mode
+(and when creating a release tarball using
+`tools/build-release-tarball`), JavaScript files are concatenated and
+minified.
 
-If you add a new JavaScript file it needs to be specified in the `JS_SPECS` dictionary defined in `zproject/settings.py` to be included in the concatenated file.
+If you add a new JavaScript file, it needs to be specified in the
+`JS_SPECS` dictionary defined in `zproject/settings.py` to be included
+in the concatenated file.
 
 Webpack/CommonJS modules
 ========================
-New JS written for Zulip can be written as CommonJS modules (bundled using `webpack <https://webpack.github.io/>`_, though this will taken care of automatically whenever ``run-dev.py`` is running). (CommonJS is the same module format that Node uses, so see `the Node documentation <https://nodejs.org/docs/latest/api/modules.html>` for more information on the syntax.)
 
-Benefits of using CommonJS modules over the `IIFE <http://benalman.com/news/2010/11/immediately-invoked-function-expression/>`_ module approach:
+New JS written for Zulip can be written as CommonJS modules (bundled
+using `webpack <https://webpack.github.io/>`_, though this will taken
+care of automatically whenever ``run-dev.py`` is running). (CommonJS
+is the same module format that Node uses, so see `the Node
+documentation <https://nodejs.org/docs/latest/api/modules.html>` for
+more information on the syntax.)
+
+Benefits of using CommonJS modules over the `IIFE
+<http://benalman.com/news/2010/11/immediately-invoked-function-expression/>`_
+module approach:
 
 * namespacing/module boilerplate will be added automatically in the bundling process
 * dependencies between modules are more explicit and easier to trace
 * no separate list of JS files needs to be maintained for concatenation and minification
 * third-party libraries can be more easily installed/versioned using npm
-* running the same code in the browser and in Node for testing is simplified (as both environments use the same module syntax)
+* running the same code in the browser and in Node for testing is
+  simplified (as both environments use the same module syntax)
+
+The entry point file for the bundle generated by webpack is
+``static/js/src/main.js``. Any modules you add will need to be
+required from this file (or one of its dependencies) in order to be
+included in the script bundle.
+
+Adding static files
+===================
+
+To add a static file to the app (JavaScript, CSS, images, etc), first
+add it to the appropriate place under ``static/``.
+
+* Third-party files should all go in ``static/third/``.  Tag the commit
+  with "[third]" when adding or modifying a third-party package.
+
+* Our own JS lives under ``static/js``; CSS lives under ``static/styles``.
+
+* JavaScript and CSS files are combined and minified in production. In
+  this case all you need to do is add the filename to PIPELINE_CSS or
+  JS_SPECS in ``zproject/settings.py``. (If you plan to only use the
+  JS/CSS within the app proper, and not on the login page or other
+  standalone pages, put it in the 'app' category.)
+
+If you want to test minified files in development, look for the
+``PIPELINE =`` line in ``zproject/settings.py`` and set it to ``True`` -- or
+just set ``DEBUG = False``.
+
+Note that ``static/html/{400,5xx}.html`` will only render properly if
+minification is enabled, since they hardcode the path
+``static/min/portico.css``.
 
-The entry point file for the bundle generated by webpack is ``static/js/src/main.js``. Any modules you add will need to be required from this file (or one of its dependencies) in order to be included in the script bundle.

docs/index.rst

@@ -11,11 +11,18 @@ Contents:
 .. toctree::
    :maxdepth: 2
 
+   integration-guide
    new-feature-tutorial
-   code-style
-   front-end-build-process
    directory-structure
+   code-style
    testing
+   markdown
+   queuing
+   schema-migrations
+   front-end-build-process
+   mypy
+   translating
+   roadmap
 
 Indices and tables
 ==================

docs/integration-guide.md

@@ -0,0 +1,178 @@
+# Integration Writing Guide
+
+Integrations are one of the most important parts of a group chat tool
+like Zulip, and we are committed to making integrating with Zulip and
+getting you integration merged upstream so everyone else can benefit
+from it as easy as possible while maintaining the high quality of the
+Zulip integrations library.
+
+Contributions to this guide are very welcome, so if you run into any
+issues following these instructions or come up with any tips or tools
+that help writing integration, please email
+zulip-devel@googlegroups.com, open an issue, or submit a pull request
+to share your ideas!
+
+## Types of integrations
+
+We have several different ways that we integrate with 3rd part
+products, ordered here by which types we prefer to write:
+
+1. Webhook integrations (examples: Freshdesk, GitHub), where the
+third-party service supports posting content to a particular URI on
+our site with data about the event.  For these, you usually just need
+to add a new handler in `zerver/views/webhooks.py` (plus
+test/document/etc.).  An example commit implementing a new webhook.
+https://github.com/zulip/zulip/pull/324.
+
+2. Python script integrations (examples: SVN, Git), where we can get
+the service to call our integration (by shelling out or otherwise),
+passing in the required data.  Our preferred model for these is to
+ship these integrations in our API release tarballs (by writing the
+integration in `api/integrations`).
+
+3. Plugin integrations (examples: Jenkins, Hubot, Trac) where the user
+needs to install a plugin into their existing software.  These are
+often more work, but for some products are the only way to integrate
+with the product at all.
+
+## General advice for writing integrations
+
+* Consider using our Zulip markup to make the output from your
+  integration especially attractive or useful (e.g.  emoji, markdown
+  emphasis, @-mentions, or `!avatar(email)`).
+
+* Use topics effectively to ensure sequential messages about the same
+  thing are threaded together; this makes for much better consumption
+  by users.  E.g. for a bug tracker integration, put the bug number in
+  the topic for all messages; for an integration like Nagios, put the
+  service in the topic.
+
+* Integrations that don't match a team's workflow can often be
+  uselessly spammy.  Give careful thought to providing options for
+  triggering Zulip messages only for certain message types, certain
+  projects, or sending different messages to different streams/topics,
+  to make it easy for teams to configure the integration to support
+  their workflow.
+
+* Sometimes it can be helpful to contact the vendor if it appears they
+  don't have an API or webhook we can use -- sometimes the right API
+  is just not properly documented.
+
+## Writing Webhook integrations
+
+New Zulip webhook integrations can take just a few hours to write,
+including tests and documentation, if you use the right process.
+Here's how we recommend doing it:
+
+* First, use http://requestb.in/ or a similar site to capture an
+  example webhook payload from the service you're integrating.  You
+  can use these captured payloads to create a set of test fixtures for
+  your integration under `zerver/fixtures`.
+
+* Then write a draft webhook handler under `zerver/views/webhooks/`;
+  there are a lot of examples in that directory.  We recommend
+  templating off a short one (like `stash.py` or `zendesk.py`), since
+  the longer ones usually just have more complex parsing which can
+  obscure what's common to all webhook integrations.  In addition to
+  writing the integration itself, you'll need to add an entry in
+  `zproject/urls.py` for your webhook; search for `webhook` in that
+  file to find the existing ones (and please add yours in the
+  alphabetically correct place).
+
+* Then write a test for your fixture in `zerver/tests/test_hooks.py`, and
+  you can iterate on the tests and webhooks handler until they work,
+  all without ever needing to post directly from the server you're
+  integrating to your Zulip development machine.  To run just the
+  tests from the test class you wrote, you can use e.g.
+
+  ```
+  test-backend zerver.tests.test_hooks.PagerDutyHookTests
+  ```
+
+  See
+  https://github.com/zulip/zulip/blob/master/README.dev.md#running-the-test-suite
+  for more details on the Zulip test runner.
+
+* Once you've gotten your webhook working and passing a test, capture
+  payloads for the other common types of posts the service's webhook
+  will make, and add tests for them; usually this part of the process
+  is pretty fast.  Webhook integration tests should all use fixtures
+  (as opposed to contacting the service), since otherwise the tests
+  can't run without Internet access and some sort of credentials for
+  the service.
+
+* Finally, write documentation for the integration (see below)!
+
+## Writing Python script and plugin integrations integrations
+
+For plugin integrations, usually you will need to consult the
+documentation for the third party software in order to learn how to
+write the integration.  But we have a few notes on how to do these:
+
+* You should always send messages by POSTing to URLs of the form
+`https://zulip.example.com/v1/messages/`, not the legacy
+`/api/v1/send_message` message sending API.
+
+* We usually build Python script integration with (at least) 2 files:
+`zulip_foo_config.py`` containing the configuration for the
+integration including the bots' API keys, plus a script that reads
+from this configuration to actually do the work (that way, it's
+possible to update the script without breaking users' configurations).
+
+* Be sure to test your integration carefully and document how to
+  install it (see notes on documentation below).
+
+* You should specify a clear HTTP User-Agent for your integration. The
+user agent should at a minimum identify the integration and version
+number, separated by a slash. If possible, you should collect platform
+information and include that in `()`s after the version number. Some
+examples of ideal UAs are:
+
+```
+ZulipDesktop/0.7.0 (Ubuntu; 14.04)
+ZulipJenkins/0.1.0 (Windows; 7.2)
+ZulipMobile/0.5.4 (Android; 4.2; maguro)
+```
+
+## Documenting your integration
+
+Every Zulip integration must be documented in
+`templates/zerver/integrations.html`.  Usually, this involves a few
+steps:
+
+* Add an `integration-lozenge` class block in the alphabetically
+  correct place in the main integration list, using the logo for the
+  integrated software.
+
+* Add an `integration-instructions` class block also in the
+  alphabetically correct place, explaining all the steps required to
+  setup the integration, including what URLs to use, etc.  If there
+  are any screens in the product involved, take a few screenshots with
+  the input fields filled out with sample values in order to make the
+  instructions really easy to follow.  For the screenshots, use
+  something like `github-bot@example.com` for the email addresses and
+  an obviously fake API key like `abcdef123456790`.
+
+* Finally, generate a message sent by the integration and take a
+  screenshot of the message to provide an example message in the
+  documentation. If your new integration is a webhook integration,
+  you can generate such a message from your test fixtures
+  using `send_webhook_fixture_message`:
+
+  ```
+  ./manage.py send_webhook_fixture_message \
+       --fixture=zerver/fixtures/pingdom/pingdom_imap_down_to_up.json \
+       '--url=/api/v1/external/pingdom?stream=stream_name&api_key=api_key'
+  ```
+
+  When generating the screenshot of a sample message, give your test
+  bot a nice name like "GitHub Bot", use the project's logo as the
+  bot's avatar, and take the screenshots showing the stream/topic bar
+  for the message, not just the message body.
+
+When writing documentation for your integration, be sure to use the
+`{{ external_api_uri }}` template variable, so that your integration
+documentation will provide the correct URL for whatever server it is
+deployed on.  If special configuration is required to set the SITE
+variable, you should document that too, inside an `{% if
+api_site_required %}` check.

docs/markdown.md

@@ -0,0 +1,135 @@
+# Zulip's markdown implementation
+
+Zulip has a special flavor of Markdown, currently called 'bugdown'
+after Zulip's original name of "humbug".
+
+Zulip has two implementations of Bugdown.  The first is based on
+Python-Markdown (`zerver/lib/bugdown/`) and is used to authoritatively
+render messages on the backend (and implements expensive features like
+querying the Twitter API to render tweets nicely).  The other is in
+javascript, based on marked (`static/js/echo.js`), and is used to
+preview and locally echo messages the moment the sender hits enter,
+without waiting for round trip from the server.  The two
+implementations are tested for compatibility via
+`zerver/tests/test_bugdown.py` and the fixtures under
+`zerver/fixtures/bugdown-data.json`.
+
+The javascript implementation knows which types of messages it can
+render correctly, and thus while there is code to rerender messages
+based on the authoritative backend rendering (which would clause a
+change in the rendering visible only to the sender shortly after a
+message is sent), this should never happen and whenever it does it is
+considered a bug.  Instead, if the frontend doesn't know how to
+correctly render a message, we simply won't echo the message for the
+sender until it's rendered by the backend.  So for example, a message
+containing a link to Twitter will not be rendered by the javascript
+implementation because it doesn't support doing the 3rd party API
+queries required to render tweets nicely.
+
+I should note that the below documentation is based on a comparison
+with original Markdown, not newer Markdown variants like CommonMark.
+
+## Zulip's Markdown philosophy
+
+Markdown is great for group chat for the same reason it's been
+successful in products ranging from blogs to wikis to bug trackers:
+it's close enough to how people try to express themselves when writing
+plain text (e.g. emails) that is helps more than getting in the way.
+
+The main issue for using Markdown in instant messaging is that the
+Markdown standard syntax used in a lot of wikis/blogs has nontrivial
+error rates, where the author needs to go back and edit the post to
+fix the formatting after typing it the first time.  While that's
+basically fine when writing a blog, it gets annoying very fast in a
+chat product; even though you can edit messages to fix formatting
+mistakes, you don't want to be doing that often.  There are basically
+2 types of error rates that are important for a product like Zulip:
+
+* What fraction of the time, if you pasted a short technical email
+that you wrote to your team and passed it through your Markdown
+implementation, would you need to change the text of your email for it
+to render in a reasonable way?  This is the "accidental Markdown
+syntax" problem, common with Markdown syntax like the italics syntax
+interacting with talking about `char *`s.
+
+* What fraction of the time do users attempting to use a particular
+Markdown syntax actually succeed at doing so correctly?  Syntax like
+required a blank line between text and the start of a bulleted list
+raise this figure substantially.
+
+Both of these are minor issues for most products using Markdown, but
+they are major problems in the instant messaging context, because one
+can't edit a message that has already been sent and users are
+generally writing quickly.  Zulip's Markdown strategy is based on the
+principles of giving users the power they need to express complicated
+ideas in a chat context while minimizing those two error rates.
+
+## Zulip's Changes to Markdown
+
+Below, we document the changes that Zulip has against stock
+Python-Markdown; some of the features we modify / disable may already
+be non-standard.
+
+### Basic syntax
+
+* Enable `nl2br</tt> extension: this means one newline creates a line
+  break (not paragraph break).
+
+* Disable italics entirely.  This resolves an issue where people were
+  using `*` and `_` and hitting it by mistake too often.  E.g. with
+  stock Markdown `You should use char * instead of void * there` would
+  trigger italics.
+
+* Allow only `**` syntax for bold, not `__` (easy to hit by mistake if
+  discussing Python `__init__` or something)
+
+* Disable special use of `\` to escape other syntax. Rendering `\\` as
+  `\` was hugely controversial, but having no escape syntax is also
+  controversial.  We may revisit this.  For now you can always put
+  things in code blocks.
+
+### Lists
+
+* Allow tacking a bulleted list or block quote onto the end of a
+  paragraph, i.e. without a blank line before it
+
+* Allow only `*` for bulleted lists, not `+` or `-` (previoulsy
+  created confusion with diff-style text sloppily not included in a
+  code block)
+
+* Disable ordered list syntax: it automatically renumbers, which can
+  be really confusing when sending a numbered list across multiple
+  messages.
+
+### Links
+
+* Enable auto-linkification, both for `http://...` and guessing at
+  things like `t.co/foo`.
+
+* Force links to be absolute. `[foo](google.com)` will go to
+  `http://google.com`, and not `http://zulip.com/google.com` which
+  is the default behavior.
+
+* Set `target="_blank"` and `title=`(the url) on every link tag so
+  clicking always opens a new window
+
+* Disable link-by-reference syntax, `[foo][bar]` ... `[bar]: http://google.com`
+
+### Code
+
+* Enable fenced code block extension, with syntax highlighting
+
+* Disable line-numbering within fenced code blocks -- the `<table>`
+  output confused our web client code.
+
+### Other
+
+* Disable headings, both `# foo` and `== foo ==` syntax: they don't
+  make much sense for chat messages.
+
+* Disabled images.
+
+* Allow embedding any avatar as a tiny (list bullet size) image.  This
+  is used primarily by version control integrations.
+
+* We added the `~~~ quote` block quote syntax.

docs/mypy.md

@@ -0,0 +1,78 @@
+# mypy Python static type checker
+
+[mypy](http://mypy-lang.org/) is a compile-time static type checker
+for Python, allowing optional, gradual typing of Python code.  Zulip
+is using mypy's Python 2 compatible syntax for type annotations, which
+means that type annotations are written inside comments that start
+with `# type: `.  Here's a brief example of the mypy syntax we're
+using in Zulip:
+
+```
+user_dict = {} # type: Dict[str, UserProfile]
+
+def get_user_profile_by_email(email):
+    # type: (str) -> UserProfile
+    ... # Actual code of the function here
+```
+
+You can learn more about it at:
+
+* [Python 2 type annotation syntax in PEP 484](https://www.python.org/dev/peps/pep-0484/#suggested-syntax-for-python-2-7-and-straddling-code)
+* [Using mypy with Python 2 code](http://mypy.readthedocs.io/en/latest/python2.html)
+
+The mypy type checker is run automatically as part of Zulip's Travis
+CI testing process.
+
+## Installing mypy
+
+If you installed Zulip's development environment correctly, mypy
+should already be installed inside the Python 3 virtualenv at
+`zulip-py3-venv` (mypy only supports Python 3).  If it isn't installed
+(e.g. because you haven't reprovisioned recently), you can run
+`tools/install-mypy` to install it.
+
+## Running mypy on Zulip's code locally
+
+To run mypy on Zulip's python code, run the command:
+
+    tools/run-mypy
+
+It will output errors in the same style of a compiler.  For example,
+if your code has a type error like this:
+
+```
+foo = 1
+foo = '1'
+```
+
+you'll get an error like this:
+
+```
+test.py: note: In function "test":
+test.py:200: error: Incompatible types in assignment (expression has type "str", variable has type "int")
+```
+
+If you need help interpreting or debugging mypy errors, please feel
+free to mention @sharmaeklavya2 or @timabbott on your pull request (or
+email zulip-devel@googlegroups.com) to get help; we'd love to both
+build a great troubleshooting guide in this doc and also help
+contribute improvements to error messages upstream.
+
+Since mypy is a new tool under rapid development and occasionally
+makes breaking changes, Zulip is using a pinned version of mypy from
+its [git repository](https://github.com/python/mypy) rather than
+tracking the (older) latest mypy release on pypi.
+
+## Excluded files
+
+Since several python files in Zulip's code don't pass mypy's checks
+(even for unannotated code) right now, a list of files to be excluded
+from the check for CI is present in tools/run-mypy.
+
+To run mypy on all python files, ignoring the exclude list, you can
+pass the `--all` option to tools/run-mypy.
+
+    tools/run-mypy --all
+
+If you type annotate some of those files, please remove them from the
+exclude list.

docs/new-feature-tutorial.rst

@@ -62,8 +62,8 @@ process.
 
 **Testing:** There are two types of frontend tests: node-based unit tests and 
 blackbox end-to-end tests. The blackbox tests are run in a headless browser 
-using Casper.js and are located in ``zerver/tests/frontend/tests/``. The unit
-tests use Node's ``assert`` module are located in ``zerver/tests/frontend/node/``.
+using Casper.js and are located in ``frontend_tests/casper_tests/``. The unit
+tests use Node's ``assert`` module are located in ``frontend_tests/node_tests/``.
 For more information on writing and running tests see the :doc:`testing 
 documentation <testing>`.
 

docs/queuing.md

@@ -0,0 +1,81 @@
+# RabbitMQ queues
+
+Zulip uses RabbitMQ to manage a system of internal queues.  These are
+used for a variety of purposes:
+
+* Asynchronously doing expensive operations like sending email
+  notifications which can take seconds per email and thus would
+  otherwise timeout when 100s are triggered at once (E.g. inviting a
+  lot of new users to a realm).
+
+* Asynchronously doing non-time-critical somewhat expensive operations
+  like updating analytics tables (e.g. UserActivityInternal) which
+  don't have any immediate runtime effect.
+
+* Communicating events to push to clients (browsers, etc.) from the
+  main Zulip Django application process to the Tornado-based events
+  system.  Example events might be that a new message was sent, a user
+  has changed their subscriptions, etc.
+
+* Processing mobile push notifications and email mirroring system
+  messages.
+
+* Processing various errors, frontend tracebacks, and slow database
+  queries in a batched fashion.
+
+* Doing markdown rendering for messages delivered to the Tornado via
+  websockets.
+
+Needless to say, the RabbitMQ-based queuing system is an important
+part of the overall Zulip architecture, since it's in critical code
+paths for everything from signing up for account, to rendering
+messages, to delivering updates to clients.
+
+We use the `pika` library to interface with RabbitMQ, using a simple
+custom integration defined in `zerver/lib/queue.py`.
+
+### Adding a new queue processor
+
+To add a new queue processor:
+
+* Define the processor in `zerver/worker/queue_processors.py` using
+  the `@assign_queue` decorator; it's pretty easy to get the template
+  for an existing similar queue processor.  This suffices to test your
+  queue worker in the Zulip development environment, though you'll
+  need to restart `tools/run-dev.py` in order to run your new queue
+  processor.  You can also run a single queue processor manually using
+  e.g. `./manage.py process_queue --queue=user_activity`.
+
+* So that supervisord will known to run the queue processor in
+  production, you will need to define a program entry for it in
+  `servers/puppet/modules/zulip/files/supervisor/conf.d/zulip.conf`
+  and add it to the `zulip-workers` group further down in the file.
+
+* For monitoring, you need to add a check that your worker is running
+  to puppet/zulip/files/cron.d/rabbitmq-numconsumers if it's a
+  one-at-a-time consumer like `user_activity_internal` or a custom
+  nagios check if it is a bulk processor like `slow_queries`.
+
+### Publishing events into a queue
+
+You can publish events to a RabbitMQ queue using the
+`queue_json_publish` function defined in `zerver/lib/queue.py`.
+
+### Clearing a RabbitMQ queue
+
+If you need to clear a queue (delete all the events in it), run
+`./manage.py purge_queue <queue_name>`, for example:
+
+```
+./manage.py purge_queue user_activity
+```
+
+You can also use the amqp tools directly.  Install `amqp-tools` from
+apt and then run:
+
+```
+amqp-delete-queue --username=zulip --password='...' --server=localhost \
+   --queue=user_presence
+```
+
+with the RabbitMQ password from `/etc/zulip/zulip-secrets.conf`.

docs/roadmap.md

@@ -0,0 +1,262 @@
+Zulip 2016 Roadmap
+==================
+
+## Introduction
+
+Zulip has received a great deal of interest and attention since it was
+released as free and open source software by Dropbox.  That attention
+has come with a lot of active development work from members of the
+Zulip community.  From when Zulip was released as open source in late
+September 2015 through today (mid-April, 2016), over 300 pull requests
+have been submitted to the various Zulip repositories (and over 250
+have been merged!), the vast majority of which are submitted by
+Zulip's users around the world (as opposed to the small core team who
+review and merge the pull requests).
+
+In any project, there can be a lot of value in periodically putting
+together a roadmap detailing the major areas where the project is
+hoping to improve.  This can be especially important in an open source
+project like Zulip where development is distributed across many people
+around the world.  This roadmap is intended to organize a list of the
+most important improvements that should to be made to Zulip in the
+relatively near future.  Our aim is to complete most of these
+improvements in 2016.
+
+This document is not meant to constrain in any way what contributions
+to Zulip will be accepted; instead, it will be used by the Zulip core
+team to prioritize our efforts, measure progress on improving the
+Zulip product, hold ourselves accountable for making Zulip improve
+rapidly, and celebrate members of the community who contribute to
+projects on the roadmap.
+
+If you're someone interested in making a larger contribution to Zulip
+and looking for somewhere to start, this roadmap is the best place to
+look for substantial projects that will definitely be of value to the
+community (if you're looking for a starter project, see the [guide to
+getting involved with
+Zulip](https://github.com/zulip/zulip#how-to-get-involved-with-contributing-to-zulip)).
+
+Without further ado, below is the Zulip 2016 roadmap.
+
+## Burning problems
+
+The top problem for the Zulip project is the state of the mobile apps.
+The Android app has started seeing rapid progress thanks to a series
+of contributions by Lisa Neigut of Recurse Center, and we believe to
+be on a good path.  The iOS app has fewer features than Android and
+has more bugs, but more importantly is in need of an experienced iOS
+developer who has time to drive the project.
+
+## Core User Experience
+
+This category includes important improvements to the core user
+experience that will benefit all users.
+
+* [Improve missed message notifications to make "reply" work nicely](https://github.com/zulip/zulip/issues/612)
+* [Add support for showing "user is typing" notifications](https://github.com/zulip/zulip/issues/150)
+* [Add pretty bubbles for recipients in the compose box](https://github.com/zulip/zulip/issues/595)
+* [Finish and merge support for pinning a few important streams](https://github.com/zulip/zulip/issues/285)
+* [Display stream descriptions more prominently](https://github.com/zulip/zulip/issues/164)
+* [Integration inline URL previews](https://github.com/zulip/zulip/issues/406)
+* [Add support for managing uploaded files](https://github.com/zulip/zulip/issues/454)
+* [Make Zulip onboarding experience smoother for teams not used to topics](https://github.com/zulip/zulip/issues/647).  That specific proposal might not be right but the issue is worth investing time in.
+
+## Ease of setup and onboarding issues
+
+This category focuses on issues users experience when installing a new
+Zulip server or setting up a new Zulip realm.
+
+* [Create a web flow for setting up a new realm / the first realm on a new server (currently, it's a command-line process)](https://github.com/zulip/zulip/issues/260)
+* [Document or better script solution to rabbitmq startup issues](https://github.com/zulip/zulip/issues/465)
+* [Add a mechanism for deleting early test messages](https://github.com/zulip/zulip/issues/135)
+* [Merge a supported way to use Zulip in Docker in production
+  implementation](https://github.com/zulip/zulip/pull/450).
+
+## Internationalization
+
+The core Zulip UI has been mostly translated into 5 languages;
+however, more work is required to make those translations actually
+displayed in the Zulip UI for the users who would benefit from them.
+
+* [Merge support for using translations in Django templates](https://github.com/zulip/zulip/pull/607)
+* [Add text in handlebars templates to translatable string database](https://github.com/zulip/zulip/issues/726)
+* [Merge support for translating text in handlebars](https://github.com/zulip/zulip/issues/726)
+* [Add text in error messages to translatable strings](https://github.com/zulip/zulip/issues/727)
+
+## User Experience at scale
+
+There are a few parts of the Zulip UI which could benefit from
+overhauls designed around making the user experience nice for large
+teams.
+
+* [Make the buddy list work better for large teams](https://github.com/zulip/zulip/issues/236)
+* [Improve @-mentioning syntax based on stronger unique identifiers](https://github.com/zulip/zulip/issues/374)
+* [Show subscriber counts on streams](https://github.com/zulip/zulip/pull/525)
+* [Make the streams page easier to navigate with 100s of streams](https://github.com/zulip/zulip/issues/563)
+* [Add support for filtering long lists of streams](https://github.com/zulip/zulip/issues/565)
+
+## Administration and management
+
+Currently, Zulip has a number of administration features that can be
+controlled only via the command line.
+
+* [Make default streams web-configurable](https://github.com/zulip/zulip/issues/665)
+* [Make realm emoji web-configurable](https://github.com/zulip/zulip/pull/543)
+* [Make realm filters web-configurable](https://github.com/zulip/zulip/pull/544)
+* [Make realm aliases web-configurable](https://github.com/zulip/zulip/pull/651)
+* [Enhance the LDAP integration and make it web-configurable](https://github.com/zulip/zulip/issues/715)
+* [Add a SAML integration for Zulip](https://github.com/zulip/zulip/issues/716)
+* [Improve administrative controls for managing streams](https://github.com/zulip/zulip/issues/425)
+
+## Scalability
+
+Zulip should support 10000 users in a realm and also support smaller
+realms in more resource-constrained environments (probably a good
+initial goal is working well with only 2GB of RAM).
+
+* [Make the Zulip Tornado service support horizontal scaling](https://github.com/zulip/zulip/issues/445)
+* [Make presence system scale well to 10000 users in a realm.](https://github.com/zulip/zulip/issues/728)
+* [Support running queue workers multithreaded in production to
+  decrease minimum memory footprint](https://github.com/zulip/zulip/issues/34)
+
+## Performance
+
+Performance is essential for a communication tool.  While some things
+are already quite good (E.g. narrowing and message sending is speedy),
+this is an area where one can always improve.  There are a few known
+performance opportunities:
+
+* [Migrate to faster jinja2 templating engine](https://github.com/zulip/zulip/issues/620)
+* [Don't load zxcvbn when it isn't needed](https://github.com/zulip/zulip/issues/263)
+* [Optimize the frontend performance of loading the Zulip webapp using profiling](https://github.com/zulip/zulip/issues/714)
+
+## Technology improvements
+
+Zulip should be making use of the best Python/Django tools available.
+
+* [Add support for Zulip running on Python 3](https://github.com/zulip/zulip/issues/256)
+* [Add support for changing users' email addresses](https://github.com/zulip/zulip/issues/734)
+* [Automatic thumbnailing of uploaded images](https://github.com/zulip/zulip/issues/432)
+* [Upgrade Zulip to use Django 1.10 once it is released.  The patches
+  needed to run Zulip were merged into mainline Django in Django 1.10,
+  so this will mean we don't need to use a fork of Django anymore.](https://github.com/zulip/zulip/issues/3)
+
+## Technical Debt
+
+While the Zulip server has a great codebase compared to most projects
+of its size, it takes work to keep it that way.
+
+* [Migrate most web routes to REST API](https://github.com/zulip/zulip/issues/611)
+* [Finish purging global variables from the Zulip javascript](https://github.com/zulip/zulip/issues/610)
+* [Finish deprecating and remove the pre-REST Zulip /send_message API](https://github.com/zulip/zulip/issues/730)
+* [Split Tornado subsystem into a separate Django app](https://github.com/zulip/zulip/issues/729)
+* [Clean up clutter in the root of the zulip.git repository](https://github.com/zulip/zulip/issues/707)
+* [Refactor zulip.css to be broken into components](https://github.com/zulip/zulip/issues/731)
+
+## Deployment and upgrade process
+
+* [Support backwards-incompatible upgrades to Python libraries](https://github.com/zulip/zulip/issues/717)
+* [Minimize the downtime required in Zulip upgrade process](https://github.com/zulip/zulip/issues/646)
+
+## Security
+
+* [Add support for 2-factor authentication on all platforms](https://github.com/zulip/zulip/pull/451)
+* [Add a retention policy feature that automatically deletes old messages](https://github.com/zulip/zulip/issues/106)
+* [Upgrade every Zulip dependency to a modern version](https://github.com/zulip/zulip/issues/717)
+* [The LOCAL_UPLOADS_DIR file uploads backend only supports world-readable uploads](https://github.com/zulip/zulip/issues/320)
+* [Add support for stronger security controls for uploaded files](https://github.com/zulip/zulip/issues/320)
+
+## Testing
+
+* [Extend Zulip's automated test coverage to include all API endpoints](https://github.com/zulip/zulip/issues/732)
+* [Build automated tests for the client API bindings](https://github.com/zulip/zulip/issues/713)
+* [Add Python static type-checking to Zulip using mypy](https://github.com/zulip/zulip/issues/733)
+* [Improve the runtime of Zulip's backend test suite](https://github.com/zulip/zulip/issues/441)
+* [Use caching to make Travis CI runtimes faster](https://github.com/zulip/zulip/issues/712)
+* [Add automated tests for the production upgrade process](https://github.com/zulip/zulip/issues/306)
+* [Improve Travis CI "production" test suite to catch more regressions](https://github.com/zulip/zulip/issues/598)
+
+## Development environment
+
+* [Migrate from jslint to eslint](https://github.com/zulip/zulip/issues/535)
+* [Figure out a nice upgrade process for Zulip Vagrant VMs](https://github.com/zulip/zulip/issues/264)
+* [Overhaul new contributor documentation](https://github.com/zulip/zulip/issues/677)
+* [Replace closure-compiler with a faster minifier toolchain](https://github.com/zulip/zulip/issues/693)
+* [Add support for building frontend features in React](https://github.com/zulip/zulip/issues/694)
+* [Use a javascript bundler like webpack](https://github.com/zulip/zulip/issues/695)
+
+## Documentation
+
+* [Significantly expand documentation of the Zulip API and integrating
+  with Zulip.](https://github.com/zulip/zulip/issues/672)
+* [Expand library of documentation on Zulip's feature set.  Currently
+  most documentation is for either developers or system administrators.](https://github.com/zulip/zulip/issues/675)
+* [Expand developer documentation with more tutorials explaining how to do
+  various types of projects.](https://github.com/zulip/zulip/issues/676)
+* [Overhaul new contributor documentation, especially on coding style,
+  to better highlight and teach the important pieces.](https://github.com/zulip/zulip/issues/677)
+* [Update all screenshots to show the current Zulip UI](https://github.com/zulip/zulip/issues/599)
+
+## Integrations
+
+Integrations are essential to Zulip.  While we currently have a
+reasonably good framework for writing new webhook integrations for
+getting notifications into Zulip, it'd be great to streamline that
+process and make bots that receive messages just as easy to build.
+
+* [Make it super easy to take screenshots for new webhook integrations](https://github.com/zulip/zulip/issues/658)
+* [Add an outgoing webhook integration system](https://github.com/zulip/zulip/issues/735)
+* [Build a framework to cut duplicated code in new webhook integrations](https://github.com/zulip/zulip/issues/660)
+* [Make setting up a new integration a smooth flow](https://github.com/zulip/zulip/issues/692)
+* [Optimize the integration writing documentation to make writing new
+   ones really easy.](https://github.com/zulip/zulip/issues/70)
+
+## Android app
+
+The Zulip Android app is ahead of the iOS app in terms of feature set,
+so this section serves to document the goals for Zulip on mobile.
+
+* [Support using a non-zulip.com server](https://github.com/zulip/zulip-android/issues/1)
+* [Support Google authentication with a non-Zulip.com server](https://github.com/zulip/zulip-android/issues/49)
+* [Add support for narrowing to @-mentions](https://github.com/zulip/zulip-android/issues/39)
+* [Support having multiple Zulip realms open simultaneously](https://github.com/zulip/zulip-android/issues/47)
+* [Build a slick development login page to simplify testing (similar to
+  the development homepage on web)](https://github.com/zulip/zulip-android/issues/48)
+* [Improve the compose box to let you see what you're replying to](https://github.com/zulip/zulip-android/issues/8)
+* [Make it easy to compose messages with mentions, emoji, etc.](https://github.com/zulip/zulip-android/issues/11)
+* [Display unread counts and improve navigation](https://github.com/zulip/zulip-android/issues/57)
+* [Hide messages sent to muted topics](https://github.com/zulip/zulip-android/issues/9)
+* [Fill out documentation to make it easy to get started](https://github.com/zulip/zulip-android/issues/58)
+
+## iOS app
+
+Most of the projects listed under Android apply here as well, but it's
+worth highlighting some areas where iOS is substantially behind
+Android.  The top priority here is recruiting a lead developer for the
+iOS app.  Once we have that resolved, we'll expand our ambitions for
+the app with more specific improvements.
+
+* [iOS app needs maintainer](https://github.com/zulip/zulip-ios/issues/12)
+* [APNS notifications are broken](https://github.com/zulip/zulip/issues/538)
+
+## Desktop apps
+
+The top goal for the desktop apps is to rebuild it in modern toolchain
+(probably Electron) so that it's easy for a wide range of developers
+to contribute to the apps.
+
+* Migrate platform from QT/webkit to Electron
+* Desktop app doesn't recover well from entering the wrong Zulip server
+* Support having multiple Zulip realms open simultaneously
+* Build an efficient process for testing and releasing new versions of
+  the desktop apps
+
+## Community
+
+These don't get GitHub issues since they're not technical projects,
+but they are important goals for the project.
+
+* Setup a Zulip server for the Zulip development community
+* Expand the number of core developers able to do code reviews
+* Expand the number of contributors regularly adding features to Zulip
+* Have a successful summer with Zulip's 3 GSOC students

docs/schema-migrations.md

@@ -0,0 +1,23 @@
+# Schema Migrations
+
+Zulip uses the [standard Django system for doing schema
+migrations](https://docs.djangoproject.com/en/1.8/topics/migrations/).
+There is some example usage in the Zulip new feature tutorial on
+readthedocs.
+
+This page documents some important issues related to writing schema
+migrations.
+
+* **Large tables**: For large tables like Message and UserMessage, you
+  want to take precautions when adding columns to the table,
+  performing data backfills, or building indexes. We have a
+  `zerver/lib/migrate.py` library to help with adding columns and
+  backfilling data. For building indexes on these tables, we should do
+  this using SQL with postgres's CONCURRENTLY keyword.
+
+* **Numbering conflicts across branches**: If you've done your schema
+  change in a branch, and meanwhile another schema change has taken
+  place, Django will now have two migrations with the same number. To
+  fix this, you can just rename the file, as long as no other
+  migrations depend on it (in which case you also need to update the
+  dependencies).

docs/testing.rst

@@ -15,7 +15,7 @@ Schema and initial data changes
 -------------------------------
 
 If you change the database schema or change the initial test data, you
-have have to regenerate the pristine test database by running
+have to regenerate the pristine test database by running
 ``tools/do-destroy-rebuild-test-database``.
 
 Wiping the test databases
@@ -53,8 +53,8 @@ it. On Ubuntu:
 Backend Django tests
 --------------------
 
-These live in ``zerver/tests.py`` and ``zerver/test_*.py``. Run them
-with ``tools/test-backend``.
+These live in ``zerver/tests/tests.py`` and
+``zerver/tests/test_*.py``. Run them with ``tools/test-backend``.
 
 Web frontend black-box casperjs tests
 -------------------------------------
@@ -117,7 +117,7 @@ below:
   collect a series of steps (each being a ``casper.then`` or
   ``casper.wait...`` call).  Then, usually at the end of the test
   file, you'll have a ``casper.run`` call which actually runs that
-  series of steps.  This means that if you If you write code in your
+  series of steps.  This means that if you write code in your
   test file outside a ``casper.then`` or ``casper.wait...`` method, it
   will actually run before all the Casper test steps that are declared
   in the file, which can lead to confusing failures where the new code
@@ -321,3 +321,47 @@ Setting up the manual testing database
 
 Will populate your local database with all the usual accounts plus some
 test messages involving Shakespeare characters.
+
+(This is run automatically as part of the development environment setup
+process.)
+
+Javascript manual testing
+-------------------------
+
+`debug.js` has some tools for profiling Javascript code, including:
+
+- `print_elapsed_time`: Wrap a function with it to print the time that
+  function takes to the javascript console.
+- `IterationProfiler`: Profile part of looping constructs (like a for
+  loop or $.each). You mark sections of the iteration body and the
+  IterationProfiler will sum the costs of those sections over all
+  iterations.
+
+Chrome has a very good debugger and inspector in its developer tools.
+Firebug for Firefox is also pretty good. They both have profilers, but
+Chrome's is a sampling profiler while Firebug's is an instrumenting
+profiler. Using them both can be helpful because they provide
+different information.
+
+Python 3 Compatibility
+======================
+
+Zulip is working on supporting Python 3, and all new code in Zulip
+should be Python 2+3 compatible.  We have converted most of the
+codebase to be compatible with Python 3 using a suite of 2to3
+conversion tools and some manual work.  In order to avoid regressions
+in that compatibility as we continue to develop new features in zulip,
+we have a special tool, `tools/check-py3`, which checks all code for
+Python 3 syntactic compatibility by running a subset of the automated
+migration tools and checking if they trigger any changes.
+`tools/check-py3` is run automatically in Zulip's Travis CI tests to
+avoid any regressions, but is not included in `test-all` since it is
+quite slow.
+
+To run `tooks/check-py3`, you need to install the `modernize` and
+`future` python packages (which are in the development environment's
+`requirements.txt` file).
+
+To run `check-py3` on just the python files in a particular directory,
+you can change the current working directory (e.g. `cd zerver/`) and
+run `check-py3` from there.

docs/translating.md

@@ -0,0 +1,19 @@
+# Translating Zulip
+
+Zulip has full support for unicode, so you can already use your
+preferred language everywhere in Zulip.
+
+To make Zulip even better for users around the world, the Zulip UI is
+being translated into a number of major languages, including Spanish,
+German, French, Chinese, Russian, and Japanese, with varying levels of
+progress.  If you speak a language other than English, your help with
+translating Zulip would be greatly appreciated!
+
+If you're interested in contributing translations to Zulip, join the
+[Zulip project on Transifex](https://www.transifex.com/zulip/zulip/)
+and ask to join any languages you'd like to contribute to (or add new
+ones).  Transifex's notification system sometimes fails to notify the
+maintainers when you ask to join a project, so please send a quick
+email to zulip-core@googlegroups.com when you request to join the
+project or add a language so that we can be sure to accept your
+request to contribute.

frontend_tests/casper_tests/02-narrow.js

@@ -20,7 +20,7 @@ common.then_send_many([
     { stream:  'Verona', subject: 'other subject',
       content: 'test message C' },
 
-    { stream:  'Venice', subject: 'frontend test',
+    { stream:  'Denmark', subject: 'frontend test',
       content: 'other message' },
 
     { recipient: 'cordelia@zulip.com, hamlet@zulip.com',
@@ -83,7 +83,7 @@ function expect_stream_subject() {
 function expect_subject() {
     common.expected_messages('zfilt', [
         'Verona > frontend test',
-        'Venice > frontend test',
+        'Denmark > frontend test',
         'Verona > frontend test'
     ], [
         '<p>test message A</p>',

frontend_tests/casper_tests/11-admin.js

@@ -68,6 +68,29 @@ casper.waitForSelector('.user_row[id="user_new-user-bot@zulip.com"]:not(.deactiv
     casper.test.assertSelectorHasText('.user_row[id="user_new-user-bot@zulip.com"]', 'Deactivate');
 });
 
+// Test custom realm emoji
+casper.waitForSelector('.admin-emoji-form', function () {
+    casper.fill('form.admin-emoji-form', {
+        'name': 'MouseFace',
+        'url': 'http://localhost:9991/static/images/integrations/logos/jenkins.png'
+    });
+    casper.click('form.admin-emoji-form input.btn');
+});
+
+casper.waitUntilVisible('div#admin-emoji-status', function () {
+    casper.test.assertSelectorHasText('div#admin-emoji-status', 'Custom emoji added!');
+});
+
+casper.waitForSelector('.emoji_row', function () {
+    casper.test.assertSelectorHasText('.emoji_row .emoji_name', 'MouseFace');
+    casper.test.assertExists('.emoji_row img[src="http://localhost:9991/static/images/integrations/logos/jenkins.png"]');
+    casper.click('.emoji_row button.delete');
+});
+
+casper.waitWhileSelector('.emoji_row', function () {
+    casper.test.assertDoesntExist('.emoji_row');
+});
+
 // TODO: Test stream deletion
 
 common.then_log_out();

frontend_tests/casperjs/bin/casperjs

@@ -1,5 +1,6 @@
 #!/usr/bin/env python2.7
 
+from __future__ import print_function
 import json
 import os
 import subprocess

frontend_tests/node_tests/echo.js

@@ -3,7 +3,8 @@ var path = require('path');
 var fs = require('fs');
 
 set_global('page_params', {realm_emoji: {
-  burrito: 'static/third/gemoji/images/emoji/burrito.png'
+  burrito: {display_url: 'static/third/gemoji/images/emoji/burrito.png',
+            source_url: 'static/third/gemoji/images/emoji/burrito.png'}
 }});
 
 add_dependencies({

frontend_tests/node_tests/filter.js

@@ -339,7 +339,7 @@ function get_predicate(operators) {
         assert_same_operators(result, operators);
     }
 
-    string ='stream:Foo topic:Bar yo';
+    string = 'stream:Foo topic:Bar yo';
     operators = [
         {operator: 'stream', operand: 'Foo'},
         {operator: 'topic', operand: 'Bar'},

frontend_tests/node_tests/message_list.js

@@ -18,8 +18,9 @@ set_global('$', function () {
 });
 
 set_global('feature_flags', {});
+set_global('Filter', function () {});
 
-var MessageList = require('js/message_list');
+var MessageList = require('js/message_list').MessageList;
 
 (function test_basics() {
     var table;
@@ -60,7 +61,7 @@ var MessageList = require('js/message_list');
     assert.equal(list.closest_id(60), 60);
     assert.equal(list.closest_id(61), 60);
 
-    assert.deepEqual(list.all(), messages);
+    assert.deepEqual(list.all_messages(), messages);
 
     global.$.Event = function (ev) {
         assert.equal(ev, 'message_selected.zulip');
@@ -95,13 +96,13 @@ var MessageList = require('js/message_list');
     list.view.clear_table = function () {};
 
     list.remove_and_rerender([{id: 60}]);
-    var removed = list.all().filter(function (msg) {
+    var removed = list.all_messages().filter(function (msg) {
         return msg.id !== 60;
     });
-    assert.deepEqual(list.all(), removed);
+    assert.deepEqual(list.all_messages(), removed);
 
     list.clear();
-    assert.deepEqual(list.all(), []);
+    assert.deepEqual(list.all_messages(), []);
 
 }());
 
@@ -165,4 +166,4 @@ var MessageList = require('js/message_list');
     assert.equal(list.closest_id(51), 50.02);
     assert.equal(list.closest_id(59), 60);
     assert.equal(list.closest_id(50.01), 50.01);
-}());
+}());

frontend_tests/node_tests/muting.js

@@ -1,6 +1,9 @@
 set_global('page_params', {
     domain: 'zulip.com'
 });
+add_dependencies({
+    unread: 'js/unread.js'
+});
 
 var muting = require('js/muting.js');
 

frontend_tests/node_tests/presence_list_performance.js

@@ -0,0 +1,121 @@
+set_global('$', function () {});
+set_global('document', {
+    hasFocus: function () {
+        return true;
+    }
+});
+set_global('feature_flags', {});
+set_global('page_params', {
+    people_list: []
+});
+
+
+add_dependencies({
+   Handlebars: 'handlebars',
+   templates: 'js/templates',
+   util: 'js/util.js',
+   compose_fade: 'js/compose_fade.js',
+   people: 'js/people.js',
+   unread: 'js/unread.js',
+   activity: 'js/activity.js'
+});
+
+var compose_fade = require('js/compose_fade.js');
+compose_fade.update_faded_users = function () {
+   return;
+};
+
+global.$ = require('jQuery');
+$.fn.expectOne = function () {
+    assert(this.length === 1);
+    return this;
+};
+
+global.use_template('user_presence_row');
+global.use_template('user_presence_rows');
+
+var people = require("js/people.js");
+var activity = require('js/activity.js');
+activity.presence_info = {
+    'alice@zulip.com': {status: activity.IDLE},
+    'fred@zulip.com': {status: activity.ACTIVE},
+    'jill@zulip.com': {status: activity.ACTIVE},
+    'mark@zulip.com': {status: activity.IDLE},
+    'norbert@zulip.com': {status: activity.ACTIVE}
+};
+
+(function test_presence_list_full_update() {
+    var users = activity.update_users();
+    assert.deepEqual(users, [
+        { name: 'Fred Flintstone',
+          email: 'fred@zulip.com',
+          num_unread: 0,
+          type: 'active',
+          type_desc: 'is active',
+          mobile: undefined },
+        { name: 'Jill Hill',
+          email: 'jill@zulip.com',
+          num_unread: 0,
+          type: 'active',
+          type_desc: 'is active',
+          mobile: undefined },
+        { name: 'Norbert Oswald',
+          email: 'norbert@zulip.com',
+          num_unread: 0,
+          type: 'active',
+          type_desc: 'is active',
+          mobile: undefined },
+        { name: 'Alice Smith',
+          email: 'alice@zulip.com',
+          num_unread: 0,
+          type: 'idle',
+          type_desc: 'is not active',
+          mobile: undefined },
+        { name: 'Marky Mark',
+          email: 'mark@zulip.com',
+          num_unread: 0,
+          type: 'idle',
+          type_desc: 'is not active',
+          mobile: undefined }
+    ]);
+}());
+
+(function test_presence_list_partial_update() {
+    var users = {
+        'alice@zulip.com': {status: 'active'}
+    };
+    activity.presence_info['alice@zulip.com'] = users['alice@zulip.com'];
+
+    users = activity.update_users(users);
+    assert.deepEqual(users, [
+        { name: 'Alice Smith',
+          email: 'alice@zulip.com',
+          num_unread: 0,
+          type: 'active',
+          type_desc: 'is active',
+          mobile: undefined }
+    ]);
+
+    // Test if user index in presence_info is the expected one
+    var all_users = activity._filter_and_sort(activity.presence_info);
+    assert.equal(all_users.indexOf('alice@zulip.com'), 0);
+
+    // Test another user
+    users = {
+        'mark@zulip.com': {status: 'active'}
+    };
+    activity.presence_info['mark@zulip.com'] = users['mark@zulip.com'];
+    users = activity.update_users(users);
+    assert.deepEqual(users, [
+        { name: 'Marky Mark',
+          email: 'mark@zulip.com',
+          num_unread: 0,
+          type: 'active',
+          type_desc: 'is active',
+          mobile: undefined }
+    ]);
+
+    all_users = activity._filter_and_sort(activity.presence_info);
+    assert.equal(all_users.indexOf('mark@zulip.com'), 3);
+
+}());

frontend_tests/node_tests/server_events.js

@@ -29,7 +29,9 @@ set_global('home_msg_list', {
     selected_id: function () {return 1;}
 });
 set_global('page_params', {test_suite: false});
-
+set_global('reload', {
+    is_in_progress: function () {return false;}
+});
 
 var server_events = require('js/server_events.js');
 

frontend_tests/node_tests/templates.js

@@ -630,6 +630,7 @@ function render(template_name, args) {
 }());
 
 (function user_presence_rows() {
+    global.use_template('user_presence_row'); // partial
     var args = {
         users: [
             {
@@ -727,6 +728,30 @@ function render(template_name, args) {
 
 }());
 
+(function admin_emoji_list() {
+    global.use_template('admin_emoji_list');
+    var args = {
+        emoji: {
+            "name": "MouseFace",
+            "display_url": "http://emojipedia-us.s3.amazonaws.com/cache/46/7f/467fe69069c408e07517621f263ea9b5.png",
+            "source_url": "http://emojipedia-us.s3.amazonaws.com/cache/46/7f/467fe69069c408e07517621f263ea9b5.png"
+        }
+    };
+
+    var html = '';
+    html += '<tbody id="admin_emoji_table">';
+    html += render('admin_emoji_list', args);
+    html += '</tbody>';
+
+    global.write_test_output('admin_emoji_list.handlebars', html);
+
+    var emoji_name = $(html).find('tr.emoji_row:first span.emoji_name');
+    var emoji_url = $(html).find('tr.emoji_row:first span.emoji_image img');
+
+    assert.equal(emoji_name.text(), 'MouseFace');
+    assert.equal(emoji_url.attr('src'), 'http://emojipedia-us.s3.amazonaws.com/cache/46/7f/467fe69069c408e07517621f263ea9b5.png');
+}());
+
 // By the end of this test, we should have compiled all our templates.  Ideally,
 // we will also have exercised them to some degree, but that's a little trickier
 // to enforce.

frontend_tests/node_tests/unread.js

@@ -8,7 +8,8 @@
 // dependencies (except _).
 
 add_dependencies({
-    muting: 'js/muting.js'
+    muting: 'js/muting.js',
+    unread: 'js/unread.js'
 });
 
 var stream_data = require('js/stream_data.js');
@@ -45,7 +46,7 @@ var zero_counts = {
     narrow.active = function () {
         return true;
     };
-    current_msg_list.all = function () {
+    current_msg_list.all_messages = function () {
         return [];
     };
 
@@ -57,7 +58,7 @@ var zero_counts = {
     narrow.active = function () {
         return false;
     };
-    current_msg_list.all = function () {
+    current_msg_list.all_messages = function () {
         return [];
     };
 
@@ -362,7 +363,7 @@ var zero_counts = {
     var message = {
         id: 15
     };
-    current_msg_list.all = function () {
+    current_msg_list.all_messages = function () {
         return [message];
     };
 

frontend_tests/run-casper

@@ -1,4 +1,5 @@
 #!/usr/bin/env python2.7
+from __future__ import print_function
 import subprocess
 import requests
 import optparse
@@ -50,7 +51,7 @@ server = subprocess.Popen(('tools/run-dev.py', '--test'),
 def assert_server_running():
     # Get the exit code of the server, or None if it is still running.
     if server.poll() is not None:
-        raise RuntimeError, 'Server died unexpectedly! Check frontend_tests/casper_tests/server.log'
+        raise RuntimeError('Server died unexpectedly! Check frontend_tests/casper_tests/server.log')
 
 def server_is_up():
     assert_server_running()
@@ -80,18 +81,18 @@ try:
         cmd += ' '.join(test_files)
     else:
         cmd += 'frontend_tests/casper_tests'
-    print "Running %s" % (cmd,)
+    print("Running %s" % (cmd,))
     ret = subprocess.call(cmd, shell=True)
 finally:
     assert_server_running()
     server.terminate()
 
 if ret != 0:
-    print >>sys.stderr, """
+    print("""
 Oops, the frontend tests failed. Tips for debugging:
  * Check the frontend test server logs at frontend_tests/casper_tests/server.log
  * Check the screenshots of failed tests at /tmp/casper-failure*.png
  * Try remote debugging the test web browser as described in docs/testing.rst
-"""
+""", file=sys.stderr)
 
 sys.exit(ret)

manage.py

@@ -9,6 +9,7 @@ if __name__ == "__main__":
         from django.core.management.base import CommandError
         raise CommandError("manage.py should not be run as root.")
     os.environ.setdefault("DJANGO_SETTINGS_MODULE", "zproject.settings")
+    os.environ.setdefault("PYTHONSTARTUP", os.path.join(os.path.dirname(__file__), "scripts/lib/pythonrc.py"))
 
     from django.conf import settings
 

provision.py

@@ -3,66 +3,96 @@ import os
 import sys
 import logging
 import platform
+import subprocess
 
-try:
-    import sh
-except ImportError:
-    import pbs as sh
+os.environ["PYTHONUNBUFFERED"] = "y"
+
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+from zulip_tools import run
 
 SUPPORTED_PLATFORMS = {
     "Ubuntu": [
         "trusty",
+        "xenial",
     ],
 }
 
-APT_DEPENDENCIES = {
-    "trusty": [
-        "closure-compiler",
-        "libfreetype6-dev",
-        "libffi-dev",
-        "memcached",
-        "rabbitmq-server",
-        "libldap2-dev",
-        "redis-server",
-        "postgresql-server-dev-all",
-        "libmemcached-dev",
-        "postgresql-9.3",
-        "python-dev",
-        "hunspell-en-us",
-        "nodejs",
-        "nodejs-legacy",
-        "python-virtualenv",
-        "supervisor",
-        "git",
-        "npm",
-        "yui-compressor",
-        "puppet",               # Used by lint-all
-        "gettext",              # Used by makemessages i18n
-    ]
-}
+VENV_PATH = "/srv/zulip-venv"
+PY3_VENV_PATH = "/srv/zulip-py3-venv"
+ZULIP_PATH = os.path.dirname(os.path.abspath(__file__))
 
-VENV_PATH="/srv/zulip-venv"
-ZULIP_PATH="/srv/zulip"
-
-if not os.path.exists(os.path.join(os.path.dirname(__file__), ".git")):
-    print("Error: No Zulip git repository present at /srv/zulip!")
+if not os.path.exists(os.path.join(ZULIP_PATH, ".git")):
+    print("Error: No Zulip git repository present!")
     print("To setup the Zulip development environment, you should clone the code")
     print("from GitHub, rather than using a Zulip production release tarball.")
     sys.exit(1)
 
-# TODO: Parse arguments properly
-if "--travis" in sys.argv or "--docker" in sys.argv:
-    ZULIP_PATH="."
+if platform.architecture()[0] == '64bit':
+    arch = 'amd64'
+elif platform.architecture()[0] == '32bit':
+    arch = "i386"
+else:
+    logging.critical("Only x86 is supported; ping zulip-devel@googlegroups.com if you want another architecture.")
+    sys.exit(1)
+
+# Ideally we wouldn't need to install a dependency here, before we
+# know the codename.
+subprocess.check_call(["sudo", "apt-get", "install", "-y", "lsb-release"])
+vendor = subprocess.check_output(["lsb_release", "-is"]).strip()
+codename = subprocess.check_output(["lsb_release", "-cs"]).strip()
+if not (vendor in SUPPORTED_PLATFORMS and codename in SUPPORTED_PLATFORMS[vendor]):
+    logging.critical("Unsupported platform: {} {}".format(vendor, codename))
+    sys.exit(1)
+
+POSTGRES_VERSION_MAP = {
+    "trusty": "9.3",
+    "xenial": "9.5",
+}
+POSTGRES_VERSION = POSTGRES_VERSION_MAP[codename]
+
+UBUNTU_COMMON_APT_DEPENDENCIES = [
+    "closure-compiler",
+    "libfreetype6-dev",
+    "libffi-dev",
+    "memcached",
+    "rabbitmq-server",
+    "libldap2-dev",
+    "redis-server",
+    "postgresql-server-dev-all",
+    "libmemcached-dev",
+    "python-dev",
+    "hunspell-en-us",
+    "nodejs",
+    "nodejs-legacy",
+    "python-virtualenv",
+    "supervisor",
+    "git",
+    "npm",
+    "yui-compressor",
+    "wget",
+    "ca-certificates",      # Explicit dependency in case e.g. wget is already installed
+    "puppet",               # Used by lint-all
+    "gettext",              # Used by makemessages i18n
+    "curl",                 # Used for fetching PhantomJS as wget occasionally fails on redirects
+    "netcat",               # Used for flushing memcached
+]
+
+APT_DEPENDENCIES = {
+    "trusty": UBUNTU_COMMON_APT_DEPENDENCIES + [
+        "postgresql-9.3",
+    ],
+    "xenial": UBUNTU_COMMON_APT_DEPENDENCIES + [
+        "postgresql-9.5",
+    ],
+}
 
 # tsearch-extras is an extension to postgres's built-in full-text search.
 # TODO: use a real APT repository
-TSEARCH_URL_BASE = "https://dl.dropboxusercontent.com/u/283158365/zuliposs/"
-TSEARCH_PACKAGE_NAME = {
-    "trusty": "postgresql-9.3-tsearch-extras"
-}
-TSEARCH_VERSION = "0.1.2"
-# TODO: this path is platform-specific!
-TSEARCH_STOPWORDS_PATH = "/usr/share/postgresql/9.3/tsearch_data/"
+TSEARCH_URL_PATTERN = "https://github.com/zulip/zulip-dist-tsearch-extras/raw/master/{}_{}_{}.deb?raw=1"
+TSEARCH_PACKAGE_NAME = "postgresql-%s-tsearch-extras" % (POSTGRES_VERSION,)
+TSEARCH_VERSION = "0.1.3"
+TSEARCH_URL = TSEARCH_URL_PATTERN.format(TSEARCH_PACKAGE_NAME, TSEARCH_VERSION, arch)
+TSEARCH_STOPWORDS_PATH = "/usr/share/postgresql/%s/tsearch_data/" % (POSTGRES_VERSION,)
 REPO_STOPWORDS_PATH = os.path.join(
     ZULIP_PATH,
     "puppet",
@@ -74,68 +104,24 @@ REPO_STOPWORDS_PATH = os.path.join(
 
 LOUD = dict(_out=sys.stdout, _err=sys.stderr)
 
-
 def main():
-    log = logging.getLogger("zulip-provisioner")
-    # TODO: support other architectures
-    if platform.architecture()[0] == '64bit':
-        arch = 'amd64'
-    else:
-        log.critical("Only amd64 is supported.")
+    run(["sudo", "apt-get", "update"])
+    run(["sudo", "apt-get", "-y", "install"] + APT_DEPENDENCIES[codename])
 
-    vendor, version, codename = platform.dist()
+    temp_deb_path = subprocess.check_output(["mktemp", "package_XXXXXX.deb", "--tmpdir"])
+    run(["wget", "-O", temp_deb_path, TSEARCH_URL])
+    run(["sudo", "dpkg", "--install", temp_deb_path])
 
-    if not (vendor in SUPPORTED_PLATFORMS and codename in SUPPORTED_PLATFORMS[vendor]):
-        log.critical("Unsupported platform: {} {}".format(vendor, codename))
+    run(["sudo", "rm", "-rf", VENV_PATH])
+    run(["sudo", "mkdir", "-p", VENV_PATH])
+    run(["sudo", "chown", "{}:{}".format(os.getuid(), os.getgid()), VENV_PATH])
 
-    with sh.sudo:
-        sh.apt_get.update(**LOUD)
-
-        sh.apt_get.install(*APT_DEPENDENCIES["trusty"], assume_yes=True, **LOUD)
-
-    temp_deb_path = sh.mktemp("package_XXXXXX.deb", tmpdir=True)
-
-    sh.wget(
-        "{}/{}_{}_{}.deb".format(
-            TSEARCH_URL_BASE,
-            TSEARCH_PACKAGE_NAME["trusty"],
-            TSEARCH_VERSION,
-            arch,
-        ),
-        output_document=temp_deb_path,
-        **LOUD
-    )
-
-    with sh.sudo:
-        sh.dpkg("--install", temp_deb_path, **LOUD)
-
-    with sh.sudo:
-        PHANTOMJS_PATH = "/srv/phantomjs"
-        PHANTOMJS_TARBALL = os.path.join(PHANTOMJS_PATH, "phantomjs-1.9.8-linux-x86_64.tar.bz2")
-        sh.mkdir("-p", PHANTOMJS_PATH, **LOUD)
-        if not os.path.exists(PHANTOMJS_TARBALL):
-            sh.wget("https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.8-linux-x86_64.tar.bz2",
-                    output_document=PHANTOMJS_TARBALL, **LOUD)
-        sh.tar("xj", directory=PHANTOMJS_PATH, file=PHANTOMJS_TARBALL, **LOUD)
-        sh.ln("-sf", os.path.join(PHANTOMJS_PATH, "phantomjs-1.9.8-linux-x86_64", "bin", "phantomjs"),
-              "/usr/local/bin/phantomjs", **LOUD)
-
-    with sh.sudo:
-        sh.rm("-rf", VENV_PATH, **LOUD)
-        sh.mkdir("-p", VENV_PATH, **LOUD)
-        sh.chown("{}:{}".format(os.getuid(), os.getgid()), VENV_PATH, **LOUD)
-
-    sh.virtualenv(VENV_PATH, **LOUD)
-
-    # Add the ./tools and ./scripts/setup directories inside the repository root to
-    # the system path; we'll reference them later.
-    orig_path = os.environ["PATH"]
-    os.environ["PATH"] = os.pathsep.join((
-            os.path.join(ZULIP_PATH, "tools"),
-            os.path.join(ZULIP_PATH, "scripts", "setup"),
-            orig_path
-    ))
+    run(["sudo", "rm", "-rf", PY3_VENV_PATH])
+    run(["sudo", "mkdir", "-p", PY3_VENV_PATH])
+    run(["sudo", "chown", "{}:{}".format(os.getuid(), os.getgid()), PY3_VENV_PATH])
 
+    run(["virtualenv", VENV_PATH])
+    run(["virtualenv", "-p", "python3", PY3_VENV_PATH])
 
     # Put Python virtualenv activation in our .bash_profile.
     with open(os.path.expanduser('~/.bash_profile'), 'w+') as bash_profile:
@@ -144,38 +130,54 @@ def main():
             "source %s\n" % (os.path.join(VENV_PATH, "bin", "activate"),),
         ])
 
-    # Switch current Python context to the virtualenv.
+    # Switch current Python context to the python3 virtualenv
+    activate_this = os.path.join(PY3_VENV_PATH, "bin", "activate_this.py")
+    execfile(activate_this, dict(__file__=activate_this))
+
+    run(["pip", "install", "--upgrade", "pip"])
+    # install requirement
+    run(["pip", "install", "--no-deps", "--requirement",
+         os.path.join(ZULIP_PATH, "tools", "py3_test_reqs.txt")])
+
+    # Switch current Python context to the python2 virtualenv.
     activate_this = os.path.join(VENV_PATH, "bin", "activate_this.py")
     execfile(activate_this, dict(__file__=activate_this))
 
-    sh.pip.install(requirement=os.path.join(ZULIP_PATH, "requirements.txt"), **LOUD)
+    run(["pip", "install", "--upgrade", "pip"])
+    run(["pip", "install", "--no-deps", "--requirement",
+         os.path.join(ZULIP_PATH, "requirements.txt")])
 
-    with sh.sudo:
-        sh.cp(REPO_STOPWORDS_PATH, TSEARCH_STOPWORDS_PATH, **LOUD)
+    run(["sudo", "cp", REPO_STOPWORDS_PATH, TSEARCH_STOPWORDS_PATH])
 
-    # npm install and management commands expect to be run from the root of the project.
+    # npm install and management commands expect to be run from the root of the
+    # project.
     os.chdir(ZULIP_PATH)
 
-    sh.npm.install(**LOUD)
-
-    os.system("tools/download-zxcvbn")
-    os.system("tools/emoji_dump/build_emoji")
-    os.system("generate_secrets.py -d")
+    run(["tools/install-phantomjs"])
+    run(["tools/download-zxcvbn"])
+    run(["tools/emoji_dump/build_emoji"])
+    run(["scripts/setup/generate_secrets.py", "-d"])
     if "--travis" in sys.argv:
-        os.system("sudo service rabbitmq-server restart")
-        os.system("sudo service redis-server restart")
-        os.system("sudo service memcached restart")
+        run(["sudo", "service", "rabbitmq-server", "restart"])
+        run(["sudo", "service", "redis-server", "restart"])
+        run(["sudo", "service", "memcached", "restart"])
     elif "--docker" in sys.argv:
-        os.system("sudo service rabbitmq-server restart")
-        os.system("sudo pg_dropcluster --stop 9.3 main")
-        os.system("sudo pg_createcluster -e utf8 --start 9.3 main")
-        os.system("sudo service redis-server restart")
-        os.system("sudo service memcached restart")
-    sh.configure_rabbitmq(**LOUD)
-    sh.postgres_init_dev_db(**LOUD)
-    sh.do_destroy_rebuild_database(**LOUD)
-    sh.postgres_init_test_db(**LOUD)
-    sh.do_destroy_rebuild_test_database(**LOUD)
+        run(["sudo", "service", "rabbitmq-server", "restart"])
+        run(["sudo", "pg_dropcluster", "--stop", POSTGRES_VERSION, "main"])
+        run(["sudo", "pg_createcluster", "-e", "utf8", "--start", POSTGRES_VERSION, "main"])
+        run(["sudo", "service", "redis-server", "restart"])
+        run(["sudo", "service", "memcached", "restart"])
+    run(["scripts/setup/configure-rabbitmq"])
+    run(["tools/postgres-init-dev-db"])
+    run(["tools/do-destroy-rebuild-database"])
+    run(["tools/postgres-init-test-db"])
+    run(["tools/do-destroy-rebuild-test-database"])
+    # Install the latest npm.
+    run(["sudo", "npm", "install", "-g", "npm"])
+    # Run npm install last because it can be flaky, and that way one
+    # only needs to rerun `npm install` to fix the installation.
+    run(["npm", "install"])
+    return 0
 
 if __name__ == "__main__":
     sys.exit(main())

puppet/apt/metadata.json

@@ -84,4 +84,4 @@
   "source": "https://github.com/puppetlabs/puppetlabs-apt",
   "project_page": "https://github.com/puppetlabs/puppetlabs-apt",
   "license": "Apache License 2.0"
-}
+}

puppet/stdlib/metadata.json

@@ -267,4 +267,4 @@
   ],
   "author": "puppetlabs",
   "name": "puppetlabs-stdlib"
-}
+}

puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_queue_worker_errors

@@ -3,6 +3,7 @@
 """
 Nagios plugin to check that none of our queue workers have reported errors.
 """
+from __future__ import print_function
 
 import sys
 sys.path.append('/home/zulip/deployments/current')

puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_rabbitmq_consumers

@@ -8,6 +8,7 @@ which is generated by bots/check-rabbitmq-consumers.
 
 It is run by cron and can be found at bots/rabbitmq-numconsumers-crontab
 """
+from __future__ import print_function
 
 import sys
 
@@ -15,12 +16,12 @@ sys.path.append('/home/zulip/deployments/current')
 from bots.cron_file_helper import nagios_from_file
 
 if len(sys.argv) < 2:
-    print "Please pass the name of the consumer file to check"
+    print("Please pass the name of the consumer file to check")
     exit(1)
 
 RESULTS_FILE = "/var/lib/nagios_state/check-rabbitmq-consumers-%s" % (sys.argv[1])
 
 ret, result = nagios_from_file(RESULTS_FILE)
 
-print result
+print(result)
 exit(ret)

puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_rabbitmq_queues

@@ -9,6 +9,7 @@ which is generated by bots/check-rabbitmq-queue.
 
 It is run by cron and can be found at bots/rabbitmq-queuesize-crontab
 """
+from __future__ import print_function
 
 import sys
 
@@ -18,5 +19,5 @@ from bots.cron_file_helper import nagios_from_file
 RESULTS_FILE = "/var/lib/nagios_state/check-rabbitmq-results"
 ret, result = nagios_from_file(RESULTS_FILE)
 
-print result
+print(result)
 exit(ret)

puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_send_receive_time

@@ -8,6 +8,8 @@ It supports both munin and nagios outputs
 It must be run on a machine that is using the live database for the
 Django ORM.
 """
+from __future__ import print_function
+from __future__ import division
 
 import datetime
 import sys
@@ -42,23 +44,22 @@ parser.add_option('--munin',
 (options, args) = parser.parse_args()
 
 if not options.nagios and not options.munin:
-    print 'No output options specified! Please provide --munin or --nagios'
+    print('No output options specified! Please provide --munin or --nagios')
     sys.exit(0)
 
 if len(args) > 2:
-    print usage
+    print(usage)
     sys.exit(0)
 
 if options.munin:
     if len(args) and args[0] == 'config':
-        print \
-"""graph_title Send-Receive times
+        print("""graph_title Send-Receive times
 graph_info The number of seconds it takes to send and receive a message from the server
 graph_args -u 5 -l 0
 graph_vlabel RTT (seconds)
 sendreceive.label Send-receive round trip time
 sendreceive.warning 3
-sendreceive.critical 5"""
+sendreceive.critical 5""")
         sys.exit(0)
 
 sys.path.append('/home/zulip/deployments/current/api')
@@ -81,9 +82,9 @@ states = {
 
 def report(state, time, msg=None):
     if msg:
-        print "%s: %s" % (state, msg)
+        print("%s: %s" % (state, msg))
     else:
-        print "%s: send time was %s" % (state, time)
+        print("%s: send time was %s" % (state, time))
     exit(states[state])
 
 def send_zulip(sender, message):
@@ -148,7 +149,7 @@ while msg_to_send not in msg_content:
 
     msg_content = [m['content'] for m in messages]
 
-print zulip_recipient.deregister(queue_id)
+print(zulip_recipient.deregister(queue_id))
 
 if options.nagios:
     if time_diff.seconds > 3:
@@ -157,6 +158,6 @@ if options.nagios:
         report('CRITICAL', time_diff)
 
 if options.munin:
-    print "sendreceive.value %s" % total_seconds(time_diff)
+    print("sendreceive.value %s" % total_seconds(time_diff))
 elif options.nagios:
     report('OK', time_diff)

puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_worker_memory

@@ -1,7 +1,7 @@
 #!/bin/bash
 # Checks for any Zulip queue workers that are leaking memory and thus have a high vsize
 datafile=$(mktemp)
-ps -o vsize,size,pid,user,command --sort -vsize "$(pgrep -f '^python /home/zulip/deployments/current/manage.py process_queue')" > "$datafile"
+ps -o vsize,size,pid,user,command --sort -vsize $(pgrep -f '^python.* /home/zulip/deployments/current/manage.py process_queue') > "$datafile"
 cat "$datafile"
 top_worker=$(cat "$datafile" | head -n2 | tail -n1)
 top_worker_memory_usage=$(echo "$top_worker" | cut -f1 -d" ")

puppet/zulip/files/nagios_plugins/zulip_nagios_server/check_postgres_replication_lag

@@ -4,6 +4,7 @@
 Nagios plugin to check the difference between the primary and
 secondary Postgres servers' xlog location.
 """
+from __future__ import print_function
 
 import subprocess
 import re
@@ -16,7 +17,7 @@ states = {
     }
 
 def report(state, msg):
-    print "%s: %s" % (state, msg)
+    print("%s: %s" % (state, msg))
     exit(states[state])
 
 def get_loc_over_ssh(host, func):
@@ -25,7 +26,7 @@ def get_loc_over_ssh(host, func):
                                         'psql -t -c "SELECT %s()"' % (func,)],
                                        stderr=subprocess.STDOUT)
     except subprocess.CalledProcessError as e:
-        report('CRITICAL', 'ssh failed: %s: %s' % (str(e),e.output))
+        report('CRITICAL', 'ssh failed: %s: %s' % (str(e), e.output))
 
 def loc_to_abs_offset(loc_str):
     m = re.match(r'^\s*([0-9a-fA-F]+)/([0-9a-fA-F]+)\s*$', loc_str)

puppet/zulip/files/nagios_plugins/zulip_nagios_server/check_website_response.sh

@@ -5,7 +5,7 @@
 # Version 1.1
 # (c) GPLv2 2011
 #
-# Special thanks to dkwiebe and Konstantine Vinogradov for suggestions and feedback 
+# Special thanks to dkwiebe and Konstantine Vinogradov for suggestions and feedback
 #
 
 
@@ -24,7 +24,7 @@ WGETOUT=/tmp/wgetoutput
 ### Functions
 # Check dependencies and paths
 checkpaths(){
-	for PATH in $NETCAT $DATE $WGET $ECHO $AWK $CKSUM $TR; do 
+	for PATH in $NETCAT $DATE $WGET $ECHO $AWK $CKSUM $TR; do
 		if [ ! -f "$PATH" ]; then
 			STATUS=UNKNOWN
 			OUTMSG="ERROR: $PATH does does not exist"
@@ -80,7 +80,7 @@ usage(){
 checkopen(){
 	# Determine PORT from scheme
 	SCHEME=`$ECHO $URL |$AWK -F: '{print $1}'| $TR [:upper:] [:lower:]`
-	
+
 	# Strip scheme out of URL
 	case $URL in
 		*://*)
@@ -88,13 +88,13 @@ checkopen(){
 		*)
 			SHORTURL=$URL;;
 	esac
-	
+
 	# Strip path out of URL
 	case $SHORTURL in
 		*/*)
 			SHORTURL=`$ECHO $SHORTURL |$AWK -F/ '{print $1}'`;;
 	esac
-	
+
 	# if no scheme check for ports in SHORTURL or else default to 80
 	case $SHORTURL in
 		*:*@*:*)
@@ -121,14 +121,14 @@ checkopen(){
 				PORT=80
 			fi;;
 	esac
-	
+
 	# Check if URL resolves and port is open
 	if ! $NETCAT -z $SHORTURL $PORT > /dev/null 2>&1; then
 		OUTMSG="URL $SHORTURL can't resolve or port $PORT not open"
 		STATUS=CRITICAL
 		output
 	fi
-	
+
 	# Check if page can be loaded and contains data
 	if [ -n "$NOCERT" ]; then
 		$WGET --no-check-certificate -q -O $WGETOUTCKSUM $URL
@@ -155,7 +155,7 @@ pageload(){
                 ENDTIME=$($DATE +%s%N)
 	fi
 	TIMEDIFF=$((($ENDTIME-$STARTTIME)/1000000))
-	if [ "$TIMEDIFF" -lt "$WARN" ]; then 
+	if [ "$TIMEDIFF" -lt "$WARN" ]; then
 		STATUS=OK
 	elif [ "$TIMEDIFF" -ge "$WARN" ] && [ "$TIMEDIFF" -lt "$CRIT" ]; then
 		STATUS=WARNING
@@ -167,7 +167,7 @@ pageload(){
 
 # Output statement and exit
 output(){
-	$ECHO "RESPONSE: $STATUS - $OUTMSG""|Response="$TIMEDIFF"ms;"$WARN";"$CRIT";0" 
+	$ECHO "RESPONSE: $STATUS - $OUTMSG""|Response="$TIMEDIFF"ms;"$WARN";"$CRIT";0"
 	if [ "$STATUS" = "OK" ]; then
 		exit 0
 	elif [ "$STATUS" = "WARNING" ]; then

puppet/zulip/files/nagios_plugins/zulip_postgres_appdb/check_fts_update_log

@@ -3,6 +3,7 @@
 """
 Nagios plugin to check the length of the FTS update log.
 """
+from __future__ import print_function
 
 import psycopg2
 
@@ -14,7 +15,7 @@ states = {
     }
 
 def report(state, num):
-    print "%s: %s rows in fts_update_log table" % (state, num)
+    print("%s: %s rows in fts_update_log table" % (state, num))
     exit(states[state])
 
 conn = psycopg2.connect("user=zulip")

puppet/zulip/files/nagios_plugins/zulip_postgres_common/check_postgres_backup

@@ -1,5 +1,6 @@
 #!/usr/bin/env python2.7
 
+from __future__ import print_function
 import dateutil.parser
 import pytz
 import subprocess
@@ -13,7 +14,7 @@ states = {
     }
 
 def report(state, msg):
-    print "%s: %s" % (state, msg)
+    print("%s: %s" % (state, msg))
     exit(states[state])
 
 if subprocess.check_output(['psql', 'postgres', '-t', '-c',

puppet/zulip/files/nginx/zulip-include-app.d/camo.conf

@@ -0,0 +1,7 @@
+# Proxies /external_content to a local installation of the camo image
+# proxy software
+location /external_content {
+    rewrite /external_content/(.*) /$1 break;
+    proxy_pass http://camo;
+    include /etc/nginx/zulip-include/proxy;
+}