Add changelog for Zulip 1.3.10 release.

This fixes an issue where the output that the lock wasn't unavailable
showed up all at once after the 5 minute timeout.

.gitattributes

@@ -15,6 +15,6 @@
 /zproject/test_settings.py export-ignore
 /zerver/fixtures export-ignore
 /zerver/tests.py export-ignore
-/zerver/tests export-ignore
+/frontend_tests export-ignore
 /node_modules export-ignore
 /humbug export-ignore

.gitignore

@@ -2,8 +2,17 @@
 *~
 /all_messages_log.*
 /event_log/*
-/server.log
+/digest.log*
+/errors.log*
+/manage.log*
+/server.log*
+/workers.log*
+/email-deliverer.log
+/email-mirror.log
+/sync_ldap_user_data.log
 /update-prod-static.log
+frontend_tests/casper_tests/server.log
+frontend_tests/casper_lib/test_credentials.js
 /prod-static
 /errors/*
 *.sw[po]
@@ -13,24 +22,22 @@ stats/
 zerver/fixtures/available-migrations
 zerver/fixtures/migration-status
 zerver/fixtures/test_data1.json
-zerver/tests/frontend/test_credentials.js
 .kdev4
 zulip.kdev4
 memcached_prefix
 coverage/
 /queue_error
-/workers.log
 .test-js-with-node.html
-digest.log
-errors.log
-manage.log
 .kateproject.d/
 .kateproject
 *.kate-swp
 event_queues.json
 .vagrant
 /zproject/dev-secrets.conf
+static/js/bundle.js
 static/third/gemoji/
 static/third/zxcvbn/
 tools/emoji_dump/bitmaps/
 tools/emoji_dump/*.ttx
+tools/phantomjs
+node_modules

.travis.yml

@@ -0,0 +1,24 @@
+before_install:
+   - nvm install 0.10
+install:
+  - tools/travis/setup-$TEST_SUITE
+cache:
+  - apt: false
+  - directories:
+    - /srv/phantomjs
+env:
+  - TEST_SUITE=frontend
+  - TEST_SUITE=backend
+  - TEST_SUITE=production
+  - TEST_SUITE=py3k
+language: python
+python:
+  - "2.7"
+# command to run tests
+script:
+  - ./tools/travis/$TEST_SUITE
+sudo: required
+services:
+- docker
+addons:
+  postgresql: "9.3"

Dockerfile

@@ -0,0 +1,12 @@
+FROM ubuntu:trusty
+
+EXPOSE 9991
+
+RUN apt-get update && apt-get install -y \
+  python-pbs \
+  wget
+
+RUN useradd -d /home/zulip -m zulip && echo 'zulip ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+USER zulip
+WORKDIR /srv/zulip

README.dev.md

@@ -0,0 +1,421 @@
+
+Installing the Zulip Development environment
+============================================
+
+You will need a machine with at least 2GB of RAM available (see
+https://github.com/zulip/zulip/issues/32 for a plan for how to
+dramatically reduce this requirement).
+
+Start by cloning this repository: `git clone https://github.com/zulip/zulip.git`
+
+Using Vagrant
+-------------
+
+This is the recommended approach for all platforms, and will install
+the Zulip development environment inside a VM or container and works
+on any platform that supports Vagrant.
+
+The best performing way to run the Zulip development environment is
+using an LXC container on a Linux host, but we support other platforms
+such as Mac via Virtualbox (but everything will be 2-3x slower).
+
+* If your host is Ubuntu 15.04 or newer, you can install and configure
+  the LXC Vagrant provider directly using apt:
+  ```
+  sudo apt-get install vagrant lxc lxc-templates cgroup-lite redir
+  vagrant plugin install vagrant-lxc
+  ```
+
+* If your host is Ubuntu 14.04, you will need to [download a newer
+  version of Vagrant](https://www.vagrantup.com/downloads.html), and
+  then do the following:
+  ```
+  sudo apt-get install lxc lxc-templates cgroup-lite redir
+  sudo dpkg -i vagrant*.deb # in directory where you downloaded vagrant
+  vagrant plugin install vagrant-lxc
+  ```
+
+* For other Linux hosts with a kernel above 3.12, [follow the Vagrant
+  LXC installation
+  instructions](https://github.com/fgrehm/vagrant-lxc) to get Vagrant
+  with LXC for your platform.
+
+* If your host is OS X or older Linux, [download VirtualBox](https://www.virtualbox.org/wiki/Downloads),
+  [download Vagrant](https://www.vagrantup.com/downloads.html), and install them both.
+
+* If you're on OS X and have VMWare, it should be possible to patch
+  Vagrantfile to use the VMWare vagrant provider which should perform
+  much better than Virtualbox.  Patches to do this by default if
+  VMWare is available are welcome!
+
+* On Windows: You can use Vagrant and Virtualbox/VMWare on Windows
+  with Cygwin, similar to the Mac setup.  Be sure to create your git
+  clone using `git clone https://github.com/zulip/zulip.git -c
+  core.autocrlf=false` to avoid Windows line endings being added to
+  files (this causes weird errors).
+
+Once that's done, simply change to your zulip directory and run
+`vagrant up` in your terminal to install the development server.  This
+will take a long time on the first run because Vagrant needs to
+download the Ubuntu Trusty base image, but later you can run `vagrant
+destroy` and then `vagrant up` again to rebuild the environment and it
+will be much faster.
+
+Once that finishes, you can run the development server as follows:
+
+```
+vagrant ssh -- -L9991:localhost:9991
+# Now inside the container
+cd /srv/zulip
+source /srv/zulip-venv/bin/activate
+./tools/run-dev.py --interface=''
+```
+
+To get shell access to the virtual machine running the server to run
+lint, management commands, etc., use `vagrant ssh`.
+
+(A small note on tools/run-dev.py: the `--interface=''` option will make
+the development server listen on all network interfaces.  While this
+is correct for the Vagrant guest sitting behind a NAT, you probably
+don't want to use that option when using run-dev.py in other environments).
+
+At this point you should [read about using the development environment](https://github.com/zulip/zulip/blob/master/README.dev.md#using-the-development-environment).
+
+
+Using provision.py without Vagrant
+----------------------------------
+
+If you'd like to install a Zulip development environment on a server
+that's already running Ubuntu 14.04 Trusty, you can do that by just
+running:
+
+```
+sudo apt-get update
+sudo apt-get install -y python-pbs
+python /srv/zulip/provision.py
+
+cd /srv/zulip
+source /srv/zulip-venv/bin/activate
+./tools/run-dev.py
+```
+
+Note that there is no supported uninstallation process without Vagrant
+(with Vagrant, you can just do `vagrant destroy` to clean up the
+development environment).
+
+By hand
+-------
+If you really want to install everything by hand, the below
+instructions should work.
+
+Install the following non-Python dependencies:
+ * libffi-dev — needed for some Python extensions
+ * postgresql 9.1 or later — our database (also install development headers)
+ * nodejs 0.10 (and npm)
+ * memcached (and headers)
+ * rabbitmq-server
+ * libldap2-dev
+ * python-dev
+ * redis-server — rate limiting
+ * tsearch-extras — better text search
+ * libfreetype6-dev - needed before you pip install Pillow to properly generate emoji PNGs
+
+### On Debian or Ubuntu systems:
+
+```
+sudo apt-get install closure-compiler libfreetype6-dev libffi-dev memcached rabbitmq-server libldap2-dev redis-server postgresql-server-dev-all libmemcached-dev python-dev hunspell-en-us nodejs nodejs-legacy npm git yui-compressor puppet gettext
+
+# If on 12.04 or wheezy:
+sudo apt-get install postgresql-9.1
+wget https://dl.dropboxusercontent.com/u/283158365/zuliposs/postgresql-9.1-tsearch-extras_0.1.2_amd64.deb
+sudo dpkg -i postgresql-9.1-tsearch-extras_0.1.2_amd64.deb
+
+# If on 14.04:
+sudo apt-get install postgresql-9.3
+wget https://dl.dropboxusercontent.com/u/283158365/zuliposs/postgresql-9.3-tsearch-extras_0.1.2_amd64.deb
+sudo dpkg -i postgresql-9.3-tsearch-extras_0.1.2_amd64.deb
+
+# If on 15.04 or jessie:
+sudo apt-get install postgresql-9.4
+wget https://dl.dropboxusercontent.com/u/283158365/zuliposs/postgresql-9.4-tsearch-extras_0.1_amd64.deb
+sudo dpkg -i postgresql-9.4-tsearch-extras_0.1_amd64.deb
+```
+
+Now continue with the "All systems" instructions below.
+
+### On Fedora 22 (experimental):
+
+These instructions are experimental and may have bugs; patches welcome!
+
+```
+sudo dnf install libffi-devel memcached rabbitmq-server openldap-devel python-devel redis postgresql-server postgresql-devel postgresql libmemcached-devel freetype-devel nodejs npm yuicompressor closure-compiler gettext
+```
+
+Now continue with the Common to Fedora/CentOS instructions below.
+
+### On CentOS 7 Core (experimental):
+
+These instructions are experimental and may have bugs; patches welcome!
+
+```
+# Add user zulip to the system (not necessary if you configured zulip as the administrator
+# user during the install process of CentOS 7).
+useradd zulip
+
+# Create a password for zulip user
+passwd zulip
+
+# Allow zulip to sudo
+visudo
+# Add this line after line `root    ALL=(ALL)       ALL`
+zulip   ALL=(ALL)       ALL
+
+# Switch to zulip user
+su zulip
+
+# Enable EPEL 7 repo so we can install rabbitmq-server, redis and other dependencies
+sudo yum install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+
+# Install dependencies
+sudo yum install libffi-devel memcached rabbitmq-server openldap-devel python-devel redis postgresql-server \
+postgresql-devel postgresql libmemcached-devel wget python-pip openssl-devel freetype-devel libjpeg-turbo-devel \
+zlib-devel nodejs yuicompressor closure-compiler gettext
+
+# We need these packages to compile tsearch-extras
+sudo yum groupinstall "Development Tools"
+
+# clone Zulip's git repo and cd into it
+cd && git clone https://github.com/zulip/zulip && cd zulip/
+
+## NEEDS TESTING: The next few DB setup items may not be required at all.
+# Initialize the postgres db
+sudo postgresql-setup initdb
+
+# Edit the postgres settings:
+sudo vi /var/lib/pgsql/data/pg_hba.conf
+
+# Change these lines:
+host    all             all             127.0.0.1/32            ident
+host    all             all             ::1/128                 ident
+# to this:
+host    all             all             127.0.0.1/32            md5
+host    all             all             ::1/128                 md5
+```
+
+Now continue with the Common to Fedora/CentOS instructions below.
+
+### Common to Fedora/CentOS instructions
+
+```
+# Build and install postgres tsearch-extras module
+wget https://launchpad.net/~tabbott/+archive/ubuntu/zulip/+files/tsearch-extras_0.1.3.tar.gz
+tar xvzf tsearch-extras_0.1.3.tar.gz
+cd ts2
+make
+sudo make install
+
+# Hack around missing dictionary files -- need to fix this to get
+# the proper dictionaries from what in debian is the hunspell-en-us package.
+sudo touch /usr/share/pgsql/tsearch_data/english.stop
+sudo touch /usr/share/pgsql/tsearch_data/en_us.dict
+sudo touch /usr/share/pgsql/tsearch_data/en_us.affix
+
+# Edit the postgres settings:
+sudo vi /var/lib/pgsql/data/pg_hba.conf
+
+# Add this line before the first uncommented line to enable password auth:
+host    all             all             127.0.0.1/32            md5
+
+# Start the services
+sudo systemctl start redis memcached rabbitmq-server postgresql
+
+# Enable automatic service startup after the system startup
+sudo systemctl enable redis rabbitmq-server memcached postgresql
+```
+
+Finally continue with the All Systems instructions below.
+
+### All Systems:
+
+```
+pip install -r requirements.txt
+npm install
+./tools/download-zxcvbn
+./tools/emoji_dump/build_emoji
+./scripts/setup/generate_secrets.py -d
+sudo cp ./puppet/zulip/files/postgresql/zulip_english.stop /usr/share/postgresql/9.3/tsearch_data/
+./scripts/setup/configure-rabbitmq
+./tools/postgres-init-dev-db
+./tools/do-destroy-rebuild-database
+./tools/postgres-init-test-db
+./tools/do-destroy-rebuild-test-database
+```
+
+To start the development server:
+
+```
+./tools/run-dev.py
+```
+
+… and visit [http://localhost:9991/](http://localhost:9991/).
+
+Using Docker
+-------------
+
+You can also use Docker to develop, first you need to install Docker in your development machine following the [instructions](https://docs.docker.com/engine/installation/). Some other interesting links for somebody new in Docker are:
+
+* [Get Started](https://docs.docker.com/linux/started/)
+* [Understand the architecture](https://docs.docker.com/engine/introduction/understanding-docker/)
+* [Docker run reference]https://docs.docker.com/engine/reference/run/()
+* [Dockerfile reference](https://docs.docker.com/engine/reference/builder/)
+
+Then you should create the Docker image based on Ubuntu Linux, first go to the directory with the Zulip source code:
+
+```
+docker build -t user/zulipdev .
+```
+
+Now you're going to install Zulip dependencies in the image:
+
+```
+docker run -itv $(pwd):/srv/zulip -p 80:9991 user/zulipdev /bin/bash
+$ /usr/bin/python /srv/zulip/provision.py --docker 
+docker ps -af ancestor=user/zulipdev
+docker commit -m "Zulip installed" <container id> user/zulipdev:v2
+```
+
+Finally you can run the docker server with:
+
+```
+docker run -itv $(pwd):/srv/zulip -p 80:9991 user/zulipdev:v2 /srv/zulip/scripts/start-dockers
+```
+
+If you want to connect to the Docker instance to build a release tarball you can use:
+
+```
+docker ps
+docker exec -it <container id> /bin/bash
+$ source /home/zulip/.bash_profile
+$ <Your commands>
+$ exit
+```
+
+To stop the server use:
+```
+docker ps
+docker kill <container id>
+```
+
+If you want to run all the tests you need to start the servers first, you can do it with:
+
+```
+docker run -itv $(pwd):/srv/zulip user/zulipdev:v2 /bin/bash
+$ scripts/test-all-docker
+```
+
+You can modify the source code in your development machine and review the results in your browser.
+
+
+Using the Development Environment
+=================================
+
+Once the development environment is running, you can visit
+<http://localhost:9991/> in your browser.  By default, the development
+server homepage just shows a list of the users that exist on the
+server and you can login as any of them by just clicking on a user.
+This setup saves time for the common case where you want to test
+something other than the login process; to test the login process
+you'll want to change AUTHENTICATION_BACKENDS in the not-PRODUCTION
+case of `zproject/settings.py` from zproject.backends.DevAuthBackend
+to use the auth method(s) you'd like to test.
+
+While developing, it's helpful to watch the `run-dev.py` console
+output, which will show any errors your Zulip development server
+encounters.
+
+When you make a change, here's a guide for what you need to do in
+order to see your change take effect in Development:
+
+* If you change Javascript or CSS, you'll just need to reload the
+browser window to see changes take effect.
+
+* If you change Python code used by the the main Django/Tornado server
+processes, these services are run on top of Django's [manage.py
+runserver](https://docs.djangoproject.com/en/1.8/ref/django-admin/#runserver-port-or-address-port),
+which will automatically restart the Zulip Django and Tornado servers
+whenever you save changes to Python code.  You can watch this happen
+in the `run-dev.py` console to make sure the backend has reloaded.
+
+* The Python queue workers don't automatically restart when you save
+changes (or when they stop running), so you will want to ctrl-C and
+then restart `run-dev.py` manually if you are testing changes to the
+queue workers or if a queue worker has crashed.
+
+* If you change the database schema, you'll need to use the standard
+Django migrations process to create and then run your migrations; see
+the [new feature
+tutorial](http://zulip.readthedocs.org/en/latest/new-feature-tutorial.html)
+for an example.  Additionally you should check out the [detailed
+testing docs](http://zulip.readthedocs.org/en/latest/testing.html) for
+how to run the tests properly after doing a migration.
+
+(In production, everything runs under supervisord and thus will
+restart if it crashes, and `upgrade-zulip` will take care of running
+migrations and then cleanly restaring the server for you).
+
+Running the test suite
+======================
+
+For more details, check out the [detailed testing
+docs](http://zulip.readthedocs.org/en/latest/testing.html).
+
+To run all the tests, do this:
+```
+./tools/test-all
+```
+
+For the Vagrant environment, you'll want to first enter the environment:
+```
+vagrant ssh
+source /srv/zulip-venv/bin/activate
+cd /srv/zulip
+```
+
+This runs the linter (`tools/lint-all`) plus all of our test suites;
+they can all be run separately (just read `tools/test-all` to see
+them).  You can also run individual tests which can save you a lot of
+time debugging a test failure, e.g.:
+
+```
+./tools/lint-all # Runs all the linters in parallel
+./tools/test-backend zerver.test_bugdown.BugdownTest.test_inline_youtube
+./tools/test-js-with-casper 10-navigation.js
+./tools/test-js-with-node # Runs all node tests but is very fast
+```
+
+The above setup instructions include the first-time setup of test
+databases, but you may need to rebuild the test database occasionally
+if you're working on new database migrations.  To do this, run:
+
+```
+./tools/postgres-init-test-db
+./tools/do-destroy-rebuild-test-database
+```
+
+Possible testing issues
+=======================
+
+- When running the test suite, if you get an error like this:
+
+  ```
+      sqlalchemy.exc.ProgrammingError: (ProgrammingError) function ts_match_locs_array(unknown, text, tsquery) does not   exist
+      LINE 2: ...ECT message_id, flags, subject, rendered_content, ts_match_l...
+                                                                   ^
+  ```
+
+  … then you need to install tsearch-extras, described
+  above. Afterwards, re-run the `init*-db` and the
+  `do-destroy-rebuild*-database` scripts.
+
+- When building the development environment using Vagrant and the LXC provider, if you encounter permissions errors, you may need to `chown -R 1000:$(whoami) /path/to/zulip` on the host before running `vagrant up` in order to ensure that the synced directory has the correct owner during provision. This issue will arise if you run `id username` on the host where `username` is the user running Vagrant and the output is anything but 1000.
+  This seems to be caused by Vagrant behavior; more information can be found here https://github.com/fgrehm/vagrant-lxc/wiki/FAQ#help-my-shared-folders-have-the-wrong-owner

README.md

@@ -10,173 +10,121 @@ previews, group private messages, audible notifications,
 missed-message emails, desktop apps, and much more.
 
 Further information on the Zulip project and its features can be found
-at https://www.zulip.org
-
-Contributing to Zulip
-=====================
-
-Zulip welcomes all forms of contributions!
-
-Before a pull request can be merged, you need to to sign the [Dropbox
-Contributor License Agreement](https://opensource.dropbox.com/cla/).
-
-Please run the tests (tools/test-all) before submitting your pull
-request and read our [commit message style
-guidelines](http://zulip.readthedocs.org/en/latest/code-style.html#commit-messages).
-
-Zulip has a growing collection of developer documentation including
-detailed documentation on coding style available on [Read The
-Docs](https://zulip.readthedocs.org/).
-
-Zulip also has a [development discussion mailing list](https://groups.google.com/forum/#!forum/zulip-devel)
-
-Feel free to send any questions or suggestions of areas where you'd
-love to see more documentation to the list!
-
-We recommend sending proposals for large features or refactorings to
-the zulip-devel list for discussion and advice before getting too deep
-into implementation.
-
-Please report any security issues you discover to support@zulip.com.
-
-Running Zulip in production
-===========================
-
-This is documented in https://zulip.org/server.html and README.prod.md.
+at https://www.zulip.org.
 
 Installing the Zulip Development environment
 ============================================
 
-Using Vagrant
--------------
+The Zulip development environment is the recommened option for folks
+interested in trying out Zulip.  This is documented in
+[README.dev.md](README.dev.md).
 
-This is the recommended approach, and is tested on OS X 10.10 as well as Ubuntu 14.04.
+Running Zulip in production
+===========================
 
-* If your host is OS X, download VirtualBox from
-  <http://download.virtualbox.org/virtualbox/4.3.30/VirtualBox-4.3.30-101610-OSX.dmg>
-  and install it.
-* If your host is Ubuntu 14.04: 
-  `sudo apt-get install vagrant lxc lxc-templates cgroup-lite redir && vagrant plugin install vagrant-lxc`
+Zulip in production only supports Ubuntu 14.04 right now, but work is
+ongoing on adding support for additional platforms. The installation
+process is documented in https://zulip.org/server.html and in more
+detail in [README.prod.md](README.prod.md).
 
-Once that's done, simply change to your zulip directory and run
-`vagrant up` in your terminal.  That will install the development
-server inside a Vagrant guest.
+Contributing to Zulip
+=====================
 
-Once that finishes, you can run the development server as follows:
+Zulip welcomes all forms of contributions!  The page documents the
+Zulip development process.
 
-```
-vagrant ssh -- -L9991:localhost:9991
-# Now inside the container
-cd /srv/zulip
-source /srv/zulip-venv/bin/activate
-./tools/run-dev.py --interface=''
-```
+* **Pull requests**. Before a pull request can be merged, you need to to sign the [Dropbox
+Contributor License Agreement](https://opensource.dropbox.com/cla/).
+Also, please skim our [commit message style
+guidelines](http://zulip.readthedocs.org/en/latest/code-style.html#commit-messages).
 
-You can now visit <http://localhost:9991/> in your browser.  To get
-shell access to the virtual machine running the server, use `vagrant ssh`.
+* **Testing**. The Zulip automated tests all run automatically when
+you submit a pull request, but you can also run them all in your
+development environment following the instructions in the [testing
+section](https://github.com/zulip/zulip#running-the-test-suite) below.
 
-(A small note on tools/run-dev.py: the `--interface=''` option will make
-the development server listen on all network interfaces.  While this
-is correct for the Vagrant guest sitting behind a NAT, you probably
-don't want to use that option when using run-dev.py in other environments).
+* **Developer Documentation**.  Zulip has a growing collection of
+developer documentation on [Read The Docs](https://zulip.readthedocs.org/).
+Recommended reading for new contributors includes the
+[directory structure](http://zulip.readthedocs.org/en/latest/directory-structure.html) and
+[new feature tutorial](http://zulip.readthedocs.org/en/latest/new-feature-tutorial.html).
 
-The run-dev.py console output will show any errors your Zulip
-development server encounters.  It runs on top of Django's "manage.py
-runserver" tool, which will automatically restart the Zulip server
-whenever you save changes to Python code.
+* **Mailing list and bug tracker** Zulip has a [development discussion
+mailing list](https://groups.google.com/forum/#!forum/zulip-devel) and
+uses [GitHub issues](https://github.com/zulip/zulip/issues).  Feel
+free to send any questions or suggestions of areas where you'd love to
+see more documentation to the list!  Please report any security issues
+you discover to support@zulip.com.
 
+* **App codebases** This repository is for the Zulip server and web app; the
+[desktop](https://github.com/zulip/zulip-desktop),
+[Android](https://github.com/zulip/zulip-android), and
+[iOS](https://github.com/zulip/zulip-ios) apps are separate
+repositories.
 
-By hand
--------
+How to get involved with contributing to Zulip
+==============================================
 
-Install the following non-Python dependencies:
- * libffi-dev — needed for some Python extensions
- * postgresql 9.1 or later — our database (also install development headers)
- * memcached (and headers)
- * rabbitmq-server
- * libldap2-dev
- * python-dev
- * redis-server — rate limiting
- * tsearch-extras — better text search
+First, subscribe to the Zulip [development discussion mailing list](https://groups.google.com/forum/#!forum/zulip-devel).
 
-On Debian or Ubuntu systems:
+The Zulip project uses a system of labels in our [issue
+tracker](https://github.com/zulip/zulip/issues) to make it easy to
+find a project if you don't have your own project idea in mind or want
+to get some experience with working on Zulip before embarking on a
+larger project you have in mind:
 
-```
-sudo apt-get install libffi-dev memcached rabbitmq-server libldap2-dev redis-server postgresql-server-dev-all libmemcached-dev
+* [Bite Size](https://github.com/zulip/zulip/labels/bite%20size):
+  Smaller projects that could be a great first contribution.
+* [Integrations](https://github.com/zulip/zulip/labels/integrations).
+  Integrate Zulip with another piece of software and contribute it
+  back to the community!  Writing an integration can be a great
+  started project.  There's some brief documentation on the best way
+  to write integrations at https://github.com/zulip/zulip/issues/70.
+* [Documentation](https://github.com/zulip/zulip/labels/documentation).
+  The Zulip project loves contributions of new documentation.
+* [Help Wanted](https://github.com/zulip/zulip/labels/help%20wanted):
+  A broader list of projects that nobody is currently working on.
+* [Platform support](https://github.com/zulip/zulip/labels/Platform%20support).
+  These are open issues about making it possible to install Zulip on a wider
+  range of platforms.
+* [Bugs](https://github.com/zulip/zulip/labels/bug). Open bugs.
+* [Feature requests](https://github.com/zulip/zulip/labels/enhancement).
+  Browsing this list can be a great way to find feature ideas to implement that
+  other Zulip users are excited about.
 
-# If on 12.04 or wheezy:
-sudo apt-get install postgresql-9.1
-wget https://dl.dropboxusercontent.com/u/283158365/zuliposs/postgresql-9.1-tsearch-extras_0.1.2_amd64.deb
-sudo dpkg -i postgresql-9.1-tsearch-extras_0.1.2_amd64.deb
+If you're excited about helping with an open issue, just post on the
+conversation thread that you're working on it.  You're encouraged to
+ask questions on how to best implement or debug your changes -- the
+Zulip maintainers are excited to answer questions to help you stay
+unblocked and working efficiently.
 
-# If on 14.04:
-sudo apt-get install postgresql-9.3
-wget https://dl.dropboxusercontent.com/u/283158365/zuliposs/postgresql-9.3-tsearch-extras_0.1.2_amd64.deb
-sudo dpkg -i postgresql-9.3-tsearch-extras_0.1.2_amd64.deb
+We also welcome suggestions of features that you feel would be
+valuable or changes that you feel would make Zulip a better open
+source project, and are happy to support you in adding new features or
+other user experience improvements to Zulip.
 
-# If on 15.04 or jessie:
-sudo apt-get install postgresql-9.4
-wget https://dl.dropboxusercontent.com/u/283158365/zuliposs/postgresql-9.4-tsearch-extras_0.1_amd64.deb
-sudo dpkg -i postgresql-9.4-tsearch-extras_0.1_amd64.deb
+If you have a new feature you'd like to add, we recommend you start by
+opening a GitHub issue about the feature idea explaining the problem
+that you're hoping to solve and that you're excited to work on it.  A
+Zulip maintainer will usually reply within a day with feedback on the
+idea, notes on any important issues or concerns, and and often tips on
+how to implement or test it.  Please feel free to ping the thread if
+you don't hear a response from the maintainers -- we try to be very
+responsive so this usually means we missed your message.
 
-# Then, all versions:
-pip install -r requirements.txt
-./scripts/setup/configure-rabbitmq
-./tools/postgres-init-db
-./tools/do-destroy-rebuild-database
-./tools/emoji_dump/build_emoji
-```
+For significant changes to the visual design, user experience, data
+model, or architecture, we highly recommend posting a mockup,
+screenshot, or description of what you have in mind to zulip-devel@ to
+get broad feedback before you spend too much time on implementation
+details.
 
-To start the development server:
+Finally, before implementing a larger feature, we highly recommend
+looking at the new feature tutorial and coding style guidelines on
+ReadTheDocs.
 
-```
-./tools/run-dev.py
-```
-
-… and hit http://localhost:9991/.
-
-
-Running the test suite
-======================
-
-One-time setup of test databases:
-
-```
-./tools/postgres-init-test-db
-./tools/do-destroy-rebuild-test-database
-```
-
-Run all tests:
-
-```
-./tools/test-all
-```
-
-This runs the linter plus all of our test suites; they can all be run
-separately (just read `tools/test-all` to see them).  You can also run
-individual tests, e.g.:
-
-```
-./tools/test-backend zerver.test_bugdown.BugdownTest.test_inline_youtube
-./tools/test-js-with-casper 10-navigation.js
-```
-
-Possible testing issues
-=======================
-
-The Casper tests are flaky on the Virtualbox environment (probably due
-to some performance-sensitive races).  Until this issue is debugged,
-you may need to rerun them to get them to pass.
-
-When running the test suite, if you get an error like this:
-
-```
-    sqlalchemy.exc.ProgrammingError: (ProgrammingError) function ts_match_locs_array(unknown, text, tsquery) does not exist
-    LINE 2: ...ECT message_id, flags, subject, rendered_content, ts_match_l...
-                                                                 ^
-```
-
-… then you need to install tsearch-extras, described above. Afterwards, re-run the `init*-db` and the `do-destroy-rebuild*-database` scripts.
+Feedback on how to make this development process more efficient, fun,
+and friendly to new contributors is very welcome!  Just shoot an email
+to the Zulip Developers list with your thoughts.
 
 License
 =======

README.prod.md

@@ -1,137 +1,886 @@
+Zulip in production
+===================
+
 This documents the process for installing Zulip in a production environment.
 
+Note that if you just want to play around with Zulip and see what it
+looks like, it is easier to install it in a development environment
+following the instructions in README.dev, since then you don't need to
+worry about setting up SSL certificates and an authentication mechanism.
+
 Recommended requirements:
 
-* Server running Ubuntu Precise or Debian Wheezy
-* At least 2 CPUs for production use
-* At least 4GB of RAM for production use
-* At least 100GB of free disk for production use
-* HTTP(S) access to the public Internet (for some features;
-  discuss with Zulip Support if this is an issue for you)
+* Server running Ubuntu Trusty
+* At least 2 CPUs for production use with 100+ users
+* At least 4GB of RAM for production use with 100+ users.  We **strongly
+  recommend against installing with less than 2GB of RAM**, as you will
+  likely experience OOM issues.  In the future we expect Zulip's RAM
+  requirements to decrease to support smaller installations (see
+  https://github.com/zulip/zulip/issues/32).
+* At least 10GB of free disk for production use (more may be required
+  if you intend to store uploaded files locally rather than in S3
+  and your team uses that feature extensively)
+* Outgoing HTTP(S) access to the public Internet.
 * SSL Certificate for the host you're putting this on
-  (e.g. https://zulip.example.com)
-* Email credentials for the service to send outgoing emails to users
-  (e.g. missed message notifications, password reminders if you're not
-  using SSO, etc.).
+  (e.g. zulip.example.com).  If you just want to see what
+  Zulip looks like, we recommend installing the development
+  environment detailed in README.md as that is easier to setup.
+* Email credentials Zulip can use to send outgoing emails to users
+  (e.g. email address confirmation emails during the signup process,
+  missed message notifications, password reminders if you're not using
+  SSO, etc.).
 
-=======================================================================
 
-How to install Zulip in production:
+Installing Zulip in production
+==============================
 
 These instructions should be followed as root.
 
 (1) Install the SSL certificates for your machine to
-  /etc/ssl/private/zulip.key
-  and
-  /etc/ssl/certs/zulip.combined-chain.crt
+  `/etc/ssl/private/zulip.key` and `/etc/ssl/certs/zulip.combined-chain.crt`.
+  If you don't know how to generate an SSL certificate, you, you can
+  do the following to generate a self-signed certificate:
 
-(2) download zulip-server.tar.gz, and unpack to it /root/zulip, e.g.
-tar -xf zulip-server-1.1.3.tar.gz
-mv zulip-server-1.1.3 /root/zulip
+  ```
+  apt-get install openssl
+  openssl genrsa -des3 -passout pass:x -out server.pass.key 4096
+  openssl rsa -passin pass:x -in server.pass.key -out zulip.key
+  rm server.pass.key
+  openssl req -new -key zulip.key -out server.csr
+  openssl x509 -req -days 365 -in server.csr -signkey zulip.key -out zulip.combined-chain.crt
+  rm server.csr
+  cp zulip.key /etc/ssl/private/zulip.key
+  cp zulip.combined-chain.crt /etc/ssl/certs/zulip.combined-chain.crt
+  ```
 
-(3) run /root/zulip/scripts/setup/install
+  You will eventually want to get a properly signed certificate (and
+  note that at present the Zulip desktop app doesn't support
+  self-signed certificates), but this will let you finish the
+  installation process.
 
-This may take a while to run, since it will install a large number of
-packages via apt.
+(2) Download [the latest built server tarball](https://www.zulip.com/dist/releases/zulip-server-latest.tar.gz)
+  and unpack it to `/root/zulip`, e.g.
+  ```
+  wget https://www.zulip.com/dist/releases/zulip-server-latest.tar.gz
+  tar -xf zulip-server-latest.tar.gz
+  mv zulip-server-1.3.6 /root/zulip
+  ```
+
+(3) Run
+  ```
+  /root/zulip/scripts/setup/install
+  ```
+  This may take a while to run, since it will install a large number of
+  packages via apt.
 
 (4) Configure the Zulip server instance by filling in the settings in
-/etc/zulip/settings.py
+  `/etc/zulip/settings.py`.  Be sure to fill in all the mandatory
+  settings, enable at least one authentication mechanism, and do the
+  configuration required for that authentication mechanism to work.
+  See the section on "Authentication" below for more detail on
+  configuring authentication mechanisms.
 
-(5) su zulip -c /home/zulip/deployments/current/scripts/setup/initialize-database
+(5) Run
+  ```
+  su zulip -c /home/zulip/deployments/current/scripts/setup/initialize-database
+  ```
+  This will report an error if you did not fill in all the mandatory
+  settings from `/etc/zulip/settings.py`.  Once this completes
+  successfully, the main installation process will be complete, and if
+  you are planning on using password authentication, you should be able
+  to visit the URL for your server and register for an account.
 
-This will report an error if you did not fill in all the mandatory
-settings from /etc/zulip/settings.py.  Once this completes
-successfully, the main installation process will be complete, and if
-you are planning on using password authentication, you should be able
-to visit the URL for your server and register for an account.
+(6) Subscribe to [the Zulip announcements Google Group](https://groups.google.com/forum/#!forum/zulip-announce)
+  to get announcements about new releases, security issues, etc.
 
-(6) Subscribe to
-https://groups.google.com/forum/#!forum/zulip-announce to get
-announcements about new releases, security issues, etc.
 
-=======================================================================
+Authentication and logging into Zulip the first time
+====================================================
 
-Maintaining Zulip in production:
+(As you read and follow the instructions in this section, if you run
+into trouble, check out the troubleshooting advice in the next major
+section.)
 
-* To upgrade to a new version, download the appropriate release
-  tarball from https://www.zulip.org, and then run as root
+Once you've finished installing Zulip, configuring your settings.py
+file, and initializing the database, it's time to login to your new
+installation.  By default, initialize-database creates 1 realm that
+you can join, the `ADMIN_DOMAIN` realm (defined in
+`/etc/zulip/settings.py`).
 
-  /home/zulip/deployments/current/scripts/upgrade-zulip <tarball>
+The `ADMIN_DOMAIN` realm is by default configured with the following settings:
+* `restricted_to_domain=True`: Only people with emails ending with @ADMIN_DOMAIN can join.
+* `invite_required=False`: An invitation is not required to join the realm.
+* `invite_by_admin_only=False`: You don't need to be an admin user to invite other users.
+* `mandatory_topics=False`: Users are not required to specify a topic when sending messages.
+
+If you would like to change these settings, you can do so using the
+Django management python shell (as the zulip user):
+
+```
+cd /home/zulip/deployments/current
+./manage.py shell
+from zerver.models import *
+r = get_realm(settings.ADMIN_DOMAIN)
+r.restricted_to_domain=False # Now anyone anywhere can login
+r.save() # save to the database
+```
+
+If you realize you set `ADMIN_DOMAIN` wrong, in addition to fixing the
+value in settings.py, you will also want to do a similar manage.py
+process to set `r.domain = "newexample.com"`.  If you've already
+changed `ADMIN_DOMAIN` in settings.py, you can use
+`Realm.objects.all()` in the management shell to find the list of
+realms and pass the domain of the realm that is not "zulip.com" to
+`get_realm`.
+
+Depending what authentication backend you're planning to use, you will
+need to do some additional setup documented in the `settings.py` template:
+
+* For Google authentication, you need to follow the configuration
+  instructions around `GOOGLE_OAUTH2_CLIENT_ID` and `GOOGLE_CLIENT_ID`.
+* For Email authentication, you will need to follow the configuration
+  instructions around outgoing SMTP from Django.
+
+You should be able to login now.  If you get an error, check
+`/var/log/zulip/errors.log` for a traceback, and consult the next
+section for advice on how to debug.  If you aren't able to figure it
+out, email zulip-help@googlegroups.com with the traceback and we'll
+try to help you out!
+
+You will likely want to make your own user account an admin user,
+which you can do via the following management command:
+
+```
+./manage.py knight username@example.com -f
+```
+
+Now that you are an administrator, you will have a special
+"Administration" tab linked to from the upper-right gear menu in the
+Zulip app that lets you deactivate other users, manage streams, change
+the Realm settings you may have edited using manage.py shell above,
+etc.
+
+You can also use `manage.py knight` with the
+`--permission=api_super_user` argument to create API super users,
+which are needed to mirror messages to streams from other users for
+the IRC and Jabber mirroring integrations (see
+`bots/irc-mirror.py` and `bots/jabber_mirror.py` for some detail on these).
+
+There are a large number of useful management commands under
+`zerver/manangement/commands/`; you can also see them listed using
+`./manage.py` with no arguments.
+
+One such command worth highlighting because it's a valuable feature
+with no UI in the Administration page is `./manage.py realm_filters`,
+which allows you to configure certain patterns in messages to be
+automatically linkified, e.g. whenever someone mentions "T1234" it
+could be auto-linkified to ticket 1234 in your team's Trac instance.
+
+Checking Zulip is healthy and debugging the services it depends on
+==================================================================
+
+You can check if the zulip application is running using:
+```
+supervisorctl status
+```
+
+And checking for errors in the Zulip errors logs under
+`/var/log/zulip/`.  That contains one log file for each service, plus
+`errors.log` (has all errors), `server.log` (logs from the Django and
+Tornado servers), and `workers.log` (combined logs from the queue
+workers).
+
+After you change configuration in `/etc/zulip/settings.py` or fix a
+misconfiguration, you will often want to restart the Zulip application.
+You can restart Zulip using:
+
+```
+supervisorctl restart all
+```
+
+Similarly, you can stop Zulip using:
+
+```
+supervisorctl stop all
+```
+
+The Zulip application uses several major services to store and cache
+data, queue messages, and otherwise support the Zulip application:
+
+* postgresql
+* rabbitmq-server
+* nginx
+* redis
+* memcached
+
+If one of these services is not installed or functioning correctly,
+Zulip will not work.  Below we detail some common configuration
+problems and how to resolve them:
+
+* An AMQPConnectionError traceback or error running rabbitmqctl
+  usually means that RabbitMQ is not running; to fix this, try:
+  ```
+  service rabbitmq-server restart
+  ```
+  If RabbitMQ fails to start, the problem is often that you are using
+  a virtual machine with broken DNS configuration; you can often
+  correct this by configuring `/etc/hosts` properly.
+
+* If your browser reports no webserver is running, that is likely
+  because nginx is not configured properly and thus failed to start.
+  nginx will fail to start if you configured SSL incorrectly or did
+  not provide SSL certificates.  To fix this, configure them properly
+  and then run:
+  ```
+  service nginx restart
+  ```
+
+If you run into additional problems, [please report
+them](https://github.com/zulip/zulip/issues) so that we can update
+these lists!  The Zulip installation scripts logs its full output to
+`/var/log/zulip/install.log`, so please include the context for any
+tracebacks from that log.
+
+
+Making your Zulip instance awesome
+==================================
+
+Once you've got Zulip setup, you'll likely want to configure it the
+way you like.  There are four big things to focus on:
+
+(1) Integrations.  We recommend setting up integrations for the major
+tools that your team works with.  For example, if you're a software
+development team, you may want to start with integrations for your
+version control, issue tracker, CI system, and monitoring tools.
+
+Spend time configuring these integrations to be how you like them --
+if an integration is spammy, you may want to change it to not send
+messages that nobody cares about (E.g. for the zulip.com trac
+integration, some teams find they only want notifications when new
+tickets are opened, commented on, or closed, and not every time
+someone edits the metadata).
+
+If Zulip doesn't have an integration you want, you can add your own!
+Most integrations are very easy to write, and even more complex
+integrations usually take less than a day's work to build.  We very
+much appreciate contributions of new integrations; there is a brief
+draft integration writing guide [here](https://github.com/zulip/zulip/issues/70).
+
+
+It can often be valuable to integrate your own internal processes to
+send notifications into Zulip; e.g. notifications of new customer
+signups, new error reports, or daily reports on the team's key
+metrics; this can often spawn discussions in response to the data.
+
+(2) Streams and Topics.  If it feels like a stream has too much
+traffic about a topic only of interest to some of the subscribers,
+consider adding or renaming streams until you feel like your team is
+working productively.
+
+Second, most users are not used to topics.  It can require a bit of
+time for everyone to get used to topics and start benefitting from
+them, but usually once a team is using them well, everyone ends up
+enthusiastic about how much topics make life easier.  Some tips on
+using topics:
+
+* When replying to an existing conversation thread, just click on the
+  message, or navigate to it with the arrow keys and hit "r" or
+  "enter" to reply on the same topic
+* When you start a new conversation topic, even if it's related to the
+  previous conversation, type a new topic in the compose box
+* You can edit topics to fix a thread that's already been started,
+  which can be helpful when onboarding new batches of users to the platform.
+
+Third, setting default streams for new users is a great way to get
+new users involved in conversations before they've accustomed
+themselves with joining streams on their own. You can use the
+[`set_default_streams`](https://github.com/zulip/zulip/blob/master/zerver/management/commands/set_default_streams.py)
+command to set default streams for users within a realm:
+
+```
+python manage.py set_default_streams --domain=example.com --streams=foo,bar,...
+```
+
+(3) Notification settings.  Zulip gives you a great deal of control
+over which messages trigger desktop notifications; you can configure
+these extensively in the `/#settings` page (get there from the gear
+menu).  If you find the desktop notifications annoying, consider
+changing the settings to only trigger desktop notifications when you
+receive a PM or are @-mentioned.
+
+(4) The mobile and desktop apps.  Currently, the Zulip Desktop app
+only supports talking to servers with a properly signed SSL
+certificate, so you may find that you get a blank screen when you
+connect to a Zulip server using a self-signed certificate.
+
+The Zulip iOS and Android apps in their respective stores don't yet
+support talking to non-zulip.com servers; the iOS app is waiting on
+Apple's app store review, while the Android app is waiting on someone
+to do the small project of adding a field to specify what Zulip server
+to talk to.
+
+These issues will likely all be addressed in the coming weeks; make
+sure to join the zulip-announce@googlegroups.com list so that you can
+receive the announcements when these become available.
+
+(5) All the other features: Hotkeys, emoji, search filters,
+@-mentions, etc.  Zulip has lots of great features, make sure your
+team knows they exist and how to use them effectively.
+
+(6) Enjoy your Zulip installation!  If you discover things that you
+wish had been documented, please contribute documentation suggestions
+either via a GitHub issue or pull request; we love even small
+contributions, and we'd love to make the Zulip documentation cover
+everything anyone might want to know about running Zulip in
+production.
+
+
+Maintaining and upgrading Zulip in production
+=============================================
+
+We recommend reading this entire section before doing your first
+upgrade.
+
+* To upgrade to a new version of the zulip server, download the
+  appropriate release tarball from
+  https://www.zulip.com/dist/releases/ and then run as root:
+  ```
+  /home/zulip/deployments/current/scripts/upgrade-zulip zulip-server-VERSION.tar.gz
+  ```
 
   The upgrade process will shut down the service, run `apt-get
-  upgrade` and any database migrations, and then bring the service
-  back up.  This will result in some brief downtime for the service,
-  which should be under 30 seconds unless there is an expensive
-  transition involved.  Unless you have tested the upgrade in advance,
-  we recommend doing upgrades at off hours.
+  upgrade`, a puppet apply, and any database migrations, and then
+  bring the service back up.  This will result in some brief downtime
+  for the service, which should be under 30 seconds unless there is an
+  expensive transition involved.  Unless you have tested the upgrade
+  in advance, we recommend doing upgrades at off hours.
 
-  You can create your own release tarballs from a copy of this
+  You can create your own release tarballs from a copy of zulip.git
   repository using `tools/build-release-tarball`.
 
-* To update your settings, simply edit /etc/zulip/settings.py and then
-  run /home/zulip/deployments/current/scripts/restart-server to
+* **Warning**: If you have modified configuration files installed by
+  Zulip (e.g. the nginx configuration), the Zulip upgrade process will
+  overwrite your configuration when it does the `puppet apply`.  You
+  can test whether this will happen assuming no upstream changes to
+  the configuration using `scripts/zulip-puppet-apply` (without the
+  `-f` option), which will do a test puppet run and output and changes
+  it would make.  Using this list, you can save a copy of any files
+  that you've modified, do the upgrade, and then restore your
+  configuration.  If you need to do this, please report the issue so
+  that we can make the Zulip puppet configuration flexible enough to
+  handle your setup.
+
+* The Zulip upgrade script automatically logs output to
+  /var/log/zulip/upgrade.log; please use those logs to include output
+  that shows all errors in any bug reports.
+
+* The Zulip upgrade process works by creating a new deployment under
+  /home/zulip/deployments/ containing a complete copy of the Zulip
+  server code, and then moving the symlinks at
+  `/home/zulip/deployments/current` and /root/zulip` as part of the
+  upgrade process.  This means that if the new version isn't working,
+  you can quickly downgrade to the old version by using
+  `/home/zulip/deployments/<date>/scripts/restart-server` to return to
+  a previous version that you've deployed (the version is specified
+  via the path to the copy of `restart-server` you call).
+
+* To update your settings, simply edit `/etc/zulip/settings.py` and then
+  run `/home/zulip/deployments/current/scripts/restart-server` to
   restart the server
 
-* You are responsible for running "apt-get upgrade" on your system on
+* You are responsible for running `apt-get upgrade` on your system on
   a regular basis to ensure that it is up to date with the latest
   security patches.
 
 * To use the Zulip API with your Zulip server, you will need to use the
-  API endpoint of e.g. "https://zulip.yourdomain.net/api".  Our Python
+  API endpoint of e.g. `https://zulip.example.com/api`.  Our Python
   API example scripts support this via the
-  "--site=https://zulip.yourdomain.net" argument.  The API bindings
-  support it via putting "site=https://zulip.yourdomain.net" in your
+  `--site=https://zulip.example.com` argument.  The API bindings
+  support it via putting `site=https://zulip.example.com` in your
   .zuliprc.
 
+  Every Zulip integration supports this sort of argument (or e.g. a
+  `ZULIP_SITE` variable in a zuliprc file or the environment), but this
+  is not yet documented for some of the integrations (the included
+  integration documentation on `/integrations` will properly document
+  how to do this for most integrations).  Pull requests welcome to
+  document this for those integrations that don't discuss this!
+
 * Similarly, you will need to instruct your users to specify the URL
   for your Zulip server when using the Zulip desktop and mobile apps.
 
 * As a measure to mitigate the impact of potential memory leaks in one
   of the Zulip daemons, the service automatically restarts itself
-  every Sunday early morning.  See /etc/cron.d/restart-zulip for the
+  every Sunday early morning.  See `/etc/cron.d/restart-zulip` for the
   precise configuration.
 
+### Backups for Zulip
 
-=======================================================================
+There are several pieces of data that you might want to back up:
 
-SSO Authentication:
+* The postgres database.  That you can back up like any postgres
+database; we have some example tooling for doing that incrementally
+into S3 using [wal-e](https://github.com/wal-e/wal-e) in
+`puppet/zulip_internal/manifests/postgres_common.pp` (that's what we
+use for zulip.com's database backups).  Note that this module isn't
+part of the Zulip server releases since it's part of the zulip.com
+configuration (see https://github.com/zulip/zulip/issues/293 for a
+ticket about fixing this to make life easier for running backups).
+
+* Any user-uploaded files.  If you're using S3 as storage for file
+uploads, this is backed up in S3, but if you have instead set
+LOCAL_UPLOADS_DIR, any files uploaded by users (including avatars)
+will be stored in that directory and you'll want to back it up.
+
+* Your Zulip configuration including secrets from /etc/zulip/.
+E.g. if you lose the value of secret_key, all users will need to login
+again when you setup a replacement server since you won't be able to
+verify their cookies; if you lose avatar_salt, any user-uploaded
+avatars will need to be re-uploaded (since avatar filenames are
+computed using a hash of avatar_salt and user's email), etc.
+
+* The logs under /var/log/zulip can be handy to have backed up, but
+they do get large on a busy server, and it's definitely
+lower-priority.
+
+### Restoration
+
+To restore from backups, the process is basically the reverse of the above:
+
+* Install new server as normal by downloading a Zulip release tarball
+  and then using `scripts/setup/install`, you don't need
+  to run the `initialize-database` second stage which puts default
+  data into the database.
+
+* Unpack to /etc/zulip the settings.py and secrets.conf files from your backups.
+
+* Restore your database from the backup using wal-e; if you ran
+  `initialize-database` anyway above, you'll want to first
+  `scripts/setup/postgres-init-db` to drop the initial database first.
+
+* If you're using local file uploads, restore those files to the path
+  specified by `settings.LOCAL_UPLOADS_DIR` and (if appropriate) any
+  logs.
+
+* Start the server using scripts/restart-server
+
+This restoration process can also be used to migrate a Zulip
+installation from one server to another.
+
+We recommend running a disaster recovery after you setup backups to
+confirm that your backups are working; you may also want to monitor
+that they are up to date using the Nagios plugin at:
+`puppet/zulip_internal/files/nagios_plugins/check_postgres_backup`.
+
+Contributions to more fully automate this process or make this section
+of the guide much more explicit and detailed are very welcome!
+
+### Postgres streaming replication
+
+Zulip has database configuration for doing with Postgres streaming
+replication ; you can see the configuration in these files:
+
+* puppet/zulip_internal/manifests/postgres_slave.pp
+* puppet/zulip_internal/manifests/postgres_master.pp
+* puppet/zulip_internal/files/postgresql/*
+
+Contribution of a step-by-step guide for setting this up (and moving
+this configuration to be available in the main `puppet/zulip/` tree)
+would be very welcome!
+
+### Using a remote postgres host
+
+This is a bit annoying to setup, but you can configure Zulip to use a
+dedicated postgres server by setting the `REMOTE_POSTGRES_HOST`
+variable in /etc/zulip/settings.py, and configuring Postgres
+certificate authentication (see
+http://www.postgresql.org/docs/9.1/static/ssl-tcp.html and
+http://www.postgresql.org/docs/9.1/static/libpq-ssl.html for
+documentation on how to set this up and deploy the certificates) to
+make the DATABASES configuration in `zproject/settings.py` work (or
+override that configuration).
+
+### Monitoring Zulip
+
+The complete Nagios configuration (sans secret keys) we used to
+monitor zulip.com is available under `puppet/zulip_internal` in the
+Zulip Git repository (those files are not installed in the release
+tarballs); there are a number of useful Nagios plugins available
+there, including:
+
+Frontend server monitoring:
+
+* check_send_receive_time (sends a test message through the system
+  between two bot users to check that end-to-end message sending works)
+* check_website_response.sh (standard HTTP check)
+
+Queue worker monitoring:
+
+* check_rabbitmq_consumers and check_rabbitmq_queues (checks for
+  rabbitmq being down or the queue workers being behind)
+* check_queue_worker_errors (checks for errors reported by the queue workers)
+* check_worker_memory (monitors for memory leaks in queue workers)
+* check_email_deliverer_backlog and check_email_deliverer_process
+  (monitors for whether outgoing emails are being sent)
+
+Database monitoring:
+
+* check_pg_replication_lag
+* check_postgres (checks the health of the postgres database)
+* check_postgres_backup (checks backups are up to date; see above)
+* check_fts_update_log (monitors for whether full-text search updates
+  are being processed)
+
+Standard server monitoring:
+
+* check_debian_packages
+
+Contributions on making it easier to monitor Zulip and maintain it in
+production, e.g.  https://github.com/zulip/zulip/issues/371, are very
+welcome!
+
+### Scalability of Zulip
+
+This section attempts to address the considerations involved with
+running Zulip with a large team (>1000 users).
+
+* We recommend using a remote postgres database (see
+  REMOTE_POSTGRES_HOST docs above) for isolation, though it is not
+  required.  In the following, we discuss a relatively simple
+  configuration with two types of servers: application servers
+  (running Django, Tornado, RabbitMQ, Redis, Memcached, etc.) and
+  database servers.
+
+* You can scale to a pretty large installation (O(~1000) concurrently
+  active users using it to chat all day) with just a single reasonably
+  large application server (e.g. AWS c3.2xlarge with 8 cores and 16GB
+  of RAM) sitting mostly idle (<10% CPU used and only 4GB of the 16GB
+  RAM actively in use).  You can probably get away with half that
+  (e.g. c3.xlarge), but ~8GB of RAM is highly recommended at scale.
+  Beyond a 1000 active users, you will eventually want to increase the
+  memory cap in `memcached.conf` from the default 512MB to avoid high
+  rates of memcached misses.
+
+* For the database server, we highly recommend SSD disks, and RAM is
+  the primary resource limitation.  We have not aggressively tested
+  for the minimum resources required, but 8 cores with 30GB of RAM
+  (e.g. AWS's m3.2xlarge) should suffice; you may be able to get away
+  with less especially on the CPU side.  The database load per user is
+  pretty optimized as long as `memcached` is working correctly.  This
+  has not been tested, but from extrapolating the load profile, it
+  should be possible to scale a Zulip installation to 10,000s of
+  active users using a single large database server without doing
+  anything complicated like sharding the database.
+
+* For reasonably high availability, it's easy to run a hot spare
+  application server and a hot spare database (using Postgres
+  streaming replication; see the section on configuring this).  Be
+  sure to check out the section on backups if you're hoping to run a
+  spare application server; in particular you probably want to use the
+  S3 backend for storing user-uploaded files and avatars and will want
+  to make sure secrets are available on the hot spare.
+
+* Zulip does not support dividing traffic for a given Zulip realm
+  between multiple application servers.  There are two issues: you
+  need to share the memcached/redis/rabbitmq instance (these should
+  can be moved to a network service shared by multiple servers with a
+  bit of configuration) and the Tornado event system for pushing to
+  browsers currently has no mechanism for multiple frontend servers
+  (or event processes) talking to each other.  One can probably get a
+  factor of 10 in a single server's scalability by [supporting
+  multiple tornado processes on a single
+  server](https://github.com/zulip/zulip/issues/372), which is also
+  likely the first part of any project to support exchanging events
+  amongst multiple servers.
+
+* The first scalability issue encountered by a very large realm (more
+  than a few thousand users), will be with the [frontend buddy list
+  perf and UI](https://github.com/zulip/zulip/issues/262).  Fixing
+  this should be a small project; the code for that part of the UI
+  layer doesn't do proper incremental updates.
+
+Questions, concerns, and bug reports about this area of Zulip are very
+welcome!  This is an area we are hoping to improve.
+
+### Security Model
+
+This section attempts to document the Zulip security model.  Since
+this is new documentation, it likely does not cover every issue; if
+there are details you're curious about, please feel free to ask
+questions on the Zulip development mailing list (or if you think
+you've found a security bug, please report it to support@zulip.com so
+we can do a responsible security announcement).
+
+#### Secure your Zulip server like your email server
+
+* It's reasonable to think about security for a Zulip server like you
+  do security for a team email server -- only trusted administrators
+  within an organization should have shell access to the server.
+
+  In particular, anyone with root access to a Zulip application server
+  or Zulip database server, or with access to the `zulip` user on a
+  Zulip application server, has complete control over the Zulip
+  installation and all of its data (so they can read messages, modify
+  history, etc.).  It would be difficult or impossible to avoid this,
+  because the server needs access to the data to support features
+  expected of a group chat system like the ability to search the
+  entire message history, and thus someone with control over the
+  server has access to that data as well.
+
+#### Encryption and Authentication
+
+* Traffic between clients (web, desktop and mobile) and the Zulip is
+  encrypted using HTTPS.  By default, all Zulip services talk to each
+  other either via a localhost connection or using an encrypted SSL
+  connection.
+
+* The preferred way to login to Zulip is using an SSO solution like
+  Google Auth, LDAP, or similar.  Zulip stores user passwords using
+  the standard PBKDF2 algorithm.  Password strength is checked and
+  weak passwords are visually discouraged using the zxcvbn library,
+  but Zulip does not by default have strong requirements on user
+  password strength.  Modify `static/js/common.js` to adjust the
+  password strength requirements (Patches welcome to make controlled
+  by an easy setting!).
+
+* Zulip requires CSRF tokens in all interactions with the web API to
+  prevent CSRF attacks.
+
+#### Messages and History
+
+* Zulip message content is rendering using a specialized Markdown
+  parser which escapes content to protect against cross-site scripting
+  attacks.
+
+* Zulip supports both public streams and private ("invite-only")
+  streams.  Any Zulip user can join any public stream in the realm
+  (and can view the complete message of any public stream history
+  without joining the stream).
+
+* Users who are not members of a private stream cannot read messages
+  on the stream, send messages to the stream, or join the stream, even
+  if they are a Zulip administrator.  However, any member of a private
+  stream can invite other users to the stream.  When a new user joins
+  a private stream, they can see future messages sent to the stream,
+  but they do not receive access to the stream's message history.
+
+* Zulip supports editing the content or topics of messages that have
+  already been sent (and even updating the topic of messages sent by
+  other users when editing the topic of the overall thread).
+
+  While edited messages are synced immediately to open browser
+  windows, editing messages is not a safe way to redact secret content
+  (e.g. a password) unintentionally shared via Zulip, because other
+  users may have seen and saved the content of the original message
+  (for example, they could have taken a screenshot immediately after
+  you sent the message, or have an API tool recording all messages
+  they receive).
+
+  Zulip stores and sends to clients the content of every historical
+  version of a message, so that future versions of Zulip could support
+  displaying the diffs between previous versions.
+
+#### Users and Bots
+
+* There are three types of users in a Zulip realm: Administrators,
+  normal users, and botsq.  Administrators have the ability to
+  deactivate and reactivate other human and bot users, delete streams,
+  add/remove administrator privileges, as well as change configuration
+  for the overall realm (e.g. whether an invitation is required to
+  join the realm).  Being a Zulip administrator does not provide the
+  ability to interact with other users' private messages or the
+  messages sent private streams to which the administrator is not
+  subscribed.  However, a Zulip administrator subscribed to a stream
+  can toggle whether that stream is public or private.
+
+* Every Zulip user has an API key, available on the settings page.
+  This API key can be used to do essentially everything the user can
+  do; for that reason, users should keep their API key safe.  Users
+  can rotate their own API key if it is accidentally compromised.
+
+* To properly remove a user's access to a Zulip team, it does not
+  suffice to change their password or deactivate their account in the
+  SSO system, since neither of those prevents authenticating with the
+  user's API key or those of bots the user has created.  Instead, you
+  should deactivate the user's account in the Zulip administration
+  interface (/#administration); this will automatically also
+  deactivate any bots the user had created.
+
+* The Zulip mobile apps authenticate to the server by sending the
+  user's password and retrieving the user's API key; the apps then use
+  the API key to authenticate all future interactions with the site.
+  Thus, if a user's phone is lost, in addition to changing passwords,
+  you should rotate the user's Zulip API key.
+
+* Zulip bots are used for integrations.  A Zulip bot can do everything
+  a normal user in the realm can do including reading other, with a
+  few exceptions (e.g. a bot cannot login to the web application or
+  create other bots).  In particular, with the API key for a Zulip
+  bot, one can read any message sent to a public stream in that bot's
+  realm.  A likely future feature for Zulip is [limited bots that can
+  only send messages](https://github.com/zulip/zulip/issues/373).
+
+* Certain Zulip bots can be marked as "API super users"; these special
+  bots have the ability to send messages that appear to have been sent
+  by another user (an important feature for implementing integrations
+  like the Jabber, IRC, and Zephyr mirrors).
+
+#### User-uploaded content
+
+* Zulip supports user-uploaded files; ideally they should be hosted
+  from a separate domain from the main Zulip server to protect against
+  various same-domain attacks (e.g. zulip-user-content.example.com)
+  using the S3 integration.
+
+  The URLs of user-uploaded files are secret; if you are using the
+  "local file upload" integration, anyone with the URL of an uploaded
+  file can access the file.  This means the local uploads integration
+  is vulnerable to a subtle attack where if a user clicks on a link in
+  a secret .PDF or .HTML file that had been uploaded to Zulip, access
+  to the file might be leaked to the other server via the Referrer
+  header (see https://github.com/zulip/zulip/issues/320).
+
+  The Zulip S3 file upload integration is relatively safe against that
+  attack, because the URLs of files presented to users don't host the
+  content.  Instead, the S3 integration checks the user has a valid
+  Zulip session in the relevant realm, and if so then redirects the
+  browser to a one-time S3 URL that expires a short time later.
+  Keeping the URL secret is still important to avoid other users in
+  the Zulip realm from being able to access the file.
+
+* Zulip supports using the Camo image proxy to proxy content like
+  inline image previews that can be inserted into the Zulip message
+  feed by other users over HTTPS.
+
+* By default, Zulip will provide image previews inline in the body of
+  messages when a message contains a link to an image.  You can
+  control this using the `INLINE_IMAGE_PREVIEW` setting.
+
+#### Final notes and security response
+
+If you find some aspect of Zulip that seems inconsistent with this
+security model, please report it to support@zulip.com so that we can
+investigate and coordinate an appropriate security release if needed.
+
+Zulip security announcements will be sent to
+zulip-announce@googlegroups.com, so you should subscribe if you are
+running Zulip in production.
+
+Remote User SSO Authentication
+==============================
 
 Zulip supports integrating with a corporate Single-Sign-On solution.
 There are a few ways to do it, but this section documents how to
 configure Zulip to use an SSO solution that best supports Apache and
-will set the REMOTE_USER variable:
+will set the `REMOTE_USER` variable:
 
-(0) Check that /etc/zulip/settings.py has
-"zproject.backends.ZulipRemoteUserBackend" as the only enabled value
-in the "AUTHENTICATION_BACKENDS" list, and that "SSO_APPEND_DOMAIN" is
+(0) Check that `/etc/zulip/settings.py` has
+`zproject.backends.ZulipRemoteUserBackend` as the only enabled value
+in the `AUTHENTICATION_BACKENDS` list, and that `SSO_APPEND_DOMAIN` is
 correct set depending on whether your SSO system uses email addresses
-or just usernames in REMOTE_USER.
+or just usernames in `REMOTE_USER`.
 
 Make sure that you've restarted the Zulip server since making this
 configuration change.
 
-(1) Edit /etc/zulip/zulip.conf and change the puppet_classes line to read:
+(1) Edit `/etc/zulip/zulip.conf` and change the `puppet_classes` line to read:
 
-puppet_classes = zulip::enterprise, zulip::apache_sso
-
-(2) As root, run
-
-/home/zulip/deployments/current/scripts/zulip-puppet-apply
+```
+puppet_classes = zulip::voyager, zulip::apache_sso
+```
 
+(2) As root, run `/home/zulip/deployments/current/scripts/zulip-puppet-apply`
 to install our SSO integration.
 
 (3) To configure our SSO integration, edit
-/etc/apache2/sites-available/zulip-sso.example and fill in the
-configuration required for your SSO service to set REMOTE_USER and
-place your completed configuration file at
+`/etc/apache2/sites-available/zulip-sso.example` and fill in the
+configuration required for your SSO service to set `REMOTE_USER` and
+place your completed configuration file at `/etc/apache2/sites-available/zulip-sso`
 
-/etc/apache2/sites-available/zulip-sso
+(4) Run `a2ensite zulip-sso` to enable the Apache integration site.
 
-(4) Run
-
-a2ensite zulip-sso
-
-To enable the Apache integration site.
-
-Now you should be able to visit https://zulip.yourdomain.net/ and
+Now you should be able to visit `https://zulip.example.com/` and
 login via the SSO solution.
+
+
+### Troubleshooting Remote User SSO
+
+This system is a little finicky to networking setup (e.g. common
+issues have to do with /etc/hosts not mapping settings.EXTERNAL_HOST
+to the Apache listening on 127.0.0.1/localhost, for example).  It can
+often help while debugging to temporarily change the Apache config in
+/etc/apache2/sites-available/zulip-sso to listen on all interfaces
+rather than just 127.0.0.1 as you debug this.  It can also be helpful
+to change /etc/nginx/zulip-include/app.d/external-sso.conf to
+proxy_pass to a more explicit URL possibly not over HTTPS when
+debugging.  The following log files can be helpful when debugging this
+setup:
+
+* /var/log/zulip/{errors.log,server.log} (the usual places)
+* /var/log/nginx/access.log (nginx access logs)
+* /var/log/apache2/zulip_auth_access.log (you may want to change
+  LogLevel to "debug" in the apache config file to make this more
+  verbose)
+
+Here's a summary of how the remote user SSO system works assuming
+you're using HTTP basic auth; this summary should help with
+understanding what's going on as you try to debug:
+
+* Since you've configured /etc/zulip/settings.py to only define the
+  zproject.backends.ZulipRemoteUserBackend, zproject/settings.py
+  configures /accounts/login/sso as HOME_NOT_LOGGED_IN, which makes
+  `https://zulip.example.com/` aka the homepage for the main Zulip
+  Django app running behind nginx redirect to /accounts/login/sso if
+  you're not logged in.
+
+* nginx proxies requests to /accounts/login/sso/ to an Apache instance
+  listening on localhost:8888 apache via the config in
+  /etc/nginx/zulip-include/app.d/external-sso.conf (using the upstream
+  localhost:8888 defined in /etc/nginx/zulip-include/upstreams).
+
+* The Apache zulip-sso site which you've enabled listens on
+  localhost:8888 and presents the htpasswd dialogue; you provide
+  correct login information and the request reaches a second Zulip
+  Django app instance that is running behind Apache with with
+  REMOTE_USER set.  That request is served by
+  `zerver.views.remote_user_sso`, which just checks the REMOTE_USER
+  variable and either logs in (sets a cookie) or registers the new
+  user (depending whether they have an account).
+
+* After succeeding, that redirects the user back to / on port 443
+  (hosted by nginx); the main Zulip Django app sees the cookie and
+  proceeds to load the site homepage with them logged in (just as if
+  they'd logged in normally via username/password).
+
+Again, most issues with this setup tend to be subtle issues with the
+hostname/DNS side of the configuration.  Suggestions for how to
+improve this SSO setup documentation are very welcome!
+
+
+Remote Postgresql database
+==========================
+
+If you want to use a remote Postgresql database, you should configure the information about the connection with the server. You need a user called "zulip" in your database server. You can configure these options in /etc/zulip/settings.py
+
+* REMOTE_POSTGRES_HOST: Name or IP address of the remote host
+* REMOTE_POSTGRES_SSLMODE: SSL Mode used to connect to the server, different options you can use are:
+  * disable: I don't care about security, and I don't want to pay the overhead of encryption.
+  * allow: I don't care about security, but I will pay the overhead of encryption if the server insists on it.
+  * prefer: I don't care about encryption, but I wish to pay the overhead of encryption if the server supports it.
+  * require: I want my data to be encrypted, and I accept the overhead. I trust that the network will make sure I always connect to the server I want.
+  * verify-ca: I want my data encrypted, and I accept the overhead. I want to be sure that I connect to a server that I trust.
+  * verify-full: I want my data encrypted, and I accept the overhead. I want to be sure that I connect to a server I trust, and that it's the one I specify.
+
+Then you should specify the password of the user zulip for the database in /etc/zulip/zulip-secrets.conf:
+
+```
+postgres_password = xxxx
+```
+
+Finally you can stop your database in the zulip server to save some memory, you can do it with:
+
+```
+sudo service postgresql stop
+sudo update-rc.d postgresql disable
+```
+

THIRDPARTY

@@ -2,7 +2,7 @@ Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
 Upstream-Name: Zulip
 Upstream-Contact: Zulip Development Discussion <zulip-devel@googlegroups.com>
 Source: https://zulip.org/
-Comment: 
+Comment:
  Unless otherwise noted, the Zulip software is distributed under the Apache
  License, Version 2.0. The software includes some works released by third
  parties under other free and open source licenses. Those works are
@@ -38,10 +38,6 @@ Files: confirmation/*
 Copyright: 2008, Jarek Zgoda <jarek.zgoda@gmail.com>
 License: BSD-3-Clause
 
-Files: node_modules/handlebars/*
-Copyright: 2011 Yehuda Katz
-License: Expat
-
 Files: puppet/apt/*
 Copyright: 2011, Evolving Web Inc.
 License: Expat
@@ -55,14 +51,14 @@ Comment: https://github.com/DavidS/puppet-common
 
 Files: puppet/stdlib/*
 Copyright: 2011, Krzysztof Wilczynski
- 2011, Puppet Labs Inc 
+ 2011, Puppet Labs Inc
 License: Apache-2.0
 
 File: puppet/zulip_internal/files/mediawiki/Auth_remoteuser.php
-Copyright: 2006 Otheus Shelling 
+Copyright: 2006 Otheus Shelling
  2007 Rusty Burchfield
- 2009 James Kinsman   
- 2010 Daniel Thomas   
+ 2009 James Kinsman
+ 2010 Daniel Thomas
  2010 Ian Ward Comfort
 License: GPL-2.0
 Comment: Not linked.
@@ -131,10 +127,6 @@ Copyright: Google, Inc.
 License: Apache-2.0
 Comment: These are actually Noto Emoji, not gemoji.
 
-Files: static/third/handlebars/handlebars.runtime.js
-Copyright: 2011 Yehuda Katz
-License: Expat
-
 Files: static/third/html5-formdata/formdata.js
 Copyright: 2010 François de Metz
 License: Expat
@@ -205,7 +197,7 @@ License: SIL-OFL-1.1
 
 Files: static/third/spectrum/*
 Copyright: 2013 Brian Grinstead
-License: Expat 
+License: Expat
 
 Files: static/third/spin/spin.js
 Copyright: 2011-2013 Felix Gnass
@@ -226,7 +218,7 @@ Copyright: 2010 C. F., Wong
 License: Expat
 
 Files: static/third/zocial/*
-Copyright: Sam Collins 
+Copyright: Sam Collins
 License: Expat
 
 Files: tools/inject-messages/othello
@@ -254,11 +246,325 @@ Files: zerver/lib/ccache.py
 Copyright: 2013 David Benjamin and Alan Huang
 License: Expat
 
-Files: zerver/tests/frontend/casperjs/*
+Files: frontend_tests/casperjs/*
 Copyright: 2011-2012 Nicolas Perriault
  Joyent, Inc. and other Node contributors
 License: Expat
 
-Files: zerver/tests/frontend/casperjs/modules/vendors/*
+Files: frontend_tests/casperjs/modules/vendors/*
 Copyright: 2011, Jeremy Ashkenas
 License: Expat
+
+
+License: Apache-2.0
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+ .
+     http://www.apache.org/licenses/LICENSE-2.0
+ .
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ .
+ On Debian systems, the full text of the Apache License version 2 can
+ be found in /usr/share/common-licenses/Apache-2.0.
+
+License: BSD-2-clause
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+    notice(s), this list of conditions and the following disclaimer
+    unmodified other than the allowable addition of one or more
+    copyright notices.
+ 2. Redistributions in binary form must reproduce the above copyright
+    notice(s), this list of conditions and the following disclaimer in
+    the documentation and/or other materials provided with the
+    distribution.
+ .
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
+ EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE
+ LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+License: BSD-3-Clause
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ .
+ 1. Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+ .
+ 2. Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in the
+    documentation and/or other materials provided with the distribution.
+ .
+ 3. Neither the name of the copyright holder nor the names of its
+    contributors may be used to endorse or promote products derived from
+    this software without specific prior written permission.
+ .
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+ CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+License: CC-0-1.0
+ Creative Commons CC0 1.0 Universal
+ CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE
+ LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN
+ ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS INFORMATION
+ ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES REGARDING THE
+ USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED HEREUNDER, AND
+ DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM THE USE OF THIS DOCUMENT
+ OR THE INFORMATION OR WORKS PROVIDED HEREUNDER.
+ .
+ Statement of Purpose
+ .
+ The laws of most jurisdictions throughout the world automatically confer
+ exclusive Copyright and Related Rights (defined below) upon the creator
+ and subsequent owner(s) (each and all, an "owner") of an original work
+ of authorship and/or a database (each, a "Work").
+ .
+ Certain owners wish to permanently relinquish those rights to a Work for
+ the purpose of contributing to a commons of creative, cultural and
+ scientific works ("Commons") that the public can reliably and without
+ fear of later claims of infringement build upon, modify, incorporate in
+ other works, reuse and redistribute as freely as possible in any form
+ whatsoever and for any purposes, including without limitation commercial
+ purposes. These owners may contribute to the Commons to promote the
+ ideal of a free culture and the further production of creative, cultural
+ and scientific works, or to gain reputation or greater distribution for
+ their Work in part through the use and efforts of others.
+ .
+ For these and/or other purposes and motivations, and without any
+ expectation of additional consideration or compensation, the person
+ associating CC0 with a Work (the "Affirmer"), to the extent that he or
+ she is an owner of Copyright and Related Rights in the Work, voluntarily
+ elects to apply CC0 to the Work and publicly distribute the Work under
+ its terms, with knowledge of his or her Copyright and Related Rights in
+ the Work and the meaning and intended legal effect of CC0 on those
+ rights.
+ .
+ 1. Copyright and Related Rights. A Work made available under CC0 may be
+ protected by copyright and related or neighboring rights ("Copyright and
+ Related Rights"). Copyright and Related Rights include, but are not
+ limited to, the following:
+ .
+ i. the right to reproduce, adapt, distribute, perform, display,
+ communicate, and translate a Work;
+ .
+ ii. moral rights retained by the original author(s) and/or performer(s);
+ .
+ iii. publicity and privacy rights pertaining to a person's image or
+ likeness depicted in a Work;
+ .
+ iv. rights protecting against unfair competition in regards to a Work,
+ subject to the limitations in paragraph 4(a), below;
+ .
+ v. rights protecting the extraction, dissemination, use and reuse of
+ data in a Work;
+ .
+ vi. database rights (such as those arising under Directive 96/9/EC of
+ the European Parliament and of the Council of 11 March 1996 on the legal
+ protection of databases, and under any national implementation thereof,
+ including any amended or successor version of such directive); and
+ .
+ vii. other similar, equivalent or corresponding rights throughout the
+ world based on applicable law or treaty, and any national
+ implementations thereof.
+ .
+ 2. Waiver. To the greatest extent permitted by, but not in contravention
+ of, applicable law, Affirmer hereby overtly, fully, permanently,
+ irrevocably and unconditionally waives, abandons, and surrenders all of
+ Affirmer's Copyright and Related Rights and associated claims and causes
+ of action, whether now known or unknown (including existing as well as
+ future claims and causes of action), in the Work (i) in all territories
+ worldwide, (ii) for the maximum duration provided by applicable law or
+ treaty (including future time extensions), (iii) in any current or
+ future medium and for any number of copies, and (iv) for any purpose
+ whatsoever, including without limitation commercial, advertising or
+ promotional purposes (the "Waiver"). Affirmer makes the Waiver for the
+ benefit of each member of the public at large and to the detriment of
+ Affirmer's heirs and successors, fully intending that such Waiver shall
+ not be subject to revocation, rescission, cancellation, termination, or
+ any other legal or equitable action to disrupt the quiet enjoyment of
+ the Work by the public as contemplated by Affirmer's express Statement
+ of Purpose.
+ .
+ 3. Public License Fallback. Should any part of the Waiver for any reason
+ be judged legally invalid or ineffective under applicable law, then the
+ Waiver shall be preserved to the maximum extent permitted taking into
+ account Affirmer's express Statement of Purpose. In addition, to the
+ extent the Waiver is so judged Affirmer hereby grants to each affected
+ person a royalty-free, non transferable, non sublicensable, non
+ exclusive, irrevocable and unconditional license to exercise Affirmer's
+ Copyright and Related Rights in the Work (i) in all territories
+ worldwide, (ii) for the maximum duration provided by applicable law or
+ treaty (including future time extensions), (iii) in any current or
+ future medium and for any number of copies, and (iv) for any purpose
+ whatsoever, including without limitation commercial, advertising or
+ promotional purposes (the "License"). The License shall be deemed
+ effective as of the date CC0 was applied by Affirmer to the Work. Should
+ any part of the License for any reason be judged legally invalid or
+ ineffective under applicable law, such partial invalidity or
+ ineffectiveness shall not invalidate the remainder of the License, and
+ in such case Affirmer hereby affirms that he or she will not (i)
+ exercise any of his or her remaining Copyright and Related Rights in the
+ Work or (ii) assert any associated claims and causes of action with
+ respect to the Work, in either case contrary to Affirmer's express
+ Statement of Purpose.
+ .
+ 4. Limitations and Disclaimers.
+ .
+ a. No trademark or patent rights held by Affirmer are waived, abandoned,
+ surrendered, licensed or otherwise affected by this document.
+ .
+ b. Affirmer offers the Work as-is and makes no representations or
+ warranties of any kind concerning the Work, express, implied, statutory
+ or otherwise, including without limitation warranties of title,
+ merchantability, fitness for a particular purpose, non infringement, or
+ the absence of latent or other defects, accuracy, or the present or
+ absence of errors, whether or not discoverable, all to the greatest
+ extent permissible under applicable law.
+ .
+ c. Affirmer disclaims responsibility for clearing rights of other
+ persons that may apply to the Work or any use thereof, including without
+ limitation any person's Copyright and Related Rights in the Work.
+ Further, Affirmer disclaims responsibility for obtaining any necessary
+ consents, permissions or other rights required for any use of the Work.
+ .
+ d. Affirmer understands and acknowledges that Creative Commons is not a
+ party to this document and has no duty or obligation with respect to
+ this CC0 or use of the Work.
+
+License: Expat
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+ .
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+ .
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+
+License: GPL-2.0
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; version 2, dated June, 1991.
+ .
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+ .
+ On Debian systems, the complete text of the GNU General Public License
+ can be found in /usr/share/common-licenses/GPL-2 file.
+
+License: SIL-OFL-1.1
+ ---------------------------------------------------------------------------
+ SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
+ ---------------------------------------------------------------------------
+ .
+ PREAMBLE
+ .
+ The goals of the Open Font License (OFL) are to stimulate worldwide development
+ of collaborative font projects, to support the font creation efforts of academic
+ and linguistic communities, and to provide a free and open framework in which
+ fonts may be shared and improved in partnership with others.
+ .
+ The OFL allows the licensed fonts to be used, studied, modified and redistributed
+ freely as long as they are not sold by themselves. The fonts, including any
+ derivative works, can be bundled, embedded, redistributed and/or sold with any
+ software provided that any reserved names are not used by derivative works. The
+ fonts and derivatives, however, cannot be released under any other type of license.
+ The requirement for fonts to remain under this license does not apply to any
+ document created using the fonts or their derivatives.
+ .
+ DEFINITIONS
+ .
+ "Font Software" refers to the set of files released by the Copyright Holder(s) under
+ this license and clearly marked as such. This may include source files, build
+ scripts and documentation.
+ .
+ "Reserved Font Name" refers to any names specified as such after the copyright
+ statement(s).
+ .
+ "Original Version" refers to the collection of Font Software components as
+ distributed by the Copyright Holder(s).
+ .
+ "Modified Version" refers to any derivative made by adding to, deleting, or
+ substituting -- in part or in whole -- any of the components of the Original Version,
+ by changing formats or by porting the Font Software to a new environment.
+ .
+ "Author" refers to any designer, engineer, programmer, technical writer or other
+ person who contributed to the Font Software.
+ .
+ PERMISSION & CONDITIONS
+ .
+ Permission is hereby granted, free of charge, to any person obtaining a copy of the
+ Font Software, to use, study, copy, merge, embed, modify, redistribute, and sell
+ modified and unmodified copies of the Font Software, subject to the following
+ conditions:
+ .
+ 1) Neither the Font Software nor any of its individual components, in Original or
+ Modified Versions, may be sold by itself.
+ .
+ 2) Original or Modified Versions of the Font Software may be bundled, redistributed
+ and/or sold with any software, provided that each copy contains the above copyright
+ notice and this license. These can be included either as stand-alone text files,
+ human-readable headers or in the appropriate machine-readable metadata fields within
+ text or binary files as long as those fields can be easily viewed by the user.
+ .
+ 3) No Modified Version of the Font Software may use the Reserved Font Name(s) unless
+ explicit written permission is granted by the corresponding Copyright Holder. This
+ restriction only applies to the primary font name as presented to the users.
+ .
+ 4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font Software shall
+ not be used to promote, endorse or advertise any Modified Version, except to
+ acknowledge the contribution(s) of the Copyright Holder(s) and the Author(s) or with
+ their explicit written permission.
+ .
+ 5) The Font Software, modified or unmodified, in part or in whole, must be distributed
+ entirely under this license, and must not be distributed under any other license. The
+ requirement for fonts to remain under this license does not apply to any document
+ created using the Font Software.
+ .
+ TERMINATION
+ .
+ This license becomes null and void if any of the above conditions are not met.
+ .
+ DISCLAIMER
+ .
+ THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+ INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+ PARTICULAR PURPOSE AND NONINFRINGEMENT OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER
+ RIGHT. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES,
+ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF THE USE OR
+ INABILITY TO USE THE FONT SOFTWARE OR FROM OTHER DEALINGS IN THE FONT SOFTWARE.

Vagrantfile

@@ -24,7 +24,7 @@ set -x
 set -e
 sudo apt-get update
 sudo apt-get install -y python-pbs
-python /srv/zulip/provision.py
+/usr/bin/python /srv/zulip/provision.py
 SCRIPT
 
   config.vm.provision "shell",

analytics/management/commands/active_user_stats.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+from __future__ import print_function
 
 from django.core.management.base import BaseCommand
 
@@ -36,7 +37,7 @@ class Command(BaseCommand):
                     user_info[last_presence.user_profile.realm.domain][bucket].append(last_presence.user_profile.email)
 
         for realm, buckets in user_info.items():
-            print("Realm %s" % realm)
+            print("Realm %s" % (realm,))
             for hr, users in sorted(buckets.items()):
                 print("\tUsers for %s: %s" % (hr, len(users)))
                 statsd.gauge("users.active.%s.%shr" %  (statsd_key(realm, True), statsd_key(hr, True)), len(users))
@@ -51,7 +52,7 @@ class Command(BaseCommand):
                 if datetime.now(activity.last_visit.tzinfo) - activity.last_visit < timedelta(hours=bucket):
                     user_info[activity.user_profile.realm.domain][bucket].append(activity.user_profile.email)
         for realm, buckets in user_info.items():
-            print("Realm %s" % realm)
+            print("Realm %s" % (realm,))
             for hr, users in sorted(buckets.items()):
                 print("\tUsers reading for %s: %s" % (hr, len(users)))
                 statsd.gauge("users.reading.%s.%shr" %  (statsd_key(realm, True), statsd_key(hr, True)), len(users))

analytics/management/commands/active_user_stats_by_day.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+from __future__ import print_function
 
 import datetime
 import pytz
@@ -19,6 +20,6 @@ class Command(BaseCommand):
             date = datetime.datetime.now() - datetime.timedelta(days=1)
         else:
             date = datetime.datetime.strptime(options["date"], "%Y-%m-%d")
-        print "Activity data for", date
-        print activity_averages_during_day(date)
-        print "Please note that the total registered user count is a total for today"
+        print("Activity data for", date)
+        print(activity_averages_during_day(date))
+        print("Please note that the total registered user count is a total for today")

analytics/management/commands/analyze_mit.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+from __future__ import print_function
 
 from optparse import make_option
 from django.core.management.base import BaseCommand
@@ -63,7 +64,7 @@ def compute_stats(log_level):
         logging.info("Top %6s | %s%%" % (size, round(top_percents[size], 1)))
 
     grand_total = sum(total_counts.values())
-    print grand_total
+    print(grand_total)
     logging.info("%15s | %s" % ("Client", "Percentage"))
     for client in total_counts.keys():
         logging.info("%15s | %s%%" % (client, round(100. * total_counts[client] / grand_total, 1)))

analytics/management/commands/analyze_user_activity.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+from __future__ import print_function
 
 from zerver.lib.statistics import seconds_usage_between
 
@@ -16,7 +17,7 @@ def analyze_activity(options):
     if options["realm"]:
         user_profile_query = user_profile_query.filter(realm__domain=options["realm"])
 
-    print "Per-user online duration:\n"
+    print("Per-user online duration:\n")
     total_duration = datetime.timedelta(0)
     for user_profile in user_profile_query:
         duration = seconds_usage_between(user_profile, day_start, day_end)
@@ -25,11 +26,11 @@ def analyze_activity(options):
             continue
 
         total_duration += duration
-        print "%-*s%s" % (37, user_profile.email, duration, )
+        print("%-*s%s" % (37, user_profile.email, duration, ))
 
-    print "\nTotal Duration:                      %s" % (total_duration,)
-    print "\nTotal Duration in minutes:           %s" % (total_duration.total_seconds() / 60.,)
-    print "Total Duration amortized to a month: %s" % (total_duration.total_seconds() * 30. / 60.,)
+    print("\nTotal Duration:                      %s" % (total_duration,))
+    print("\nTotal Duration in minutes:           %s" % (total_duration.total_seconds() / 60.,))
+    print("Total Duration amortized to a month: %s" % (total_duration.total_seconds() * 30. / 60.,))
 
 class Command(BaseCommand):
     help = """Report analytics of user activity on a per-user and realm basis.
@@ -42,7 +43,7 @@ It will correctly not count server-initiated reloads in the activity statistics.
 
 The duration flag can be used to control how many days to show usage duration for
 
-Usage: python manage.py analyze_user_activity [--realm=zulip.com] [--date=2013-09-10] [--duration=1]
+Usage: python2.7 manage.py analyze_user_activity [--realm=zulip.com] [--date=2013-09-10] [--duration=1]
 
 By default, if no date is selected 2013-09-10 is used. If no realm is provided, information
 is shown for all realms"""

analytics/management/commands/client_activity.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+from __future__ import print_function
 
 from django.core.management.base import BaseCommand
 from django.db.models import Count
@@ -13,9 +14,9 @@ class Command(BaseCommand):
 
 Usage examples:
 
-python manage.py client_activity
-python manage.py client_activity zulip.com
-python manage.py client_activity jesstess@zulip.com"""
+python2.7 manage.py client_activity
+python2.7 manage.py client_activity zulip.com
+python2.7 manage.py client_activity jesstess@zulip.com"""
 
     def add_arguments(self, parser):
         parser.add_argument('arg', metavar='<arg>', type=str, nargs='?', default=None,
@@ -48,8 +49,8 @@ python manage.py client_activity jesstess@zulip.com"""
         counts.sort()
 
         for count in counts:
-            print "%25s %15d" % (count[1], count[0])
-        print "Total:", total
+            print("%25s %15d" % (count[1], count[0]))
+        print("Total:", total)
 
 
     def handle(self, *args, **options):
@@ -70,5 +71,5 @@ python manage.py client_activity jesstess@zulip.com"""
                     self.compute_activity(UserActivity.objects.filter(
                             user_profile__realm=realm))
                 except Realm.DoesNotExist:
-                    print "Unknown user or domain %s" % (arg,)
+                    print("Unknown user or domain %s" % (arg,))
                     exit(1)

analytics/management/commands/realm_stats.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+from __future__ import print_function
 
 import datetime
 import pytz
@@ -6,7 +7,7 @@ import pytz
 from django.core.management.base import BaseCommand
 from django.db.models import Count
 from zerver.models import UserProfile, Realm, Stream, Message, Recipient, UserActivity, \
-    Subscription, UserMessage
+    Subscription, UserMessage, get_realm
 
 MOBILE_CLIENT_LIST = ["Android", "ios"]
 HUMAN_CLIENT_LIST = MOBILE_CLIENT_LIST + ["website"]
@@ -65,52 +66,51 @@ class Command(BaseCommand):
             fraction = 0.0
         else:
             fraction = numerator / float(denominator)
-        print "%.2f%% of" % (fraction * 100,), text
+        print("%.2f%% of" % (fraction * 100,), text)
 
     def handle(self, *args, **options):
         if options['realms']:
             try:
-                realms = [Realm.objects.get(domain=domain) for domain in options['realms']]
-            except Realm.DoesNotExist, e:
-                print e
+                realms = [get_realm(domain) for domain in options['realms']]
+            except Realm.DoesNotExist as e:
+                print(e)
                 exit(1)
         else:
             realms = Realm.objects.all()
 
         for realm in realms:
-            print realm.domain
+            print(realm.domain)
 
             user_profiles = UserProfile.objects.filter(realm=realm, is_active=True)
             active_users = self.active_users(realm)
             num_active = len(active_users)
 
-            print "%d active users (%d total)" % (num_active, len(user_profiles))
+            print("%d active users (%d total)" % (num_active, len(user_profiles)))
             streams = Stream.objects.filter(realm=realm).extra(
                 tables=['zerver_subscription', 'zerver_recipient'],
                 where=['zerver_subscription.recipient_id = zerver_recipient.id',
                        'zerver_recipient.type = 2',
                        'zerver_recipient.type_id = zerver_stream.id',
                        'zerver_subscription.active = true']).annotate(count=Count("name"))
-            print "%d streams" % (streams.count(),)
+            print("%d streams" % (streams.count(),))
 
             for days_ago in (1, 7, 30):
-                print "In last %d days, users sent:" % (days_ago,)
+                print("In last %d days, users sent:" % (days_ago,))
                 sender_quantities = [self.messages_sent_by(user, days_ago) for user in user_profiles]
                 for quantity in sorted(sender_quantities, reverse=True):
-                    print quantity,
-                print ""
+                    print(quantity, end=' ')
+                print("")
 
-                print "%d stream messages" % (self.stream_messages(realm, days_ago),)
-                print "%d one-on-one private messages" % (self.private_messages(realm, days_ago),)
-                print "%d messages sent via the API" % (self.api_messages(realm, days_ago),)
-                print "%d group private messages" % (self.group_private_messages(realm, days_ago),)
+                print("%d stream messages" % (self.stream_messages(realm, days_ago),))
+                print("%d one-on-one private messages" % (self.private_messages(realm, days_ago),))
+                print("%d messages sent via the API" % (self.api_messages(realm, days_ago),))
+                print("%d group private messages" % (self.group_private_messages(realm, days_ago),))
 
-            num_notifications_enabled = len(filter(lambda x: x.enable_desktop_notifications == True,
-                                                   active_users))
+            num_notifications_enabled = len([x for x in active_users if x.enable_desktop_notifications == True])
             self.report_percentage(num_notifications_enabled, num_active,
                                    "active users have desktop notifications enabled")
 
-            num_enter_sends = len(filter(lambda x: x.enter_sends, active_users))
+            num_enter_sends = len([x for x in active_users if x.enter_sends])
             self.report_percentage(num_enter_sends, num_active,
                                    "active users have enter-sends")
 
@@ -124,8 +124,8 @@ class Command(BaseCommand):
             starrers = UserMessage.objects.filter(user_profile__in=user_profiles,
                                                   flags=UserMessage.flags.starred).values(
                 "user_profile").annotate(count=Count("user_profile"))
-            print "%d users have starred %d messages" % (
-                len(starrers), sum([elt["count"] for elt in starrers]))
+            print("%d users have starred %d messages" % (
+                len(starrers), sum([elt["count"] for elt in starrers])))
 
             active_user_subs = Subscription.objects.filter(
                 user_profile__in=user_profiles, active=True)
@@ -133,20 +133,20 @@ class Command(BaseCommand):
             # Streams not in home view
             non_home_view = active_user_subs.filter(in_home_view=False).values(
                 "user_profile").annotate(count=Count("user_profile"))
-            print "%d users have %d streams not in home view" % (
-                len(non_home_view), sum([elt["count"] for elt in non_home_view]))
+            print("%d users have %d streams not in home view" % (
+                len(non_home_view), sum([elt["count"] for elt in non_home_view])))
 
             # Code block markup
             markup_messages = human_messages.filter(
                 sender__realm=realm, content__contains="~~~").values(
                 "sender").annotate(count=Count("sender"))
-            print "%d users have used code block markup on %s messages" % (
-                len(markup_messages), sum([elt["count"] for elt in markup_messages]))
+            print("%d users have used code block markup on %s messages" % (
+                len(markup_messages), sum([elt["count"] for elt in markup_messages])))
 
             # Notifications for stream messages
             notifications = active_user_subs.filter(notifications=True).values(
                 "user_profile").annotate(count=Count("user_profile"))
-            print "%d users receive desktop notifications for %d streams" % (
-                len(notifications), sum([elt["count"] for elt in notifications]))
+            print("%d users receive desktop notifications for %d streams" % (
+                len(notifications), sum([elt["count"] for elt in notifications])))
 
-            print ""
+            print("")

analytics/management/commands/stream_stats.py

@@ -1,8 +1,9 @@
 from __future__ import absolute_import
+from __future__ import print_function
 
 from django.core.management.base import BaseCommand
 from django.db.models import Q
-from zerver.models import Realm, Stream, Message, Subscription, Recipient
+from zerver.models import Realm, Stream, Message, Subscription, Recipient, get_realm
 
 class Command(BaseCommand):
     help = "Generate statistics on the streams for a realm."
@@ -14,27 +15,27 @@ class Command(BaseCommand):
     def handle(self, *args, **options):
         if options['realms']:
             try:
-                realms = [Realm.objects.get(domain=domain) for domain in options['realms']]
-            except Realm.DoesNotExist, e:
-                print e
+                realms = [get_realm(domain) for domain in options['realms']]
+            except Realm.DoesNotExist as e:
+                print(e)
                 exit(1)
         else:
             realms = Realm.objects.all()
 
         for realm in realms:
-            print realm.domain
-            print "------------"
-            print "%25s %15s %10s" % ("stream", "subscribers", "messages")
+            print(realm.domain)
+            print("------------")
+            print("%25s %15s %10s" % ("stream", "subscribers", "messages"))
             streams = Stream.objects.filter(realm=realm).exclude(Q(name__istartswith="tutorial-"))
             invite_only_count = 0
             for stream in streams:
                 if stream.invite_only:
                     invite_only_count += 1
                     continue
-                print "%25s" % (stream.name,),
+                print("%25s" % (stream.name,), end=' ')
                 recipient = Recipient.objects.filter(type=Recipient.STREAM, type_id=stream.id)
-                print "%10d" % (len(Subscription.objects.filter(recipient=recipient, active=True)),),
+                print("%10d" % (len(Subscription.objects.filter(recipient=recipient, active=True)),), end=' ')
                 num_messages = len(Message.objects.filter(recipient=recipient))
-                print "%12d" % (num_messages,)
-            print "%d invite-only streams" % (invite_only_count,)
-            print ""
+                print("%12d" % (num_messages,))
+            print("%d invite-only streams" % (invite_only_count,))
+            print("")

analytics/management/commands/user_stats.py

@@ -1,10 +1,12 @@
 from __future__ import absolute_import
+from __future__ import print_function
 
 import datetime
 import pytz
 
 from django.core.management.base import BaseCommand
-from zerver.models import UserProfile, Realm, Stream, Message
+from zerver.models import UserProfile, Realm, Stream, Message, get_realm
+from six.moves import range
 
 class Command(BaseCommand):
     help = "Generate statistics on user activity."
@@ -21,21 +23,21 @@ class Command(BaseCommand):
     def handle(self, *args, **options):
         if options['realms']:
             try:
-                realms = [Realm.objects.get(domain=domain) for domain in options['realms']]
-            except Realm.DoesNotExist, e:
-                print e
+                realms = [get_realm(domain) for domain in options['realms']]
+            except Realm.DoesNotExist as e:
+                print(e)
                 exit(1)
         else:
             realms = Realm.objects.all()
 
         for realm in realms:
-            print realm.domain
+            print(realm.domain)
             user_profiles = UserProfile.objects.filter(realm=realm, is_active=True)
-            print "%d users" % (len(user_profiles),)
-            print "%d streams" % (len(Stream.objects.filter(realm=realm)),)
+            print("%d users" % (len(user_profiles),))
+            print("%d streams" % (len(Stream.objects.filter(realm=realm)),))
 
             for user_profile in user_profiles:
-                print "%35s" % (user_profile.email,),
+                print("%35s" % (user_profile.email,), end=' ')
                 for week in range(10):
-                    print "%5d" % (self.messages_sent_by(user_profile, week)),
-                print ""
+                    print("%5d" % (self.messages_sent_by(user_profile, week)), end=' ')
+                print("")

analytics/views.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 from django.db import connection
 from django.template import RequestContext, loader
 from django.utils.html import mark_safe
@@ -15,6 +16,10 @@ import itertools
 import time
 import re
 import pytz
+from six.moves import filter
+from six.moves import map
+from six.moves import range
+from six.moves import zip
 eastern_tz = pytz.timezone('US/Eastern')
 
 def make_table(title, cols, rows, has_row_class=False):
@@ -22,7 +27,7 @@ def make_table(title, cols, rows, has_row_class=False):
     if not has_row_class:
         def fix_row(row):
             return dict(cells=row, row_class=None)
-        rows = map(fix_row, rows)
+        rows = list(map(fix_row, rows))
 
     data = dict(title=title, cols=cols, rows=rows)
 
@@ -37,7 +42,7 @@ def dictfetchall(cursor):
     "Returns all rows from a cursor as a dict"
     desc = cursor.description
     return [
-        dict(zip([col[0] for col in desc], row))
+        dict(list(zip([col[0] for col in desc], row)))
         for row in cursor.fetchall()
     ]
 
@@ -226,7 +231,7 @@ def realm_summary_table(realm_minutes):
     def meets_goal(row):
         return row['active_user_count'] >= 5
 
-    num_active_sites = len(filter(meets_goal, rows))
+    num_active_sites = len(list(filter(meets_goal, rows)))
 
     # create totals
     total_active_user_count = 0
@@ -375,7 +380,7 @@ def ad_hoc_queries():
         cursor = connection.cursor()
         cursor.execute(query)
         rows = cursor.fetchall()
-        rows = map(list, rows)
+        rows = list(map(list, rows))
         cursor.close()
 
         def fix_rows(i, fixup_func):
@@ -609,7 +614,7 @@ def raw_user_activity_table(records):
                 format_date_for_activity_reports(record.last_visit)
         ]
 
-    rows = map(row, records)
+    rows = list(map(row, records))
     title = 'Raw Data'
     return make_table(title, cols, rows)
 
@@ -816,7 +821,7 @@ def get_realm_activity(request, realm):
     all_user_records = {}
 
     try:
-        admins = Realm.objects.get(domain=realm).get_admin_users()
+        admins = get_realm(realm).get_admin_users()
     except Realm.DoesNotExist:
         return HttpResponseNotFound("Realm %s does not exist" % (realm,))
 

api/README.md

@@ -31,11 +31,24 @@ file is as follows:
     key=<api key from the web interface>
     email=<your email address>
     site=<your Zulip server's URI>
+    insecure=<true or false, true means do not verify the server certificate>
+    cert_bundle=<path to a file containing CA or server certificates to trust>
+
+If omitted, these settings have the following defaults:
+
+    site=https://api.zulip.com
+    insecure=false
+    cert_bundle=<the default CA bundle trusted by Python>
 
 Alternatively, you may explicitly use "--user" and "--api-key" in our
 examples, which is especially useful if you are running several bots
-which share a home directory.  There is also a "--site" option for
-setting the Zulip server on the command line.
+which share a home directory.
+
+The command line equivalents for other configuration options are:
+
+    --site=<your Zulip server's URI>
+    --insecure
+    --cert-bundle=<file>
 
 You can obtain your Zulip API key, create bots, and manage bots all
 from your Zulip [settings page](https://zulip.com/#settings).
@@ -101,3 +114,46 @@ Alternatively, if you don't want to use your ~/.zuliprc file:
         --api-key a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 \
         hamlet@example.com cordelia@example.com -m \
         "Conscience doth make cowards of us all."
+
+#### Working with an untrusted server certificate
+
+If your server has either a self-signed certificate, or a certificate signed
+by a CA that you don't wish to globally trust then by default the API will
+fail with an SSL verification error.
+
+You can add `insecure=true` to your .zuliprc file.
+
+    [api]
+    site=https://zulip.example.com
+    insecure=true
+
+This disables verification of the server certificate, so connections are
+encrypted but unauthenticated. This is not secure, but may be good enough
+for a development environment.
+
+
+You can explicitly trust the server certificate using `cert_bundle=<filename>`
+in your .zuliprc file.
+
+    [api]
+    site=https://zulip.example.com
+    cert_bundle=/home/bots/certs/zulip.example.com.crt
+
+You can also explicitly trust a different set of Certificate Authorities from
+the default bundle that is trusted by Python. For example to trust a company
+internal CA.
+
+    [api]
+    site=https://zulip.example.com
+    cert_bundle=/home/bots/certs/example.com.ca-bundle
+
+Save the server certificate (or the CA certificate) in its own file,
+converting to PEM format first if necessary.
+Verify that the certificate you have saved is the same as the one on the
+server.
+
+The `cert_bundle` option trusts the server / CA certificate only for
+interaction with the zulip site, and is relatively secure.
+
+Note that a certificate bundle is merely one or more certificates combined
+into a single file.

api/bin/zulip-send

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 # -*- coding: utf-8 -*-
 # zulip-send -- Sends a message to the specified recipients.
 

api/examples/create-user

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python2.7
 # -*- coding: utf-8 -*-
 
 # Copyright © 2012-2014 Zulip, Inc.

api/examples/edit-message

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 # -*- coding: utf-8 -*-
 
 # Copyright © 2012 Zulip, Inc.

api/examples/get-public-streams

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 # -*- coding: utf-8 -*-
 
 # Copyright © 2012 Zulip, Inc.

api/examples/list-members

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 # -*- coding: utf-8 -*-
 
 # Copyright © 2014 Zulip, Inc.

api/examples/list-subscriptions

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 # -*- coding: utf-8 -*-
 
 # Copyright © 2012 Zulip, Inc.

api/examples/print-events

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 # -*- coding: utf-8 -*-
 
 # Copyright © 2012 Zulip, Inc.

api/examples/print-messages

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 # -*- coding: utf-8 -*-
 
 # Copyright © 2012 Zulip, Inc.

api/examples/print-next-message

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 # -*- coding: utf-8 -*-
 
 # Copyright © 2012 Zulip, Inc.

api/examples/send-message

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 # -*- coding: utf-8 -*-
 
 # Copyright © 2012 Zulip, Inc.

api/examples/subscribe

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 # -*- coding: utf-8 -*-
 
 # Copyright © 2012 Zulip, Inc.

api/examples/unsubscribe

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 # -*- coding: utf-8 -*-
 
 # Copyright © 2012 Zulip, Inc.

api/integrations/asana/zulip_asana_config.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 # -*- coding: utf-8 -*-
 #
 # Copyright © 2014 Zulip, Inc.

api/integrations/asana/zulip_asana_mirror

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 # -*- coding: utf-8 -*-
 #
 # Asana integration for Zulip

api/integrations/basecamp/zulip_basecamp_config.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 # -*- coding: utf-8 -*-
 #
 # Copyright © 2014 Zulip, Inc.

api/integrations/basecamp/zulip_basecamp_mirror

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 # -*- coding: utf-8 -*-
 #
 # Zulip mirror of Basecamp activity
@@ -49,7 +49,7 @@ client = zulip.Client(
     site=config.ZULIP_SITE,
     api_key=config.ZULIP_API_KEY,
     client="ZulipBasecamp/" + VERSION)
-user_agent = "Basecamp To Zulip Mirroring script (support@zulip.com)"
+user_agent = "Basecamp To Zulip Mirroring script (zulip-devel@googlegroups.com)"
 htmlParser = HTMLParser()
 
 # find some form of JSON loader/dumper, with a preference order for speed.

api/integrations/codebase/zulip_codebase_config.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 # -*- coding: utf-8 -*-
 #
 # Copyright © 2014 Zulip, Inc.

api/integrations/codebase/zulip_codebase_mirror

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 # -*- coding: utf-8 -*-
 #
 # Zulip mirror of Codebase HQ activity
@@ -58,7 +58,7 @@ client = zulip.Client(
     site=config.ZULIP_SITE,
     api_key=config.ZULIP_API_KEY,
     client="ZulipCodebase/" + VERSION)
-user_agent = "Codebase To Zulip Mirroring script (support@zulip.com)"
+user_agent = "Codebase To Zulip Mirroring script (zulip-devel@googlegroups.com)"
 
 # find some form of JSON loader/dumper, with a preference order for speed.
 json_implementations = ['ujson', 'cjson', 'simplejson', 'json']
@@ -110,7 +110,7 @@ def handle_event(event):
         project_name = raw_props.get('name')
         project_repo_type = raw_props.get('scm_type')
 
-        url = make_url("projects/%s" % project_link)
+        url = make_url("projects/%s" % (project_link,))
         scm = "of type %s" % (project_repo_type,) if project_repo_type else ""
 
 

api/integrations/git/post-receive

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 # -*- coding: utf-8 -*-
 #
 # Zulip notification post-receive hook.

api/integrations/git/zulip_git_config.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 # -*- coding: utf-8 -*-
 #
 # Copyright © 2014 Zulip, Inc.

api/integrations/hg/zulip-changegroup.py

@@ -1,8 +1,8 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 # -*- coding: utf-8 -*-
 #
 # Zulip hook for Mercurial changeset pushes.
-# Copyright © 2012-2014 Zulip, Inc.
+# Copyright © 2012-2014 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -25,8 +25,10 @@
 #
 # This hook is called when changesets are pushed to the master repository (ie
 # `hg push`). See https://zulip.com/integrations for installation instructions.
+from __future__ import absolute_import
 
 import zulip
+from six.moves import range
 
 VERSION = "0.9"
 

api/integrations/nagios/nagios-notify-zulip

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 import optparse
 import zulip
 

api/integrations/perforce/zulip_change-commit.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 # -*- coding: utf-8 -*-
 #
 # Copyright © 2012-2014 Zulip, Inc.
@@ -33,6 +33,7 @@ For example:
   1234 //depot/security/src/
 
 '''
+from __future__ import print_function
 
 import os
 import sys
@@ -59,12 +60,12 @@ try:
     changelist = int(sys.argv[1])
     changeroot = sys.argv[2]
 except IndexError:
-    print >> sys.stderr, "Wrong number of arguments.\n\n",
-    print >> sys.stderr,  __doc__
+    print("Wrong number of arguments.\n\n", end=' ', file=sys.stderr)
+    print(__doc__, file=sys.stderr)
     sys.exit(-1)
 except ValueError:
-    print >> sys.stderr, "First argument must be an integer.\n\n",
-    print >> sys.stderr, __doc__
+    print("First argument must be an integer.\n\n", end=' ', file=sys.stderr)
+    print(__doc__, file=sys.stderr)
     sys.exit(-1)
 
 metadata = git_p4.p4_describe(changelist)

api/integrations/perforce/zulip_perforce_config.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 # -*- coding: utf-8 -*-
 #
 # Copyright © 2014 Zulip, Inc.

api/integrations/rss/rss-bot

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 # -*- coding: utf-8 -*-
 #
 # RSS integration for Zulip

api/integrations/svn/post-commit

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 # -*- coding: utf-8 -*-
 #
 # Zulip notification post-commit hook.

api/integrations/svn/zulip_svn_config.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 # -*- coding: utf-8 -*-
 #
 # Copyright © 2014 Zulip, Inc.

api/integrations/trac/zulip_trac.py

@@ -107,8 +107,8 @@ class ZulipPlugin(Component):
         field_changes = []
         for key in old_values.keys():
             if key == "description":
-                content += '- Changed %s from %s to %s' % (key, markdown_block(old_values.get(key)),
-                                                           markdown_block(ticket.values.get(key)))
+                content += '- Changed %s from %s\n\nto %s' % (key, markdown_block(old_values.get(key)),
+                                                              markdown_block(ticket.values.get(key)))
             elif old_values.get(key) == "":
                 field_changes.append('%s: => **%s**' % (key, ticket.values.get(key)))
             elif ticket.values.get(key) == "":

api/integrations/trac/zulip_trac_config.py

@@ -34,13 +34,13 @@ TRAC_BASE_TICKET_URL = "https://trac.example.com/ticket"
 # and annoying.  We solve this issue by only sending a notification
 # for changes to the fields listed below.
 #
-# Total list of possible fields is:
+# TRAC_NOTIFY_FIELDS lets you specify which fields will trigger a
+# Zulip notification in response to a trac update; you should change
+# this list to match your team's workflow.  The complete list of
+# possible fields is:
+#
 # (priority, milestone, cc, owner, keywords, component, severity,
 #  type, versions, description, resolution, summary, comment)
-#
-# The following is the list of fields which can be changed without
-# triggering a Zulip notification; change these to match your team's
-# workflow.
 TRAC_NOTIFY_FIELDS = ["description", "summary", "resolution", "comment", "owner"]
 
 ## If properly installed, the Zulip API should be in your import

api/integrations/twitter/twitter-bot

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 # -*- coding: utf-8 -*-
 #
 # Twitter integration for Zulip

api/integrations/twitter/twitter-search-bot

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 # -*- coding: utf-8 -*-
 #
 # Twitter search integration for Zulip

api/setup.py

@@ -1,6 +1,7 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 # -*- coding: utf-8 -*-
 
+from __future__ import print_function
 import os
 import sys
 
@@ -9,8 +10,8 @@ import itertools
 def version():
     version_py = os.path.join(os.path.dirname(__file__), "zulip", "__init__.py")
     with open(version_py) as in_handle:
-        version_line = itertools.dropwhile(lambda x: not x.startswith("__version__"),
-                                           in_handle).next()
+        version_line = next(itertools.dropwhile(lambda x: not x.startswith("__version__"),
+                                           in_handle))
     version = version_line.split('=')[-1].strip().replace('"', '')
     return version
 
@@ -26,7 +27,7 @@ package_info = dict(
     version=version(),
     description='Bindings for the Zulip message API',
     author='Zulip, Inc.',
-    author_email='support@zulip.com',
+    author_email='zulip-devel@googlegroups.com',
     classifiers=[
         'Development Status :: 3 - Alpha',
         'Environment :: Web Environment',
@@ -60,13 +61,13 @@ except ImportError:
     try:
         import simplejson
     except ImportError:
-        print >>sys.stderr, "simplejson is not installed"
+        print("simplejson is not installed", file=sys.stderr)
         sys.exit(1)
     try:
         import requests
         assert(LooseVersion(requests.__version__) >= LooseVersion('0.12.1'))
     except (ImportError, AssertionError):
-        print >>sys.stderr, "requests >=0.12.1 is not installed"
+        print("requests >=0.12.1 is not installed", file=sys.stderr)
         sys.exit(1)
 
 

api/zulip/__init__.py

@@ -20,6 +20,8 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+from __future__ import print_function
+from __future__ import absolute_import
 import simplejson
 import requests
 import time
@@ -33,8 +35,9 @@ import urllib
 import random
 from distutils.version import LooseVersion
 
-from ConfigParser import SafeConfigParser
+from six.moves.configparser import SafeConfigParser
 import logging
+import six
 
 
 __version__ = "0.2.4"
@@ -87,7 +90,7 @@ class RandomExponentialBackoff(CountingBackoff):
         try:
             logger.warning(message)
         except NameError:
-            print message
+            print(message)
         time.sleep(delay)
 
 def _default_client():
@@ -117,6 +120,19 @@ def generate_option_group(parser, prefix=''):
                      default=None,
                      dest="zulip_client",
                      help=optparse.SUPPRESS_HELP)
+    group.add_option('--insecure',
+                     action='store_true',
+                     dest='insecure',
+                     help='''Do not verify the server certificate.
+                          The https connection will not be secure.''')
+    group.add_option('--cert-bundle',
+                     action='store',
+                     dest='cert_bundle',
+                     help='''Specify a file containing either the
+                          server certificate, or a set of trusted
+                          CA certificates. This will be used to
+                          verify the server's identity. All
+                          certificates should be PEM encoded.''')
     return group
 
 def init_from_options(options, client=None):
@@ -126,7 +142,8 @@ def init_from_options(options, client=None):
         client = _default_client()
     return Client(email=options.zulip_email, api_key=options.zulip_api_key,
                   config_file=options.zulip_config_file, verbose=options.verbose,
-                  site=options.zulip_site, client=client)
+                  site=options.zulip_site, client=client,
+                  cert_bundle=options.cert_bundle, insecure=options.insecure)
 
 def get_default_config_filename():
     config_file = os.path.join(os.environ["HOME"], ".zuliprc")
@@ -138,15 +155,14 @@ def get_default_config_filename():
 class Client(object):
     def __init__(self, email=None, api_key=None, config_file=None,
                  verbose=False, retry_on_errors=True,
-                 site=None, client=None):
+                 site=None, client=None,
+                 cert_bundle=None, insecure=None):
         if client is None:
             client = _default_client()
-        if None in (api_key, email):
-            if config_file is None:
-                config_file = get_default_config_filename()
-            if not os.path.exists(config_file):
-                raise RuntimeError("api_key or email not specified and %s does not exist"
-                                   % (config_file,))
+
+        if config_file is None:
+            config_file = get_default_config_filename()
+        if os.path.exists(config_file):
             config = SafeConfigParser()
             with file(config_file, 'r') as f:
                 config.readfp(f, config_file)
@@ -156,6 +172,22 @@ class Client(object):
                 email = config.get("api", "email")
             if site is None and config.has_option("api", "site"):
                 site = config.get("api", "site")
+            if cert_bundle is None and config.has_option("api", "cert_bundle"):
+                cert_bundle = config.get("api", "cert_bundle")
+            if insecure is None and config.has_option("api", "insecure"):
+                # Be quite strict about what is accepted so that users don't
+                # disable security unintentionally.
+                insecure_setting = config.get("api", "insecure").lower()
+                if insecure_setting == "true":
+                    insecure = True
+                elif insecure_setting == "false":
+                    insecure = False
+                else:
+                    raise RuntimeError("insecure is set to '%s', it must be 'true' or 'false' if it is used in %s"
+                                       % (insecure_setting, config_file))
+        elif None in (api_key, email):
+            raise RuntimeError("api_key or email not specified and %s does not exist"
+                               % (config_file,))
 
         self.api_key = api_key
         self.email = email
@@ -175,6 +207,17 @@ class Client(object):
         self.retry_on_errors = retry_on_errors
         self.client_name = client
 
+        if insecure:
+            self.tls_verification=False
+        elif cert_bundle is not None:
+            if not os.path.isfile(cert_bundle):
+                raise RuntimeError("tls bundle '%s' does not exist"
+                                   %(cert_bundle,))
+            self.tls_verification=cert_bundle
+        else:
+            # Default behavior: verify against system CA certificates
+            self.tls_verification=True
+
     def get_user_agent(self):
         vendor = ''
         vendor_version = ''
@@ -203,7 +246,7 @@ class Client(object):
         request = {}
 
         for (key, val) in orig_request.iteritems():
-            if not (isinstance(val, str) or isinstance(val, unicode)):
+            if not (isinstance(val, str) or isinstance(val, six.text_type)):
                 request[key] = simplejson.dumps(val)
             else:
                 request[key] = val
@@ -233,9 +276,9 @@ class Client(object):
         def end_error_retry(succeeded):
             if query_state["had_error_retry"] and self.verbose:
                 if succeeded:
-                    print "Success!"
+                    print("Success!")
                 else:
-                    print "Failed!"
+                    print("Failed!")
 
         while True:
             try:
@@ -249,7 +292,7 @@ class Client(object):
                         urlparse.urljoin(self.base_url, url),
                         auth=requests.auth.HTTPBasicAuth(self.email,
                                                          self.api_key),
-                        verify=True, timeout=90,
+                        verify=self.tls_verification, timeout=90,
                         headers={"User-agent": self.get_user_agent()},
                         **kwargs)
 
@@ -311,7 +354,7 @@ class Client(object):
             else:
                 req_url = url
             return self.do_api_query(request, API_VERSTRING + req_url, method=method, **query_kwargs)
-        call.func_name = name
+        call.__name__ = name
         setattr(cls, name, call)
 
     def call_on_each_event(self, callback, event_types=None, narrow=[]):
@@ -324,7 +367,7 @@ class Client(object):
 
                 if 'error' in res.get('result'):
                     if self.verbose:
-                        print "Server returned error:\n%s" % res['msg']
+                        print("Server returned error:\n%s" % res['msg'])
                     time.sleep(1)
                 else:
                     return (res['queue_id'], res['last_event_id'])
@@ -338,13 +381,13 @@ class Client(object):
             if 'error' in res.get('result'):
                 if res["result"] == "http-error":
                     if self.verbose:
-                        print "HTTP error fetching events -- probably a server restart"
+                        print("HTTP error fetching events -- probably a server restart")
                 elif res["result"] == "connection-error":
                     if self.verbose:
-                        print "Connection error fetching events -- probably server is temporarily down?"
+                        print("Connection error fetching events -- probably server is temporarily down?")
                 else:
                     if self.verbose:
-                        print "Server returned error:\n%s" % res["msg"]
+                        print("Server returned error:\n%s" % res["msg"])
                     if res["msg"].startswith("Bad event queue id:"):
                         # Our event queue went away, probably because
                         # we were asleep or the server restarted

assets/favicon/generate

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 import xml.etree.ElementTree as ET
 import subprocess
 

bin/get-django-setting

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 from __future__ import absolute_import
 
 import os

bin/log-management-command

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 import sys
 import logging
 import os

bots/check-mirroring

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 import sys
 import time
 import optparse

bots/check-rabbitmq-consumers

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 
 import sys
 import time

bots/check-rabbitmq-queue

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 
 import sys
 import re

bots/cron_file_helper.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 import time
 
 def nagios_from_file(results_file):

bots/gcal-bot

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 import sys
 import time
 import datetime

bots/irc-mirror.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python2.7
 #
 # EXPERIMENTAL
 # IRC <=> Zulip mirroring bot
@@ -6,6 +6,7 @@
 # Setup: First, you need to install python-irc version 8.5.3
 # (https://bitbucket.org/jaraco/irc)
 
+from __future__ import print_function
 import irc.bot
 import irc.strings
 from irc.client import ip_numstr_to_quad, ip_quad_to_numstr
@@ -53,12 +54,12 @@ class IRCBot(irc.bot.SingleServerIRCBot):
             return
 
         # Forward the PM to Zulip
-        print zulip_client.send_message({
+        print(zulip_client.send_message({
                 "sender": sender,
                 "type": "private",
                 "to": "username@example.com",
                 "content": content,
-                })
+                }))
 
     def on_pubmsg(self, c, e):
         content = e.arguments[0]
@@ -68,14 +69,14 @@ class IRCBot(irc.bot.SingleServerIRCBot):
             return
 
         # Forward the stream message to Zulip
-        print zulip_client.send_message({
+        print(zulip_client.send_message({
                 "forged": "yes",
                 "sender": sender,
                 "type": "stream",
                 "to": stream,
                 "subject": "IRC",
                 "content": content,
-                })
+                }))
 
     def on_dccmsg(self, c, e):
         c.privmsg("You said: " + e.arguments[0])
@@ -92,11 +93,11 @@ class IRCBot(irc.bot.SingleServerIRCBot):
                 return
             self.dcc_connect(address, port)
 
-usage = """python irc-mirror.py --server=IRC_SERVER --channel=<CHANNEL> --nick-prefix=<NICK> [optional args]
+usage = """python2.7 irc-mirror.py --server=IRC_SERVER --channel=<CHANNEL> --nick-prefix=<NICK> [optional args]
 
 Example:
 
-python irc-mirror.py --irc-server=127.0.0.1 --channel='#test' --nick-prefix=username
+python2.7 irc-mirror.py --irc-server=127.0.0.1 --channel='#test' --nick-prefix=username
   --site=https://zulip.example.com --user=irc-bot@example.com
   --api-key=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 

bots/jabber_mirror.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 # Copyright (C) 2014 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person
@@ -21,6 +21,7 @@
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
+from __future__ import print_function
 import sys
 import subprocess
 import os
@@ -39,7 +40,7 @@ args.extend(sys.argv[1:])
 
 backoff = RandomExponentialBackoff(timeout_success_equivalent=300)
 while backoff.keep_going():
-    print "Starting Jabber mirroring bot"
+    print("Starting Jabber mirroring bot")
     try:
         ret = subprocess.call(args)
     except:
@@ -51,9 +52,9 @@ while backoff.keep_going():
 
     backoff.fail()
 
-print ""
-print ""
-print "ERROR: The Jabber mirroring bot is unable to continue mirroring Jabber."
-print "Please contact support@zulip.com if you need assistance."
-print ""
+print("")
+print("")
+print("ERROR: The Jabber mirroring bot is unable to continue mirroring Jabber.")
+print("Please contact zulip-devel@googlegroups.com if you need assistance.")
+print("")
 sys.exit(1)

bots/jabber_mirror_backend.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python2.7
 #
 # Copyright (C) 2013 Permabit, Inc.
 # Copyright (C) 2013--2014 Zulip, Inc.
@@ -44,7 +44,7 @@ import optparse
 
 from sleekxmpp import ClientXMPP, InvalidJID, JID
 from sleekxmpp.exceptions import IqError, IqTimeout
-from ConfigParser import SafeConfigParser
+from six.moves.configparser import SafeConfigParser
 import os, sys, zulip, getpass
 import re
 

bots/log2zulip

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python2.7
 import subprocess
 import os
 import sys

bots/process_ccache

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python2.7
 import sys
 import subprocess
 import base64

bots/summarize_stream.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 # This is hacky code to analyze data on our support stream.  The main
 # reusable bits are get_recent_messages and get_words.
 
@@ -31,7 +32,7 @@ def analyze_messages(msgs, word_count, email_count):
         if False:
             if ' ack' in msg['content']:
                 name = msg['sender_full_name'].split()[0]
-                print 'ACK', name
+                print('ACK', name)
         m = re.search('ticket (Z....).*email: (\S+).*~~~(.*)', msg['content'], re.M | re.S)
         if m:
             ticket, email, req = m.groups()
@@ -40,9 +41,9 @@ def analyze_messages(msgs, word_count, email_count):
                 word_count[word] += 1
             email_count[email] += 1
         if False:
-            print
+            print()
             for k, v in msg.items():
-                print '%-20s: %s' % (k, v)
+                print('%-20s: %s' % (k, v))
 
 def generate_support_stats():
     client = zulip.Client()
@@ -64,16 +65,16 @@ def generate_support_stats():
 
     if True:
         words = word_count.keys()
-        words = filter(lambda w: word_count[w] >= 10, words)
-        words = filter(lambda w: len(w) >= 5, words)
+        words = [w for w in words if word_count[w] >= 10]
+        words = [w for w in words if len(w) >= 5]
         words = sorted(words, key=lambda w: word_count[w], reverse=True)
         for word in words:
-            print word, word_count[word]
+            print(word, word_count[word])
 
     if False:
         emails = email_count.keys()
         emails = sorted(emails, key=lambda w: email_count[w], reverse=True)
         for email in emails:
-            print email, email_count[email]
+            print(email, email_count[email])
 
 generate_support_stats()

bots/sync-public-streams

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 import sys
 import os
 import logging

bots/tddium-notify-humbug

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 # Copyright (C) 2012 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person

bots/zephyr_mirror.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 # Copyright (C) 2012 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person
@@ -21,13 +21,15 @@
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
+from __future__ import absolute_import
+from __future__ import print_function
 import sys
 import subprocess
 import os
 import traceback
 import signal
 
-from zephyr_mirror_backend import parse_args
+from .zephyr_mirror_backend import parse_args
 
 def die(signal, frame):
     # We actually want to exit, so run os._exit (so as not to be caught and restarted)
@@ -52,30 +54,30 @@ if options.forward_class_messages and not options.noshard:
     if options.on_startup_command is not None:
         subprocess.call([options.on_startup_command])
     from zerver.lib.parallel import run_parallel
-    print "Starting parallel zephyr class mirroring bot"
+    print("Starting parallel zephyr class mirroring bot")
     jobs = list("0123456789abcdef")
     def run_job(shard):
         subprocess.call(args + ["--shard=%s" % (shard,)])
         return 0
     for (status, job) in run_parallel(run_job, jobs, threads=16):
-        print "A mirroring shard died!"
+        print("A mirroring shard died!")
         pass
     sys.exit(0)
 
 backoff = RandomExponentialBackoff(timeout_success_equivalent=300)
 while backoff.keep_going():
-    print "Starting zephyr mirroring bot"
+    print("Starting zephyr mirroring bot")
     try:
         subprocess.call(args)
     except:
         traceback.print_exc()
     backoff.fail()
 
-print ""
-print ""
-print "ERROR: The Zephyr mirroring bot is unable to continue mirroring Zephyrs."
-print "This is often caused by failing to maintain unexpired Kerberos tickets"
-print "or AFS tokens.  See https://zulip.com/zephyr for documentation on how to"
-print "maintain unexpired Kerberos tickets and AFS tokens."
-print ""
+print("")
+print("")
+print("ERROR: The Zephyr mirroring bot is unable to continue mirroring Zephyrs.")
+print("This is often caused by failing to maintain unexpired Kerberos tickets")
+print("or AFS tokens.  See https://zulip.com/zephyr for documentation on how to")
+print("maintain unexpired Kerberos tickets and AFS tokens.")
+print("")
 sys.exit(1)

bots/zephyr_mirror_backend.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 # Copyright (C) 2012 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person
@@ -20,8 +20,11 @@
 # ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
+from __future__ import absolute_import
 
 import sys
+from six.moves import map
+from six.moves import range
 try:
     import simplejson
 except ImportError:
@@ -41,8 +44,8 @@ import select
 
 DEFAULT_SITE = "https://api.zulip.com"
 
-class States:
-    Startup, ZulipToZephyr, ZephyrToZulip, ChildSending = range(4)
+class States(object):
+    Startup, ZulipToZephyr, ZephyrToZulip, ChildSending = list(range(4))
 CURRENT_STATE = States.Startup
 
 def to_zulip_username(zephyr_username):

changelog.md

@@ -0,0 +1,53 @@
+# Change Log
+
+All notable changes to this project will be documented in this file.
+
+[Unreleased]
+
+[1.3.10]
+- Added new integration for Travis CI.
+- Added settings option to control maximum file upload size.
+- Added support for running Zulip development environment in Docker.
+- Added easy configuration support for a remote postgres database.
+- Added extensive documentation on scalability, backups, and security.
+- Recent private message threads are now displayed expanded similar to
+  the pre-existing recent topics feature.
+- Made it possible to set LDAP and EMAIL_HOST passwords in
+  /etc/zulip/secrets.conf.
+- Improved the styling for the Administration page and added tabs.
+- Substantially improved loading performance on slow networks by enabling
+  GZIP compression on more assets.
+- Changed the page title in narrowed views to include the current narrow.
+- Fixed several backend performance issues affecting very large realms.
+- Fixed bugs where draft compose content might be lost when reloading site.
+- Fixed support for disabling the "zulip" notifications stream.
+- Fixed missing step in postfix_localmail installation instructions.
+- Fixed several bugs/inconveniences in the production upgrade process.
+- Fixed realm restrictions for servers with a unique, open realm.
+- Substantially cleaned up console logging from run-dev.py.
+
+[1.3.9] - 2015-11-16
+- Fixed buggy #! lines in upgrade scripts.
+
+[1.3.8] - 2015-11-15
+- Added options to the Python api for working with untrusted server certificates.
+- Added a lot of documentation on the development environment and testing.
+- Added partial support for translating the Zulip UI.
+- Migrated installing Node dependencies to use npm.
+- Fixed LDAP integration breaking autocomplete of @-mentions.
+- Fixed admin panel reactivation/deactivation of bots.
+- Fixed inaccurate documentation for downloading the desktop apps.
+- Fixed various minor bugs in production installation process.
+- Fixed security issue where recent history on private streams might
+  be visible to new users (to the Zulip team) who were invited with that
+  private stream as one of their initial streams
+  (https://github.com/zulip/zulip/issues/230).
+- Major preliminary progress towards supporting Python 3.
+
+[1.3.7] - 2015-10-19
+- Turn off desktop and audible notifications for streams by default.
+- Added support for the LDAP authentication integration creating new users.
+- Added new endpoint to support Google auth on mobile.
+- Fixed desktop notifications in modern Firefox.
+- Fixed several installation issues for both production and development environments.
+- Improved documentation for outgoing SMTP and the email mirror integration.

confirmation/models.py

@@ -105,7 +105,7 @@ class Confirmation(models.Model):
 
     objects = ConfirmationManager()
 
-    class Meta:
+    class Meta(object):
         verbose_name = _('confirmation email')
         verbose_name_plural = _('confirmation emails')
 

docs/code-style.rst

@@ -352,8 +352,8 @@ styles (separate lines for each selector)::
 Python
 ------
 
--  Scripts should start with ``#!/usr/bin/env python`` and not
-   ``#!/usr/bin/python``. See commit ``437d4aee`` for an explanation of
+-  Scripts should start with ``#!/usr/bin/env python2.7`` and not
+   ``#!/usr/bin/env python2.7``. See commit ``437d4aee`` for an explanation of
    why. Don't put such a line on a Python file unless it's meaningful to
    run it as a script. (Some libraries can also be run as scripts, e.g.
    to run a test suite.)

docs/directory-structure.rst

@@ -76,11 +76,11 @@ Tests
 =====
 
 +------------------------+-----------------------------------+
-| ``zerver/test*.py``             | Backend tests            |       |
+| ``zerver/test*.py``             | Backend tests            |
 +------------------------+-----------------------------------+
-| ``zerver/tests/frontend/node``  | Node Frontend unit tests |
+| ``frontend_tests/node``         | Node Frontend unit tests |
 +------------------------+-----------------------------------+
-| ``zerver/tests/frontend/tests`` | Casper frontend tests    |
+| ``frontend_tests/tests``        | Casper frontend tests    |
 +------------------------+-----------------------------------+
 
 Documentation

docs/front-end-build-process.rst

@@ -0,0 +1,26 @@
+=======================
+Front End Build Process
+=======================
+
+This page documents additional information that may be useful when developing new features for Zulip that require front-end changes. For a more general overview, see the new feature tutorial. The code style documentation also has relevant information about how Zulip's code is structured.
+
+Primary build process
+=====================
+
+Most of the exisiting JS in Zulip is written in IIFE-wrapped modules, one per file in the `static/js` directory. When running Zulip in development mode each file is loaded seperately. In production mode (and when creating a release tarball) JavaScript files are concatenated and minified.
+
+If you add a new JavaScript file it needs to be specified in the `JS_SPECS` dictionary defined in `zproject/settings.py` to be included in the concatenated file.
+
+Webpack/CommonJS modules
+========================
+New JS written for Zulip can be written as CommonJS modules (bundled using `webpack <https://webpack.github.io/>`_, though this will taken care of automatically whenever ``run-dev.py`` is running). (CommonJS is the same module format that Node uses, so see `the Node documentation <https://nodejs.org/docs/latest/api/modules.html>` for more information on the syntax.)
+
+Benefits of using CommonJS modules over the `IIFE <http://benalman.com/news/2010/11/immediately-invoked-function-expression/>`_ module approach:
+
+* namespacing/module boilerplate will be added automatically in the bundling process
+* dependencies between modules are more explicit and easier to trace
+* no separate list of JS files needs to be maintained for concatenation and minification
+* third-party libraries can be more easily installed/versioned using npm
+* running the same code in the browser and in Node for testing is simplified (as both environments use the same module syntax)
+
+The entry point file for the bundle generated by webpack is ``static/js/src/main.js``. Any modules you add will need to be required from this file (or one of its dependencies) in order to be included in the script bundle.

docs/html_unescape.py

@@ -1,4 +1,5 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python2.7
+from __future__ import print_function
 
 # Remove HTML entity escaping left over from MediaWiki->rST conversion.
 

docs/index.rst

@@ -13,6 +13,7 @@ Contents:
 
    new-feature-tutorial
    code-style
+   front-end-build-process
    directory-structure
    testing
 

docs/new-feature-tutorial.rst

@@ -2,84 +2,215 @@
 New Feature Tutorial
 ====================
 
-.. attention::
-   This tutorial is an unfinished work -- contributions welcome!
+The changes needed to add a new feature will vary, of course, but this document
+provides a general outline of what you may need to do, as well as an example of
+the specific steps needed to add a new feature: adding a new option to the 
+application that is dynamically synced through the data system in real-time to
+all browsers the user may have open.
 
-The changes needed to add a new feature will vary, of course.  We give an
-example here that illustrates some of the common steps needed.  We describe
-the process of adding a new setting for admins that restricts inviting new
-users to admins only.
-
-Backend Changes
+General Process
 ===============
 
 Adding a field to the database
 ------------------------------
 
-The server accesses the underlying database in `zerver/models.py`.  Add
-a new field in the appropriate class, `realm_invite_by_admins_only`
-in the `Realm` class in this case.
+**Update the model:** The server accesses the underlying database in `zerver/
+models.py`. Add a new field in the appropriate class.
 
-Once you do so, you need to create the migration and run it; the
-process is documented at:
-https://docs.djangoproject.com/en/1.8/topics/migrations/
+**Create and run the migration:** To create and apply a migration, run: ::
 
-Once you've run the migration, to test your changes, you'll want to
-restart memcached on your development server (``/etc/init.d/memcached restart``) and
-then restart ``run-dev.py`` to avoid interacting with cached objects.
+./manage.py makemigrations
+./manage.py migrate
+
+**Test your changes:** Once you've run the migration, restart memcached on your 
+development server (``/etc/init.d/memcached restart``) and then restart 
+``run-dev.py`` to avoid interacting with cached objects.
 
 Backend changes
 ---------------
 
-You should add code in `zerver/lib/actions.py` to interact with the database,
-that actually updates the relevant field.  In this case, `do_set_realm_invite_by_admins_only`
-is a function that actually updates the field in the database, and sends
-an event announcing that this change has been made.
+**Database interaction:** Add any necessary code for updating and interacting
+with the database in ``zerver/lib/actions.py``. It should update the database and 
+send an event announcing the change.
 
-You then need update the `fetch_initial_state_data` and `apply_events` functions
-in `zerver/lib/actions.py` to update the state based on the event you just created.
-In this case, we add a line
+**Application state:** Modify the ``fetch_initial_state_data`` and ``apply_events`` 
+functions in ``zerver/lib/actions.py`` to update the state based on the event you 
+just created.
 
-::
+**Backend implementation:** Make any other modifications to the backend required for 
+your change.
 
-  state['realm_invite_by_admins_only'] = user_profile.realm.invite_by_admins_only`
-
-to the `fetch_initial_state_data` function.  The `apply_events` function
-doesn't need to be updated since
-
-::
-
-   elif event['type'] == 'realm':
-       field = 'realm_' + event['property']
-       state[field] = event['value']
-
-already took care of our event.
-
-Then update `zerver/views/__init__.py` to actually call your function.
-In the dictionary which sets the javascript `page_params` dictionary,
-add a value for your feature.
-
-::
-
-   realm_invite_by_admins_only = register_ret['realm_invite_by_admins_only']
-
-Perhaps your new option controls some other backend rendering: in our case
-we test for this option in the `home` method for adding a variable to the response.
-The functions in this file control the generation of various pages served
-(along with the Django templates).
-Our new feature also shows up in the administration tab (as a checkbox),
-so we need to update the `update_realm` function.
-
-
-Finally, add tests for your backend changes; at the very least you
-should add a test of your event data flowing through the system in
-``test_events.py``.
+**Testing:** At the very least, add a test of your event data flowing through 
+the system in ``test_events.py``.
 
 
 Frontend changes
 ----------------
 
-You need to change various things on the front end.  In this case, the relevant files
-are `static/js/server_events.js`, `static/js/admin.js`, `static/styles/zulip.css
-and `static/templates/admin_tab.handlebars`.
+**JavaScript:** Zulip's JavaScript is located in the directory ``static/js/``. 
+The exact files you may need to change depend on your feature. If you've added a 
+new event that is sent to clients, be sure to add a handler for it to
+``static/js/server_events.js``.
+
+**CSS:** The primary CSS file is ``static/styles/zulip.css``. If your new 
+feature requires UI changes, you may need to add additional CSS to this file.
+
+**Templates:** The initial page structure is rendered via Django templates 
+located in ``template/server``. For JavaScript, Zulip uses Handlebars templates located in
+``static/templates``. Templates are precompiled as part of the build/deploy
+process.
+
+**Testing:** There are two types of frontend tests: node-based unit tests and 
+blackbox end-to-end tests. The blackbox tests are run in a headless browser 
+using Casper.js and are located in ``zerver/tests/frontend/tests/``. The unit
+tests use Node's ``assert`` module are located in ``zerver/tests/frontend/node/``.
+For more information on writing and running tests see the :doc:`testing 
+documentation <testing>`.
+
+Example Feature
+===============
+
+This example describes the process of adding a new setting to Zulip:
+a flag that restricts inviting new users to admins only (the default behavior
+is that any user can invite other users). It is based on an actual Zulip feature,
+and you can review `the original commit in the Zulip git repo <https://github.com/zulip/zulip/commit/5b7f3466baee565b8e5099bcbd3e1ccdbdb0a408>`_.
+(Note that Zulip has since been upgraded from Django 1.6 to 1.8, so the migration
+format has changed.)
+
+First, update the database and model to store the new setting. Add a 
+new boolean field, ``realm_invite_by_admins_only``, to the Realm model in
+``zerver/models.py``.
+
+Then create a Django migration that adds a new field, ``invite_by_admins_only``,
+to the ``zerver_realm`` table.
+
+In ``zerver/lib/actions.py``, create a new function named 
+``do_set_realm_invite_by_admins_only``. This function will update the database
+and trigger an event to notify clients when this setting changes. In this case 
+there was an exisiting ``realm|update`` event type which was used for setting 
+similar flags on the Realm model, so it was possible to add a new property to 
+that event rather than creating a new one. The property name matches the 
+database field to make it easy to understand what it indicates.
+
+The second argument to ``send_event`` is the list of users whose browser 
+sessions should be notified. Depending on the setting, this can be a single user
+(if the setting is a personal one, like time display format), only members in a
+particular stream or all active users in a realm. ::
+
+  # zerver/lib/actions.py
+
+  def do_set_realm_invite_by_admins_only(realm, invite_by_admins_only):
+    realm.invite_by_admins_only = invite_by_admins_only
+    realm.save(update_fields=['invite_by_admins_only'])
+    event = dict(
+      type="realm",
+      op="update",
+      property='invite_by_admins_only',
+      value=invite_by_admins_only,
+    )
+    send_event(event, active_user_ids(realm))
+    return {}
+
+You then need to add code that will handle the event and update the application
+state. In ``zerver/lib/actions.py`` update the ``fetch_initial_state`` and
+``apply_events`` functions. ::
+
+  def fetch_initial_state_data(user_profile, event_types, queue_id):
+    # ...
+    state['realm_invite_by_admins_only'] = user_profile.realm.invite_by_admins_only`
+
+In this case you don't need to change ``apply_events`` because there is already
+code that will correctly handle the realm update event type: ::
+
+  def apply_events(state, events, user_profile):
+    for event in events:
+      # ...
+      elif event['type'] == 'realm':
+         field = 'realm_' + event['property']
+         state[field] = event['value']
+
+You then need to add a view for clients to access that will call the newly-added
+``actions.py`` code to update the database. This example feature adds a new
+parameter that should be sent to clients when the application loads and be
+accessible via JavaScript, and there is already a view that does this for
+related flags: ``update_realm``. So in this case, we can add out code to the
+exisiting view instead of creating a new one. ::
+
+  # zerver/views/__init__.py
+
+  def home(request):
+    # ...
+    page_params = dict(
+      # ...
+      realm_invite_by_admins_only = register_ret['realm_invite_by_admins_only'],
+      # ...
+    )
+
+Since this feature also adds a checkbox to the admin page, and adds a new
+property the Realm model that can be modified from there, you also need to make
+changes to the ``update_realm`` function in the same file: ::
+
+  # zerver/views/__init__.py
+
+  def update_realm(request, user_profile,
+    name=REQ(validator=check_string, default=None),
+    restricted_to_domain=REQ(validator=check_bool, default=None),
+    invite_by_admins_only=REQ(validator=check_bool,default=None)):
+
+    # ...
+
+    if invite_by_admins_only is not None and
+      realm.invite_by_admins_only != invite_by_admins_only:
+        do_set_realm_invite_by_admins_only(realm, invite_by_admins_only)
+        data['invite_by_admins_only'] = invite_by_admins_only
+
+Then make the required front end changes: in this case a checkbox needs to be
+added to the admin page (and its value added to the data sent back to server
+when a realm is updated) and the change event needs to be handled on the client.
+
+To add the checkbox to the admin page, modify the relevant template,
+``static/templates/admin_tab.handlebars`` (omitted here since it is relatively
+straightforward). Then add code to handle changes to the new form control in
+``static/js/admin.js``. ::
+
+  var url = "/json/realm";
+  var new_invite_by_admins_only =
+    $("#id_realm_invite_by_admins_only").prop("checked");
+  data[invite_by_admins_only] = JSON.stringify(new_invite_by_admins_only);
+
+  channel.patch({
+    url: url,
+    data: data,
+    success: function (data) {
+      # ...
+      if (data.invite_by_admins_only) {
+        ui.report_success("New users must be invited by an admin!", invite_by_admins_only_status);
+      } else {
+        ui.report_success("Any user may now invite new users!", invite_by_admins_only_status);
+      }
+      # ...
+    }
+  });
+
+Finally, update ``server_events.js`` to handle related events coming from the
+server. ::
+
+  # static/js/server_events.js
+
+  function get_events_success(events) {
+    # ...
+    var dispatch_event = function dispatch_event(event) {
+        switch (event.type) {
+        # ...
+        case 'realm':
+          if (event.op === 'update' && event.property === 'invite_by_admins_only') {
+            page_params.realm_invite_by_admins_only = event.value;
+        }
+    }
+  }
+
+Any code needed to update the UI should be placed in ``dispatch_event`` callback
+(rather than the ``channel.patch``) function. This ensures the appropriate code
+will run even if the changes are made in another browser window. In this example
+most of the changes are on the backend, so no UI updates are required.
 

docs/testing.rst

@@ -56,10 +56,10 @@ Backend Django tests
 These live in ``zerver/tests.py`` and ``zerver/test_*.py``. Run them
 with ``tools/test-backend``.
 
-Web frontend black-box tests
-----------------------------
+Web frontend black-box casperjs tests
+-------------------------------------
 
-These live in ``zerver/tests/frontend/tests/``. This is a "black box"
+These live in ``frontend_tests/casper_tests/``. This is a "black box"
 test; we load the frontend in a real (headless) browser, from a real dev
 server, and simulate UI interactions like sending messages, narrowing,
 etc.
@@ -67,8 +67,63 @@ etc.
 Since this is interacting with a real dev server, it can catch backend
 bugs as well.
 
-You can run this with ``./zerver/tests/frontend/run``. You will need
-`PhantomJS <http://phantomjs.org/>`__ 1.7.0 or later.
+You can run this with ``./tools/test-js-with-casper`` or as
+``./tools/test-js-with-casper 05-settings.js`` to run a single test
+file from ``frontend_tests/casper_tests/``.
+
+Writing Casper tests
+~~~~~~~~~~~~~~~~~~~~
+
+Probably the easiest way to learn how to write Casper tests is to
+study some of the existing test files.  There are a few tips that can
+be useful for writing Casper tests in addition to the debugging notes
+below:
+
+- Run just the file containing your new tests as described above to
+  have a fast debugging cycle.
+- With frontend tests in general, it's very important to write your
+  code to wait for the right events.  Before essentially every action
+  you take on the page, you'll want to use ``waitForSelector``,
+  ``waitUntilVisible``, or a similar function to make sure the page or
+  elemant is ready before you interact with it. For instance, if you
+  want to click a button that you can select via ``#btn-submit``, and
+  then check that it causes ``success-elt`` to appear, you'll want to
+  write something like:
+
+  ::
+
+    casper.waitForSelector("#btn-submit", function () {
+       casper.click('#btn-submit')
+       casper.test.assertExists("#success-elt");
+     });
+
+  This will ensure that the element is present before the interaction
+  is attempted.  The various wait functions supported in Casper are
+  documented in the Casper here:
+  http://docs.casperjs.org/en/latest/modules/casper.html#waitforselector
+  and the various assert statements available are documented here:
+  http://docs.casperjs.org/en/latest/modules/tester.html#the-tester-prototype
+- Casper uses CSS3 selectors; you can often save time by testing and
+  debugigng your selectors on the relevant page of the Zulip
+  development app in the Chrome javascript console by using
+  e.g. ``$$("#settings-dropdown")``.
+- The test suite uses a smaller set of default user accounts and other
+  data initialized in the database than the development environment;
+  to see what differs check out the section related to
+  ``options["test_suite"]`` in
+  ``zilencer/management/commands/populate_db.py``.
+- Casper effectively runs your test file in two phases -- first it
+  runs the code in the test file, which for most test files will just
+  collect a series of steps (each being a ``casper.then`` or
+  ``casper.wait...`` call).  Then, usually at the end of the test
+  file, you'll have a ``casper.run`` call which actually runs that
+  series of steps.  This means that if you If you write code in your
+  test file outside a ``casper.then`` or ``casper.wait...`` method, it
+  will actually run before all the Casper test steps that are declared
+  in the file, which can lead to confusing failures where the new code
+  you write in between two ``casper.then`` blocks actually runs before
+  either of them.  See this for more details about how Casper works:
+  http://docs.casperjs.org/en/latest/faq.html#how-does-then-and-the-step-stack-work
 
 Debugging Casper.JS
 ~~~~~~~~~~~~~~~~~~~
@@ -78,7 +133,7 @@ is not perfect. Here are some steps for using it and gotchas you might
 want to know.
 
 To turn on remote debugging, pass ``--remote-debug`` to the
-``./zerver/frontend/tests/run`` script. This will run the tests with
+``./frontend_tests/tests/run`` script. This will run the tests with
 port ``7777`` open for remote debugging. You can now connect to
 ``localhost:7777`` in a Webkit browser. Somewhat recent versions of
 Chrome or Safari might be required.
@@ -97,6 +152,23 @@ you set a breakpoint and it is hit, the inspector will pause and you can
 do your normal JS debugging. You can also put breakpoints in the Zulip
 webpage itself if you wish to inspect the state of the Zulip frontend.
 
+If you need to use print debugging in casper, you can do using
+``casper.log``; see http://docs.casperjs.org/en/latest/logging.html
+for details.
+
+An additional debugging technique is to enable verbose mode in the
+Casper tests; you can do this by adding to the top of the relevant
+test file the following:
+
+  ::
+
+     var casper = require('casper').create({
+        verbose: true,
+        logLevel: "debug"
+     });
+
+This can sometimes give insight into exactly what's happening.
+
 Web frontend unit tests
 -----------------------
 
@@ -114,7 +186,7 @@ bottom of ``foobar.js``:
 This makes ``foobar.js`` follow the CommonJS module pattern, so it can
 be required in Node.js, which runs our tests.
 
-Now create ``zerver/tests/frontend/node/foobar.js``. At the top, require
+Now create ``frontend_tests/node_tests/foobar.js``. At the top, require
 the `Node.js assert module <http://nodejs.org/api/assert.html>`__, and
 the module you're testing, like so:
 
@@ -140,12 +212,12 @@ asserts, the *actual* value comes first, the *expected* value second.
      }());
 
 The test runner (index.js) automatically runs all .js files in the
-zerver/tests/frontend/node directory.
+frontend_tests/node directory.
 
 .. _handling-dependencies:
 
-Handling dependencies in tests
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Handling dependencies in unit tests
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The following scheme helps avoid tests leaking globals between each
 other.

frontend_tests/casper_lib/.gitignore

@@ -0,0 +1,2 @@
+/server.log
+/test_credentials.js

frontend_tests/casper_lib/common.js

@@ -2,7 +2,7 @@ var common = (function () {
 
 var exports = {};
 
-var test_credentials = require('../test_credentials.js').test_credentials;
+var test_credentials = require('../casper_lib/test_credentials.js').test_credentials;
 
 function timestamp() {
     return new Date().getTime();
@@ -129,11 +129,11 @@ exports.then_send_message = function (type, params) {
         else {
             casper.test.assertTrue(false, "send_message got valid message type");
         }
-        casper.fill('form[action^="/json/send_message"]', params);
+        casper.fill('form[action^="/json/messages"]', params);
         casper.click('#compose-send-button');
     });
     casper.waitFor(function emptyComposeBox() {
-        return casper.getFormValues('form[action^="/json/send_message"]').content === '';
+        return casper.getFormValues('form[action^="/json/messages"]').content === '';
     }, function () {
         last_send_or_update = timestamp();
     });

frontend_tests/casper_tests/00-login.js

@@ -1,4 +1,4 @@
-var common = require('../common.js').common;
+var common = require('../casper_lib/common.js').common;
 
 // Start of test script.
 casper.start('http://localhost:9981/', common.initialize_casper);

frontend_tests/casper_tests/01-site.js

@@ -10,7 +10,7 @@
 // For example, utils.dump() prints an Object with nice formatting.
 var utils = require('utils');
 
-var common = require('../common.js').common;
+var common = require('../casper_lib/common.js').common;
 
 common.start_and_log_in();
 

frontend_tests/casper_tests/02-narrow.js

@@ -1,4 +1,4 @@
-var common = require('../common.js').common;
+var common = require('../casper_lib/common.js').common;
 
 common.start_and_log_in();
 
@@ -125,12 +125,20 @@ function expect_all_pm() {
     ]);
 }
 
+function check_narrow_title(title) {
+    return function () {
+        // need to get title tag from HTML
+        // test if it's equal to some string passed in to function
+        casper.test.assertSelectorHasText('title', title, 'Got expected narrow title');
+    };
+}
+
 function un_narrow() {
     casper.then(common.un_narrow);
     casper.then(expect_home);
+    casper.then(check_narrow_title('home - Zulip Dev - Zulip'));
 }
 
-
 // Narrow by clicking links.
 
 common.wait_for_receive(function () {
@@ -141,6 +149,7 @@ common.wait_for_receive(function () {
 casper.waitUntilVisible('#zfilt', function () {
     expect_stream();
 });
+casper.then(check_narrow_title('Verona - Zulip Dev - Zulip'));
 un_narrow();
 
 casper.waitUntilVisible('#zhome', function () {
@@ -148,6 +157,7 @@ casper.waitUntilVisible('#zhome', function () {
     casper.test.info('Narrowing by clicking subject');
     casper.click('*[title="Narrow to stream \\\"Verona\\\", topic \\\"frontend test\\\""]');
 });
+casper.then(check_narrow_title('frontend test - Zulip Dev - Zulip'));
 
 casper.waitUntilVisible('#zfilt', function () {
     expect_stream_subject();
@@ -163,6 +173,7 @@ casper.waitUntilVisible('#zhome', function () {
     casper.click('*[title="Narrow to your private messages with Cordelia Lear, King Hamlet"]');
 });
 
+casper.then(check_narrow_title('private - Zulip Dev - Zulip'));
 
 casper.waitUntilVisible('#zfilt', function () {
     expect_huddle();
@@ -205,32 +216,39 @@ function do_search(str, item) {
     });
 }
 
-function search_and_check(str, item, check) {
+function search_and_check(str, item, check, narrow_title) {
     do_search(str, item);
-
     casper.then(check);
+    casper.then(check_narrow_title(narrow_title));
     un_narrow();
 }
 
 casper.waitUntilVisible('#zhome', expect_home);
 
 // Test stream / recipient autocomplete in the search bar
-search_and_check('Verona',   'Narrow to stream',  expect_stream);
-search_and_check('Cordelia', 'Narrow to private', expect_1on1);
+search_and_check('Verona', 'Narrow to stream', expect_stream,
+                 'Verona - Zulip Dev - Zulip');
+search_and_check('Cordelia', 'Narrow to private', expect_1on1,
+                'private - Zulip Dev - Zulip');
 
 // Test operators
-search_and_check('stream:verona',                       'Narrow', expect_stream);
-search_and_check('stream:verona subject:frontend+test', 'Narrow', expect_stream_subject);
-search_and_check('subject:frontend+test',               'Narrow', expect_subject);
-
+search_and_check('stream:Verona', 'Narrow', expect_stream,
+                'Verona - Zulip Dev - Zulip');
+search_and_check('stream:Verona subject:frontend+test', 'Narrow', expect_stream_subject,
+                'frontend test - Zulip Dev - Zulip');
+search_and_check('subject:frontend+test', 'Narrow', expect_subject,
+                'home - Zulip Dev - Zulip');
 
 // Narrow by clicking the left sidebar.
 casper.then(function () {
     casper.test.info('Narrowing with left sidebar');
 });
 casper.thenClick('#stream_filters [data-name="Verona"]  a', expect_stream);
+casper.then(check_narrow_title('Verona - Zulip Dev - Zulip'));
 casper.thenClick('#global_filters [data-name="home"]    a', expect_home);
+casper.then(check_narrow_title('home - Zulip Dev - Zulip'));
 casper.thenClick('#global_filters [data-name="private"] a', expect_all_pm);
+casper.then(check_narrow_title('private - Zulip Dev - Zulip'));
 un_narrow();
 
 

frontend_tests/casper_tests/03-compose.js

@@ -1,4 +1,4 @@
-var common = require('../common.js').common;
+var common = require('../casper_lib/common.js').common;
 
 common.start_and_log_in();
 

frontend_tests/casper_tests/04-subscriptions.js

@@ -1,4 +1,4 @@
-var common = require('../common.js').common;
+var common = require('../casper_lib/common.js').common;
 
 common.start_and_log_in();
 

frontend_tests/casper_tests/05-settings.js

@@ -1,5 +1,5 @@
-var common = require('../common.js').common;
-var test_credentials = require('../test_credentials.js').test_credentials;
+var common = require('../casper_lib/common.js').common;
+var test_credentials = require('../casper_lib/test_credentials.js').test_credentials;
 
 common.start_and_log_in();
 

frontend_tests/casper_tests/07-stars.js

@@ -1,4 +1,4 @@
-var common = require('../common.js').common;
+var common = require('../casper_lib/common.js').common;
 
 function star_count() {
     return casper.evaluate(function () {

frontend_tests/casper_tests/09-edit.js

@@ -1,4 +1,4 @@
-var common = require('../common.js').common;
+var common = require('../casper_lib/common.js').common;
 
 common.start_and_log_in();
 

frontend_tests/casper_tests/10-navigation.js

@@ -1,4 +1,4 @@
-var common = require('../common.js').common;
+var common = require('../casper_lib/common.js').common;
 
 // Test basic tab navigation.
 

frontend_tests/casper_tests/11-admin.js

@@ -0,0 +1,77 @@
+var common = require('../casper_lib/common.js').common;
+var test_credentials = require('../casper_lib/test_credentials.js').test_credentials;
+
+common.start_and_log_in();
+
+casper.then(function () {
+    casper.test.info('Administration page');
+    casper.click('a[href^="#administration"]');
+    casper.test.assertUrlMatch(/^http:\/\/[^\/]+\/#administration/, 'URL suggests we are on administration page');
+    casper.test.assertExists('#administration.tab-pane.active', 'Administration page is active');
+});
+
+// Test user deactivation and reactivation
+casper.waitForSelector('.user_row[id="user_cordelia@zulip.com"]', function () {
+    casper.test.assertSelectorHasText('.user_row[id="user_cordelia@zulip.com"]', 'Deactivate');
+    casper.click('.user_row[id="user_cordelia@zulip.com"] .deactivate');
+    casper.test.assertTextExists('Deactivate cordelia@zulip.com', 'Deactivate modal has right user');
+    casper.test.assertTextExists('Deactivate now', 'Deactivate now button available');
+    casper.click('#do_deactivate_user_button');
+});
+
+casper.waitForSelector('.user_row[id="user_cordelia@zulip.com"].deactivated_user', function () {
+    casper.test.assertSelectorHasText('.user_row[id="user_cordelia@zulip.com"]', 'Reactivate');
+    casper.click('.user_row[id="user_cordelia@zulip.com"] .reactivate');
+});
+
+casper.waitForSelector('.user_row[id="user_cordelia@zulip.com"]:not(.deactivated_user)', function () {
+    casper.test.assertSelectorHasText('.user_row[id="user_cordelia@zulip.com"]', 'Deactivate');
+});
+
+// Test Deactivated users section of admin page
+casper.waitForSelector('.user_row[id="user_cordelia@zulip.com"]', function () {
+    casper.test.assertSelectorHasText('.user_row[id="user_cordelia@zulip.com"]', 'Deactivate');
+    casper.click('.user_row[id="user_cordelia@zulip.com"] .deactivate');
+    casper.test.assertTextExists('Deactivate cordelia@zulip.com', 'Deactivate modal has right user');
+    casper.test.assertTextExists('Deactivate now', 'Deactivate now button available');
+    casper.click('#do_deactivate_user_button');
+});
+
+casper.then(function () {
+    // Leave the page and return
+    casper.click('#settings-dropdown');
+    casper.click('a[href^="#subscriptions"]');
+    casper.click('#settings-dropdown');
+    casper.click('a[href^="#administration"]');
+
+    casper.waitForSelector('.user_row[id="user_cordelia@zulip.com"]', function () {
+        casper.test.assertSelectorHasText('#admin_deactivated_users_table .user_row[id="user_cordelia@zulip.com"]', 'Reactivate');
+        casper.click('#admin_deactivated_users_table .user_row[id="user_cordelia@zulip.com"] .reactivate');
+    });
+
+    casper.waitForSelector('#admin_deactivated_users_table .user_row[id="user_cordelia@zulip.com"]:not(.deactivated_user)', function () {
+        casper.test.assertSelectorHasText('#admin_deactivated_users_table .user_row[id="user_cordelia@zulip.com"]', 'Deactivate');
+    });
+});
+
+// Test bot deactivation and reactivation
+casper.waitForSelector('.user_row[id="user_new-user-bot@zulip.com"]', function () {
+    casper.test.assertSelectorHasText('.user_row[id="user_new-user-bot@zulip.com"]', 'Deactivate');
+    casper.click('.user_row[id="user_new-user-bot@zulip.com"] .deactivate');
+});
+
+casper.waitForSelector('.user_row[id="user_new-user-bot@zulip.com"].deactivated_user', function () {
+    casper.test.assertSelectorHasText('.user_row[id="user_new-user-bot@zulip.com"]', 'Reactivate');
+    casper.click('.user_row[id="user_new-user-bot@zulip.com"] .reactivate');
+});
+casper.waitForSelector('.user_row[id="user_new-user-bot@zulip.com"]:not(.deactivated_user)', function () {
+    casper.test.assertSelectorHasText('.user_row[id="user_new-user-bot@zulip.com"]', 'Deactivate');
+});
+
+// TODO: Test stream deletion
+
+common.then_log_out();
+
+casper.run(function () {
+    casper.test.done();
+});