diff --git a/puppet/zulip/files/nginx/zulip-include-common/location-sockjs b/puppet/zulip/files/nginx/zulip-include-common/location-sockjs
index 74b6fb5bdf..ce2852d528 100644
--- a/puppet/zulip/files/nginx/zulip-include-common/location-sockjs
+++ b/puppet/zulip/files/nginx/zulip-include-common/location-sockjs
@@ -2,5 +2,6 @@
 include /etc/nginx/zulip-include/proxy_longpolling;
 
 proxy_set_header Upgrade         $http_upgrade;
+# This should override the Connection setting in zulip-include/proxy
 proxy_set_header Connection      $connection_upgrade;
 proxy_set_header X-Real-IP       $remote_addr;
diff --git a/puppet/zulip/files/nginx/zulip-include-common/proxy b/puppet/zulip/files/nginx/zulip-include-common/proxy
index c25364042b..017a169824 100644
--- a/puppet/zulip/files/nginx/zulip-include-common/proxy
+++ b/puppet/zulip/files/nginx/zulip-include-common/proxy
@@ -1,4 +1,7 @@
 proxy_http_version 1.1;
+# Clearing the Connection header is required for keepalives from the load balancer
+# http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive
+proxy_set_header Connection "";
 proxy_set_header Host $host;
 proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 proxy_next_upstream off;
diff --git a/puppet/zulip_internal/files/nginx/sites-available/loadbalancer b/puppet/zulip_internal/files/nginx/sites-available/loadbalancer
index 15b565655b..edfef52150 100644
--- a/puppet/zulip_internal/files/nginx/sites-available/loadbalancer
+++ b/puppet/zulip_internal/files/nginx/sites-available/loadbalancer
@@ -1,9 +1,13 @@
 upstream staging {
     server staging.zulip.net:443;
+
+    keepalive 10000;
 }
 
 upstream prod {
     server prod0.zulip.net:443;
+
+    keepalive 10000;
 }
 
 server {