upload: Support S3-compatible S3 hosting providers.

Previously, we were hardcoding the domain s3.amazonaws.com.  Given
that we already have an interface for configuring the host in
/etc/zulip/boto.cfg (which in turn, automatically configures boto), we
just need to actually use the value configured in boto for what S3
hostname to use.

Includes incomplete edits to the documentation.

docs/production/upload-backends.md

@@ -12,8 +12,8 @@ running quickly. You can later migrate the uploads to S3 by
 [following the instructions here](#migrating-from-local-uploads-to-amazon-s3-backend).
 
 We also support an `S3` backend, which uses the Python `boto` library
-to upload files to Amazon S3 (and, with some work, it should be
+to upload files to Amazon S3 (or an S3-compatible block storage
-possible to use any other storage provider compatible with `boto`).
+provider supported by the `boto` library).
 
 ## S3 backend configuration
 
@@ -27,24 +27,26 @@ two buckets because the "user avatars" bucket is generally configured
 as world-readable, whereas the "uploaded files" one is not.
 
 1. Set `s3_key` and `s3_secret_key` in /etc/zulip/zulip-secrets.conf
-to be the S3 access and secret keys for the IAM account.
+   to be the S3 access and secret keys for the IAM account.
 
 1. Set the `S3_AUTH_UPLOADS_BUCKET` and `S3_AVATAR_BUCKET` settings in
-`/etc/zulip/settings.py` to be the names of the S3 buckets you
+   `/etc/zulip/settings.py` to be the names of the S3 buckets you
-created (e.g. `exampleinc-zulip-uploads`).
+   created (e.g. `exampleinc-zulip-uploads`).
 
 1. Comment out the `LOCAL_UPLOADS_DIR` setting in
-`/etc/zulip/settings.py` (add a `#` at the start of the line).
+   `/etc/zulip/settings.py` (add a `#` at the start of the line).
 
-1. In some AWS regions, you need to explicitly
+1. If you are using a non-AWS block storage provider, or certain AWS
-    [configure boto](http://boto.cloudhackers.com/en/latest/boto_config_tut.html)
+   regions, you may need to explicitly
-    to use AWS's SIGv4 signature format (because AWS has stopped
+   [configure boto](http://boto.cloudhackers.com/en/latest/boto_config_tut.html).
-    supporting the older v3 format in those regions).  You can do this
+   For AWS, you may need to use AWS's SIGv4 signature format (because AWS has stopped
+    supporting the older v3 format in those regions); for other
+   providers, you may just need to set the hostname.  You can do this
     by adding an `/etc/zulip/boto.cfg` containing the following:
     ```
     [s3]
     use-sigv4 = True
-    # Edit to provide your S3 bucket's AWS region here.
+    # Edit to provide your bucket's AWS region or hostname here.
     host = s3.eu-central-1.amazonaws.com
     ```
 

zerver/lib/upload.py

@@ -322,9 +322,11 @@ def get_realm_for_filename(path: str) -> Optional[int]:
     return get_user_profile_by_id(key.metadata["user_profile_id"]).realm_id
 
 class S3UploadBackend(ZulipUploadBackend):
+    def __init__(self) -> None:
+        self.connection = S3Connection(settings.S3_KEY, settings.S3_SECRET_KEY)
+
     def delete_file_from_s3(self, path_id: str, bucket_name: str) -> bool:
-        conn = S3Connection(settings.S3_KEY, settings.S3_SECRET_KEY)
+        bucket = get_bucket(self.connection, bucket_name)
-        bucket = get_bucket(conn, bucket_name)
 
         # check if file exists
         key = bucket.get_key(path_id)
@@ -415,9 +417,7 @@ class S3UploadBackend(ZulipUploadBackend):
         self.delete_file_from_s3(path_id, bucket_name)
 
     def get_avatar_key(self, file_name: str) -> Key:
-        conn = S3Connection(settings.S3_KEY, settings.S3_SECRET_KEY)
+        bucket = get_bucket(self.connection, settings.S3_AVATAR_BUCKET)
-        bucket_name = settings.S3_AVATAR_BUCKET
-        bucket = get_bucket(conn, bucket_name)
 
         key = bucket.get_key(file_name)
         return key
@@ -436,7 +436,8 @@ class S3UploadBackend(ZulipUploadBackend):
         bucket = settings.S3_AVATAR_BUCKET
         medium_suffix = "-medium.png" if medium else ""
         # ?x=x allows templates to append additional parameters with &s
-        return "https://%s.s3.amazonaws.com/%s%s?x=x" % (bucket, hash_key, medium_suffix)
+        return "https://%s.%s/%s%s?x=x" % (bucket, self.connection.DefaultHost,
+                                           hash_key, medium_suffix)
 
     def upload_realm_icon_image(self, icon_file: File, user_profile: UserProfile) -> None:
         content_type = guess_type(icon_file.name)[0]
@@ -466,7 +467,8 @@ class S3UploadBackend(ZulipUploadBackend):
     def get_realm_icon_url(self, realm_id: int, version: int) -> str:
         bucket = settings.S3_AVATAR_BUCKET
         # ?x=x allows templates to append additional parameters with &s
-        return "https://%s.s3.amazonaws.com/%s/realm/icon.png?version=%s" % (bucket, realm_id, version)
+        return "https://%s.%s/%s/realm/icon.png?version=%s" % (
+            bucket, self.connection.DefaultHost, realm_id, version)
 
     def upload_realm_logo_image(self, logo_file: File, user_profile: UserProfile,
                                 night: bool) -> None:
@@ -505,15 +507,15 @@ class S3UploadBackend(ZulipUploadBackend):
             file_name = 'logo.png'
         else:
             file_name = 'night_logo.png'
-        return "https://%s.s3.amazonaws.com/%s/realm/%s?version=%s" % (bucket, realm_id, file_name, version)
+        return "https://%s.%s/%s/realm/%s?version=%s" % (
+            bucket, self.connection.DefaultHost, realm_id, file_name, version)
 
     def ensure_medium_avatar_image(self, user_profile: UserProfile) -> None:
         file_path = user_avatar_path(user_profile)
         s3_file_name = file_path
 
         bucket_name = settings.S3_AVATAR_BUCKET
-        conn = S3Connection(settings.S3_KEY, settings.S3_SECRET_KEY)
+        bucket = get_bucket(self.connection, bucket_name)
-        bucket = get_bucket(conn, bucket_name)
         key = bucket.get_key(file_path + ".original")
         image_data = key.get_contents_as_string()
 
@@ -533,8 +535,7 @@ class S3UploadBackend(ZulipUploadBackend):
         s3_file_name = file_path
 
         bucket_name = settings.S3_AVATAR_BUCKET
-        conn = S3Connection(settings.S3_KEY, settings.S3_SECRET_KEY)
+        bucket = get_bucket(self.connection, bucket_name)
-        bucket = get_bucket(conn, bucket_name)
         key = bucket.get_key(file_path + ".original")
         image_data = key.get_contents_as_string()
 
@@ -577,7 +578,7 @@ class S3UploadBackend(ZulipUploadBackend):
         bucket = settings.S3_AVATAR_BUCKET
         emoji_path = RealmEmoji.PATH_ID_TEMPLATE.format(realm_id=realm_id,
                                                         emoji_file_name=emoji_file_name)
-        return "https://%s.s3.amazonaws.com/%s" % (bucket, emoji_path)
+        return "https://%s.%s/%s" % (bucket, self.connection.DefaultHost, emoji_path)
 
 
 ### Local