s3: Support non-AWS S3 providers which do not support request checksums.

(cherry picked from commit aeed907c50)

docs/production/upload-backends.md

@@ -52,6 +52,10 @@ backend. To enable this backend, you need to do the following:
    For certain AWS regions, you may need to set the `S3_REGION`
    setting to your default AWS region's code (e.g., `"eu-central-1"`).
 
+1. Non-AWS block storage providers may need `S3_SKIP_CHECKSUM = True`; you
+   should try without this at first, but enable it if you see exceptions
+   involving `XAmzContentSHA256Mismatch`.
+
 1. Finally, restart the Zulip server so that your settings changes
    take effect
    (`/home/zulip/deployments/current/scripts/restart-server`).

zerver/lib/upload/s3.py

@@ -56,6 +56,9 @@ if settings.S3_SKIP_PROXY is True:  # nocoverage
 def get_bucket(bucket_name: str, authed: bool = True) -> "Bucket":
     import boto3
 
+    checksum: Literal["when_required", "when_supported"] = (
+        "when_required" if settings.S3_SKIP_CHECKSUM else "when_supported"
+    )
     return boto3.resource(
         "s3",
         aws_access_key_id=settings.S3_KEY if authed else None,
@@ -65,6 +68,7 @@ def get_bucket(bucket_name: str, authed: bool = True) -> "Bucket":
         config=Config(
             signature_version=None if authed else botocore.UNSIGNED,
             s3={"addressing_style": settings.S3_ADDRESSING_STYLE},
+            request_checksum_calculation=checksum,
         ),
     ).Bucket(bucket_name)
 

zerver/management/commands/runtusd.py

@@ -77,4 +77,6 @@ class Command(BaseCommand):
                 env_vars["AWS_REGION"] = boto3.client("s3").meta.region_name
             else:
                 env_vars["AWS_REGION"] = settings.S3_REGION
+            if settings.S3_SKIP_CHECKSUM:
+                env_vars["AWS_REQUEST_CHECKSUM_CALCULATION"] = "when_required"
         os.execvpe("tusd", tusd_args, env_vars)

zproject/default_settings.py

@@ -166,6 +166,7 @@ S3_UPLOADS_STORAGE_CLASS: Literal[
     "STANDARD_IA",
 ] = "STANDARD"
 S3_AVATAR_PUBLIC_URL_PREFIX: str | None = None
+S3_SKIP_CHECKSUM: bool = False
 LOCAL_UPLOADS_DIR: str | None = None
 LOCAL_AVATARS_DIR: str | None = None
 LOCAL_FILES_DIR: str | None = None

zproject/prod_settings_template.py

@@ -818,6 +818,7 @@ LOCAL_UPLOADS_DIR = "/home/zulip/uploads"
 # S3_ADDRESSING_STYLE = "auto"
 # S3_SKIP_PROXY = True
 # S3_UPLOADS_STORAGE_CLASS = "STANDARD"
+# S3_SKIP_CHECKSUM = False
 
 ## Maximum allowed size of uploaded files, in megabytes. Set
 ## MAX_FILE_UPLOAD_SIZE to 0 to disable file uploads completely