paulmataruso/zulip
1
0
Fork 0
mirror of https://github.com/zulip/zulip.git synced 2025-11-10 17:07:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

export_files_from_s3: Move checking for s3 oject's metadata to a separate function.

Browse Source 
This refactor makes upgrading boto to boto3 easier.
Based on 24bf813e8a
This commit is contained in:
rht
2018-12-07 16:12:09 +00:00
committed by Tim Abbott
parent 1dc741b963
commit 1cecf0f142
1 changed files with 19 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
zerver/lib/export.py
View File

@@ -1,5 +1,6 @@
import datetime import datetime
from boto.s3.connection import S3Connection from boto.s3.connection import S3Connection
from boto.s3.key import Key # for mypy
from django.apps import apps from django.apps import apps
from django.conf import settings from django.conf import settings
from django.db import connection from django.db import connection
@@ -1064,6 +1065,23 @@ def export_uploads_and_avatars(realm: Realm, output_dir: Path) -> None:
output_dir=emoji_output_dir, output_dir=emoji_output_dir,
processing_emoji=True) processing_emoji=True)
def _check_key_metadata(email_gateway_bot: Optional[UserProfile],
key: Key, processing_avatars: bool,
realm: Realm, user_ids: Set[int]) -> None:
# Helper function for export_files_from_s3
if 'realm_id' in key.metadata and key.metadata['realm_id'] != str(realm.id):
if email_gateway_bot is None or key.metadata['user_profile_id'] != str(email_gateway_bot.id):
raise AssertionError("Key metadata problem: %s %s / %s" % (key.name, key.metadata, realm.id))
# Email gateway bot sends messages, potentially including attachments, cross-realm.
print("File uploaded by email gateway bot: %s / %s" % (key.name, key.metadata))
elif processing_avatars:
if 'user_profile_id' not in key.metadata:
raise AssertionError("Missing user_profile_id in key metadata: %s" % (key.metadata,))
if int(key.metadata['user_profile_id']) not in user_ids:
raise AssertionError("Wrong user_profile_id in key metadata: %s" % (key.metadata,))
elif 'realm_id' not in key.metadata:
raise AssertionError("Missing realm_id in key metadata: %s" % (key.metadata,))
def export_files_from_s3(realm: Realm, bucket_name: str, output_dir: Path, def export_files_from_s3(realm: Realm, bucket_name: str, output_dir: Path,
processing_avatars: bool=False, processing_avatars: bool=False,
processing_emoji: bool=False) -> None: processing_emoji: bool=False) -> None:
@@ -1099,18 +1117,7 @@ def export_files_from_s3(realm: Realm, bucket_name: str, output_dir: Path,
key = bucket.get_key(bkey.name) key = bucket.get_key(bkey.name)
# This can happen if an email address has moved realms # This can happen if an email address has moved realms
if 'realm_id' in key.metadata and key.metadata['realm_id'] != str(realm.id): _check_key_metadata(email_gateway_bot, key, processing_avatars, realm, user_ids)
if email_gateway_bot is None or key.metadata['user_profile_id'] != str(email_gateway_bot.id):
raise AssertionError("Key metadata problem: %s %s / %s" % (key.name, key.metadata, realm.id))
# Email gateway bot sends messages, potentially including attachments, cross-realm.
print("File uploaded by email gateway bot: %s / %s" % (key.name, key.metadata))
elif processing_avatars:
if 'user_profile_id' not in key.metadata:
raise AssertionError("Missing user_profile_id in key metadata: %s" % (key.metadata,))
if int(key.metadata['user_profile_id']) not in user_ids:
raise AssertionError("Wrong user_profile_id in key metadata: %s" % (key.metadata,))
elif 'realm_id' not in key.metadata:
raise AssertionError("Missing realm_id in key metadata: %s" % (key.metadata,))
record = dict(s3_path=key.name, bucket=bucket_name, record = dict(s3_path=key.name, bucket=bucket_name,
size=key.size, last_modified=key.last_modified, size=key.size, last_modified=key.last_modified,