paulmataruso/zulip
1
0
Fork 0
mirror of https://github.com/zulip/zulip.git synced 2025-11-05 14:35:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: Pass object_key into export_files_from_s3.

Browse Source 
This makes it easier to read the calling code and see
the big picture of how the four asset types are
organized.

I also handle uploads first, to be similar to the local
code.

This code is well tested--you can modify any of the callers
to pass in a wrong value of `object_key` and get a failing
test.
This commit is contained in:
Steve Howell
2021-12-06 15:29:20 +00:00
committed by Tim Abbott
parent 45b49b2b70
commit f4354c896b
1 changed files with 24 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
zerver/lib/export.py
View File

@@ -1310,15 +1310,32 @@ def export_uploads_and_avatars(realm: Realm, output_dir: Path) -> None:
else: else:
# Some bigger installations will have their data stored on S3. # Some bigger installations will have their data stored on S3.
export_files_from_s3( export_files_from_s3(
realm, settings.S3_AVATAR_BUCKET, output_dir=avatars_output_dir, processing_avatars=True realm,
) bucket_name=settings.S3_AUTH_UPLOADS_BUCKET,
export_files_from_s3(realm, settings.S3_AUTH_UPLOADS_BUCKET, output_dir=uploads_output_dir) object_prefix=f"{realm.id}/",
export_files_from_s3( output_dir=uploads_output_dir,
realm, settings.S3_AVATAR_BUCKET, output_dir=emoji_output_dir, processing_emoji=True
) )
export_files_from_s3( export_files_from_s3(
realm, realm,
settings.S3_AVATAR_BUCKET, bucket_name=settings.S3_AVATAR_BUCKET,
object_prefix=f"{realm.id}/",
output_dir=avatars_output_dir,
processing_avatars=True,
)
export_files_from_s3(
realm,
bucket_name=settings.S3_AVATAR_BUCKET,
object_prefix=f"{realm.id}/emoji/images/",
output_dir=emoji_output_dir,
processing_emoji=True,
)
export_files_from_s3(
realm,
bucket_name=settings.S3_AVATAR_BUCKET,
object_prefix=f"{realm.id}/realm/",
output_dir=realm_icons_output_dir, output_dir=realm_icons_output_dir,
processing_realm_icon_and_logo=True, processing_realm_icon_and_logo=True,
) )
@@ -1422,6 +1439,7 @@ def _save_s3_object_to_file(
def export_files_from_s3( def export_files_from_s3(
realm: Realm, realm: Realm,
bucket_name: str, bucket_name: str,
object_prefix: str,
output_dir: Path, output_dir: Path,
processing_avatars: bool = False, processing_avatars: bool = False,
processing_emoji: bool = False, processing_emoji: bool = False,
@@ -1441,13 +1459,6 @@ def export_files_from_s3(
avatar_hash_values.add(avatar_path + ".original") avatar_hash_values.add(avatar_path + ".original")
user_ids.add(user_profile.id) user_ids.add(user_profile.id)
if processing_realm_icon_and_logo:
object_prefix = f"{realm.id}/realm/"
elif processing_emoji:
object_prefix = f"{realm.id}/emoji/images/"
else:
object_prefix = f"{realm.id}/"
if settings.EMAIL_GATEWAY_BOT is not None: if settings.EMAIL_GATEWAY_BOT is not None:
internal_realm = get_realm(settings.SYSTEM_BOT_REALM) internal_realm = get_realm(settings.SYSTEM_BOT_REALM)
email_gateway_bot: Optional[UserProfile] = get_system_bot( email_gateway_bot: Optional[UserProfile] = get_system_bot(