mirror of
https://github.com/zulip/zulip.git
synced 2025-11-18 21:48:30 +00:00
export_files_from_s3: Move saving s3 object to local file to a separate function.
This refactor makes upgrading boto to boto3 easier.
Based on 24bf813e8a
This commit is contained in:
@@ -1108,6 +1108,26 @@ def _get_exported_s3_record(
|
|||||||
|
|
||||||
return record
|
return record
|
||||||
|
|
||||||
|
def _save_s3_object_to_file(
|
||||||
|
key: Key,
|
||||||
|
output_dir: str,
|
||||||
|
processing_avatars: bool,
|
||||||
|
processing_emoji: bool) -> None:
|
||||||
|
|
||||||
|
# Helper function for export_files_from_s3
|
||||||
|
if processing_avatars or processing_emoji:
|
||||||
|
filename = os.path.join(output_dir, key.name)
|
||||||
|
else:
|
||||||
|
fields = key.name.split('/')
|
||||||
|
if len(fields) != 3:
|
||||||
|
raise AssertionError("Suspicious key with invalid format %s" % (key.name))
|
||||||
|
filename = os.path.join(output_dir, key.name)
|
||||||
|
|
||||||
|
dirname = os.path.dirname(filename)
|
||||||
|
if not os.path.exists(dirname):
|
||||||
|
os.makedirs(dirname)
|
||||||
|
key.get_contents_to_filename(filename)
|
||||||
|
|
||||||
def export_files_from_s3(realm: Realm, bucket_name: str, output_dir: Path,
|
def export_files_from_s3(realm: Realm, bucket_name: str, output_dir: Path,
|
||||||
processing_avatars: bool=False,
|
processing_avatars: bool=False,
|
||||||
processing_emoji: bool=False) -> None:
|
processing_emoji: bool=False) -> None:
|
||||||
@@ -1147,18 +1167,7 @@ def export_files_from_s3(realm: Realm, bucket_name: str, output_dir: Path,
|
|||||||
record = _get_exported_s3_record(bucket_name, key, processing_avatars, processing_emoji)
|
record = _get_exported_s3_record(bucket_name, key, processing_avatars, processing_emoji)
|
||||||
|
|
||||||
record['path'] = key.name
|
record['path'] = key.name
|
||||||
if processing_avatars or processing_emoji:
|
_save_s3_object_to_file(key, output_dir, processing_avatars, processing_emoji)
|
||||||
filename = os.path.join(output_dir, key.name)
|
|
||||||
else:
|
|
||||||
fields = key.name.split('/')
|
|
||||||
if len(fields) != 3:
|
|
||||||
raise AssertionError("Suspicious key with invalid format %s" % (key.name))
|
|
||||||
filename = os.path.join(output_dir, key.name)
|
|
||||||
|
|
||||||
dirname = os.path.dirname(filename)
|
|
||||||
if not os.path.exists(dirname):
|
|
||||||
os.makedirs(dirname)
|
|
||||||
key.get_contents_to_filename(filename)
|
|
||||||
|
|
||||||
records.append(record)
|
records.append(record)
|
||||||
count += 1
|
count += 1
|
||||||
|
|||||||
Reference in New Issue
Block a user