export_files_from_s3: Move saving s3 object to local file to a separate function.

This refactor makes upgrading boto to boto3 easier.
Based on 24bf813e8a
This commit is contained in:
rht
2018-12-07 16:26:48 +00:00
committed by Tim Abbott
parent 0ddb242583
commit a0dbcde063

View File

@@ -1108,6 +1108,26 @@ def _get_exported_s3_record(
return record return record
def _save_s3_object_to_file(
key: Key,
output_dir: str,
processing_avatars: bool,
processing_emoji: bool) -> None:
# Helper function for export_files_from_s3
if processing_avatars or processing_emoji:
filename = os.path.join(output_dir, key.name)
else:
fields = key.name.split('/')
if len(fields) != 3:
raise AssertionError("Suspicious key with invalid format %s" % (key.name))
filename = os.path.join(output_dir, key.name)
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
os.makedirs(dirname)
key.get_contents_to_filename(filename)
def export_files_from_s3(realm: Realm, bucket_name: str, output_dir: Path, def export_files_from_s3(realm: Realm, bucket_name: str, output_dir: Path,
processing_avatars: bool=False, processing_avatars: bool=False,
processing_emoji: bool=False) -> None: processing_emoji: bool=False) -> None:
@@ -1147,18 +1167,7 @@ def export_files_from_s3(realm: Realm, bucket_name: str, output_dir: Path,
record = _get_exported_s3_record(bucket_name, key, processing_avatars, processing_emoji) record = _get_exported_s3_record(bucket_name, key, processing_avatars, processing_emoji)
record['path'] = key.name record['path'] = key.name
if processing_avatars or processing_emoji: _save_s3_object_to_file(key, output_dir, processing_avatars, processing_emoji)
filename = os.path.join(output_dir, key.name)
else:
fields = key.name.split('/')
if len(fields) != 3:
raise AssertionError("Suspicious key with invalid format %s" % (key.name))
filename = os.path.join(output_dir, key.name)
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
os.makedirs(dirname)
key.get_contents_to_filename(filename)
records.append(record) records.append(record)
count += 1 count += 1