paulmataruso/zulip
1
0
Fork 0
mirror of https://github.com/zulip/zulip.git synced 2025-11-03 13:33:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

export: Support export of Custom emojis.

Browse Source 
Export of RealmEmoji should also include the image
file of those emojis.

Here, we export emojis both for local and S3 backend
in a method with is similar to attachments and avatars.

Added tests for the same.
This commit is contained in:
Rhea Parekh
2018-05-27 00:48:54 +05:30
committed by Tim Abbott
parent 7198cc3899
commit 468afe4840
2 changed files with 65 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
zerver/lib/export.py
View File

@@ -840,8 +840,9 @@ def write_message_partial_for_query(realm: Realm, message_query: Any, dump_file_
def export_uploads_and_avatars(realm: Realm, output_dir: Path) -> None:
uploads_output_dir = os.path.join(output_dir, 'uploads')
avatars_output_dir = os.path.join(output_dir, 'avatars')
emoji_output_dir = os.path.join(output_dir, 'emoji')
for output_dir in (uploads_output_dir, avatars_output_dir):
for output_dir in (uploads_output_dir, avatars_output_dir, emoji_output_dir):
if not os.path.exists(output_dir):
os.makedirs(output_dir)
@@ -853,6 +854,9 @@ def export_uploads_and_avatars(realm: Realm, output_dir: Path) -> None:
export_avatars_from_local(realm,
local_dir=os.path.join(settings.LOCAL_UPLOADS_DIR, "avatars"),
output_dir=avatars_output_dir)
export_emoji_from_local(realm,
local_dir=os.path.join(settings.LOCAL_UPLOADS_DIR, "avatars"),
output_dir=emoji_output_dir)
else:
# Some bigger installations will have their data stored on S3.
export_files_from_s3(realm,
@@ -862,9 +866,14 @@ def export_uploads_and_avatars(realm: Realm, output_dir: Path) -> None:
export_files_from_s3(realm,
settings.S3_AUTH_UPLOADS_BUCKET,
output_dir=uploads_output_dir)
export_files_from_s3(realm,
settings.S3_AVATAR_BUCKET,
output_dir=emoji_output_dir,
processing_emoji=True)
def export_files_from_s3(realm: Realm, bucket_name: str, output_dir: Path,
processing_avatars: bool=False) -> None:
processing_avatars: bool=False,
processing_emoji: bool=False) -> None:
conn = S3Connection(settings.S3_KEY, settings.S3_SECRET_KEY)
bucket = conn.get_bucket(bucket_name, validate=True)
records = []
@@ -880,6 +889,8 @@ def export_files_from_s3(realm: Realm, bucket_name: str, output_dir: Path,
avatar_hash_values.add(avatar_path)
avatar_hash_values.add(avatar_path + ".original")
user_ids.add(user_profile.id)
if processing_emoji:
bucket_list = bucket.list(prefix="%s/emoji/images/" % (realm.id,))
else:
bucket_list = bucket.list(prefix="%s/" % (realm.id,))
@@ -919,7 +930,7 @@ def export_files_from_s3(realm: Realm, bucket_name: str, output_dir: Path,
record['realm_id'] = user_profile.realm_id
record['user_profile_email'] = user_profile.email
if processing_avatars:
if processing_avatars or processing_emoji:
filename = os.path.join(output_dir, key.name)
record['path'] = key.name
else:
@@ -1014,6 +1025,34 @@ def export_avatars_from_local(realm: Realm, local_dir: Path, output_dir: Path) -
with open(os.path.join(output_dir, "records.json"), "w") as records_file:
ujson.dump(records, records_file, indent=4)
def export_emoji_from_local(realm: Realm, local_dir: Path, output_dir: Path) -> None:
count = 0
records = []
for realm_emoji in RealmEmoji.objects.filter(realm_id=realm.id):
emoji_path = RealmEmoji.PATH_ID_TEMPLATE.format(
realm_id=realm.id,
emoji_file_name=realm_emoji.file_name
)
local_path = os.path.join(local_dir, emoji_path)
output_path = os.path.join(output_dir, emoji_path)
os.makedirs(os.path.dirname(output_path), exist_ok=True)
subprocess.check_call(["cp", "-a", local_path, output_path])
record = dict(realm_id=realm.id,
author=realm_emoji.author.id,
path=emoji_path,
s3_path=emoji_path,
file_name=realm_emoji.file_name,
name=realm_emoji.name,
deactivated=realm_emoji.deactivated)
records.append(record)
count += 1
if (count % 100 == 0):
logging.info("Finished %s" % (count,))
with open(os.path.join(output_dir, "records.json"), "w") as records_file:
ujson.dump(records, records_file, indent=4)
def do_write_stats_file_for_realm_export(output_dir: Path) -> None:
stats_file = os.path.join(output_dir, 'stats.txt')
realm_file = os.path.join(output_dir, 'realm.json')

24
zerver/tests/test_export.py
View File

@@ -21,6 +21,7 @@ from zerver.lib.export import (
from zerver.lib.upload import (
claim_attachment,
upload_message_file,
upload_emoji_image,
)
from zerver.lib.utils import (
query_chunker,
@@ -34,10 +35,15 @@ from zerver.lib.test_runner import slow
from zerver.models import (
Message,
Realm,
RealmEmoji,
Recipient,
UserMessage,
)
from zerver.lib.test_helpers import (
get_test_image_file,
)
def rm_tree(path: str) -> None:
if os.path.exists(path):
shutil.rmtree(path)
@@ -207,9 +213,10 @@ class ExportTest(ZulipTestCase):
result['attachment'] = read_file('attachment.json')
result['message'] = read_file('message.json')
result['uploads_dir'] = os.path.join(output_dir, 'uploads')
result['emoji_dir'] = os.path.join(output_dir, 'emoji')
return result
def test_attachment(self) -> None:
def test_attachment_and_emoji(self) -> None:
message = Message.objects.all()[0]
user_profile = message.sender
url = upload_message_file(u'dummy.txt', len(b'zulip!'), u'text/plain', b'zulip!', user_profile)
@@ -222,6 +229,9 @@ class ExportTest(ZulipTestCase):
)
realm = Realm.objects.get(string_id='zulip')
with get_test_image_file('img.png') as img_file:
upload_emoji_image(img_file, '1.png', user_profile)
full_data = self._export_realm(realm)
data = full_data['attachment']
@@ -233,9 +243,17 @@ class ExportTest(ZulipTestCase):
with open(fn) as f:
self.assertEqual(f.read(), 'zulip!')
fn = os.path.join(full_data['emoji_dir'],
RealmEmoji.PATH_ID_TEMPLATE.format(realm_id=realm.id, emoji_file_name='1.png'))
fn = fn.replace('1.png', '')
self.assertEqual('1.png', os.listdir(fn)[0])
def test_zulip_realm(self) -> None:
realm = Realm.objects.get(string_id='zulip')
realm_emoji = RealmEmoji.objects.get(realm=realm)
realm_emoji.delete()
full_data = self._export_realm(realm)
realm_emoji.save()
data = full_data['realm']
self.assertEqual(len(data['zerver_userprofile_crossrealm']), 0)
@@ -278,7 +296,11 @@ class ExportTest(ZulipTestCase):
hamlet = self.example_user('hamlet')
user_ids = set([cordelia.id, hamlet.id])
realm_emoji = RealmEmoji.objects.get(realm=realm)
realm_emoji.delete()
full_data = self._export_realm(realm, exportable_user_ids=user_ids)
realm_emoji.save()
data = full_data['realm']
exported_user_emails = get_set('zerver_userprofile', 'email')
self.assertIn(self.example_email('cordelia'), exported_user_emails)