avatar: Move avatar data from email based to user id based.

This commit introduces a migration for moving avatars from email based
to user id based storage.

This is in responce to change in behaviour of user_avatar_path to
return path comprising of realm id and a hash based on user id. Also
we fix test_helpers accordingly.

Fixes #3776.

zerver/lib/avatar_hash.py

@@ -21,16 +21,20 @@ def gravatar_hash(email):
     # not error out on it.
     return make_safe_digest(email.lower(), hashlib.md5)
 
-def user_avatar_hash(email):
+# WARNING: If this method is changed, you may need to do a
+# corresponding update to zerver/migrations/0060_move_avatars_to_be_uid_based.py
+def user_avatar_hash(uid):
     # type: (Text) -> Text
     # Salting the user_key may be overkill, but it prevents us from
     # basically mimicking Gravatar's hashing scheme, which could lead
     # to some abuse scenarios like folks using us as a free Gravatar
     # replacement.
-    user_key = email.lower() + settings.AVATAR_SALT
+    user_key = uid + settings.AVATAR_SALT
     return make_safe_digest(user_key, hashlib.sha1)
 
+# WARNING: If this method is changed, you will may to do a
+# corresponding update to zerver/migrations/0060_move_avatars_to_be_uid_based.py
 def user_avatar_path(user_profile):
     # type: (UserProfile) -> Text
-    user_email_hash = user_avatar_hash(str(user_profile.email))
+    user_id_hash = user_avatar_hash(str(user_profile.id))
-    return ('%s' % (user_email_hash))
+    return '%s/%s' % (str(user_profile.realm_id), user_id_hash)

zerver/lib/test_helpers.py

@@ -185,6 +185,7 @@ def avatar_disk_path(user_profile, medium=False):
     # type: (UserProfile, bool) -> str
     avatar_url_path = avatar_url(user_profile, medium)
     avatar_disk_path = os.path.join(settings.LOCAL_UPLOADS_DIR, "avatars",
+                                    avatar_url_path.split("/")[-2],
                                     avatar_url_path.split("/")[-1].split("?")[0])
     return avatar_disk_path
 

zerver/migrations/0060_move_avatars_to_be_uid_based.py

@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.5 on 2017-02-27 17:03
+from __future__ import unicode_literals
+
+from django.db import migrations
+from django.db.backends.postgresql_psycopg2.schema import DatabaseSchemaEditor
+from django.db.migrations.state import StateApps
+from django.core.files.uploadedfile import SimpleUploadedFile
+from django.conf import settings
+from zerver.lib.avatar_hash import user_avatar_hash, user_avatar_path
+from boto.s3.bucket import Bucket
+from boto.s3.key import Key
+from boto.s3.connection import S3Connection
+from typing import Text
+import requests
+import os
+
+
+def mkdirs(path):
+    # type: (Text) -> None
+    dirname = os.path.dirname(path)
+    if not os.path.isdir(dirname):
+        os.makedirs(dirname)
+
+def move_local_file(type, path_src, path_dst):
+    # type: (Text, Text, Text) -> None
+    src_file_path = os.path.join(settings.LOCAL_UPLOADS_DIR, type, path_src)
+    dst_file_path = os.path.join(settings.LOCAL_UPLOADS_DIR, type, path_dst)
+    mkdirs(dst_file_path)
+    os.rename(src_file_path, dst_file_path)
+
+def move_avatars_to_be_uid_based(apps, schema_editor):
+    # type: (StateApps, DatabaseSchemaEditor) -> None
+    user_profile_model = apps.get_model('zerver', 'UserProfile')
+    if settings.LOCAL_UPLOADS_DIR is not None:
+        for user_profile in user_profile_model.objects.filter(avatar_source=u"U"):
+            src_file_name = user_avatar_hash(user_profile.email)
+            dst_file_name = user_avatar_path(user_profile)
+            move_local_file('avatars', src_file_name + '.original', dst_file_name + '.original')
+            move_local_file('avatars', src_file_name + '-medium.png', dst_file_name + '-medium.png')
+            move_local_file('avatars', src_file_name + '.png', dst_file_name + '.png')
+    else:
+        conn = S3Connection(settings.S3_KEY, settings.S3_SECRET_KEY)
+        bucket_name = settings.S3_AVATAR_BUCKET
+        bucket = conn.get_bucket(bucket_name, validate=False)
+        for user_profile in user_profile_model.objects.filter(avatar_source=u"U"):
+            bucket.copy_key(user_avatar_path(user_profile) + ".original",
+                            bucket,
+                            user_avatar_hash(user_profile.email) + ".original")
+            bucket.copy_key(user_avatar_path(user_profile) + "-medium.png",
+                            bucket,
+                            user_avatar_hash(user_profile.email) + "-medium.png")
+            bucket.copy_key(user_avatar_path(user_profile),
+                            bucket,
+                            user_avatar_hash(user_profile.email))
+
+        # From an error handling sanity perspective, it's best to
+        # start deleting after everything is copied, so that recovery
+        # from failures is easy (just rerun one loop or the other).
+        for user_profile in user_profile_model.objects.filter(avatar_source=u"U"):
+            bucket.delete_key(user_avatar_hash(user_profile.email) + ".original")
+            bucket.delete_key(user_avatar_hash(user_profile.email) + "-medium.png")
+            bucket.delete_key(user_avatar_hash(user_profile.email))
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('zerver', '0059_userprofile_quota'),
+    ]
+
+    operations = [
+        migrations.RunPython(move_avatars_to_be_uid_based)
+    ]