python3: Add missing utf-8 encoding/decoding in various places.

zerver/lib/bugdown/__init__.py

@@ -1,5 +1,6 @@
 from __future__ import absolute_import
 
+import codecs
 import markdown
 import logging
 import traceback
@@ -242,9 +243,11 @@ class InlineHttpsProcessor(markdown.treeprocessors.Treeprocessor):
             if not url.startswith("http://"):
                 # Don't rewrite images on our own site (e.g. emoji).
                 continue
-            digest = hmac.new(settings.CAMO_KEY, url, hashlib.sha1).hexdigest()
+            encoded_url = url.encode("utf-8")
-            encoded_url = url.encode("hex")
+            encoded_camo_key = settings.CAMO_KEY.encode("utf-8")
-            img.set("src", "%s%s/%s" % (settings.CAMO_URI, digest, encoded_url))
+            digest = hmac.new(encoded_camo_key, encoded_url, hashlib.sha1).hexdigest()
+            hex_encoded_url = codecs.encode(encoded_url, "hex")
+            img.set("src", "%s%s/%s" % (settings.CAMO_URI, digest, hex_encoded_url))
 
 class InlineInterestingLinkProcessor(markdown.treeprocessors.Treeprocessor):
     TWITTER_MAX_IMAGE_HEIGHT = 400

zerver/lib/create_user.py

@@ -11,8 +11,8 @@ from six.moves import range
 
 def random_api_key():
     choices = string.ascii_letters + string.digits
-    altchars = ''.join([choices[ord(os.urandom(1)) % 62] for _ in range(2)])
+    altchars = ''.join([choices[ord(os.urandom(1)) % 62] for _ in range(2)]).encode("utf-8")
-    return base64.b64encode(os.urandom(24), altchars=altchars)
+    return base64.b64encode(os.urandom(24), altchars=altchars).decode("utf-8")
 
 # create_user_profile is based on Django's User.objects.create_user,
 # except that we don't save to the database so it can used in

zerver/lib/initial_password.py

@@ -10,7 +10,8 @@ def initial_password(email):
        created by populate_db."""
 
     if settings.INITIAL_PASSWORD_SALT is not None:
-        digest = hashlib.sha256(settings.INITIAL_PASSWORD_SALT + email).digest()
+        encoded_key = (settings.INITIAL_PASSWORD_SALT + email).encode("utf-8")
+        digest = hashlib.sha256(encoded_key).digest()
         return base64.b64encode(digest)[:16]
     else:
         # None as a password for a user tells Django to set an unusable password

zerver/middleware.py

@@ -96,7 +96,7 @@ def write_log_line(log_data, path, method, remote_ip, email, client_name,
         # Remove non-ascii chars from path (there should be none, if there are it's
         # because someone manually entered a nonexistant path), as UTF-8 chars make
         # statsd sad when it sends the key name over the socket
-        statsd_path = statsd_path.encode('ascii', errors='ignore')
+        statsd_path = statsd_path.encode('ascii', errors='ignore').decode("ascii")
     blacklisted_requests = ['do_confirm', 'send_confirm',
                             'eventslast_event_id', 'webreq.content', 'avatar', 'user_uploads',
                             'password.reset', 'static', 'json.bots', 'json.users', 'json.streams',

zerver/models.py

@@ -664,10 +664,10 @@ def linebreak(string):
     return string.replace('\n\n', '<p/>').replace('\n', '<br/>')
 
 def extract_message_dict(message_str):
-    return ujson.loads(zlib.decompress(message_str))
+    return ujson.loads(zlib.decompress(message_str).decode("utf-8"))
 
 def stringify_message_dict(message_dict):
-    return zlib.compress(ujson.dumps(message_dict))
+    return zlib.compress(ujson.dumps(message_dict).encode("utf-8"))
 
 def to_dict_cache_key_id(message_id, apply_markdown):
     return 'message_dict:%d:%d' % (message_id, apply_markdown)