import: Migrate from json to ujson for better perf.

We expect to get better memory performace from
ujson than json.

We also do a better job of closing file handles.

This likely fixes #10377.

zerver/data_import/gitter.py

@@ -1,12 +1,11 @@
 import os
-import ujson
 import dateutil.parser
 import random
 import requests
-import json
 import logging
 import shutil
 import subprocess
+import ujson
 
 from django.conf import settings
 from django.forms.models import model_to_dict
@@ -231,7 +230,8 @@ def do_convert_data(gitter_data_file: str, output_dir: str, threads: int=6) -> N
         raise Exception("Output directory should be empty!")
 
     # Read data from the gitter file
-    gitter_data = json.load(open(gitter_data_file))
+    with open(gitter_data_file, "r") as fp:
+        gitter_data = ujson.load(fp)
 
     realm, avatar_list, user_map = gitter_workspace_to_realm(
         domain_name, gitter_data, realm_subdomain)

zerver/data_import/hipchat.py

@@ -1,11 +1,11 @@
 import base64
 import dateutil
 import glob
-import json
 import logging
 import os
 import shutil
 import subprocess
+import ujson
 
 from typing import Any, Callable, Dict, List, Set
 
@@ -82,8 +82,8 @@ def untar_input_file(tar_file: str) -> str:
 def read_user_data(data_dir: str) -> List[ZerverFieldsT]:
     fn = 'users.json'
     data_file = os.path.join(data_dir, fn)
-    data = json.load(open(data_file))
-    return data
+    with open(data_file, "r") as fp:
+        return ujson.load(fp)
 
 def convert_user_data(user_handler: UserHandler,
                       raw_data: List[ZerverFieldsT],
@@ -184,7 +184,8 @@ def convert_avatar_data(avatar_folder: str,
 def read_room_data(data_dir: str) -> List[ZerverFieldsT]:
     fn = 'rooms.json'
     data_file = os.path.join(data_dir, fn)
-    data = json.load(open(data_file))
+    with open(data_file) as f:
+        data = ujson.load(f)
     return data
 
 def convert_room_data(raw_data: List[ZerverFieldsT], realm_id: int) -> List[ZerverFieldsT]:
@@ -296,7 +297,8 @@ def write_emoticon_data(realm_id: int,
 
     fn = 'emoticons.json'
     data_file = os.path.join(data_dir, fn)
-    data = json.load(open(data_file))
+    with open(data_file) as f:
+        data = ujson.load(f)
 
     flat_data = [
         dict(
@@ -431,7 +433,8 @@ def process_message_file(realm_id: int,
                          attachment_handler: AttachmentHandler) -> None:
 
     def get_raw_messages(fn: str) -> List[ZerverFieldsT]:
-        data = json.load(open(fn))
+        with open(fn) as f:
+            data = ujson.load(f)
 
         flat_data = [
             d[message_key]

zerver/data_import/import_util.py

@@ -1,9 +1,9 @@
-import json
 import random
 import requests
 import shutil
 import logging
 import os
+import ujson
 
 from typing import List, Dict, Any, Optional, Set
 from django.forms.models import model_to_dict
@@ -509,4 +509,5 @@ def process_emojis(zerver_realmemoji: List[ZerverFieldsT], emoji_dir: str,
 def create_converted_data_files(data: Any, output_dir: str, file_path: str) -> None:
     output_file = output_dir + file_path
     os.makedirs(os.path.dirname(output_file), exist_ok=True)
-    json.dump(data, open(output_file, 'w'), indent=4)
+    with open(output_file, 'w') as fp:
+        ujson.dump(data, fp, indent=4)

zerver/data_import/slack.py

@@ -1,5 +1,4 @@
 import os
-import json
 import ujson
 import hashlib
 import sys
@@ -569,7 +568,7 @@ def channel_message_to_zerver_message(realm_id: int, users: List[ZerverFieldsT],
                 message['text'], users, added_channels, added_users)
         except Exception:
             print("Slack message unexpectedly missing text representation:")
-            print(json.dumps(message, indent=4))
+            print(ujson.dumps(message, indent=4))
             continue
         rendered_content = None
 
@@ -793,8 +792,9 @@ def do_convert_data(slack_zip_file: str, output_dir: str, token: str, threads: i
     logging.info("Zulip data dump created at %s" % (output_dir))
 
 def get_data_file(path: str) -> Any:
-    data = json.load(open(path))
-    return data
+    with open(path, "r") as fp:
+        data = ujson.load(fp)
+        return data
 
 def get_slack_api_data(token: str, slack_api_url: str, get_param: str) -> Any:
     data = requests.get('%s?token=%s' % (slack_api_url, token))