From 2d4b09f59d0dc8518b86711de4d199ad8a757665 Mon Sep 17 00:00:00 2001
From: Steve Howell <showell@zulipchat.com>
Date: Mon, 15 Oct 2018 12:24:13 +0000
Subject: [PATCH] utils: Add process_list_in_batches().

---
 zerver/lib/import_realm.py | 14 ++++++--------
 zerver/lib/utils.py        | 12 ++++++++++++
 2 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/zerver/lib/import_realm.py b/zerver/lib/import_realm.py
index ff30a47521..d5e804eca8 100644
--- a/zerver/lib/import_realm.py
+++ b/zerver/lib/import_realm.py
@@ -23,7 +23,7 @@ from zerver.lib.message import save_message_rendered_content
 from zerver.lib.bugdown import version as bugdown_version
 from zerver.lib.upload import random_name, sanitize_name, \
     S3UploadBackend, LocalUploadBackend, guess_type
-from zerver.lib.utils import generate_api_key
+from zerver.lib.utils import generate_api_key, process_list_in_batches
 from zerver.models import UserProfile, Realm, Client, Huddle, Stream, \
     UserMessage, Subscription, Message, RealmEmoji, \
     RealmDomain, Recipient, get_user_profile_by_id, \
@@ -435,15 +435,13 @@ def bulk_import_user_message_data(data: TableData, dump_file_id: int) -> None:
         ]
         bulk_insert_ums(ums)
 
-    offset = 0
     chunk_size = 10000
 
-    while True:
-        items = lst[offset:offset+chunk_size]
-        if not items:
-            break
-        process_batch(items)
-        offset += chunk_size
+    process_list_in_batches(
+        lst=lst,
+        chunk_size=chunk_size,
+        process_batch=process_batch,
+    )
 
     logging.info("Successfully imported %s from %s[%s]." % (model, table, dump_file_id))
 
diff --git a/zerver/lib/utils.py b/zerver/lib/utils.py
index 8860784a0e..a0a0ef8e6e 100644
--- a/zerver/lib/utils.py
+++ b/zerver/lib/utils.py
@@ -177,6 +177,18 @@ def query_chunker(queries: List[Any],
 
         yield [row for row_id, i, row in tup_chunk]
 
+def process_list_in_batches(lst: List[Any],
+                            chunk_size: int,
+                            process_batch: Callable[[List[Any]], None]) -> None:
+    offset = 0
+
+    while True:
+        items = lst[offset:offset+chunk_size]
+        if not items:
+            break
+        process_batch(items)
+        offset += chunk_size
+
 def split_by(array: List[Any], group_size: int, filler: Any) -> List[List[Any]]:
     """
     Group elements into list of size `group_size` and fill empty cells with