Use bulk requests for updating memcached in get_old_messages.

Otherwise we end up doing 1000 requests to memcached, which can be
quite expensive.

(imported from commit be247f63b5fb88c6f4a45326261b66ea67fe1028)

zephyr/lib/cache.py

@@ -74,6 +74,13 @@ def cache_get_many(keys, cache_name=None):
         cache_backend = get_cache(cache_name)
     return cache_backend.get_many(keys)
 
+def cache_set_many(items, cache_name=None):
+    if cache_name is None:
+        cache_backend = djcache
+    else:
+        cache_backend = get_cache(cache_name)
+    return cache_backend.set_many(items)
+
 def cache(func):
     """Decorator which applies Django caching to a function.
 

zephyr/models.py

@@ -253,6 +253,9 @@ class Message(models.Model):
 
     @cache_with_key(to_dict_cache_key, timeout=3600*24)
     def to_dict(self, apply_markdown, rendered_content=None):
+        return self.to_dict_uncached(apply_markdown, rendered_content)
+
+    def to_dict_uncached(self, apply_markdown, rendered_content=None):
         display_recipient = get_display_recipient(self.recipient)
         if self.recipient.type == Recipient.STREAM:
             display_type = "stream"

zephyr/views.py

@@ -47,7 +47,7 @@ from zephyr.lib.query import last_n
 from zephyr.lib.avatar import gravatar_hash
 from zephyr.lib.response import json_success, json_error, json_response, json_method_not_allowed
 from zephyr.lib.timestamp import datetime_to_timestamp
-from zephyr.lib.cache import cache_with_key, cache_get_many
+from zephyr.lib.cache import cache_with_key, cache_get_many, cache_set_many
 from zephyr.lib.unminify import SourceMap
 from zephyr.lib.queue import queue_json_publish
 from zephyr.lib.utils import statsd
@@ -774,18 +774,23 @@ def get_old_messages_backend(request, user_profile,
 
     bulk_messages = cache_get_many([to_dict_cache_key(message, apply_markdown)
                                     for message in messages])
+    items_for_memcached = {}
+    for message in messages:
+        key = to_dict_cache_key(message, apply_markdown)
+        if bulk_messages.get(key) is None:
+            elt = message.to_dict_uncached(apply_markdown)
+            items_for_memcached[key] = (elt,)
+            bulk_messages[key] = (elt,)
+    if len(items_for_memcached) > 0:
+        cache_set_many(items_for_memcached)
+
     message_list = []
     for message in messages:
         if include_history:
             flags_dict = {'flags': ["read", "historical"]}
         if message.id in user_messages:
             flags_dict = user_messages[message.id].flags_dict()
-
+        elt = bulk_messages.get(to_dict_cache_key(message, apply_markdown))[0]
-        data = bulk_messages.get(to_dict_cache_key(message, apply_markdown))
-        if data is None:
-            elt = message.to_dict(apply_markdown)
-        else:
-            elt = data[0]
         message_list.append(dict(elt, **flags_dict))
 
     statsd.incr('loaded_old_messages', len(message_list))