From a6d3bbfc63f1c5c8d17c17cef8a935537685aedb Mon Sep 17 00:00:00 2001 From: Tim Abbott Date: Sat, 2 Feb 2019 11:48:16 -0800 Subject: [PATCH] zilencer: Add client-size rate limiting of analytics upload. This should help both by avoiding high memory usage causing OOM kills on the client, as well as timeouts causing an exception email to be sent. --- zerver/lib/remote_server.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/zerver/lib/remote_server.py b/zerver/lib/remote_server.py index 53e8b24777..0814b99ab1 100644 --- a/zerver/lib/remote_server.py +++ b/zerver/lib/remote_server.py @@ -92,12 +92,16 @@ def send_json_to_push_bouncer(method: str, endpoint: str, post_data: Dict[str, A def build_analytics_data(realm_count_query: Any, installation_count_query: Any) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: + # We limit the batch size on the client side to avoid OOM kills timeouts, etc. + MAX_CLIENT_BATCH_SIZE = 10000 data = {} data['analytics_realmcount'] = [ - model_to_dict(realm_count) for realm_count in realm_count_query.order_by("id") + model_to_dict(realm_count) for realm_count in + realm_count_query.order_by("id")[0:MAX_CLIENT_BATCH_SIZE] ] data['analytics_installationcount'] = [ - model_to_dict(count) for count in installation_count_query.order_by("id") + model_to_dict(count) for count in + installation_count_query.order_by("id")[0:MAX_CLIENT_BATCH_SIZE] ] floatify_datetime_fields(data, 'analytics_realmcount')