paulmataruso/zulip
1
0
Fork 0
mirror of https://github.com/zulip/zulip.git synced 2025-11-01 04:23:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

import/export: Use separate analytics.json for analytics data.

Browse Source 
This helps keep the realm.json small and easy to process; previously,
almost the entire size of that file was the analytics data.

We implement this by refactoring the analytics Config objects into a
separate subroutine that writes to a separate file, plus the
corresponding import code.

Manual testing was performed by exporting the 'analytics' realm, and
importing back to a newly created 'test' realm.  The 'test' realm was
then exported and the json files were inspected.  The data appeared
consistent with no abnormalities.

Fixes: #11220.
This commit is contained in:
Wyatt Hoodes
2019-01-29 21:54:29 -10:00
committed by Tim Abbott
parent bb6a75c3dc
commit 9c68a97472
2 changed files with 87 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
zerver/lib/import_realm.py
View File

@@ -897,24 +897,6 @@ def do_import_realm(import_dir: Path, subdomain: str, processes: int=1) -> Realm
update_model_ids(UserPresence, data, 'user_presence')
bulk_import_model(data, UserPresence)
if 'analytics_realmcount' in data:
fix_datetime_fields(data, 'analytics_realmcount')
re_map_foreign_keys(data, 'analytics_realmcount', 'realm', related_table="realm")
update_model_ids(RealmCount, data, 'analytics_realmcount')
bulk_import_model(data, RealmCount)
fix_datetime_fields(data, 'analytics_usercount')
re_map_foreign_keys(data, 'analytics_usercount', 'realm', related_table="realm")
re_map_foreign_keys(data, 'analytics_usercount', 'user', related_table="user_profile")
update_model_ids(UserCount, data, 'analytics_usercount')
bulk_import_model(data, UserCount)
fix_datetime_fields(data, 'analytics_streamcount')
re_map_foreign_keys(data, 'analytics_streamcount', 'realm', related_table="realm")
re_map_foreign_keys(data, 'analytics_streamcount', 'stream', related_table="stream")
update_model_ids(StreamCount, data, 'analytics_streamcount')
bulk_import_model(data, StreamCount)
fix_datetime_fields(data, 'zerver_useractivity')
re_map_foreign_keys(data, 'zerver_useractivity', 'user_profile', related_table="user_profile")
re_map_foreign_keys(data, 'zerver_useractivity', 'client', related_table='client')
@@ -999,6 +981,9 @@ def do_import_realm(import_dir: Path, subdomain: str, processes: int=1) -> Realm
import_attachments(data)
# Import the analytics file.
import_analytics_data(realm=realm, import_dir=import_dir)
if settings.BILLING_ENABLED:
do_change_plan_type(realm, Realm.LIMITED)
return realm
@@ -1220,3 +1205,30 @@ def import_attachments(data: TableData) -> None:
cursor.executemany(sql_template, tups)
logging.info('Successfully imported M2M table %s' % (m2m_table_name,))
def import_analytics_data(realm: Realm, import_dir: Path) -> None:
analytics_filename = os.path.join(import_dir, "analytics.json")
if not os.path.exists(analytics_filename):
return
logging.info("Importing analytics data from %s" % (analytics_filename,))
with open(analytics_filename) as f:
data = ujson.load(f)
# Process the data through the fixer functions.
fix_datetime_fields(data, 'analytics_realmcount')
re_map_foreign_keys(data, 'analytics_realmcount', 'realm', related_table="realm")
update_model_ids(RealmCount, data, 'analytics_realmcount')
bulk_import_model(data, RealmCount)
fix_datetime_fields(data, 'analytics_usercount')
re_map_foreign_keys(data, 'analytics_usercount', 'realm', related_table="realm")
re_map_foreign_keys(data, 'analytics_usercount', 'user', related_table="user_profile")
update_model_ids(UserCount, data, 'analytics_usercount')
bulk_import_model(data, UserCount)
fix_datetime_fields(data, 'analytics_streamcount')
re_map_foreign_keys(data, 'analytics_streamcount', 'realm', related_table="realm")
re_map_foreign_keys(data, 'analytics_streamcount', 'stream', related_table="stream")
update_model_ids(StreamCount, data, 'analytics_streamcount')
bulk_import_model(data, StreamCount)