events: Rewrite system for managing realm exports.

This feature is intended to cover all of our ways of exporting a
realm, not just the initial "public export" feature, so we should name
things appropriately for that goal.

Additionally, we don't want to include data exports in page_params;
the original implementation was actually buggy and would have.

zerver/lib/actions.py

@@ -38,6 +38,7 @@ from zerver.lib.email_mirror_helpers import encode_email_address, encode_email_a
 from zerver.lib.emoji import emoji_name_to_emoji_code, get_emoji_file_name
 from zerver.lib.exceptions import StreamDoesNotExistError, \
     StreamWithIDDoesNotExistError
+from zerver.lib.export import get_realm_exports_serialized
 from zerver.lib.hotspots import get_next_hotspots
 from zerver.lib.message import (
     access_message,
@@ -5670,8 +5671,8 @@ def get_zoom_video_call_url(realm: Realm) -> str:
 
     return response['join_url']
 
-def notify_export_completed(user_profile: UserProfile, public_url: str) -> None:
+def notify_export_completed(user_profile: UserProfile) -> None:
     # In the future, we may want to send this event to all realm admins.
-    event = dict(type='realm_exported',
+    event = dict(type='realm_export',
-                 public_url=public_url)
+                 exports=get_realm_exports_serialized(user_profile))
     send_event(user_profile.realm, event, [user_profile.id])

zerver/lib/events.py

@@ -751,6 +751,10 @@ def apply_event(state: Dict[str, Any],
                                       if realm_domain['domain'] != event['domain']]
     elif event['type'] == "realm_emoji":
         state['realm_emoji'] = event['realm_emoji']
+    elif event['type'] == 'realm_export':
+        # These realm export events are only available to
+        # administrators, and aren't included in page_params.
+        pass
     elif event['type'] == "alert_words":
         state['alert_words'] = event['alert_words']
     elif event['type'] == "muted_topics":
@@ -812,8 +816,6 @@ def apply_event(state: Dict[str, Any],
             user_status.pop(user_id, None)
 
         state['user_status'] = user_status
-    elif event['type'] == 'realm_exported':
-        pass
     else:
         raise AssertionError("Unexpected event type %s" % (event['type'],))
 

zerver/lib/export.py

@@ -1702,7 +1702,7 @@ def export_realm_wrapper(realm: Realm, output_dir: str,
         print("Successfully deleted the tarball at %s" % (tarball_path,))
     return public_url
 
-def get_public_exports_serialized(user: UserProfile) -> List[Dict[str, Any]]:
+def get_realm_exports_serialized(user: UserProfile) -> List[Dict[str, Any]]:
     all_exports = RealmAuditLog.objects.filter(realm=user.realm,
                                                event_type=RealmAuditLog.REALM_EXPORTED)
     exports_dict = {}

zerver/tests/test_events.py

@@ -2759,19 +2759,26 @@ class EventsRegisterTest(ZulipTestCase):
         error = schema_checker('events[0]', events[0])
         self.assert_on_error(error)
 
-    def test_public_export_notify_admins(self) -> None:
+    def test_realm_export_notify_admins(self) -> None:
+        # TODO: This test is completely busted because the
+        # RealmAuditLog table is empty in it, so it's testing an event
+        # containing an empty list.
         schema_checker = self.check_events_dict([
-            ('type', equals('realm_exported')),
+            ('type', equals('realm_export')),
-            ('public_url', check_string),
+            ('exports', check_list(check_dict_only([
+                ('id', check_string),
+                ('event_time', check_string),
+                ('acting_user_id', check_int),
+                ('path', check_string),
+            ])))
         ])
 
         # Traditionally, we'd be testing the endpoint, but that
         # requires somewhat annoying mocking setup for what to do with
         # the export tarball.
         events = self.do_test(
-            lambda: notify_export_completed(self.user_profile,
+            lambda: notify_export_completed(self.user_profile),
-                                            "http://localhost:9991/path/to/export.tar.gz"),
+            state_change_expected=False, num_events=1)
-            state_change_expected=False)
         error = schema_checker('events[0]', events[0])
         self.assert_on_error(error)
 

zerver/tests/test_realm_export.py

@@ -8,7 +8,7 @@ from zerver.lib.exceptions import JsonableError
 from zerver.lib.test_helpers import use_s3_backend, create_s3_buckets
 
 from zerver.models import RealmAuditLog
-from zerver.views.public_export import public_only_realm_export
+from zerver.views.realm_export import export_realm
 import zerver.lib.upload
 
 import os
@@ -24,7 +24,7 @@ class RealmExportTest(ZulipTestCase):
         user = self.example_user('hamlet')
         self.login(user.email)
         with self.assertRaises(JsonableError):
-            public_only_realm_export(self.client_post, user)
+            export_realm(self.client_post, user)
 
     @use_s3_backend
     def test_endpoint_s3(self) -> None:
@@ -57,7 +57,7 @@ class RealmExportTest(ZulipTestCase):
         result = self.client_get('/json/export/realm')
         self.assert_json_success(result)
 
-        export_dict = result.json()['public_exports']
+        export_dict = result.json()['exports']
         self.assertEqual(export_dict[0]['path'], path_id)
         self.assertEqual(export_dict[0]['acting_user_id'], admin.id)
         self.assert_length(export_dict,
@@ -98,7 +98,7 @@ class RealmExportTest(ZulipTestCase):
         result = self.client_get('/json/export/realm')
         self.assert_json_success(result)
 
-        export_dict = result.json()['public_exports']
+        export_dict = result.json()['exports']
         self.assertEqual(export_dict[0]['path'], path_id)
         self.assertEqual(export_dict[0]['acting_user_id'], admin.id)
         self.assert_length(export_dict,
@@ -126,5 +126,5 @@ class RealmExportTest(ZulipTestCase):
                                          event_time=timezone_now()))
         RealmAuditLog.objects.bulk_create(exports)
 
-        result = public_only_realm_export(self.client_post, admin)
+        result = export_realm(self.client_post, admin)
         self.assert_json_error(result, 'Exceeded rate limit.')

zerver/views/realm_export.py

@@ -8,10 +8,11 @@ from zerver.decorator import require_realm_admin
 from zerver.models import RealmAuditLog, UserProfile
 from zerver.lib.queue import queue_json_publish
 from zerver.lib.response import json_error, json_success
-from zerver.lib.export import get_public_exports_serialized
+from zerver.lib.export import get_realm_exports_serialized
 
 @require_realm_admin
-def public_only_realm_export(request: HttpRequest, user: UserProfile) -> HttpResponse:
+def export_realm(request: HttpRequest, user: UserProfile) -> HttpResponse:
+    # Currently only supports public-data-only exports.
     event_type = RealmAuditLog.REALM_EXPORTED
     event_time = timezone_now()
     realm = user.realm
@@ -32,7 +33,7 @@ def public_only_realm_export(request: HttpRequest, user: UserProfile) -> HttpRes
                                        acting_user=user)
     # Using the deferred_work queue processor to avoid
     # killing the process after 60s
-    event = {'type': event_type,
+    event = {'type': "realm_export",
              'time': event_time,
              'realm_id': realm.id,
              'user_profile_id': user.id,
@@ -41,6 +42,6 @@ def public_only_realm_export(request: HttpRequest, user: UserProfile) -> HttpRes
     return json_success()
 
 @require_realm_admin
-def get_public_exports(request: HttpRequest, user: UserProfile) -> HttpResponse:
+def get_realm_exports(request: HttpRequest, user: UserProfile) -> HttpResponse:
-    public_exports = get_public_exports_serialized(user)
+    realm_exports = get_realm_exports_serialized(user)
-    return json_success({"public_exports": public_exports})
+    return json_success({"exports": realm_exports})

zerver/worker/queue_processors.py

@@ -610,7 +610,7 @@ class DeferredWorker(QueueProcessingWorker):
                 (stream, recipient, sub) = access_stream_by_id(user_profile, stream_id,
                                                                require_active=False)
                 do_mark_stream_messages_as_read(user_profile, client, stream)
-        elif event['type'] == 'realm_exported':
+        elif event['type'] == 'realm_export':
             realm = Realm.objects.get(id=event['realm_id'])
             output_dir = tempfile.mkdtemp(prefix="zulip-export-")
 
@@ -637,6 +637,5 @@ class DeferredWorker(QueueProcessingWorker):
             )
 
             # For future frontend use, also notify administrator
-            # clients that the export happened, including sending the
+            # clients that the export happened.
-            # url.
+            notify_export_completed(user_profile)
-            notify_export_completed(user_profile, public_url)

zproject/urls.py

@@ -36,7 +36,7 @@ import zerver.views.realm
 import zerver.views.digest
 import zerver.views.messages
 from zerver.context_processors import latest_info_context
-import zerver.views.public_export
+import zerver.views.realm_export
 
 from zerver.lib.rest import rest_dispatch
 
@@ -398,10 +398,10 @@ v1_api_and_json_patterns = [
     url(r'^calls/create$', rest_dispatch,
         {'GET': 'zerver.views.video_calls.get_zoom_url'}),
 
-    # Used for public-only realm exporting
+    # Used realm data exporting
     url(r'^export/realm$', rest_dispatch,
-        {'POST': 'zerver.views.public_export.public_only_realm_export',
+        {'POST': 'zerver.views.realm_export.export_realm',
-         'GET': 'zerver.views.public_export.get_public_exports'}),
+         'GET': 'zerver.views.realm_export.get_realm_exports'}),
 ]
 
 # These views serve pages (HTML). As such, their internationalization