diff --git a/static/js/settings_exports.js b/static/js/settings_exports.js index 42d7213197..79e687bd70 100644 --- a/static/js/settings_exports.js +++ b/static/js/settings_exports.js @@ -38,6 +38,8 @@ exports.populate_exports_table = function (exports) { new XDate(data.export_time * 1000) ), url: data.export_url, + failed: data.failed_timestamp, + pending: data.pending, }, }); } diff --git a/static/templates/admin_export_list.hbs b/static/templates/admin_export_list.hbs index 4986589bd6..d6baf45bc0 100644 --- a/static/templates/admin_export_list.hbs +++ b/static/templates/admin_export_list.hbs @@ -6,11 +6,18 @@ {{event_time}} + + {{#if url}} + {{t 'Complete' }} + {{else if failed}} + {{t 'Failed' }} + {{else if pending}} + {{t 'Pending' }} + {{/if}} + {{#if url}} {{t 'Download' }} - {{else}} - {{t 'The export URL is not yet available... Check back soon.' }} {{/if}} diff --git a/static/templates/settings/data_exports_admin.hbs b/static/templates/settings/data_exports_admin.hbs index 110a2478d2..64e88a42a8 100644 --- a/static/templates/settings/data_exports_admin.hbs +++ b/static/templates/settings/data_exports_admin.hbs @@ -35,6 +35,7 @@ {{t "Requesting user" }} {{t "Time" }} + {{t "Status" }} {{t "File" }} {{t "Actions" }} diff --git a/zerver/lib/actions.py b/zerver/lib/actions.py index 846e4333d2..d31a8b984c 100644 --- a/zerver/lib/actions.py +++ b/zerver/lib/actions.py @@ -5798,8 +5798,12 @@ def do_delete_realm_export(user_profile: UserProfile, export: RealmAuditLog) -> export_extra_data = export.extra_data assert export_extra_data is not None export_data = ujson.loads(export_extra_data) + export_path = export_data.get('export_path') + + if export_path: + # Allow removal even if the export failed. + delete_export_tarball(export_path) - delete_export_tarball(export_data.get('export_path')) export_data.update({'deleted_timestamp': timezone_now().timestamp()}) export.extra_data = ujson.dumps(export_data) export.save(update_fields=['extra_data']) diff --git a/zerver/lib/export.py b/zerver/lib/export.py index 60e037c740..90908924bf 100644 --- a/zerver/lib/export.py +++ b/zerver/lib/export.py @@ -1763,14 +1763,30 @@ def get_realm_exports_serialized(user: UserProfile) -> List[Dict[str, Any]]: event_type=RealmAuditLog.REALM_EXPORTED) exports_dict = {} for export in all_exports: - export_data = ujson.loads(export.extra_data) - export_url = zerver.lib.upload.upload_backend.get_export_tarball_url( - user.realm, export_data['export_path']) + pending = True + export_url = None + deleted_timestamp = None + failed_timestamp = None + + if export.extra_data is not None: + pending = False + + export_data = ujson.loads(export.extra_data) + deleted_timestamp = export_data.get('deleted_timestamp') + failed_timestamp = export_data.get('failed_timestamp') + export_path = export_data.get('export_path') + + if export_path: + export_url = zerver.lib.upload.upload_backend.get_export_tarball_url( + user.realm, export_path) + exports_dict[export.id] = dict( id=export.id, export_time=export.event_time.timestamp(), acting_user_id=export.acting_user.id, export_url=export_url, - deleted_timestamp=export_data.get('deleted_timestamp'), + deleted_timestamp=deleted_timestamp, + failed_timestamp=failed_timestamp, + pending=pending, ) return sorted(exports_dict.values(), key=lambda export_dict: export_dict['id']) diff --git a/zerver/tests/test_events.py b/zerver/tests/test_events.py index f059616213..bca1e3b527 100644 --- a/zerver/tests/test_events.py +++ b/zerver/tests/test_events.py @@ -2813,6 +2813,19 @@ class EventsRegisterTest(ZulipTestCase): self.assert_on_error(error) def test_notify_realm_export(self) -> None: + pending_schema_checker = self.check_events_dict([ + ('type', equals('realm_export')), + ('exports', check_list(check_dict_only([ + ('id', check_int), + ('export_time', check_float), + ('acting_user_id', check_int), + ('export_url', equals(None)), + ('deleted_timestamp', equals(None)), + ('failed_timestamp', equals(None)), + ('pending', check_bool), + ]))), + ]) + schema_checker = self.check_events_dict([ ('type', equals('realm_export')), ('exports', check_list(check_dict_only([ @@ -2821,6 +2834,8 @@ class EventsRegisterTest(ZulipTestCase): ('acting_user_id', check_int), ('export_url', check_string), ('deleted_timestamp', equals(None)), + ('failed_timestamp', equals(None)), + ('pending', check_bool), ]))), ]) @@ -2832,10 +2847,14 @@ class EventsRegisterTest(ZulipTestCase): with stdout_suppressed(): events = self.do_test( lambda: self.client_post('/json/export/realm'), - state_change_expected=True, num_events=2) + state_change_expected=True, num_events=3) - # The first event is a message from notification-bot. - error = schema_checker('events[1]', events[1]) + # We first notify when an export is initiated, + error = pending_schema_checker('events[0]', events[0]) + self.assert_on_error(error) + + # The second event is then a message from notification-bot. + error = schema_checker('events[2]', events[2]) self.assert_on_error(error) # Now we check the deletion of the export. @@ -2847,6 +2866,8 @@ class EventsRegisterTest(ZulipTestCase): ('acting_user_id', check_int), ('export_url', check_string), ('deleted_timestamp', check_float), + ('failed_timestamp', equals(None)), + ('pending', check_bool), ]))), ]) @@ -2858,6 +2879,49 @@ class EventsRegisterTest(ZulipTestCase): error = deletion_schema_checker('events[0]', events[0]) self.assert_on_error(error) + def test_notify_realm_export_on_failure(self) -> None: + pending_schema_checker = self.check_events_dict([ + ('type', equals('realm_export')), + ('exports', check_list(check_dict_only([ + ('id', check_int), + ('export_time', check_float), + ('acting_user_id', check_int), + ('export_url', equals(None)), + ('deleted_timestamp', equals(None)), + ('failed_timestamp', equals(None)), + ('pending', check_bool), + ]))), + ]) + + failed_schema_checker = self.check_events_dict([ + ('type', equals('realm_export')), + ('exports', check_list(check_dict_only([ + ('id', check_int), + ('export_time', check_float), + ('acting_user_id', check_int), + ('export_url', equals(None)), + ('deleted_timestamp', equals(None)), + ('failed_timestamp', check_float), + ('pending', check_bool), + ]))), + ]) + + do_change_is_admin(self.user_profile, True) + self.login_user(self.user_profile) + + with mock.patch('zerver.lib.export.do_export_realm', + side_effect=Exception("test")): + with stdout_suppressed(): + events = self.do_test( + lambda: self.client_post('/json/export/realm'), + state_change_expected=False, num_events=2) + + error = pending_schema_checker('events[0]', events[0]) + self.assert_on_error(error) + + error = failed_schema_checker('events[1]', events[1]) + self.assert_on_error(error) + class FetchInitialStateDataTest(ZulipTestCase): # Non-admin users don't have access to all bots def test_realm_bots_non_admin(self) -> None: diff --git a/zerver/views/realm_export.py b/zerver/views/realm_export.py index 5f3f67e18a..4b7b777fc8 100644 --- a/zerver/views/realm_export.py +++ b/zerver/views/realm_export.py @@ -12,7 +12,7 @@ from zerver.models import RealmAuditLog, UserProfile from zerver.lib.queue import queue_json_publish from zerver.lib.response import json_error, json_success from zerver.lib.export import get_realm_exports_serialized -from zerver.lib.actions import do_delete_realm_export +from zerver.lib.actions import do_delete_realm_export, notify_realm_export import ujson @@ -51,6 +51,10 @@ def export_realm(request: HttpRequest, user: UserProfile) -> HttpResponse: event_type=event_type, event_time=event_time, acting_user=user) + + # Allow for UI updates on a pending export + notify_realm_export(user) + # Using the deferred_work queue processor to avoid # killing the process after 60s event = {'type': "realm_export", diff --git a/zerver/worker/queue_processors.py b/zerver/worker/queue_processors.py index a8ad186f67..0022c2d381 100644 --- a/zerver/worker/queue_processors.py +++ b/zerver/worker/queue_processors.py @@ -13,6 +13,7 @@ import socket from django.conf import settings from django.db import connection +from django.utils.timezone import now as timezone_now from zerver.models import \ get_client, get_system_bot, PreregistrationUser, \ get_user_profile_by_id, Message, Realm, UserMessage, UserProfile, \ @@ -680,14 +681,26 @@ class DeferredWorker(QueueProcessingWorker): start = time.time() realm = Realm.objects.get(id=event['realm_id']) output_dir = tempfile.mkdtemp(prefix="zulip-export-") + export_event = RealmAuditLog.objects.get(id=event['id']) + user_profile = get_user_profile_by_id(event['user_profile_id']) + + try: + public_url = export_realm_wrapper(realm=realm, output_dir=output_dir, + threads=6, upload=True, public_only=True, + delete_after_upload=True) + except Exception: + export_event.extra_data = ujson.dumps(dict( + failed_timestamp=timezone_now().timestamp() + )) + export_event.save(update_fields=['extra_data']) + logging.error("Data export for %s failed after %s" % ( + user_profile.realm.string_id, time.time() - start)) + notify_realm_export(user_profile) + return - public_url = export_realm_wrapper(realm=realm, output_dir=output_dir, - threads=6, upload=True, public_only=True, - delete_after_upload=True) assert public_url is not None # Update the extra_data field now that the export is complete. - export_event = RealmAuditLog.objects.get(id=event['id']) export_event.extra_data = ujson.dumps(dict( export_path=urllib.parse.urlparse(public_url).path, )) @@ -695,7 +708,6 @@ class DeferredWorker(QueueProcessingWorker): # Send a private message notification letting the user who # triggered the export know the export finished. - user_profile = get_user_profile_by_id(event['user_profile_id']) content = "Your data export is complete and has been uploaded here:\n\n%s" % ( public_url,) internal_send_private_message(