migrations: Fix possible 0257_fix_has_link_attribute.py failure.

While it should be an invariant that message.rendered_content is never
None for a row saved to the database, it is possible for that
invariant to be violated, likely including due to bugs in previous
versions of data import/export tools.

While it'd be ideal for such messages to be rendered to fix the
invariant, it doesn't make sense for this has_link migration to crash
because of such a corrupted row, so we apply the similar policy we
already have for rendered_content="".

docs/overview/changelog.md

@@ -1,11 +1,9 @@
 # Version History
 
-All notable changes to the Zulip server are documented in this file.
-
-## Unreleased
-
-This section lists notable unreleased changes; it is generally updated
-in bursts.
+This page contains the release history for the Zulip 3.x stable
+release series. See the [current Zulip changelog][latest-changelog]
+for newer release series, or the [commit log][commit-log] for an
+up-to-date list of raw changes.
 
 ### 3.4 -- April 14, 2021
 
@@ -1910,3 +1908,7 @@ easily read them all when upgrading across multiple releases.
 * [Upgrade notes for 1.9.0](#upgrade-notes-for-1-9-0)
 * [Upgrade notes for 1.8.0](#upgrade-notes-for-1-8-0)
 * [Upgrade notes for 1.7.0](#upgrade-notes-for-1-7-0)
+
+[docker-zulip]: https://github.com/zulip/docker-zulip
+[commit-log]: https://github.com/zulip/zulip/commits/master
+[latest-changelog]: https://zulip.readthedocs.io/en/latest/overview/changelog.html

docs/production/install.md

@@ -3,7 +3,7 @@
 You'll need an Ubuntu or Debian system that satisfies
 [the installation requirements](../production/requirements.md). Alternatively,
 you can use a preconfigured
-[Digital Ocean droplet](https://marketplace.digitalocean.com/apps/zulip), or
+[DigitalOcean droplet](https://marketplace.digitalocean.com/apps/zulip?refcode=3ee45da8ee26), or
 Zulip's
 [experimental Docker image](../production/deployment.html#zulip-in-docker).
 

version.py

@@ -1,6 +1,6 @@
 import os
 
-ZULIP_VERSION = "3.4"
+ZULIP_VERSION = "3.4+git"
 # Add information on number of commits and commit hash to version, if available
 zulip_git_version_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'zulip-git-version')
 if os.path.exists(zulip_git_version_file):

zerver/data_import/slack.py

@@ -1055,7 +1055,7 @@ def do_convert_data(slack_zip_file: str, output_dir: str, token: str, threads: i
     realm_id = 0
     domain_name = settings.EXTERNAL_HOST
 
-    log_token_warning(token)
+    check_token_access(token)
 
     slack_data_dir = slack_zip_file.replace('.zip', '')
     if not os.path.exists(slack_data_dir):
@@ -1124,17 +1124,29 @@ def get_data_file(path: str) -> Any:
         data = ujson.load(fp)
         return data
 
-def log_token_warning(token: str) -> None:
-    if not token.startswith("xoxp-"):
-        logging.info('Not a Slack legacy token.\n'
-                     '  This token might not have all the needed scopes. We need the following scopes:\n'
-                     '  - emoji:read\n  - users:read\n  - users:read.email\n  - team:read')
+def check_token_access(token: str) -> None:
+    if token.startswith("xoxp-"):
+        logging.info('This is a Slack user token, which grants all rights the user has!')
+    elif token.startswith("xoxb-"):
+        data = requests.get(
+            "https://slack.com/api/team.info", headers={"Authorization": "Bearer {}".format(token)}
+        )
+        has_scopes = set(data.headers.get("x-oauth-scopes", "").split(","))
+        required_scopes = set(['emoji:read', 'users:read', 'users:read.email', 'team:read'])
+        missing_scopes = required_scopes - has_scopes
+        if missing_scopes:
+            raise ValueError("Slack token is missing the following required scopes: {}".format(sorted(missing_scopes)))
+    else:
+        raise Exception("Unknown token type -- must start with xoxb- or xoxp-")
 
 
 def get_slack_api_data(slack_api_url: str, get_param: str, **kwargs: Any) -> Any:
     if not kwargs.get("token"):
         raise AssertionError("Slack token missing in kwargs")
-    data = requests.get(f"{slack_api_url}?{urlencode(kwargs)}")
+    token = kwargs.pop("token")
+    data = requests.get(
+        slack_api_url, headers={"Authorization": "Bearer {}".format(token)}, **kwargs
+    )
 
     if data.status_code == requests.codes.ok:
         result = data.json()

zerver/management/commands/sync_ldap_user_data.py

@@ -39,8 +39,8 @@ def sync_ldap_user_data(user_profiles: List[UserProfile], deactivation_protectio
                 raise Exception(error_msg)
             for string_id in realms:
                 if not UserProfile.objects.filter(is_bot=False, is_active=True, realm__string_id=string_id,
-                                                  role__gte=UserProfile.ROLE_REALM_ADMINISTRATOR).exists():
-                    error_msg = ("Ldap sync would have deactivated all administrators of realm %s. " +
+                                                  role=UserProfile.ROLE_REALM_OWNER).exists():
+                    error_msg = ("Ldap sync would have deactivated all owners of realm %s. " +
                                  "This is most likely due " +
                                  "to a misconfiguration of ldap settings. Rolling back...\n" +
                                  "Use the --force option if the mass deactivation is intended.")

zerver/migrations/0257_fix_has_link_attribute.py

@@ -12,9 +12,9 @@ BATCH_SIZE = 1000
 def process_batch(apps: StateApps, id_start: int, id_end: int, last_id: int) -> None:
     Message = apps.get_model('zerver', 'Message')
     for message in Message.objects.filter(id__gte=id_start, id__lte=id_end).order_by("id"):
-        if message.rendered_content == "":
+        if message.rendered_content in ["", None]:
             # There have been bugs in the past that made it possible
-            # for a message to have "" as its rendered_content; we
+            # for a message to have "" or None as its rendered_content; we
             # need to skip those because lxml won't process them.
             #
             # They should safely already have the correct state

zerver/tests/test_slack_importer.py

@@ -58,14 +58,16 @@ class MockResponse:
         return self.json_data
 
 # This method will be used by the mock to replace requests.get
-def mocked_requests_get(*args: List[str], **kwargs: List[str]) -> MockResponse:
-    if args[0] == 'https://slack.com/api/users.list?token=xoxp-valid-token':
-        return MockResponse({"ok": True, "members": "user_data"}, 200)
-    elif args[0] == 'https://slack.com/api/users.list?token=xoxp-invalid-token':
-        return MockResponse({"ok": False, "error": "invalid_auth"}, 200)
-    else:
+def mocked_requests_get(*args: str, headers: Dict[str, str]) -> MockResponse:
+    if args != ("https://slack.com/api/users.list",):
         return MockResponse(None, 404)
 
+    if (headers.get("Authorization") != "Bearer xoxp-valid-token"):
+        return MockResponse({"ok": False, "error": "invalid_auth"}, 200)
+
+    return MockResponse({"ok": True, "members": "user_data"}, 200)
+
+
 class SlackImporter(ZulipTestCase):
     logger = logging.getLogger()
     # set logger to a higher level to suppress 'logger.INFO' outputs
@@ -754,7 +756,7 @@ class SlackImporter(ZulipTestCase):
 
         test_realm_subdomain = 'test-slack-import'
         output_dir = os.path.join(settings.DEPLOY_ROOT, "var", "test-slack-importer-data")
-        token = 'valid-token'
+        token = 'xoxp-valid-token'
 
         # If the test fails, the 'output_dir' would not be deleted and hence it would give an
         # error when we run the tests next time, as 'do_convert_data' expects an empty 'output_dir'