python: Remove now-unnecessary str_utils library.

This library was absolutely essential as part of our Python 2->3
migration process, but all of its calls should be either no-ops or
encode/decode operations.

Note also that the library has been wrong since the incorrect
refactoring in 1f9244e060.

Fixes #10807.

scripts/setup/generate_secrets.py

@@ -16,7 +16,6 @@ from django.utils.crypto import get_random_string
 import argparse
 import uuid
 import configparser
-from zerver.lib.str_utils import force_str
 from zerver.lib.utils import generate_random_token
 
 os.chdir(os.path.join(os.path.dirname(__file__), '..', '..'))
@@ -111,7 +110,7 @@ def generate_secrets(development=False):
     out = open(OUTPUT_SETTINGS_FILENAME, 'a')
     # Write a newline at the start, in case there was no newline at
     # the end of the file due to human editing.
-    out.write("\n" + force_str("".join(lines)))
+    out.write("\n" + "".join(lines))
     out.close()
 
     print("Generated new secrets in %s." % (OUTPUT_SETTINGS_FILENAME,))

tools/test-backend

@@ -84,7 +84,6 @@ not_yet_fully_covered = {
     'zerver/lib/rate_limiter.py',
     'zerver/lib/sqlalchemy_utils.py',
     'zerver/lib/storage.py',
-    'zerver/lib/str_utils.py',
     'zerver/lib/stream_recipient.py',
     'zerver/lib/timeout.py',
     'zerver/lib/unminify.py',

zerver/lib/email_mirror.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Union, cast
 
 import logging
 import re
@@ -17,7 +17,6 @@ from zerver.lib.queue import queue_json_publish
 from zerver.lib.redis_utils import get_redis_client
 from zerver.lib.upload import upload_message_file
 from zerver.lib.utils import generate_random_token
-from zerver.lib.str_utils import force_text
 from zerver.lib.send_email import FromAddress
 from zerver.models import Stream, Recipient, \
     get_user_profile_by_id, get_display_recipient, get_personal_recipient, \
@@ -324,7 +323,7 @@ def process_message(message: message.Message, rcpt_to: Optional[str]=None, pre_c
         subject_header = "(no topic)"
     encoded_subject, encoding = decode_header(subject_header)[0]
     if encoding is None:
-        subject = force_text(encoded_subject)  # encoded_subject has type str when encoding is None
+        subject = cast(str, encoded_subject)  # encoded_subject has type str when encoding is None
     else:
         try:
             subject = encoded_subject.decode(encoding)

zerver/lib/storage.py

@@ -8,8 +8,6 @@ from django.conf import settings
 from django.contrib.staticfiles.storage import ManifestStaticFilesStorage
 from pipeline.storage import PipelineMixin
 
-from zerver.lib.str_utils import force_str
-
 class AddHeaderMixin:
     def post_process(self, paths: Dict[str, Tuple['ZulipStorage', str]], dry_run: bool=False,
                      **kwargs: Any) -> List[Tuple[str, str, bool]]:
@@ -38,7 +36,7 @@ class AddHeaderMixin:
             storage.delete(path)
 
             with storage.open(path, 'w') as new_file:
-                new_file.write(force_str(header + orig_contents, encoding=settings.FILE_CHARSET))
+                new_file.write(header + orig_contents)
 
             ret_dict[path] = (path, path, True)
 

zerver/lib/str_utils.py

@@ -1,52 +0,0 @@
-"""
-String Utilities:
-
-This module helps in converting strings from one type to another.
-
-Currently we have strings of 3 semantic types:
-
-1.  text strings: These strings are used to represent all textual data,
-    like people's names, stream names, content of messages, etc.
-    These strings can contain non-ASCII characters, so its type should be
-    typing.str (which is `str` in python 3 and `unicode` in python 2).
-
-2.  binary strings: These strings are used to represent binary data.
-    This should be of type `bytes`
-
-3.  native strings: These strings are for internal use only.  Strings of
-    this type are not meant to be stored in database, displayed to end
-    users, etc.  Things like exception names, parameter names, attribute
-    names, etc should be native strings.  These strings should only
-    contain ASCII characters and they should have type `str`.
-
-There are 3 utility functions provided for converting strings from one type
-to another - force_text, force_bytes, force_str
-
-Interconversion between text strings and binary strings can be done by
-using encode and decode appropriately or by using the utility functions
-force_text and force_bytes.
-
-It is recommended to use the utility functions for other string conversions.
-"""
-
-from typing import Any, Dict, Mapping, Union, TypeVar
-
-def force_text(s: Union[str, bytes], encoding: str='utf-8') -> str:
-    """converts a string to a text string"""
-    if isinstance(s, str):
-        return s
-    elif isinstance(s, bytes):
-        return s.decode(encoding)
-    else:
-        raise TypeError("force_text expects a string type")
-
-def force_str(s: Union[str, bytes], encoding: str='utf-8') -> str:
-    """converts a string to a native string"""
-    if isinstance(s, str):
-        return s
-    elif isinstance(s, str):
-        return s.encode(encoding)
-    elif isinstance(s, bytes):
-        return s.decode(encoding)
-    else:
-        raise TypeError("force_str expects a string type")

zerver/lib/zephyr.py

@@ -2,8 +2,6 @@ import re
 import traceback
 import DNS
 
-from zerver.lib.str_utils import force_str
-
 def compute_mit_user_fullname(email: str) -> str:
     try:
         # Input is either e.g. username@mit.edu or user|CROSSREALM.INVALID@mit.edu
@@ -12,7 +10,7 @@ def compute_mit_user_fullname(email: str) -> str:
             answer = DNS.dnslookup(
                 "%s.passwd.ns.athena.mit.edu" % (match_user.group(1),),
                 DNS.Type.TXT)
-            hesiod_name = force_str(answer[0][0]).split(':')[4].split(',')[0].strip()
+            hesiod_name = answer[0][0].split(':')[4].split(',')[0].strip()
             if hesiod_name != "":
                 return hesiod_name
         elif match_user:

zerver/management/commands/create_stream.py

@@ -5,7 +5,6 @@ from typing import Any
 
 from zerver.lib.actions import create_stream_if_needed
 from zerver.lib.management import ZulipBaseCommand
-from zerver.lib.str_utils import force_text
 
 class Command(ZulipBaseCommand):
     help = """Create a stream, and subscribe all active users (excluding bots).
@@ -22,6 +21,5 @@ the command."""
         realm = self.get_realm(options)
         assert realm is not None  # Should be ensured by parser
 
-        encoding = sys.getfilesystemencoding()
         stream_name = options['stream_name']
-        create_stream_if_needed(realm, force_text(stream_name, encoding))
+        create_stream_if_needed(realm, stream_name)

zerver/management/commands/makemessages.py

@@ -41,8 +41,6 @@ from django.core.management.commands import makemessages
 from django.template.base import BLOCK_TAG_END, BLOCK_TAG_START
 from django.utils.translation import template
 
-from zerver.lib.str_utils import force_text
-
 strip_whitespace_right = re.compile("(%s-?\\s*(trans|pluralize).*?-%s)\\s+" % (
                                     BLOCK_TAG_START, BLOCK_TAG_END), re.U)
 strip_whitespace_left = re.compile("\\s+(%s-\\s*(endtrans|pluralize).*?-?%s)" % (
@@ -249,7 +247,7 @@ class Command(makemessages.Command):
                 old_strings = {}
 
             new_strings = {
-                force_text(k): v
+                k: v
                 for k, v in self.get_new_strings(old_strings,
                                                  translation_strings,
                                                  locale).items()

zerver/management/commands/rename_stream.py

@@ -5,7 +5,6 @@ from typing import Any
 
 from zerver.lib.actions import do_rename_stream
 from zerver.lib.management import ZulipBaseCommand
-from zerver.lib.str_utils import force_text
 from zerver.models import get_stream
 
 class Command(ZulipBaseCommand):
@@ -23,7 +22,6 @@ class Command(ZulipBaseCommand):
         assert realm is not None  # Should be ensured by parser
         old_name = options['old_name']
         new_name = options['new_name']
-        encoding = sys.getfilesystemencoding()
 
-        stream = get_stream(force_text(old_name, encoding), realm)
+        stream = get_stream(old_name, realm)
-        do_rename_stream(stream, force_text(new_name, encoding))
+        do_rename_stream(stream, new_name)

zerver/views/zephyr.py

@@ -7,7 +7,6 @@ from zerver.decorator import authenticated_json_view
 from zerver.lib.ccache import make_ccache
 from zerver.lib.request import has_request_variables, REQ, JsonableError
 from zerver.lib.response import json_success, json_error
-from zerver.lib.str_utils import force_str
 from zerver.lib.users import get_api_key
 from zerver.models import UserProfile
 
@@ -48,9 +47,9 @@ def webathena_kerberos_login(request: HttpRequest, user_profile: UserProfile,
         api_key = get_api_key(user_profile)
         subprocess.check_call(["ssh", settings.PERSONAL_ZMIRROR_SERVER, "--",
                                "/home/zulip/python-zulip-api/zulip/integrations/zephyr/process_ccache",
-                               force_str(user),
+                               user,
-                               force_str(api_key),
+                               api_key,
-                               force_str(base64.b64encode(ccache))])
+                               base64.b64encode(ccache).decode("utf-8")])
     except Exception:
         logging.exception("Error updating the user's ccache")
         return json_error(_("We were unable to setup mirroring for you"))

zerver/worker/queue_processors.py

@@ -40,7 +40,6 @@ from zerver.tornado.socket import req_redis_key, respond_send_message
 from confirmation.models import Confirmation, create_confirmation_link
 from zerver.lib.db import reset_queries
 from zerver.lib.redis_utils import get_redis_client
-from zerver.lib.str_utils import force_str
 from zerver.context_processors import common_context
 from zerver.lib.outgoing_webhook import do_rest_call, get_outgoing_webhook_service_handler
 from zerver.models import get_bot_services
@@ -479,8 +478,7 @@ class DigestWorker(QueueProcessingWorker):  # nocoverage
 @assign_queue('email_mirror')
 class MirrorWorker(QueueProcessingWorker):
     def consume(self, event: Mapping[str, Any]) -> None:
-        message = force_str(event["message"])
+        mirror_email(email.message_from_string(event["message"]),
-        mirror_email(email.message_from_string(message),
                      rcpt_to=event["rcpt_to"], pre_checked=True)
 
 @assign_queue('test', queue_type="test")