Switch all urllib/urlparse usage to six.moves.urllib.

This provides Python 2+3 compatibility for our use of urllib.

Also add a test to avoid future regressions.

api/integrations/asana/zulip_asana_mirror

@@ -37,7 +37,7 @@ import json
 import logging
 import os
 import time
-import urllib2
+from six.moves import urllib
 
 import sys
 
@@ -74,8 +74,8 @@ def fetch_from_asana(path):
     headers = {"Authorization": "Basic %s" % auth}
 
     url = "https://app.asana.com/api/1.0" + path
-    request = urllib2.Request(url, None, headers)
-    result = urllib2.urlopen(request)
+    request = urllib.request.Request(url, None, headers)
+    result = urllib.request.urlopen(request)
 
     return json.load(result)
 

api/integrations/rss/rss-bot

@@ -32,7 +32,7 @@ import optparse
 import os
 import sys
 import time
-import urlparse
+from six.moves import urllib
 
 import feedparser
 import zulip
@@ -169,7 +169,7 @@ client = zulip.Client(email=opts.email, api_key=opts.api_key,
 first_message = True
 
 for feed_url in feed_urls:
-    feed_file = os.path.join(opts.data_dir, urlparse.urlparse(feed_url).netloc)
+    feed_file = os.path.join(opts.data_dir, urllib.parse.urlparse(feed_url).netloc)
 
     try:
         with open(feed_file, "r") as f:

api/zulip/__init__.py

@@ -26,16 +26,15 @@ import simplejson
 import requests
 import time
 import traceback
-import urlparse
 import sys
 import os
 import optparse
 import platform
-import urllib
 import random
 from distutils.version import LooseVersion
 
 from six.moves.configparser import SafeConfigParser
+from six.moves import urllib
 import logging
 import six
 
@@ -289,7 +288,7 @@ class Client(object):
                 kwargs = {kwarg: query_state["request"]}
                 res = requests.request(
                         method,
-                        urlparse.urljoin(self.base_url, url),
+                        urllib.parse.urljoin(self.base_url, url),
                         auth=requests.auth.HTTPBasicAuth(self.email,
                                                          self.api_key),
                         verify=self.tls_verification, timeout=90,
@@ -468,7 +467,7 @@ Client._register('list_subscriptions', method='GET', url='users/me/subscriptions
 Client._register('add_subscriptions', url='users/me/subscriptions', make_request=_mk_subs)
 Client._register('remove_subscriptions', method='PATCH', url='users/me/subscriptions', make_request=_mk_rm_subs)
 Client._register('get_subscribers', method='GET',
-                 computed_url=lambda request: 'streams/%s/members' % (urllib.quote(request['stream'], safe=''),),
+                 computed_url=lambda request: 'streams/%s/members' % (urllib.parse.quote(request['stream'], safe=''),),
                  make_request=_kwargs_to_dict)
 Client._register('render_message', method='GET', url='messages/render')
 Client._register('create_user', method='POST', url='users')

bots/gcal-bot

@@ -3,7 +3,7 @@ import sys
 import time
 import datetime
 import optparse
-import urlparse
+from six.moves import urllib
 import itertools
 import traceback
 import os
@@ -56,13 +56,13 @@ except ImportError:
     parser.error('Install python-gdata')
 
 def get_calendar_url():
-    parts = urlparse.urlparse(options.calendar)
+    parts = urllib.parse.urlparse(options.calendar)
     pat = os.path.split(parts.path)
     if pat[1] != 'basic':
         parser.error('The --calendar URL should be the XML "Private Address" ' +
                      'from your calendar settings')
-    return urlparse.urlunparse((parts.scheme, parts.netloc, pat[0] + '/full',
-                                '', 'futureevents=true&orderby=startdate', ''))
+    return urllib.parse.urlunparse((parts.scheme, parts.netloc, pat[0] + '/full',
+                                   '', 'futureevents=true&orderby=startdate', ''))
 
 calendar_url = get_calendar_url()
 

tools/travis/py3k

@@ -24,6 +24,7 @@ lib2to3.fixes.fix_types
 lib2to3.fixes.fix_ws_comma
 lib2to3.fixes.fix_xreadlines
 libfuturize.fixes.fix_absolute_import
+libfuturize.fixes.fix_future_standard_library_urllib
 libfuturize.fixes.fix_next_call
 libfuturize.fixes.fix_print_with_import
 libfuturize.fixes.fix_raise

zerver/lib/bugdown/__init__.py

@@ -3,7 +3,7 @@ from __future__ import absolute_import
 import markdown
 import logging
 import traceback
-import urlparse
+from six.moves import urllib
 import re
 import os.path
 import glob
@@ -13,7 +13,7 @@ import time
 import six.moves.html_parser
 import httplib2
 import itertools
-import urllib
+from six.moves import urllib
 import xml.etree.cElementTree as etree
 
 import hashlib
@@ -220,7 +220,7 @@ def fetch_open_graph_image(url):
     return {'image': image, 'title': title, 'desc': desc}
 
 def get_tweet_id(url):
-    parsed_url = urlparse.urlparse(url)
+    parsed_url = urllib.parse.urlparse(url)
     if not (parsed_url.netloc == 'twitter.com' or parsed_url.netloc.endswith('.twitter.com')):
         return False
     to_match = parsed_url.path
@@ -258,7 +258,7 @@ class InlineInterestingLinkProcessor(markdown.treeprocessors.Treeprocessor):
     def is_image(self, url):
         if not settings.INLINE_IMAGE_PREVIEW:
             return False
-        parsed_url = urlparse.urlparse(url)
+        parsed_url = urllib.parse.urlparse(url)
         # List from http://support.google.com/chromeos/bin/answer.py?hl=en&answer=183093
         for ext in [".bmp", ".gif", ".jpg", "jpeg", ".png", ".webp"]:
             if parsed_url.path.lower().endswith(ext):
@@ -266,7 +266,7 @@ class InlineInterestingLinkProcessor(markdown.treeprocessors.Treeprocessor):
         return False
 
     def dropbox_image(self, url):
-        parsed_url = urlparse.urlparse(url)
+        parsed_url = urllib.parse.urlparse(url)
         if (parsed_url.netloc == 'dropbox.com' or parsed_url.netloc.endswith('.dropbox.com')):
             is_album = parsed_url.path.startswith('/sc/') or parsed_url.path.startswith('/photos/')
             # Only allow preview Dropbox shared links
@@ -309,7 +309,7 @@ class InlineInterestingLinkProcessor(markdown.treeprocessors.Treeprocessor):
             image_info['is_image'] = True
             parsed_url_list = list(parsed_url)
             parsed_url_list[4] = "dl=1" # Replaces query
-            image_info["image"] = urlparse.urlunparse(parsed_url_list)
+            image_info["image"] = urllib.parse.urlunparse(parsed_url_list)
 
             return image_info
         return None
@@ -364,7 +364,7 @@ class InlineInterestingLinkProcessor(markdown.treeprocessors.Treeprocessor):
                 to_linkify.append({
                     'start': match.start(),
                     'end': match.end(),
-                    'url': 'https://twitter.com/' + urllib.quote(screen_name),
+                    'url': 'https://twitter.com/' + urllib.parse.quote(screen_name),
                     'text': mention_string,
                 })
         # Build dicts for media
@@ -625,7 +625,7 @@ def sanitize_url(url):
     See the docstring on markdown.inlinepatterns.LinkPattern.sanitize_url.
     """
     try:
-        parts = urlparse.urlparse(url.replace(' ', '%20'))
+        parts = urllib.parse.urlparse(url.replace(' ', '%20'))
         scheme, netloc, path, params, query, fragment = parts
     except ValueError:
         # Bad url - so bad it couldn't be parsed.
@@ -637,10 +637,10 @@ def sanitize_url(url):
         scheme = 'mailto'
     elif scheme == '' and netloc == '' and len(path) > 0 and path[0] == '/':
         # Allow domain-relative links
-        return urlparse.urlunparse(('', '', path, params, query, fragment))
+        return urllib.parse.urlunparse(('', '', path, params, query, fragment))
     elif (scheme, netloc, path, params, query) == ('', '', '', '', '') and len(fragment) > 0:
         # Allow fragment links
-        return urlparse.urlunparse(('', '', '', '', '', fragment))
+        return urllib.parse.urlunparse(('', '', '', '', '', fragment))
 
     # Zulip modification: If scheme is not specified, assume http://
     # We re-enter sanitize_url because netloc etc. need to be re-parsed.
@@ -663,7 +663,7 @@ def sanitize_url(url):
     # Upstream code scans path, parameters, and query for colon characters
     # because
     #
-    #    some aliases [for javascript:] will appear to urlparse() to have
+    #    some aliases [for javascript:] will appear to urllib.parse to have
     #    no scheme. On top of that relative links (i.e.: "foo/bar.html")
     #    have no scheme.
     #
@@ -671,7 +671,7 @@ def sanitize_url(url):
     # the colon check, which would also forbid a lot of legitimate URLs.
 
     # Url passes all tests. Return url as-is.
-    return urlparse.urlunparse((scheme, netloc, path, params, query, fragment))
+    return urllib.parse.urlunparse((scheme, netloc, path, params, query, fragment))
 
 def url_to_a(url, text = None):
     a = markdown.util.etree.Element('a')

zerver/lib/notifications.py

@@ -13,7 +13,7 @@ import datetime
 import re
 import subprocess
 import ujson
-import urllib
+from six.moves import urllib
 from collections import defaultdict
 
 def unsubscribe_token(user_profile):
@@ -35,7 +35,7 @@ def hashchange_encode(string):
     # Do the same encoding operation as hashchange.encodeHashComponent on the
     # frontend.
     # `safe` has a default value of "/", but we want those encoded, too.
-    return urllib.quote(
+    return urllib.parse.quote(
         string.encode("utf-8"), safe="").replace(".", "%2E").replace("%", ".")
 
 def pm_narrow_url(participants):

zerver/lib/test_helpers.py

@@ -33,7 +33,7 @@ import os
 import re
 import time
 import ujson
-import urllib
+from six.moves import urllib
 
 from contextlib import contextmanager
 import six
@@ -201,13 +201,13 @@ class POSTRequestMock(object):
 class AuthedTestCase(TestCase):
     # Helper because self.client.patch annoying requires you to urlencode
     def client_patch(self, url, info={}, **kwargs):
-        info = urllib.urlencode(info)
+        info = urllib.parse.urlencode(info)
         return self.client.patch(url, info, **kwargs)
     def client_put(self, url, info={}, **kwargs):
-        info = urllib.urlencode(info)
+        info = urllib.parse.urlencode(info)
         return self.client.put(url, info, **kwargs)
     def client_delete(self, url, info={}, **kwargs):
-        info = urllib.urlencode(info)
+        info = urllib.parse.urlencode(info)
         return self.client.delete(url, info, **kwargs)
 
     def login(self, email, password=None):

zerver/management/commands/runtornado.py

@@ -149,10 +149,10 @@ class AsyncDjangoHandler(tornado.web.RequestHandler, base.BaseHandler):
     def get(self):
         from tornado.wsgi import WSGIContainer
         from django.core.handlers.wsgi import WSGIRequest, get_script_name
-        import urllib
+        from six.moves import urllib
 
         environ  = WSGIContainer.environ(self.request)
-        environ['PATH_INFO'] = urllib.unquote(environ['PATH_INFO'])
+        environ['PATH_INFO'] = urllib.parse.unquote(environ['PATH_INFO'])
         request  = WSGIRequest(environ)
         request._tornado_handler     = self
 

zerver/test_external.py

@@ -20,8 +20,7 @@ from zerver.lib.test_runner import slow
 
 import time
 import ujson
-import urllib
-import urllib2
+from six.moves import urllib
 
 from boto.s3.connection import S3Connection
 from boto.s3.key import Key
@@ -68,7 +67,7 @@ class S3Test(AuthedTestCase):
         response = self.client.get(uri)
         redirect_url = response['Location']
 
-        self.assertEquals("zulip!", urllib2.urlopen(redirect_url).read().strip())
+        self.assertEquals("zulip!", urllib.request.urlopen(redirect_url).read().strip())
 
     def test_multiple_upload_failure(self):
         """
@@ -99,7 +98,7 @@ class S3Test(AuthedTestCase):
         conn = S3Connection(settings.S3_KEY, settings.S3_SECRET_KEY)
         for uri in self.test_uris:
             key = Key(conn.get_bucket(settings.S3_BUCKET))
-            key.name = urllib2.urlparse.urlparse(uri).path[1:]
+            key.name = urllib.parse.urlparse(uri).path[1:]
             key.delete()
             self.test_uris.remove(uri)
 
@@ -212,7 +211,7 @@ class GCMTokenTests(AuthedTestCase):
         result = self.client.post('/json/users/me/android_gcm_reg_id', {'token':token})
         self.assert_json_success(result)
 
-        result = self.client.delete('/json/users/me/android_gcm_reg_id', urllib.urlencode({'token': token}))
+        result = self.client.delete('/json/users/me/android_gcm_reg_id', urllib.parse.urlencode({'token': token}))
         self.assert_json_success(result)
 
     def test_change_user(self):

zerver/test_hooks.py

@@ -4,7 +4,7 @@ from zerver.lib.test_runner import slow
 from zerver.models import Message
 
 import ujson
-import urllib
+from six.moves import urllib
 
 class JiraHookTests(AuthedTestCase):
 
@@ -846,7 +846,7 @@ class TravisHookTests(AuthedTestCase):
         """
         email = "hamlet@zulip.com"
         api_key = self.get_api_key(email)
-        body = urllib.urlencode({'payload': self.fixture_data("travis", "build", file_type="json")})
+        body = urllib.parse.urlencode({'payload': self.fixture_data("travis", "build", file_type="json")})
 
         stream = "travis"
         url = "/api/v1/external/travis?stream=%s&topic=builds&api_key=%s" % (stream, api_key)

zerver/test_signup.py

@@ -25,7 +25,7 @@ from zerver.lib.session_user import get_session_dict_user
 import re
 import ujson
 
-from urlparse import urlparse
+from six.moves import urllib
 from six.moves import range
 
 
@@ -471,7 +471,7 @@ class EmailUnsubscribeTests(AuthedTestCase):
 
         unsubscribe_link = one_click_unsubscribe_link(user_profile,
                                                       "missed_messages")
-        result = self.client.get(urlparse(unsubscribe_link).path)
+        result = self.client.get(urllib.parse.urlparse(unsubscribe_link).path)
 
         self.assertEqual(result.status_code, 200)
         # Circumvent user_profile caching.
@@ -493,7 +493,7 @@ class EmailUnsubscribeTests(AuthedTestCase):
 
         # Simulate unsubscribing from the welcome e-mails.
         unsubscribe_link = one_click_unsubscribe_link(user_profile, "welcome")
-        result = self.client.get(urlparse(unsubscribe_link).path)
+        result = self.client.get(urllib.parse.urlparse(unsubscribe_link).path)
 
         # The welcome email jobs are no longer scheduled.
         self.assertEqual(result.status_code, 200)
@@ -519,7 +519,7 @@ class EmailUnsubscribeTests(AuthedTestCase):
 
         # Simulate unsubscribing from digest e-mails.
         unsubscribe_link = one_click_unsubscribe_link(user_profile, "digest")
-        result = self.client.get(urlparse(unsubscribe_link).path)
+        result = self.client.get(urllib.parse.urlparse(unsubscribe_link).path)
 
         # The setting is toggled off, and scheduled jobs have been removed.
         self.assertEqual(result.status_code, 200)

zerver/test_subs.py

@@ -29,9 +29,8 @@ from zerver.lib.actions import (
 
 import random
 import ujson
-import urllib
 import six
-from six.moves import range
+from six.moves import range, urllib
 
 
 class StreamAdminTest(AuthedTestCase):
@@ -796,6 +795,11 @@ class SubscriptionAPITest(AuthedTestCase):
         invitee = "iago@zulip.com"
         invitee_full_name = 'Iago'
 
+        current_stream = self.get_streams(invitee)[0]
+        notifications_stream = Stream.objects.get(name=current_stream, realm=self.realm)
+        self.realm.notifications_stream = notifications_stream
+        self.realm.save()
+
         invite_streams = ['strange ) \\ test']
         result = self.common_subscribe_to_streams(
             invitee,
@@ -810,7 +814,7 @@ class SubscriptionAPITest(AuthedTestCase):
         msg = Message.objects.latest('id')
         self.assertEqual(msg.sender_id,
                          get_user_profile_by_email('notification-bot@zulip.com').id)
-        expected_msg = "Hi there!  %s just created a new stream '%s'. " \
+        expected_msg = "%s just created a new stream `%s`. " \
                        "!_stream_subscribe_button(strange \\) \\\\ test)" % (
                                                           invitee_full_name,
                                                           invite_streams[0])
@@ -881,7 +885,7 @@ class SubscriptionAPITest(AuthedTestCase):
                         "subscribed you to the %sstream [%s](#narrow/stream/%s)."
                         % (self.user_profile.full_name,
                            '**invite-only** ' if invite_only else '',
-                           streams[0], urllib.quote(streams[0].encode('utf-8'))))
+                           streams[0], urllib.parse.quote(streams[0].encode('utf-8'))))
 
         if not Stream.objects.get(name=streams[0]).invite_only:
             expected_msg += ("\nYou can see historical content on a "

zerver/views/__init__.py

@@ -58,7 +58,7 @@ import datetime
 import ujson
 import simplejson
 import re
-import urllib
+from six.moves import urllib
 import base64
 import time
 import logging
@@ -124,7 +124,7 @@ def accounts_register(request):
             # Other users should not already exist at all.
             user_email_is_unique(email)
     except ValidationError:
-        return HttpResponseRedirect(reverse('django.contrib.auth.views.login') + '?email=' + urllib.quote_plus(email))
+        return HttpResponseRedirect(reverse('django.contrib.auth.views.login') + '?email=' + urllib.parse.quote_plus(email))
 
     name_validated = False
     full_name = None
@@ -380,7 +380,7 @@ def maybe_send_to_registration(request, email, full_name=''):
             '?full_name=',
             # urllib does not handle Unicode, so coerece to encoded byte string
             # Explanation: http://stackoverflow.com/a/5605354/90777
-            urllib.quote_plus(full_name.encode('utf8')))))
+            urllib.parse.quote_plus(full_name.encode('utf8')))))
     else:
         return render_to_response('zerver/accounts_home.html', {'form': form},
                                   context_instance=RequestContext(request))
@@ -461,7 +461,7 @@ def start_google_oauth2(request):
         'scope': 'profile email',
         'state': csrf_state,
     }
-    return redirect(uri + urllib.urlencode(prams))
+    return redirect(uri + urllib.parse.urlencode(prams))
 
 # Workaround to support the Python-requests 1.0 transition of .json
 # from a property to a function
@@ -652,7 +652,7 @@ def accounts_home(request):
             # Note: We don't check for uniqueness
             is_inactive(email)
         except ValidationError:
-            return HttpResponseRedirect(reverse('django.contrib.auth.views.login') + '?email=' + urllib.quote_plus(email))
+            return HttpResponseRedirect(reverse('django.contrib.auth.views.login') + '?email=' + urllib.parse.quote_plus(email))
     else:
         form = create_homepage_form(request)
     return render_to_response('zerver/accounts_home.html',

zerver/views/streams.py

@@ -23,7 +23,7 @@ from zerver.models import UserProfile, Stream, Subscription, \
 
 from collections import defaultdict
 import ujson
-import urllib
+from six.moves import urllib
 
 from zerver.lib.rest import rest_dispatch as _rest_dispatch
 rest_dispatch = csrf_exempt((lambda request, *args, **kwargs: _rest_dispatch(request, globals(), *args, **kwargs)))
@@ -237,7 +237,7 @@ def filter_stream_authorization(user_profile, streams):
 
 def stream_link(stream_name):
     "Escapes a stream name to make a #narrow/stream/stream_name link"
-    return "#narrow/stream/%s" % (urllib.quote(stream_name.encode('utf-8')),)
+    return "#narrow/stream/%s" % (urllib.parse.quote(stream_name.encode('utf-8')),)
 
 def stream_button(stream_name):
     stream_name = stream_name.replace('\\', '\\\\')