paulmataruso/zulip
1
0
Fork 0
mirror of https://github.com/zulip/zulip.git synced 2025-11-04 14:03:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

zerver/lib: Use python 3 syntax for typing.

Browse Source 
Edited by tabbott to improve various line-wrapping decisions.
This commit is contained in:
rht
2017-11-05 11:15:10 +01:00
committed by Tim Abbott
parent 229a8b38c0
commit ee546a33a3
8 changed files with 523 additions and 746 deletions
Download Patch File Download Diff File Expand all files Collapse all files

670
zerver/lib/actions.py
View File

File diff suppressed because it is too large Load Diff

6
zerver/lib/bot_storage.py
View File

@@ -9,15 +9,13 @@ from typing import Text, Optional, List, Tuple
class StateError(Exception):
pass
def get_bot_storage(bot_profile, key):
# type: (UserProfile, Text) -> Text
def get_bot_storage(bot_profile: UserProfile, key: Text) -> Text:
try:
return BotStorageData.objects.get(bot_profile=bot_profile, key=key).value
except BotStorageData.DoesNotExist:
raise StateError("Key does not exist.")
def get_bot_storage_size(bot_profile, key=None):
# type: (UserProfile, Optional[Text]) -> int
def get_bot_storage_size(bot_profile: UserProfile, key: Optional[Text]=None) -> int:
if key is None:
return BotStorageData.objects.filter(bot_profile=bot_profile) \
.annotate(key_size=Length('key'), value_size=Length('value')) \

251
zerver/lib/bugdown/__init__.py
View File

@@ -61,11 +61,6 @@ FullNameInfo = TypedDict('FullNameInfo', {
version = 1
_T = TypeVar('_T')
# We need to avoid this running at runtime, but mypy will see this.
# The problem is that under python 2, Element isn't exactly a type,
# which means that at runtime Union causes this to blow up.
if False:
# mypy requires the Optional to be inside Union
ElementStringNone = Union[Element, Optional[Text]]
AVATAR_REGEX = r'!avatar\((?P<email>[^)]*)\)'
@@ -82,8 +77,7 @@ STREAM_LINK_REGEX = r"""
class BugdownRenderingException(Exception):
pass
def url_embed_preview_enabled_for_realm(message):
# type: (Optional[Message]) -> bool
def url_embed_preview_enabled_for_realm(message: Optional[Message]) -> bool:
if message is not None:
realm = message.get_realm() # type: Optional[Realm]
else:
@@ -95,8 +89,7 @@ def url_embed_preview_enabled_for_realm(message):
return True
return realm.inline_url_embed_preview
def image_preview_enabled_for_realm():
# type: () -> bool
def image_preview_enabled_for_realm() -> bool:
global current_message
if current_message is not None:
realm = current_message.get_realm() # type: Optional[Realm]
@@ -108,8 +101,7 @@ def image_preview_enabled_for_realm():
return True
return realm.inline_image_preview
def list_of_tlds():
# type: () -> List[Text]
def list_of_tlds() -> List[Text]:
# HACK we manually blacklist a few domains
blacklist = ['PY\n', "MD\n"]
@@ -120,8 +112,9 @@ def list_of_tlds():
tlds.sort(key=len, reverse=True)
return tlds
def walk_tree(root, processor, stop_after_first=False):
# type: (Element, Callable[[Element], Optional[_T]], bool) -> List[_T]
def walk_tree(root: Element,
processor: Callable[[Element], Optional[_T]],
stop_after_first: bool=False) -> List[_T]:
results = []
queue = deque([root])
@@ -166,8 +159,7 @@ def add_a(root, url, link, title=None, desc=None,
desc_div.set("class", "message_inline_image_desc")
def add_embed(root, link, extracted_data):
# type: (Element, Text, Dict[Text, Any]) -> None
def add_embed(root: Element, link: Text, extracted_data: Dict[Text, Any]) -> None:
container = markdown.util.etree.SubElement(root, "div")
container.set("class", "message_embed")
@@ -206,8 +198,7 @@ def add_embed(root, link, extracted_data):
@cache_with_key(lambda tweet_id: tweet_id, cache_name="database", with_statsd_key="tweet_data")
def fetch_tweet_data(tweet_id):
# type: (Text) -> Optional[Dict[Text, Any]]
def fetch_tweet_data(tweet_id: Text) -> Optional[Dict[Text, Any]]:
if settings.TEST_SUITE:
from . import testing_mocks
res = testing_mocks.twitter(tweet_id)
@@ -266,8 +257,7 @@ HEAD_END_RE = re.compile('^/head[ >]')
META_START_RE = re.compile('^meta[ >]')
META_END_RE = re.compile('^/meta[ >]')
def fetch_open_graph_image(url):
# type: (Text) -> Optional[Dict[str, Any]]
def fetch_open_graph_image(url: Text) -> Optional[Dict[str, Any]]:
in_head = False
# HTML will auto close meta tags, when we start the next tag add
# a closing tag if it has not been closed yet.
@@ -333,8 +323,7 @@ def fetch_open_graph_image(url):
desc = og_desc.get('content')
return {'image': image, 'title': title, 'desc': desc}
def get_tweet_id(url):
# type: (Text) -> Optional[Text]
def get_tweet_id(url: Text) -> Optional[Text]:
parsed_url = urllib.parse.urlparse(url)
if not (parsed_url.netloc == 'twitter.com' or parsed_url.netloc.endswith('.twitter.com')):
return None
@@ -350,8 +339,7 @@ def get_tweet_id(url):
return tweet_id_match.group("tweetid")
class InlineHttpsProcessor(markdown.treeprocessors.Treeprocessor):
def run(self, root):
# type: (Element) -> None
def run(self, root: Element) -> None:
# Get all URLs from the blob
found_imgs = walk_tree(root, lambda e: e if e.tag == "img" else None)
for img in found_imgs:
@@ -365,14 +353,12 @@ class InlineInterestingLinkProcessor(markdown.treeprocessors.Treeprocessor):
TWITTER_MAX_IMAGE_HEIGHT = 400
TWITTER_MAX_TO_PREVIEW = 3
def __init__(self, md, bugdown):
# type: (markdown.Markdown, Bugdown) -> None
def __init__(self, md: markdown.Markdown, bugdown: 'Bugdown') -> None:
# Passing in bugdown for access to config to check if realm is zulip.com
self.bugdown = bugdown
markdown.treeprocessors.Treeprocessor.__init__(self, md)
def get_actual_image_url(self, url):
# type: (Text) -> Text
def get_actual_image_url(self, url: Text) -> Text:
# Add specific per-site cases to convert image-preview urls to image urls.
# See https://github.com/zulip/zulip/issues/4658 for more information
parsed_url = urllib.parse.urlparse(url)
@@ -386,8 +372,7 @@ class InlineInterestingLinkProcessor(markdown.treeprocessors.Treeprocessor):
return url
def is_image(self, url):
# type: (Text) -> bool
def is_image(self, url: Text) -> bool:
if not image_preview_enabled_for_realm():
return False
parsed_url = urllib.parse.urlparse(url)
@@ -397,8 +382,7 @@ class InlineInterestingLinkProcessor(markdown.treeprocessors.Treeprocessor):
return True
return False
def dropbox_image(self, url):
# type: (Text) -> Optional[Dict[str, Any]]
def dropbox_image(self, url: Text) -> Optional[Dict[str, Any]]:
# TODO: The returned Dict could possibly be a TypedDict in future.
parsed_url = urllib.parse.urlparse(url)
if (parsed_url.netloc == 'dropbox.com' or parsed_url.netloc.endswith('.dropbox.com')):
@@ -443,8 +427,7 @@ class InlineInterestingLinkProcessor(markdown.treeprocessors.Treeprocessor):
return image_info
return None
def youtube_id(self, url):
# type: (Text) -> Optional[Text]
def youtube_id(self, url: Text) -> Optional[Text]:
if not image_preview_enabled_for_realm():
return None
# Youtube video id extraction regular expression from http://pastebin.com/KyKAFv1s
@@ -457,16 +440,17 @@ class InlineInterestingLinkProcessor(markdown.treeprocessors.Treeprocessor):
return None
return match.group(2)
def youtube_image(self, url):
# type: (Text) -> Optional[Text]
def youtube_image(self, url: Text) -> Optional[Text]:
yt_id = self.youtube_id(url)
if yt_id is not None:
return "https://i.ytimg.com/vi/%s/default.jpg" % (yt_id,)
return None
def twitter_text(self, text, urls, user_mentions, media):
# type: (Text, List[Dict[Text, Text]], List[Dict[Text, Any]], List[Dict[Text, Any]]) -> Element
def twitter_text(self, text: Text,
urls: List[Dict[Text, Text]],
user_mentions: List[Dict[Text, Any]],
media: List[Dict[Text, Any]]) -> Element:
"""
Use data from the twitter API to turn links, mentions and media into A
tags. Also convert unicode emojis to images.
@@ -542,8 +526,7 @@ class InlineInterestingLinkProcessor(markdown.treeprocessors.Treeprocessor):
to_process.sort(key=lambda x: x['start'])
p = current_node = markdown.util.etree.Element('p')
def set_text(text):
# type: (Text) -> None
def set_text(text: Text) -> None:
"""
Helper to set the text or the tail of the current_node
"""
@@ -571,8 +554,7 @@ class InlineInterestingLinkProcessor(markdown.treeprocessors.Treeprocessor):
set_text(text[current_index:])
return p
def twitter_link(self, url):
# type: (Text) -> Optional[Element]
def twitter_link(self, url: Text) -> Optional[Element]:
tweet_id = get_tweet_id(url)
if tweet_id is None:
@@ -641,16 +623,14 @@ class InlineInterestingLinkProcessor(markdown.treeprocessors.Treeprocessor):
logging.warning(traceback.format_exc())
return None
def get_url_data(self, e):
# type: (Element) -> Optional[Tuple[Text, Text]]
def get_url_data(self, e: Element) -> Optional[Tuple[Text, Text]]:
if e.tag == "a":
if e.text is not None:
return (e.get("href"), e.text)
return (e.get("href"), e.get("href"))
return None
def is_only_element(self, root, url):
# type: (Element, str) -> bool
def is_only_element(self, root: Element, url: str) -> bool:
# Check if the url is the only content of the message.
if not len(root) == 1:
@@ -668,8 +648,7 @@ class InlineInterestingLinkProcessor(markdown.treeprocessors.Treeprocessor):
return True
def run(self, root):
# type: (Element) -> None
def run(self, root: Element) -> None:
# Get all URLs from the blob
found_urls = walk_tree(root, self.get_url_data)
@@ -735,8 +714,7 @@ class InlineInterestingLinkProcessor(markdown.treeprocessors.Treeprocessor):
class Avatar(markdown.inlinepatterns.Pattern):
def handleMatch(self, match):
# type: (Match[Text]) -> Optional[Element]
def handleMatch(self, match: Match[Text]) -> Optional[Element]:
img = markdown.util.etree.Element('img')
email_address = match.group('email')
email = email_address.strip().lower()
@@ -753,8 +731,7 @@ class Avatar(markdown.inlinepatterns.Pattern):
img.set('alt', email)
return img
def possible_avatar_emails(content):
# type: (Text) -> Set[Text]
def possible_avatar_emails(content: Text) -> Set[Text]:
emails = set()
for regex in [AVATAR_REGEX, GRAVATAR_REGEX]:
matches = re.findall(regex, content)
@@ -819,8 +796,7 @@ unicode_emoji_regex = '(?P<syntax>['\
# For more information, please refer to the following article:
# http://crocodillon.com/blog/parsing-emoji-unicode-in-javascript
def make_emoji(codepoint, display_string):
# type: (Text, Text) -> Element
def make_emoji(codepoint: Text, display_string: Text) -> Element:
# Replace underscore in emoji's title with space
title = display_string[1:-1].replace("_", " ")
span = markdown.util.etree.Element('span')
@@ -829,8 +805,7 @@ def make_emoji(codepoint, display_string):
span.text = display_string
return span
def make_realm_emoji(src, display_string):
# type: (Text, Text) -> Element
def make_realm_emoji(src: Text, display_string: Text) -> Element:
elt = markdown.util.etree.Element('img')
elt.set('src', src)
elt.set('class', 'emoji')
@@ -838,8 +813,7 @@ def make_realm_emoji(src, display_string):
elt.set("title", display_string[1:-1].replace("_", " "))
return elt
def unicode_emoji_to_codepoint(unicode_emoji):
# type: (Text) -> Text
def unicode_emoji_to_codepoint(unicode_emoji: Text) -> Text:
codepoint = hex(ord(unicode_emoji))[2:]
# Unicode codepoints are minimum of length 4, padded
# with zeroes if the length is less than zero.
@@ -848,8 +822,7 @@ def unicode_emoji_to_codepoint(unicode_emoji):
return codepoint
class UnicodeEmoji(markdown.inlinepatterns.Pattern):
def handleMatch(self, match):
# type: (Match[Text]) -> Optional[Element]
def handleMatch(self, match: Match[Text]) -> Optional[Element]:
orig_syntax = match.group('syntax')
codepoint = unicode_emoji_to_codepoint(orig_syntax)
if codepoint in codepoint_to_name:
@@ -859,8 +832,7 @@ class UnicodeEmoji(markdown.inlinepatterns.Pattern):
return None
class Emoji(markdown.inlinepatterns.Pattern):
def handleMatch(self, match):
# type: (Match[Text]) -> Optional[Element]
def handleMatch(self, match: Match[Text]) -> Optional[Element]:
orig_syntax = match.group("syntax")
name = orig_syntax[1:-1]
@@ -877,15 +849,13 @@ class Emoji(markdown.inlinepatterns.Pattern):
else:
return None
def content_has_emoji_syntax(content):
# type: (Text) -> bool
def content_has_emoji_syntax(content: Text) -> bool:
return re.search(EMOJI_REGEX, content) is not None
class StreamSubscribeButton(markdown.inlinepatterns.Pattern):
# This markdown extension has required javascript in
# static/js/custom_markdown.js
def handleMatch(self, match):
# type: (Match[Text]) -> Element
def handleMatch(self, match: Match[Text]) -> Element:
stream_name = match.group('stream_name')
stream_name = stream_name.replace('\\)', ')').replace('\\\\', '\\')
@@ -907,8 +877,7 @@ class ModalLink(markdown.inlinepatterns.Pattern):
A pattern that allows including in-app modal links in messages.
"""
def handleMatch(self, match):
# type: (Match[Text]) -> Element
def handleMatch(self, match: Match[Text]) -> Element:
relative_url = match.group('relative_url')
text = match.group('text')
@@ -920,8 +889,7 @@ class ModalLink(markdown.inlinepatterns.Pattern):
return a_tag
class Tex(markdown.inlinepatterns.Pattern):
def handleMatch(self, match):
# type: (Match[Text]) -> Element
def handleMatch(self, match: Match[Text]) -> Element:
rendered = render_tex(match.group('body'), is_inline=True)
if rendered is not None:
return etree.fromstring(rendered.encode('utf-8'))
@@ -932,8 +900,7 @@ class Tex(markdown.inlinepatterns.Pattern):
return span
upload_title_re = re.compile("^(https?://[^/]*)?(/user_uploads/\\d+)(/[^/]*)?/[^/]*/(?P<filename>[^/]*)$")
def url_filename(url):
# type: (Text) -> Text
def url_filename(url: Text) -> Text:
"""Extract the filename if a URL is an uploaded file, or return the original URL"""
match = upload_title_re.match(url)
if match:
@@ -941,16 +908,14 @@ def url_filename(url):
else:
return url
def fixup_link(link, target_blank=True):
# type: (markdown.util.etree.Element, bool) -> None
def fixup_link(link: markdown.util.etree.Element, target_blank: bool=True) -> None:
"""Set certain attributes we want on every link."""
if target_blank:
link.set('target', '_blank')
link.set('title', url_filename(link.get('href')))
def sanitize_url(url):
# type: (Text) -> Optional[Text]
def sanitize_url(url: Text) -> Optional[Text]:
"""
Sanitize a url against xss attacks.
See the docstring on markdown.inlinepatterns.LinkPattern.sanitize_url.
@@ -1004,8 +969,7 @@ def sanitize_url(url):
# Url passes all tests. Return url as-is.
return urllib.parse.urlunparse((scheme, netloc, path, params, query, fragment))
def url_to_a(url, text = None):
# type: (Text, Optional[Text]) -> Union[Element, Text]
def url_to_a(url: Text, text: Optional[Text]=None) -> Union[Element, Text]:
a = markdown.util.etree.Element('a')
href = sanitize_url(url)
@@ -1032,8 +996,7 @@ def url_to_a(url, text = None):
return a
class VerbosePattern(markdown.inlinepatterns.Pattern):
def __init__(self, pattern):
# type: (Text) -> None
def __init__(self, pattern: Text) -> None:
markdown.inlinepatterns.Pattern.__init__(self, ' ')
# HACK: we just had python-markdown compile an empty regex.
@@ -1044,8 +1007,7 @@ class VerbosePattern(markdown.inlinepatterns.Pattern):
re.DOTALL | re.UNICODE | re.VERBOSE)
class AutoLink(VerbosePattern):
def handleMatch(self, match):
# type: (Match[Text]) -> ElementStringNone
def handleMatch(self, match: Match[Text]) -> ElementStringNone:
url = match.group('url')
return url_to_a(url)
@@ -1058,8 +1020,7 @@ class UListProcessor(markdown.blockprocessors.UListProcessor):
TAG = 'ul'
RE = re.compile('^[ ]{0,3}[*][ ]+(.*)')
def __init__(self, parser):
# type: (Any) -> None
def __init__(self, parser: Any) -> None:
# HACK: Set the tab length to 2 just for the initialization of
# this class, so that bulleted lists (and only bulleted lists)
@@ -1074,8 +1035,7 @@ class ListIndentProcessor(markdown.blockprocessors.ListIndentProcessor):
Based on markdown.blockprocessors.ListIndentProcessor, but with 2-space indent
"""
def __init__(self, parser):
# type: (Any) -> None
def __init__(self, parser: Any) -> None:
# HACK: Set the tab length to 2 just for the initialization of
# this class, so that bulleted lists (and only bulleted lists)
@@ -1095,8 +1055,7 @@ class BugdownUListPreprocessor(markdown.preprocessors.Preprocessor):
LI_RE = re.compile('^[ ]{0,3}[*][ ]+(.*)', re.MULTILINE)
HANGING_ULIST_RE = re.compile('^.+\\n([ ]{0,3}[*][ ]+.*)', re.MULTILINE)
def run(self, lines):
# type: (List[Text]) -> List[Text]
def run(self, lines: List[Text]) -> List[Text]:
""" Insert a newline between a paragraph and ulist if missing """
inserts = 0
fence = None
@@ -1123,8 +1082,7 @@ class BugdownUListPreprocessor(markdown.preprocessors.Preprocessor):
class LinkPattern(markdown.inlinepatterns.Pattern):
""" Return a link element from the given match. """
def handleMatch(self, m):
# type: (Match[Text]) -> Optional[Element]
def handleMatch(self, m: Match[Text]) -> Optional[Element]:
href = m.group(9)
if not href:
return None
@@ -1141,8 +1099,7 @@ class LinkPattern(markdown.inlinepatterns.Pattern):
fixup_link(el, target_blank = (href[:1] != '#'))
return el
def prepare_realm_pattern(source):
# type: (Text) -> Text
def prepare_realm_pattern(source: Text) -> Text:
""" Augment a realm filter so it only matches after start-of-string,
whitespace, or opening delimiters, won't match if there are word
characters directly after, and saves what was matched as "name". """
@@ -1153,20 +1110,19 @@ def prepare_realm_pattern(source):
class RealmFilterPattern(markdown.inlinepatterns.Pattern):
""" Applied a given realm filter to the input """
def __init__(self, source_pattern, format_string, markdown_instance=None):
# type: (Text, Text, Optional[markdown.Markdown]) -> None
def __init__(self, source_pattern: Text,
format_string: Text,
markdown_instance: Optional[markdown.Markdown]=None) -> None:
self.pattern = prepare_realm_pattern(source_pattern)
self.format_string = format_string
markdown.inlinepatterns.Pattern.__init__(self, self.pattern, markdown_instance)
def handleMatch(self, m):
# type: (Match[Text]) -> Union[Element, Text]
def handleMatch(self, m: Match[Text]) -> Union[Element, Text]:
return url_to_a(self.format_string % m.groupdict(),
m.group("name"))
class UserMentionPattern(markdown.inlinepatterns.Pattern):
def handleMatch(self, m):
# type: (Match[Text]) -> Optional[Element]
def handleMatch(self, m: Match[Text]) -> Optional[Element]:
match = m.group(2)
if current_message and db_data is not None:
@@ -1202,8 +1158,7 @@ class UserMentionPattern(markdown.inlinepatterns.Pattern):
return None
class UserGroupMentionPattern(markdown.inlinepatterns.Pattern):
def handleMatch(self, m):
# type: (Match[Text]) -> Optional[Element]
def handleMatch(self, m: Match[Text]) -> Optional[Element]:
match = m.group(2)
if current_message and db_data is not None:
@@ -1226,15 +1181,13 @@ class UserGroupMentionPattern(markdown.inlinepatterns.Pattern):
return None
class StreamPattern(VerbosePattern):
def find_stream_by_name(self, name):
# type: (Match[Text]) -> Optional[Dict[str, Any]]
def find_stream_by_name(self, name: Match[Text]) -> Optional[Dict[str, Any]]:
if db_data is None:
return None
stream = db_data['stream_names'].get(name)
return stream
def handleMatch(self, m):
# type: (Match[Text]) -> Optional[Element]
def handleMatch(self, m: Match[Text]) -> Optional[Element]:
name = m.group('stream_name')
if current_message:
@@ -1254,14 +1207,12 @@ class StreamPattern(VerbosePattern):
return el
return None
def possible_linked_stream_names(content):
# type: (Text) -> Set[Text]
def possible_linked_stream_names(content: Text) -> Set[Text]:
matches = re.findall(STREAM_LINK_REGEX, content, re.VERBOSE)
return set(matches)
class AlertWordsNotificationProcessor(markdown.preprocessors.Preprocessor):
def run(self, lines):
# type: (Iterable[Text]) -> Iterable[Text]
def run(self, lines: Iterable[Text]) -> Iterable[Text]:
if current_message and db_data is not None:
# We check for alert words here, the set of which are
# dependent on which users may see this message.
@@ -1292,8 +1243,7 @@ class AlertWordsNotificationProcessor(markdown.preprocessors.Preprocessor):
# Markdown link, breaking up the link. This is a monkey-patch, but it
# might be worth sending a version of this change upstream.
class AtomicLinkPattern(LinkPattern):
def handleMatch(self, m):
# type: (Match[Text]) -> Optional[Element]
def handleMatch(self, m: Match[Text]) -> Optional[Element]:
ret = LinkPattern.handleMatch(self, m)
if ret is None:
return None
@@ -1307,8 +1257,7 @@ DEFAULT_BUGDOWN_KEY = -1
ZEPHYR_MIRROR_BUGDOWN_KEY = -2
class Bugdown(markdown.Extension):
def __init__(self, *args, **kwargs):
# type: (*Any, **Union[bool, int, List[Any]]) -> None
def __init__(self, *args: Any, **kwargs: Union[bool, int, List[Any]]) -> None:
# define default configs
self.config = {
"realm_filters": [kwargs['realm_filters'],
@@ -1320,8 +1269,7 @@ class Bugdown(markdown.Extension):
super().__init__(*args, **kwargs)
def extendMarkdown(self, md, md_globals):
# type: (markdown.Markdown, Dict[str, Any]) -> None
def extendMarkdown(self, md: markdown.Markdown, md_globals: Dict[str, Any]) -> None:
del md.preprocessors['reference']
if self.getConfig('code_block_processor_disabled'):
@@ -1476,13 +1424,11 @@ md_engines = {} # type: Dict[Tuple[int, bool], markdown.Markdown]
realm_filter_data = {} # type: Dict[int, List[Tuple[Text, Text, int]]]
class EscapeHtml(markdown.Extension):
def extendMarkdown(self, md, md_globals):
# type: (markdown.Markdown, Dict[str, Any]) -> None
def extendMarkdown(self, md: markdown.Markdown, md_globals: Dict[str, Any]) -> None:
del md.preprocessors['html_block']
del md.inlinePatterns['html']
def make_md_engine(realm_filters_key, email_gateway):
# type: (int, bool) -> None
def make_md_engine(realm_filters_key: int, email_gateway: bool) -> None:
md_engine_key = (realm_filters_key, email_gateway)
if md_engine_key in md_engines:
del md_engines[md_engine_key]
@@ -1503,8 +1449,7 @@ def make_md_engine(realm_filters_key, email_gateway):
realm=realm_filters_key,
code_block_processor_disabled=email_gateway)])
def subject_links(realm_filters_key, subject):
# type: (int, Text) -> List[Text]
def subject_links(realm_filters_key: int, subject: Text) -> List[Text]:
matches = [] # type: List[Text]
realm_filters = realm_filters_for_realm(realm_filters_key)
@@ -1515,8 +1460,7 @@ def subject_links(realm_filters_key, subject):
matches += [realm_filter[1] % m.groupdict()]
return matches
def maybe_update_markdown_engines(realm_filters_key, email_gateway):
# type: (Optional[int], bool) -> None
def maybe_update_markdown_engines(realm_filters_key: Optional[int], email_gateway: bool) -> None:
# If realm_filters_key is None, load all filters
global realm_filter_data
if realm_filters_key is None:
@@ -1551,8 +1495,7 @@ def maybe_update_markdown_engines(realm_filters_key, email_gateway):
# We also use repr() to improve reproducibility, and to escape terminal control
# codes, which can do surprisingly nasty things.
_privacy_re = re.compile('\\w', flags=re.UNICODE)
def privacy_clean_markdown(content):
# type: (Text) -> Text
def privacy_clean_markdown(content: Text) -> Text:
return repr(_privacy_re.sub('x', content))
@@ -1565,16 +1508,14 @@ current_message = None # type: Optional[Message]
# threads themselves, as well.
db_data = None # type: Optional[Dict[Text, Any]]
def log_bugdown_error(msg):
# type: (str) -> None
def log_bugdown_error(msg: str) -> None:
"""We use this unusual logging approach to log the bugdown error, in
order to prevent AdminZulipHandler from sending the santized
original markdown formatting into another Zulip message, which
could cause an infinite exception loop."""
logging.getLogger('').error(msg)
def get_email_info(realm_id, emails):
# type: (int, Set[Text]) -> Dict[Text, FullNameInfo]
def get_email_info(realm_id: int, emails: Set[Text]) -> Dict[Text, FullNameInfo]:
if not emails:
return dict()
@@ -1598,8 +1539,7 @@ def get_email_info(realm_id, emails):
}
return dct
def get_full_name_info(realm_id, full_names):
# type: (int, Set[Text]) -> Dict[Text, FullNameInfo]
def get_full_name_info(realm_id: int, full_names: Set[Text]) -> Dict[Text, FullNameInfo]:
if not full_names:
return dict()
@@ -1626,8 +1566,7 @@ def get_full_name_info(realm_id, full_names):
return dct
class MentionData:
def __init__(self, realm_id, content):
# type: (int, Text) -> None
def __init__(self, realm_id: int, content: Text) -> None:
full_names = possible_mentions(content)
self.full_name_info = get_full_name_info(realm_id, full_names)
self.user_ids = {
@@ -1645,12 +1584,10 @@ class MentionData:
user_profile_id = info['user_profile_id']
self.user_group_members[group_id].append(user_profile_id)
def get_user(self, name):
# type: (Text) -> Optional[FullNameInfo]
def get_user(self, name: Text) -> Optional[FullNameInfo]:
return self.full_name_info.get(name.lower(), None)
def get_user_ids(self):
# type: () -> Set[int]
def get_user_ids(self) -> Set[int]:
"""
Returns the user IDs that might have been mentioned by this
content. Note that because this data structure has not parsed
@@ -1659,16 +1596,13 @@ class MentionData:
"""
return self.user_ids
def get_user_group(self, name):
# type: (Text) -> Optional[UserGroup]
def get_user_group(self, name: Text) -> Optional[UserGroup]:
return self.user_group_name_info.get(name.lower(), None)
def get_group_members(self, user_group_id):
# type: (int) -> List[int]
def get_group_members(self, user_group_id: int) -> List[int]:
return self.user_group_members.get(user_group_id, [])
def get_user_group_name_info(realm_id, user_group_names):
# type: (int, Set[Text]) -> Dict[Text, UserGroup]
def get_user_group_name_info(realm_id: int, user_group_names: Set[Text]) -> Dict[Text, UserGroup]:
if not user_group_names:
return dict()
@@ -1677,8 +1611,7 @@ def get_user_group_name_info(realm_id, user_group_names):
dct = {row.name.lower(): row for row in rows}
return dct
def get_stream_name_info(realm, stream_names):
# type: (Realm, Set[Text]) -> Dict[Text, FullNameInfo]
def get_stream_name_info(realm: Realm, stream_names: Set[Text]) -> Dict[Text, FullNameInfo]:
if not stream_names:
return dict()
@@ -1703,9 +1636,13 @@ def get_stream_name_info(realm, stream_names):
return dct
def do_convert(content, message=None, message_realm=None, possible_words=None, sent_by_bot=False,
mention_data=None, email_gateway=False):
# type: (Text, Optional[Message], Optional[Realm], Optional[Set[Text]], Optional[bool], Optional[MentionData], Optional[bool]) -> Text
def do_convert(content: Text,
message: Optional[Message]=None,
message_realm: Optional[Realm]=None,
possible_words: Optional[Set[Text]]=None,
sent_by_bot: Optional[bool]=False,
mention_data: Optional[MentionData]=None,
email_gateway: Optional[bool]=False) -> Text:
"""Convert Markdown to HTML, with Zulip-specific settings and hacks."""
# This logic is a bit convoluted, but the overall goal is to support a range of use cases:
# * Nothing is passed in other than content -> just run default options (e.g. for docs)
@@ -1803,30 +1740,30 @@ bugdown_time_start = 0.0
bugdown_total_time = 0.0
bugdown_total_requests = 0
def get_bugdown_time():
# type: () -> float
def get_bugdown_time() -> float:
return bugdown_total_time
def get_bugdown_requests():
# type: () -> int
def get_bugdown_requests() -> int:
return bugdown_total_requests
def bugdown_stats_start():
# type: () -> None
def bugdown_stats_start() -> None:
global bugdown_time_start
bugdown_time_start = time.time()
def bugdown_stats_finish():
# type: () -> None
def bugdown_stats_finish() -> None:
global bugdown_total_time
global bugdown_total_requests
global bugdown_time_start
bugdown_total_requests += 1
bugdown_total_time += (time.time() - bugdown_time_start)
def convert(content, message=None, message_realm=None, possible_words=None, sent_by_bot=False,
mention_data=None, email_gateway=False):
# type: (Text, Optional[Message], Optional[Realm], Optional[Set[Text]], Optional[bool], Optional[MentionData], Optional[bool]) -> Text
def convert(content: Text,
message: Optional[Message]=None,
message_realm: Optional[Realm]=None,
possible_words: Optional[Set[Text]]=None,
sent_by_bot: Optional[bool]=False,
mention_data: Optional[MentionData]=None,
email_gateway: Optional[bool]=False) -> Text:
bugdown_stats_start()
ret = do_convert(content, message, message_realm,
possible_words, sent_by_bot, mention_data, email_gateway)

81
zerver/lib/bugdown/fenced_code.py
View File

@@ -110,8 +110,7 @@ LANG_TAG = ' class="%s"'
class FencedCodeExtension(markdown.Extension):
def extendMarkdown(self, md, md_globals):
# type: (markdown.Markdown, Dict[str, Any]) -> None
def extendMarkdown(self, md: markdown.Markdown, md_globals: Dict[str, Any]) -> None:
""" Add FencedBlockPreprocessor to the Markdown instance. """
md.registerExtension(self)
@@ -127,41 +126,34 @@ class FencedCodeExtension(markdown.Extension):
class FencedBlockPreprocessor(markdown.preprocessors.Preprocessor):
def __init__(self, md):
# type: (markdown.Markdown) -> None
def __init__(self, md: markdown.Markdown) -> None:
markdown.preprocessors.Preprocessor.__init__(self, md)
self.checked_for_codehilite = False
self.codehilite_conf = {} # type: Dict[str, List[Any]]
def run(self, lines):
# type: (Iterable[Text]) -> List[Text]
def run(self, lines: Iterable[Text]) -> List[Text]:
""" Match and store Fenced Code Blocks in the HtmlStash. """
output = [] # type: List[Text]
class BaseHandler:
def handle_line(self, line):
# type: (Text) -> None
def handle_line(self, line: Text) -> None:
raise NotImplementedError()
def done(self):
# type: () -> None
def done(self) -> None:
raise NotImplementedError()
processor = self
handlers = [] # type: List[BaseHandler]
def push(handler):
# type: (BaseHandler) -> None
def push(handler: BaseHandler) -> None:
handlers.append(handler)
def pop():
# type: () -> None
def pop() -> None:
handlers.pop()
def check_for_new_fence(output, line):
# type: (MutableSequence[Text], Text) -> None
def check_for_new_fence(output: MutableSequence[Text], line: Text) -> None:
m = FENCE_RE.match(line)
if m:
fence = m.group('fence')
@@ -172,20 +164,16 @@ class FencedBlockPreprocessor(markdown.preprocessors.Preprocessor):
output.append(line)
class OuterHandler(BaseHandler):
def __init__(self, output):
# type: (MutableSequence[Text]) -> None
def __init__(self, output: MutableSequence[Text]) -> None:
self.output = output
def handle_line(self, line):
# type: (Text) -> None
def handle_line(self, line: Text) -> None:
check_for_new_fence(self.output, line)
def done(self):
# type: () -> None
def done(self) -> None:
pop()
def generic_handler(output, fence, lang):
# type: (MutableSequence[Text], Text, Text) -> BaseHandler
def generic_handler(output: MutableSequence[Text], fence: Text, lang: Text) -> BaseHandler:
if lang in ('quote', 'quoted'):
return QuoteHandler(output, fence)
elif lang in ('math', 'tex', 'latex'):
@@ -194,22 +182,19 @@ class FencedBlockPreprocessor(markdown.preprocessors.Preprocessor):
return CodeHandler(output, fence, lang)
class CodeHandler(BaseHandler):
def __init__(self, output, fence, lang):
# type: (MutableSequence[Text], Text, Text) -> None
def __init__(self, output: MutableSequence[Text], fence: Text, lang: Text) -> None:
self.output = output
self.fence = fence
self.lang = lang
self.lines = [] # type: List[Text]
def handle_line(self, line):
# type: (Text) -> None
def handle_line(self, line: Text) -> None:
if line.rstrip() == self.fence:
self.done()
else:
self.lines.append(line.rstrip())
def done(self):
# type: () -> None
def done(self) -> None:
text = '\n'.join(self.lines)
text = processor.format_code(self.lang, text)
text = processor.placeholder(text)
@@ -220,21 +205,18 @@ class FencedBlockPreprocessor(markdown.preprocessors.Preprocessor):
pop()
class QuoteHandler(BaseHandler):
def __init__(self, output, fence):
# type: (MutableSequence[Text], Text) -> None
def __init__(self, output: MutableSequence[Text], fence: Text) -> None:
self.output = output
self.fence = fence
self.lines = [] # type: List[Text]
def handle_line(self, line):
# type: (Text) -> None
def handle_line(self, line: Text) -> None:
if line.rstrip() == self.fence:
self.done()
else:
check_for_new_fence(self.lines, line)
def done(self):
# type: () -> None
def done(self) -> None:
text = '\n'.join(self.lines)
text = processor.format_quote(text)
processed_lines = text.split('\n')
@@ -244,21 +226,18 @@ class FencedBlockPreprocessor(markdown.preprocessors.Preprocessor):
pop()
class TexHandler(BaseHandler):
def __init__(self, output, fence):
# type: (MutableSequence[Text], Text) -> None
def __init__(self, output: MutableSequence[Text], fence: Text) -> None:
self.output = output
self.fence = fence
self.lines = [] # type: List[Text]
def handle_line(self, line):
# type: (Text) -> None
def handle_line(self, line: Text) -> None:
if line.rstrip() == self.fence:
self.done()
else:
self.lines.append(line)
def done(self):
# type: () -> None
def done(self) -> None:
text = '\n'.join(self.lines)
text = processor.format_tex(text)
text = processor.placeholder(text)
@@ -284,8 +263,7 @@ class FencedBlockPreprocessor(markdown.preprocessors.Preprocessor):
output.append('')
return output
def format_code(self, lang, text):
# type: (Text, Text) -> Text
def format_code(self, lang: Text, text: Text) -> Text:
if lang:
langclass = LANG_TAG % (lang,)
else:
@@ -318,8 +296,7 @@ class FencedBlockPreprocessor(markdown.preprocessors.Preprocessor):
return code
def format_quote(self, text):
# type: (Text) -> Text
def format_quote(self, text: Text) -> Text:
paragraphs = text.split("\n\n")
quoted_paragraphs = []
for paragraph in paragraphs:
@@ -327,8 +304,7 @@ class FencedBlockPreprocessor(markdown.preprocessors.Preprocessor):
quoted_paragraphs.append("\n".join("> " + line for line in lines if line != ''))
return "\n\n".join(quoted_paragraphs)
def format_tex(self, text):
# type: (Text) -> Text
def format_tex(self, text: Text) -> Text:
paragraphs = text.split("\n\n")
tex_paragraphs = []
for paragraph in paragraphs:
@@ -340,12 +316,10 @@ class FencedBlockPreprocessor(markdown.preprocessors.Preprocessor):
escape(paragraph) + '</span>')
return "\n\n".join(tex_paragraphs)
def placeholder(self, code):
# type: (Text) -> Text
def placeholder(self, code: Text) -> Text:
return self.markdown.htmlStash.store(code, safe=True)
def _escape(self, txt):
# type: (Text) -> Text
def _escape(self, txt: Text) -> Text:
""" basic html escaping """
txt = txt.replace('&', '&amp;')
txt = txt.replace('<', '&lt;')
@@ -354,8 +328,7 @@ class FencedBlockPreprocessor(markdown.preprocessors.Preprocessor):
return txt
def makeExtension(*args, **kwargs):
# type: (*Any, **Union[bool, None, Text]) -> FencedCodeExtension
def makeExtension(*args: Any, **kwargs: None) -> FencedCodeExtension:
return FencedCodeExtension(*args, **kwargs)
if __name__ == "__main__":

103
zerver/lib/cache.py
View File

@@ -33,29 +33,24 @@ remote_cache_time_start = 0.0
remote_cache_total_time = 0.0
remote_cache_total_requests = 0
def get_remote_cache_time():
# type: () -> float
def get_remote_cache_time() -> float:
return remote_cache_total_time
def get_remote_cache_requests():
# type: () -> int
def get_remote_cache_requests() -> int:
return remote_cache_total_requests
def remote_cache_stats_start():
# type: () -> None
def remote_cache_stats_start() -> None:
global remote_cache_time_start
remote_cache_time_start = time.time()
def remote_cache_stats_finish():
# type: () -> None
def remote_cache_stats_finish() -> None:
global remote_cache_total_time
global remote_cache_total_requests
global remote_cache_time_start
remote_cache_total_requests += 1
remote_cache_total_time += (time.time() - remote_cache_time_start)
def get_or_create_key_prefix():
# type: () -> Text
def get_or_create_key_prefix() -> Text:
if settings.CASPER_TESTS:
# This sets the prefix for the benefit of the Casper tests.
#
@@ -99,32 +94,27 @@ def get_or_create_key_prefix():
KEY_PREFIX = get_or_create_key_prefix() # type: Text
def bounce_key_prefix_for_testing(test_name):
# type: (Text) -> None
def bounce_key_prefix_for_testing(test_name: Text) -> None:
global KEY_PREFIX
KEY_PREFIX = test_name + ':' + Text(os.getpid()) + ':'
# We are taking the hash of the KEY_PREFIX to decrease the size of the key.
# Memcached keys should have a length of less than 256.
KEY_PREFIX = hashlib.sha1(KEY_PREFIX.encode('utf-8')).hexdigest()
def get_cache_backend(cache_name):
# type: (Optional[str]) -> BaseCache
def get_cache_backend(cache_name: Optional[str]) -> BaseCache:
if cache_name is None:
return djcache
return caches[cache_name]
def get_cache_with_key(keyfunc, cache_name=None):
# type: (Any, Optional[str]) -> Any
def get_cache_with_key(keyfunc: Any, cache_name: Optional[str]=None) -> Any:
"""
The main goal of this function getting value from the cache like in the "cache_with_key".
A cache value can contain any data including the "None", so
here used exception for case if value isn't found in the cache.
"""
def decorator(func):
# type: (Callable[..., Any]) -> (Callable[..., Any])
def decorator(func: Callable[..., Any]) -> (Callable[..., Any]):
@wraps(func)
def func_with_caching(*args, **kwargs):
# type: (*Any, **Any) -> Callable[..., Any]
def func_with_caching(*args: Any, **kwargs: Any) -> Callable[..., Any]:
key = keyfunc(*args, **kwargs)
val = cache_get(key, cache_name=cache_name)
if val is not None:
@@ -144,11 +134,9 @@ def cache_with_key(keyfunc, cache_name=None, timeout=None, with_statsd_key=None)
for avoiding collisions with other uses of this decorator or
other uses of caching."""
def decorator(func):
# type: (Callable[..., ReturnT]) -> Callable[..., ReturnT]
def decorator(func: Callable[..., ReturnT]) -> Callable[..., ReturnT]:
@wraps(func)
def func_with_caching(*args, **kwargs):
# type: (*Any, **Any) -> ReturnT
def func_with_caching(*args: Any, **kwargs: Any) -> ReturnT:
key = keyfunc(*args, **kwargs)
val = cache_get(key, cache_name=cache_name)
@@ -180,31 +168,28 @@ def cache_with_key(keyfunc, cache_name=None, timeout=None, with_statsd_key=None)
return decorator
def cache_set(key, val, cache_name=None, timeout=None):
# type: (Text, Any, Optional[str], Optional[int]) -> None
def cache_set(key: Text, val: Any, cache_name: Optional[str]=None, timeout: Optional[int]=None) -> None:
remote_cache_stats_start()
cache_backend = get_cache_backend(cache_name)
cache_backend.set(KEY_PREFIX + key, (val,), timeout=timeout)
remote_cache_stats_finish()
def cache_get(key, cache_name=None):
# type: (Text, Optional[str]) -> Any
def cache_get(key: Text, cache_name: Optional[str]=None) -> Any:
remote_cache_stats_start()
cache_backend = get_cache_backend(cache_name)
ret = cache_backend.get(KEY_PREFIX + key)
remote_cache_stats_finish()
return ret
def cache_get_many(keys, cache_name=None):
# type: (List[Text], Optional[str]) -> Dict[Text, Any]
def cache_get_many(keys: List[Text], cache_name: Optional[str]=None) -> Dict[Text, Any]:
keys = [KEY_PREFIX + key for key in keys]
remote_cache_stats_start()
ret = get_cache_backend(cache_name).get_many(keys)
remote_cache_stats_finish()
return dict([(key[len(KEY_PREFIX):], value) for key, value in ret.items()])
def cache_set_many(items, cache_name=None, timeout=None):
# type: (Dict[Text, Any], Optional[str], Optional[int]) -> None
def cache_set_many(items: Dict[Text, Any], cache_name: Optional[str]=None,
timeout: Optional[int]=None) -> None:
new_items = {}
for key in items:
new_items[KEY_PREFIX + key] = items[key]
@@ -213,14 +198,12 @@ def cache_set_many(items, cache_name=None, timeout=None):
get_cache_backend(cache_name).set_many(items, timeout=timeout)
remote_cache_stats_finish()
def cache_delete(key, cache_name=None):
# type: (Text, Optional[str]) -> None
def cache_delete(key: Text, cache_name: Optional[str]=None) -> None:
remote_cache_stats_start()
get_cache_backend(cache_name).delete(KEY_PREFIX + key)
remote_cache_stats_finish()
def cache_delete_many(items, cache_name=None):
# type: (Iterable[Text], Optional[str]) -> None
def cache_delete_many(items: Iterable[Text], cache_name: Optional[str]=None) -> None:
remote_cache_stats_start()
get_cache_backend(cache_name).delete_many(
KEY_PREFIX + item for item in items)
@@ -289,8 +272,7 @@ def generic_bulk_cached_fetch(
return dict((object_id, cached_objects[cache_keys[object_id]]) for object_id in object_ids
if cache_keys[object_id] in cached_objects)
def cache(func):
# type: (Callable[..., ReturnT]) -> Callable[..., ReturnT]
def cache(func: Callable[..., ReturnT]) -> Callable[..., ReturnT]:
"""Decorator which applies Django caching to a function.
Uses a key based on the function's name, filename, and
@@ -299,20 +281,17 @@ def cache(func):
func_uniqifier = '%s-%s' % (func.__code__.co_filename, func.__name__)
@wraps(func)
def keyfunc(*args, **kwargs):
# type: (*Any, **Any) -> str
def keyfunc(*args: Any, **kwargs: Any) -> str:
# Django complains about spaces because memcached rejects them
key = func_uniqifier + repr((args, kwargs))
return key.replace('-', '--').replace(' ', '-s')
return cache_with_key(keyfunc)(func)
def display_recipient_cache_key(recipient_id):
# type: (int) -> Text
def display_recipient_cache_key(recipient_id: int) -> Text:
return u"display_recipient_dict:%d" % (recipient_id,)
def user_profile_by_email_cache_key(email):
# type: (Text) -> Text
def user_profile_by_email_cache_key(email: Text) -> Text:
# See the comment in zerver/lib/avatar_hash.py:gravatar_hash for why we
# are proactively encoding email addresses even though they will
# with high likelihood be ASCII-only for the foreseeable future.
@@ -326,16 +305,13 @@ def user_profile_cache_key(email, realm):
# type: (Text, Realm) -> Text
return user_profile_cache_key_id(email, realm.id)
def bot_profile_cache_key(email):
# type: (Text) -> Text
def bot_profile_cache_key(email: Text) -> Text:
return u"bot_profile:%s" % (make_safe_digest(email.strip()))
def user_profile_by_id_cache_key(user_profile_id):
# type: (int) -> Text
def user_profile_by_id_cache_key(user_profile_id: int) -> Text:
return u"user_profile_by_id:%s" % (user_profile_id,)
def user_profile_by_api_key_cache_key(api_key):
# type: (Text) -> Text
def user_profile_by_api_key_cache_key(api_key: Text) -> Text:
return u"user_profile_by_api_key:%s" % (api_key,)
# TODO: Refactor these cache helpers into another file that can import
@@ -346,12 +322,10 @@ realm_user_dict_fields = [
'avatar_source', 'avatar_version', 'is_active',
'is_realm_admin', 'is_bot', 'realm_id', 'timezone'] # type: List[str]
def realm_user_dicts_cache_key(realm_id):
# type: (int) -> Text
def realm_user_dicts_cache_key(realm_id: int) -> Text:
return u"realm_user_dicts:%s" % (realm_id,)
def active_user_ids_cache_key(realm_id):
# type: (int) -> Text
def active_user_ids_cache_key(realm_id: int) -> Text:
return u"active_user_ids:%s" % (realm_id,)
bot_dict_fields = ['id', 'full_name', 'short_name', 'bot_type', 'email',
@@ -366,8 +340,7 @@ def bot_dicts_in_realm_cache_key(realm):
# type: (Realm) -> Text
return u"bot_dicts_in_realm:%s" % (realm.id,)
def get_stream_cache_key(stream_name, realm_id):
# type: (Text, int) -> Text
def get_stream_cache_key(stream_name: Text, realm_id: int) -> Text:
return u"stream_by_realm_and_name:%s:%s" % (
realm_id, make_safe_digest(stream_name.strip().lower()))
@@ -392,13 +365,11 @@ def delete_display_recipient_cache(user_profile):
# Called by models.py to flush the user_profile cache whenever we save
# a user_profile object
def flush_user_profile(sender, **kwargs):
# type: (Any, **Any) -> None
def flush_user_profile(sender: Any, **kwargs: Any) -> None:
user_profile = kwargs['instance']
delete_user_profile_caches([user_profile])
def changed(fields):
# type: (List[str]) -> bool
def changed(fields: List[str]) -> bool:
if kwargs.get('update_fields') is None:
# adds/deletes should invalidate the cache
return True
@@ -434,8 +405,7 @@ def flush_user_profile(sender, **kwargs):
# Called by models.py to flush various caches whenever we save
# a Realm object. The main tricky thing here is that Realm info is
# generally cached indirectly through user_profile objects.
def flush_realm(sender, **kwargs):
# type: (Any, **Any) -> None
def flush_realm(sender: Any, **kwargs: Any) -> None:
realm = kwargs['instance']
users = realm.get_active_users()
delete_user_profile_caches(users)
@@ -452,8 +422,7 @@ def realm_alert_words_cache_key(realm):
# Called by models.py to flush the stream cache whenever we save a stream
# object.
def flush_stream(sender, **kwargs):
# type: (Any, **Any) -> None
def flush_stream(sender: Any, **kwargs: Any) -> None:
from zerver.models import UserProfile
stream = kwargs['instance']
items_for_remote_cache = {}
@@ -466,15 +435,13 @@ def flush_stream(sender, **kwargs):
Q(default_events_register_stream=stream)).exists():
cache_delete(bot_dicts_in_realm_cache_key(stream.realm))
def to_dict_cache_key_id(message_id):
# type: (int) -> Text
def to_dict_cache_key_id(message_id: int) -> Text:
return 'message_dict:%d' % (message_id,)
def to_dict_cache_key(message):
# type: (Message) -> Text
return to_dict_cache_key_id(message.id)
def flush_message(sender, **kwargs):
# type: (Any, **Any) -> None
def flush_message(sender: Any, **kwargs: Any) -> None:
message = kwargs['instance']
cache_delete(to_dict_cache_key_id(message.id))

65
zerver/lib/notifications.py
View File

@@ -32,8 +32,7 @@ import ujson
import urllib
from collections import defaultdict
def one_click_unsubscribe_link(user_profile, email_type):
# type: (UserProfile, str) -> str
def one_click_unsubscribe_link(user_profile: UserProfile, email_type: str) -> str:
"""
Generate a unique link that a logged-out user can visit to unsubscribe from
Zulip e-mails without having to first log in.
@@ -42,33 +41,28 @@ def one_click_unsubscribe_link(user_profile, email_type):
Confirmation.UNSUBSCRIBE,
url_args = {'email_type': email_type})
def hash_util_encode(string):
# type: (Text) -> Text
def hash_util_encode(string: Text) -> Text:
# Do the same encoding operation as hash_util.encodeHashComponent on the
# frontend.
# `safe` has a default value of "/", but we want those encoded, too.
return urllib.parse.quote(
string.encode("utf-8"), safe=b"").replace(".", "%2E").replace("%", ".")
def pm_narrow_url(realm, participants):
# type: (Realm, List[Text]) -> Text
def pm_narrow_url(realm: Realm, participants: List[Text]) -> Text:
participants.sort()
base_url = u"%s/#narrow/pm-with/" % (realm.uri,)
return base_url + hash_util_encode(",".join(participants))
def stream_narrow_url(realm, stream):
# type: (Realm, Text) -> Text
def stream_narrow_url(realm: Realm, stream: Text) -> Text:
base_url = u"%s/#narrow/stream/" % (realm.uri,)
return base_url + hash_util_encode(stream)
def topic_narrow_url(realm, stream, topic):
# type: (Realm, Text, Text) -> Text
def topic_narrow_url(realm: Realm, stream: Text, topic: Text) -> Text:
base_url = u"%s/#narrow/stream/" % (realm.uri,)
return u"%s%s/topic/%s" % (base_url, hash_util_encode(stream),
hash_util_encode(topic))
def relative_to_full_url(base_url, content):
# type: (Text, Text) -> Text
def relative_to_full_url(base_url: Text, content: Text) -> Text:
# Convert relative URLs to absolute URLs.
fragment = lxml.html.fromstring(content)
@@ -101,10 +95,8 @@ def relative_to_full_url(base_url, content):
return content
def fix_emojis(content, base_url, emojiset):
# type: (Text, Text, Text) -> Text
def make_emoji_img_elem(emoji_span_elem):
# type: (Any) -> Dict[str, Any]
def fix_emojis(content: Text, base_url: Text, emojiset: Text) -> Text:
def make_emoji_img_elem(emoji_span_elem: Any) -> Dict[str, Any]:
# Convert the emoji spans to img tags.
classes = emoji_span_elem.get('class')
match = re.search('emoji-(?P<emoji_code>\S+)', classes)
@@ -138,8 +130,7 @@ def fix_emojis(content, base_url, emojiset):
content = lxml.html.tostring(fragment).decode('utf-8')
return content
def build_message_list(user_profile, messages):
# type: (UserProfile, List[Message]) -> List[Dict[str, Any]]
def build_message_list(user_profile: UserProfile, messages: List[Message]) -> List[Dict[str, Any]]:
"""
Builds the message list object for the missed message email template.
The messages are collapsed into per-recipient and per-sender blocks, like
@@ -147,22 +138,19 @@ def build_message_list(user_profile, messages):
"""
messages_to_render = [] # type: List[Dict[str, Any]]
def sender_string(message):
# type: (Message) -> Text
def sender_string(message: Message) -> Text:
if message.recipient.type in (Recipient.STREAM, Recipient.HUDDLE):
return message.sender.full_name
else:
return ''
def fix_plaintext_image_urls(content):
# type: (Text) -> Text
def fix_plaintext_image_urls(content: Text) -> Text:
# Replace image URLs in plaintext content of the form
# [image name](image url)
# with a simple hyperlink.
return re.sub(r"\[(\S*)\]\((\S*)\)", r"\2", content)
def build_message_payload(message):
# type: (Message) -> Dict[str, Text]
def build_message_payload(message: Message) -> Dict[str, Text]:
plain = message.content
plain = fix_plaintext_image_urls(plain)
# There's a small chance of colliding with non-Zulip URLs containing
@@ -181,14 +169,12 @@ def build_message_list(user_profile, messages):
return {'plain': plain, 'html': html}
def build_sender_payload(message):
# type: (Message) -> Dict[str, Any]
def build_sender_payload(message: Message) -> Dict[str, Any]:
sender = sender_string(message)
return {'sender': sender,
'content': [build_message_payload(message)]}
def message_header(user_profile, message):
# type: (UserProfile, Message) -> Dict[str, Any]
def message_header(user_profile: UserProfile, message: Message) -> Dict[str, Any]:
disp_recipient = get_display_recipient(message.recipient)
if message.recipient.type == Recipient.PERSONAL:
header = u"You and %s" % (message.sender.full_name,)
@@ -264,8 +250,9 @@ def build_message_list(user_profile, messages):
return messages_to_render
@statsd_increment("missed_message_reminders")
def do_send_missedmessage_events_reply_in_zulip(user_profile, missed_messages, message_count):
# type: (UserProfile, List[Message], int) -> None
def do_send_missedmessage_events_reply_in_zulip(user_profile: UserProfile,
missed_messages: List[Message],
message_count: int) -> None:
"""
Send a reminder email to a user if she's missed some PMs by being offline.
@@ -384,8 +371,7 @@ def do_send_missedmessage_events_reply_in_zulip(user_profile, missed_messages, m
user_profile.last_reminder = timezone_now()
user_profile.save(update_fields=['last_reminder'])
def handle_missedmessage_emails(user_profile_id, missed_email_events):
# type: (int, Iterable[Dict[str, Any]]) -> None
def handle_missedmessage_emails(user_profile_id: int, missed_email_events: Iterable[Dict[str, Any]]) -> None:
message_ids = [event.get('message_id') for event in missed_email_events]
user_profile = get_user_profile_by_id(user_profile_id)
@@ -429,29 +415,25 @@ def handle_missedmessage_emails(user_profile_id, missed_email_events):
message_count_by_recipient_subject[recipient_subject],
)
def clear_scheduled_invitation_emails(email):
# type: (str) -> None
def clear_scheduled_invitation_emails(email: str) -> None:
"""Unlike most scheduled emails, invitation emails don't have an
existing user object to key off of, so we filter by address here."""
items = ScheduledEmail.objects.filter(address__iexact=email,
type=ScheduledEmail.INVITATION_REMINDER)
items.delete()
def clear_scheduled_emails(user_id, email_type=None):
# type: (int, Optional[int]) -> None
def clear_scheduled_emails(user_id: int, email_type: Optional[int]=None) -> None:
items = ScheduledEmail.objects.filter(user_id=user_id)
if email_type is not None:
items = items.filter(type=email_type)
items.delete()
def log_digest_event(msg):
# type: (Text) -> None
def log_digest_event(msg: Text) -> None:
import logging
logging.basicConfig(filename=settings.DIGEST_LOG_PATH, level=logging.INFO)
logging.info(msg)
def enqueue_welcome_emails(user):
# type: (UserProfile) -> None
def enqueue_welcome_emails(user: UserProfile) -> None:
from zerver.context_processors import common_context
if settings.WELCOME_EMAIL_SENDER is not None:
# line break to avoid triggering lint rule
@@ -476,8 +458,7 @@ def enqueue_welcome_emails(user):
"zerver/emails/followup_day2", to_user_id=user.id, from_name=from_name,
from_address=from_address, context=context, delay=datetime.timedelta(days=1))
def convert_html_to_markdown(html):
# type: (Text) -> Text
def convert_html_to_markdown(html: Text) -> Text:
# On Linux, the tool installs as html2markdown, and there's a command called
# html2text that does something totally different. On OSX, the tool installs
# as html2text.

79
zerver/lib/push_notifications.py
View File

@@ -44,12 +44,10 @@ else: # nocoverage -- Not convenient to add test for this.
DeviceToken = Union[PushDeviceToken, RemotePushDeviceToken]
# We store the token as b64, but apns-client wants hex strings
def b64_to_hex(data):
# type: (bytes) -> Text
def b64_to_hex(data: bytes) -> Text:
return binascii.hexlify(base64.b64decode(data)).decode('utf-8')
def hex_to_b64(data):
# type: (Text) -> bytes
def hex_to_b64(data: Text) -> bytes:
return base64.b64encode(binascii.unhexlify(data.encode('utf-8')))
#
@@ -58,8 +56,7 @@ def hex_to_b64(data):
_apns_client = None # type: APNsClient
def get_apns_client():
# type: () -> APNsClient
def get_apns_client() -> APNsClient:
global _apns_client
if _apns_client is None:
# NB if called concurrently, this will make excess connections.
@@ -69,8 +66,7 @@ def get_apns_client():
use_sandbox=settings.APNS_SANDBOX)
return _apns_client
def modernize_apns_payload(data):
# type: (Dict[str, Any]) -> Dict[str, Any]
def modernize_apns_payload(data: Dict[str, Any]) -> Dict[str, Any]:
'''Take a payload in an unknown Zulip version's format, and return in current format.'''
# TODO this isn't super robust as is -- if a buggy remote server
# sends a malformed payload, we are likely to raise an exception.
@@ -96,8 +92,8 @@ def modernize_apns_payload(data):
APNS_MAX_RETRIES = 3
@statsd_increment("apple_push_notification")
def send_apple_push_notification(user_id, devices, payload_data):
# type: (int, List[DeviceToken], Dict[str, Any]) -> None
def send_apple_push_notification(user_id: int, devices: List[DeviceToken],
payload_data: Dict[str, Any]) -> None:
logging.info("APNs: Sending notification for user %d to %d devices",
user_id, len(devices))
payload = APNsPayload(**modernize_apns_payload(payload_data))
@@ -107,8 +103,7 @@ def send_apple_push_notification(user_id, devices, payload_data):
for device in devices:
# TODO obviously this should be made to actually use the async
def attempt_send():
# type: () -> Optional[str]
def attempt_send() -> Optional[str]:
stream_id = client.send_notification_async(
device.token, payload, topic='org.zulip.Zulip',
expiration=expiration)
@@ -144,15 +139,14 @@ if settings.ANDROID_GCM_API_KEY: # nocoverage
else:
gcm = None
def send_android_push_notification_to_user(user_profile, data):
# type: (UserProfile, Dict[str, Any]) -> None
def send_android_push_notification_to_user(user_profile: UserProfile, data: Dict[str, Any]) -> None:
devices = list(PushDeviceToken.objects.filter(user=user_profile,
kind=PushDeviceToken.GCM))
send_android_push_notification(devices, data)
@statsd_increment("android_push_notification")
def send_android_push_notification(devices, data, remote=False):
# type: (List[DeviceToken], Dict[str, Any], bool) -> None
def send_android_push_notification(devices: List[DeviceToken], data: Dict[str, Any],
remote: bool=False) -> None:
if not gcm:
logging.warning("Skipping sending a GCM push notification since "
"PUSH_NOTIFICATION_BOUNCER_URL and ANDROID_GCM_API_KEY are both unset")
@@ -218,12 +212,12 @@ def send_android_push_notification(devices, data, remote=False):
# Sending to a bouncer
#
def uses_notification_bouncer():
# type: () -> bool
def uses_notification_bouncer() -> bool:
return settings.PUSH_NOTIFICATION_BOUNCER_URL is not None
def send_notifications_to_bouncer(user_profile_id, apns_payload, gcm_payload):
# type: (int, Dict[str, Any], Dict[str, Any]) -> None
def send_notifications_to_bouncer(user_profile_id: int,
apns_payload: Dict[str, Any],
gcm_payload: Dict[str, Any]) -> None:
post_data = {
'user_id': user_profile_id,
'apns_payload': apns_payload,
@@ -231,8 +225,7 @@ def send_notifications_to_bouncer(user_profile_id, apns_payload, gcm_payload):
}
send_json_to_push_bouncer('POST', 'notify', post_data)
def send_json_to_push_bouncer(method, endpoint, post_data):
# type: (str, str, Dict[str, Any]) -> None
def send_json_to_push_bouncer(method: str, endpoint: str, post_data: Dict[str, Any]) -> None:
send_to_push_bouncer(
method,
endpoint,
@@ -243,8 +236,10 @@ def send_json_to_push_bouncer(method, endpoint, post_data):
class PushNotificationBouncerException(Exception):
pass
def send_to_push_bouncer(method, endpoint, post_data, extra_headers=None):
# type: (str, str, Union[Text, Dict[str, Any]], Optional[Dict[str, Any]]) -> None
def send_to_push_bouncer(method: str,
endpoint: str,
post_data: Union[Text, Dict[str, Any]],
extra_headers: Optional[Dict[str, Any]]=None) -> None:
"""While it does actually send the notice, this function has a lot of
code and comments around error handling for the push notifications
bouncer. There are several classes of failures, each with its own
@@ -310,15 +305,16 @@ def send_to_push_bouncer(method, endpoint, post_data, extra_headers=None):
# Managing device tokens
#
def num_push_devices_for_user(user_profile, kind = None):
# type: (UserProfile, Optional[int]) -> PushDeviceToken
def num_push_devices_for_user(user_profile: UserProfile, kind: Optional[int]=None) -> PushDeviceToken:
if kind is None:
return PushDeviceToken.objects.filter(user=user_profile).count()
else:
return PushDeviceToken.objects.filter(user=user_profile, kind=kind).count()
def add_push_device_token(user_profile, token_str, kind, ios_app_id=None):
# type: (UserProfile, bytes, int, Optional[str]) -> None
def add_push_device_token(user_profile: UserProfile,
token_str: bytes,
kind: int,
ios_app_id: Optional[str]=None) -> None:
logging.info("New push device: %d %r %d %r",
user_profile.id, token_str, kind, ios_app_id)
@@ -357,8 +353,7 @@ def add_push_device_token(user_profile, token_str, kind, ios_app_id=None):
else:
logging.info("New push device created.")
def remove_push_device_token(user_profile, token_str, kind):
# type: (UserProfile, bytes, int) -> None
def remove_push_device_token(user_profile: UserProfile, token_str: bytes, kind: int) -> None:
# If we're sending things to the push notification bouncer
# register this user with them here
@@ -383,8 +378,7 @@ def remove_push_device_token(user_profile, token_str, kind):
# Push notifications in general
#
def get_alert_from_message(message):
# type: (Message) -> Text
def get_alert_from_message(message: Message) -> Text:
"""
Determine what alert string to display based on the missed messages.
"""
@@ -401,10 +395,8 @@ def get_alert_from_message(message):
else:
return "New Zulip mentions and private messages from %s" % (sender_str,)
def get_mobile_push_content(rendered_content):
# type: (Text) -> Text
def get_text(elem):
# type: (LH.HtmlElement) -> Text
def get_mobile_push_content(rendered_content: Text) -> Text:
def get_text(elem: LH.HtmlElement) -> Text:
# Convert default emojis to their unicode equivalent.
classes = elem.get("class", "")
if "emoji" in classes:
@@ -421,8 +413,7 @@ def get_mobile_push_content(rendered_content):
return elem.text or ""
def process(elem):
# type: (LH.HtmlElement) -> Text
def process(elem: LH.HtmlElement) -> Text:
plain_text = get_text(elem)
for child in elem:
plain_text += process(child)
@@ -436,8 +427,7 @@ def get_mobile_push_content(rendered_content):
plain_text = process(elem)
return plain_text
def truncate_content(content):
# type: (Text) -> Text
def truncate_content(content: Text) -> Text:
# We use unicode character 'HORIZONTAL ELLIPSIS' (U+2026) instead
# of three dots as this saves two extra characters for textual
# content. This function will need to be updated to handle unicode
@@ -446,8 +436,7 @@ def truncate_content(content):
return content
return content[:200] + "…"
def get_apns_payload(message):
# type: (Message) -> Dict[str, Any]
def get_apns_payload(message: Message) -> Dict[str, Any]:
text_content = get_mobile_push_content(message.rendered_content)
truncated_content = truncate_content(text_content)
return {
@@ -464,8 +453,7 @@ def get_apns_payload(message):
}
}
def get_gcm_payload(user_profile, message):
# type: (UserProfile, Message) -> Dict[str, Any]
def get_gcm_payload(user_profile: UserProfile, message: Message) -> Dict[str, Any]:
text_content = get_mobile_push_content(message.rendered_content)
truncated_content = truncate_content(text_content)
@@ -492,8 +480,7 @@ def get_gcm_payload(user_profile, message):
return android_data
@statsd_increment("push_notifications")
def handle_push_notification(user_profile_id, missed_message):
# type: (int, Dict[str, Any]) -> None
def handle_push_notification(user_profile_id: int, missed_message: Dict[str, Any]) -> None:
"""
missed_message is the event received by the
zerver.worker.queue_processors.PushNotificationWorker.consume function.

12
zerver/lib/retention.py
View File

@@ -10,8 +10,7 @@ from zerver.models import Realm, Message, UserMessage, ArchivedMessage, Archived
from typing import Any, Dict, Optional, Generator
def get_realm_expired_messages(realm):
# type: (Any) -> Optional[Dict[str, Any]]
def get_realm_expired_messages(realm: Any) -> Optional[Dict[str, Any]]:
expired_date = timezone_now() - timedelta(days=realm.message_retention_days)
expired_messages = Message.objects.order_by('id').filter(sender__realm=realm,
pub_date__lt=expired_date)
@@ -20,8 +19,7 @@ def get_realm_expired_messages(realm):
return {'realm_id': realm.id, 'expired_messages': expired_messages}
def get_expired_messages():
# type: () -> Generator[Any, None, None]
def get_expired_messages() -> Generator[Any, None, None]:
# Get all expired messages by Realm.
realms = Realm.objects.order_by('string_id').filter(
deactivated=False, message_retention_days__isnull=False)
@@ -31,8 +29,7 @@ def get_expired_messages():
yield realm_expired_messages
def move_attachment_message_to_archive_by_message(message_id):
# type: (int) -> None
def move_attachment_message_to_archive_by_message(message_id: int) -> None:
# Move attachments messages relation table data to archive.
query = """
INSERT INTO zerver_archivedattachment_messages (id, archivedattachment_id,
@@ -50,8 +47,7 @@ def move_attachment_message_to_archive_by_message(message_id):
@transaction.atomic
def move_message_to_archive(message_id):
# type: (int) -> None
def move_message_to_archive(message_id: int) -> None:
msg = list(Message.objects.filter(id=message_id).values())
if not msg:
raise Message.DoesNotExist