diff --git a/zerver/lib/actions.py b/zerver/lib/actions.py index 42a37dabe5..d7bc663188 100644 --- a/zerver/lib/actions.py +++ b/zerver/lib/actions.py @@ -136,8 +136,7 @@ STREAM_ASSIGNMENT_COLORS = [ "#c6a8ad", "#e7cc4d", "#c8bebf", "#a47462"] # Store an event in the log for re-importing messages -def log_event(event): - # type: (MutableMapping[str, Any]) -> None +def log_event(event: MutableMapping[str, Any]) -> None: if settings.EVENT_LOG_DIR is None: return @@ -155,9 +154,7 @@ def log_event(event): with open(template % ('events',), 'a') as log: log.write(ujson.dumps(event) + '\n') -def can_access_stream_user_ids(stream): - # type: (Stream) -> Set[int] - +def can_access_stream_user_ids(stream: Stream) -> Set[int]: # return user ids of users who can access the attributes of # a stream, such as its name/description if stream.is_public(): @@ -165,15 +162,12 @@ def can_access_stream_user_ids(stream): else: return private_stream_user_ids(stream.id) -def private_stream_user_ids(stream_id): - # type: (int) -> Set[int] - +def private_stream_user_ids(stream_id: int) -> Set[int]: # TODO: Find similar queries elsewhere and de-duplicate this code. subscriptions = get_active_subscriptions_for_stream_id(stream_id) return {sub['user_profile_id'] for sub in subscriptions.values('user_profile_id')} -def bot_owner_user_ids(user_profile): - # type: (UserProfile) -> Set[int] +def bot_owner_user_ids(user_profile: UserProfile) -> Set[int]: is_private_bot = ( user_profile.default_sending_stream and user_profile.default_sending_stream.invite_only or @@ -186,13 +180,11 @@ def bot_owner_user_ids(user_profile): users.add(user_profile.bot_owner_id) return users -def realm_user_count(realm): - # type: (Realm) -> int +def realm_user_count(realm: Realm) -> int: return UserProfile.objects.filter(realm=realm, is_active=True, is_bot=False).count() -def get_topic_history_for_stream(user_profile, recipient): - # type: (UserProfile, Recipient) -> List[Dict[str, Any]] - +def get_topic_history_for_stream(user_profile: UserProfile, + recipient: Recipient) -> List[Dict[str, Any]]: query = ''' SELECT "zerver_message"."subject" as topic, @@ -277,8 +269,7 @@ def send_signup_message(sender, admin_realm_signup_notifications_stream, user_pr ) ) -def notify_new_user(user_profile, internal=False): - # type: (UserProfile, bool) -> None +def notify_new_user(user_profile: UserProfile, internal: bool=False) -> None: if settings.NEW_USER_BOT is not None: send_signup_message(settings.NEW_USER_BOT, "signups", user_profile, internal) statsd.gauge("users.signups.%s" % (user_profile.realm.string_id), 1, delta=True) @@ -287,8 +278,7 @@ def notify_new_user(user_profile, internal=False): # from being sent after the user is created. clear_scheduled_invitation_emails(user_profile.email) -def add_new_user_history(user_profile, streams): - # type: (UserProfile, Iterable[Stream]) -> None +def add_new_user_history(user_profile: UserProfile, streams: Iterable[Stream]) -> None: """Give you the last 1000 messages on your public streams, so you have something to look at in your home view once you finish the tutorial.""" @@ -393,8 +383,7 @@ def process_new_human_user(user_profile, prereg_user=None, newsletter_data=None, }, lambda event: None) -def notify_created_user(user_profile): - # type: (UserProfile) -> None +def notify_created_user(user_profile: UserProfile) -> None: event = dict(type="realm_user", op="add", person=dict(email=user_profile.email, user_id=user_profile.id, @@ -405,11 +394,8 @@ def notify_created_user(user_profile): is_bot=user_profile.is_bot)) send_event(event, active_user_ids(user_profile.realm_id)) -def notify_created_bot(user_profile): - # type: (UserProfile) -> None - - def stream_name(stream): - # type: (Optional[Stream]) -> Optional[Text] +def notify_created_bot(user_profile: UserProfile) -> None: + def stream_name(stream: Optional[Stream]) -> Optional[Text]: if not stream: return None return stream.name @@ -470,8 +456,7 @@ def do_create_user(email, password, realm, full_name, short_name, default_stream_groups=default_stream_groups) return user_profile -def do_activate_user(user_profile): - # type: (UserProfile) -> None +def do_activate_user(user_profile: UserProfile) -> None: user_profile.is_active = True user_profile.is_mirror_dummy = False user_profile.set_unusable_password() @@ -488,8 +473,7 @@ def do_activate_user(user_profile): notify_created_user(user_profile) -def do_reactivate_user(user_profile, acting_user=None): - # type: (UserProfile, Optional[UserProfile]) -> None +def do_reactivate_user(user_profile: UserProfile, acting_user: Optional[UserProfile]=None) -> None: # Unlike do_activate_user, this is meant for re-activating existing users, # so it doesn't reset their password, etc. user_profile.is_active = True @@ -507,13 +491,11 @@ def do_reactivate_user(user_profile, acting_user=None): if user_profile.is_bot: notify_created_bot(user_profile) -def active_humans_in_realm(realm): - # type: (Realm) -> Sequence[UserProfile] +def active_humans_in_realm(realm: Realm) -> Sequence[UserProfile]: return UserProfile.objects.filter(realm=realm, is_active=True, is_bot=False) -def do_set_realm_property(realm, name, value): - # type: (Realm, str, Union[Text, bool, int]) -> None +def do_set_realm_property(realm: Realm, name: str, value: Any) -> None: """Takes in a realm object, the name of an attribute to update, and the value to update. """ @@ -533,8 +515,8 @@ def do_set_realm_property(realm, name, value): send_event(event, active_user_ids(realm.id)) -def do_set_realm_authentication_methods(realm, authentication_methods): - # type: (Realm, Dict[str, bool]) -> None +def do_set_realm_authentication_methods(realm: Realm, + authentication_methods: Dict[str, bool]) -> None: for key, value in list(authentication_methods.items()): index = getattr(realm.authentication_methods, key).number realm.authentication_methods.set_bit(index, int(value)) @@ -548,8 +530,9 @@ def do_set_realm_authentication_methods(realm, authentication_methods): send_event(event, active_user_ids(realm.id)) -def do_set_realm_message_editing(realm, allow_message_editing, message_content_edit_limit_seconds): - # type: (Realm, bool, int) -> None +def do_set_realm_message_editing(realm: Realm, + allow_message_editing: bool, + message_content_edit_limit_seconds: int) -> None: realm.allow_message_editing = allow_message_editing realm.message_content_edit_limit_seconds = message_content_edit_limit_seconds realm.save(update_fields=['allow_message_editing', 'message_content_edit_limit_seconds']) @@ -562,8 +545,7 @@ def do_set_realm_message_editing(realm, allow_message_editing, message_content_e ) send_event(event, active_user_ids(realm.id)) -def do_set_realm_notifications_stream(realm, stream, stream_id): - # type: (Realm, Stream, int) -> None +def do_set_realm_notifications_stream(realm: Realm, stream: Stream, stream_id: int) -> None: realm.notifications_stream = stream realm.save(update_fields=['notifications_stream']) event = dict( @@ -574,8 +556,8 @@ def do_set_realm_notifications_stream(realm, stream, stream_id): ) send_event(event, active_user_ids(realm.id)) -def do_set_realm_signup_notifications_stream(realm, stream, stream_id): - # type: (Realm, Stream, int) -> None +def do_set_realm_signup_notifications_stream(realm: Realm, stream: Stream, + stream_id: int) -> None: realm.signup_notifications_stream = stream realm.save(update_fields=['signup_notifications_stream']) event = dict( @@ -586,8 +568,7 @@ def do_set_realm_signup_notifications_stream(realm, stream, stream_id): ) send_event(event, active_user_ids(realm.id)) -def do_deactivate_realm(realm): - # type: (Realm) -> None +def do_deactivate_realm(realm: Realm) -> None: """ Deactivate this realm. Do NOT deactivate the users -- we need to be able to tell the difference between users that were intentionally deactivated, @@ -607,13 +588,13 @@ def do_deactivate_realm(realm): delete_user_sessions(user) clear_scheduled_emails(user.id) -def do_reactivate_realm(realm): - # type: (Realm) -> None +def do_reactivate_realm(realm: Realm) -> None: realm.deactivated = False realm.save(update_fields=["deactivated"]) -def do_deactivate_user(user_profile, acting_user=None, _cascade=True): - # type: (UserProfile, Optional[UserProfile], bool) -> None +def do_deactivate_user(user_profile: UserProfile, + acting_user: Optional[UserProfile]=None, + _cascade: bool=True) -> None: if not user_profile.is_active: return @@ -649,8 +630,7 @@ def do_deactivate_user(user_profile, acting_user=None, _cascade=True): for profile in bot_profiles: do_deactivate_user(profile, acting_user=acting_user, _cascade=False) -def do_deactivate_stream(stream, log=True): - # type: (Stream, bool) -> None +def do_deactivate_stream(stream: Stream, log: bool=True) -> None: # Get the affected user ids *before* we deactivate everybody. affected_user_ids = can_access_stream_user_ids(stream) @@ -694,8 +674,7 @@ def do_deactivate_stream(stream, log=True): streams=[stream_dict]) send_event(event, affected_user_ids) -def do_change_user_email(user_profile, new_email): - # type: (UserProfile, Text) -> None +def do_change_user_email(user_profile: UserProfile, new_email: Text) -> None: delete_user_profile_caches([user_profile]) user_profile.email = new_email @@ -710,8 +689,7 @@ def do_change_user_email(user_profile, new_email): modified_user=user_profile, event_type='user_email_changed', event_time=event_time) -def do_start_email_change_process(user_profile, new_email): - # type: (UserProfile, Text) -> None +def do_start_email_change_process(user_profile: UserProfile, new_email: Text) -> None: old_email = user_profile.email user_profile.email = new_email obj = EmailChangeStatus.objects.create(new_email=new_email, old_email=old_email, @@ -729,16 +707,13 @@ def do_start_email_change_process(user_profile, new_email): from_name='Zulip Account Security', from_address=FromAddress.NOREPLY, context=context) -def compute_irc_user_fullname(email): - # type: (NonBinaryStr) -> NonBinaryStr +def compute_irc_user_fullname(email: NonBinaryStr) -> NonBinaryStr: return email.split("@")[0] + " (IRC)" -def compute_jabber_user_fullname(email): - # type: (NonBinaryStr) -> NonBinaryStr +def compute_jabber_user_fullname(email: NonBinaryStr) -> NonBinaryStr: return email.split("@")[0] + " (XMPP)" -def compute_mit_user_fullname(email): - # type: (NonBinaryStr) -> NonBinaryStr +def compute_mit_user_fullname(email: NonBinaryStr) -> NonBinaryStr: try: # Input is either e.g. username@mit.edu or user|CROSSREALM.INVALID@mit.edu match_user = re.match(r'^([a-zA-Z0-9_.-]+)(\|.+)?@mit\.edu$', email.lower()) @@ -760,8 +735,8 @@ def compute_mit_user_fullname(email): @cache_with_key(lambda realm, email, f: user_profile_by_email_cache_key(email), timeout=3600*24*7) -def create_mirror_user_if_needed(realm, email, email_to_fullname): - # type: (Realm, Text, Callable[[Text], Text]) -> UserProfile +def create_mirror_user_if_needed(realm: Realm, email: Text, + email_to_fullname: Callable[[Text], Text]) -> UserProfile: try: return get_user(email, realm) except UserProfile.DoesNotExist: @@ -773,8 +748,7 @@ def create_mirror_user_if_needed(realm, email, email_to_fullname): except IntegrityError: return get_user(email, realm) -def send_welcome_bot_response(message): - # type: (MutableMapping[str, Any]) -> None +def send_welcome_bot_response(message: MutableMapping[str, Any]) -> None: welcome_bot = get_system_bot(settings.WELCOME_BOT) human_recipient = get_personal_recipient(message['message'].sender.id) if Message.objects.filter(sender=welcome_bot, recipient=human_recipient).count() < 2: @@ -784,8 +758,12 @@ def send_welcome_bot_response(message): "Feel free to continue using this space to practice your new messaging " "skills. Or, try clicking on some of the stream names to your left!") -def render_incoming_message(message, content, user_ids, realm, mention_data=None, email_gateway=False): - # type: (Message, Text, Set[int], Realm, Optional[bugdown.MentionData], Optional[bool]) -> Text +def render_incoming_message(message: Message, + content: Text, + user_ids: Set[int], + realm: Realm, + mention_data: Optional[bugdown.MentionData]=None, + email_gateway: Optional[bool]=False) -> Text: realm_alert_words = alert_words_in_realm(realm) try: rendered_content = render_markdown( @@ -801,8 +779,7 @@ def render_incoming_message(message, content, user_ids, realm, mention_data=None raise JsonableError(_('Unable to render message')) return rendered_content -def get_typing_user_profiles(recipient, sender_id): - # type: (Recipient, int) -> List[UserProfile] +def get_typing_user_profiles(recipient: Recipient, sender_id: int) -> List[UserProfile]: if recipient.type == Recipient.STREAM: ''' We don't support typing indicators for streams because they @@ -836,8 +813,10 @@ RecipientInfoResult = TypedDict('RecipientInfoResult', { 'service_bot_tuples': List[Tuple[int, int]], }) -def get_recipient_info(recipient, sender_id, stream_topic, possibly_mentioned_user_ids=None): - # type: (Recipient, int, Optional[StreamTopicTarget], Optional[Set[int]]) -> RecipientInfoResult +def get_recipient_info(recipient: Recipient, + sender_id: int, + stream_topic: Optional[StreamTopicTarget], + possibly_mentioned_user_ids: Optional[Set[int]]=None) -> RecipientInfoResult: if recipient.type == Recipient.STREAM: # Anybody calling us w/r/t a stream message needs to supply # stream_topic. We may eventually want to have different versions @@ -917,8 +896,7 @@ def get_recipient_info(recipient, sender_id, stream_topic, possibly_mentioned_us # this `else` clause and just `assert(user_ids)`. rows = [] - def get_ids_for(f): - # type: (Callable[[Dict[str, Any]], bool]) -> Set[int] + def get_ids_for(f: Callable[[Dict[str, Any]], bool]) -> Set[int]: """Only includes users on the explicit message to line""" return { row['id'] @@ -926,8 +904,7 @@ def get_recipient_info(recipient, sender_id, stream_topic, possibly_mentioned_us if f(row) } & message_to_user_id_set - def is_service_bot(row): - # type: (Dict[str, Any]) -> bool + def is_service_bot(row: Dict[str, Any]) -> bool: return row['is_bot'] and (row['bot_type'] in UserProfile.SERVICE_BOT_TYPES) active_user_ids = get_ids_for(lambda r: True) @@ -1030,8 +1007,8 @@ def get_service_bot_events(sender, service_bot_tuples, mentioned_user_ids, return event_dict -def do_send_messages(messages_maybe_none, email_gateway=False): - # type: (Sequence[Optional[MutableMapping[str, Any]]], Optional[bool]) -> List[int] +def do_send_messages(messages_maybe_none: Sequence[Optional[MutableMapping[str, Any]]], + email_gateway: Optional[bool]=False) -> List[int]: # Filter out messages which didn't pass internal_prep_message properly messages = [message for message in messages_maybe_none if message is not None] @@ -1273,18 +1250,18 @@ class UserMessageLite: is optimized for the simple use case of inserting a bunch of rows into zerver_usermessage. ''' - def __init__(self, user_profile_id, message_id): - # type: (int, int) -> None + def __init__(self, user_profile_id: int, message_id: int) -> None: self.user_profile_id = user_profile_id self.message_id = message_id self.flags = 0 - def flags_list(self): - # type: () -> List[str] + def flags_list(self) -> List[str]: return UserMessage.flags_list_for_flags(self.flags) -def create_user_messages(message, um_eligible_user_ids, long_term_idle_user_ids, mentioned_user_ids): - # type: (Message, Set[int], Set[int], Set[int]) -> List[UserMessageLite] +def create_user_messages(message: Message, + um_eligible_user_ids: Set[int], + long_term_idle_user_ids: Set[int], + mentioned_user_ids: Set[int]) -> List[UserMessageLite]: ums_to_create = [] for user_profile_id in um_eligible_user_ids: @@ -1320,8 +1297,7 @@ def create_user_messages(message, um_eligible_user_ids, long_term_idle_user_ids, return user_messages -def bulk_insert_ums(ums): - # type: (List[UserMessageLite]) -> None +def bulk_insert_ums(ums: List[UserMessageLite]) -> None: ''' Doing bulk inserts this way is much faster than using Django, since we don't have any ORM overhead. Profiling with 1000 @@ -1344,8 +1320,8 @@ def bulk_insert_ums(ums): with connection.cursor() as cursor: cursor.execute(query) -def notify_reaction_update(user_profile, message, reaction, op): - # type: (UserProfile, Message, Reaction, Text) -> None +def notify_reaction_update(user_profile: UserProfile, message: Message, + reaction: Reaction, op: Text) -> None: user_dict = {'user_id': user_profile.id, 'email': user_profile.email, 'full_name': user_profile.full_name} @@ -1375,8 +1351,7 @@ def notify_reaction_update(user_profile, message, reaction, op): ums = UserMessage.objects.filter(message=message.id) send_event(event, [um.user_profile_id for um in ums]) -def do_add_reaction_legacy(user_profile, message, emoji_name): - # type: (UserProfile, Message, Text) -> None +def do_add_reaction_legacy(user_profile: UserProfile, message: Message, emoji_name: Text) -> None: (emoji_code, reaction_type) = emoji_name_to_emoji_code(user_profile.realm, emoji_name) reaction = Reaction(user_profile=user_profile, message=message, emoji_name=emoji_name, emoji_code=emoji_code, @@ -1384,24 +1359,23 @@ def do_add_reaction_legacy(user_profile, message, emoji_name): reaction.save() notify_reaction_update(user_profile, message, reaction, "add") -def do_remove_reaction_legacy(user_profile, message, emoji_name): - # type: (UserProfile, Message, Text) -> None +def do_remove_reaction_legacy(user_profile: UserProfile, message: Message, emoji_name: Text) -> None: reaction = Reaction.objects.filter(user_profile=user_profile, message=message, emoji_name=emoji_name).get() reaction.delete() notify_reaction_update(user_profile, message, reaction, "remove") -def do_add_reaction(user_profile, message, emoji_name, emoji_code, reaction_type): - # type: (UserProfile, Message, Text, Text, Text) -> None +def do_add_reaction(user_profile: UserProfile, message: Message, + emoji_name: Text, emoji_code: Text, reaction_type: Text) -> None: reaction = Reaction(user_profile=user_profile, message=message, emoji_name=emoji_name, emoji_code=emoji_code, reaction_type=reaction_type) reaction.save() notify_reaction_update(user_profile, message, reaction, "add") -def do_remove_reaction(user_profile, message, emoji_code, reaction_type): - # type: (UserProfile, Message, Text, Text) -> None +def do_remove_reaction(user_profile: UserProfile, message: Message, + emoji_code: Text, reaction_type: Text) -> None: reaction = Reaction.objects.filter(user_profile=user_profile, message=message, emoji_code=emoji_code, @@ -1409,8 +1383,7 @@ def do_remove_reaction(user_profile, message, emoji_code, reaction_type): reaction.delete() notify_reaction_update(user_profile, message, reaction, "remove") -def do_send_typing_notification(notification): - # type: (Dict[str, Any]) -> None +def do_send_typing_notification(notification: Dict[str, Any]) -> None: recipient_user_profiles = get_typing_user_profiles(notification['recipient'], notification['sender'].id) # Only deliver the notification to active user recipients @@ -1429,16 +1402,16 @@ def do_send_typing_notification(notification): # check_send_typing_notification: # Checks the typing notification and sends it -def check_send_typing_notification(sender, notification_to, operator): - # type: (UserProfile, Sequence[Text], Text) -> None +def check_send_typing_notification(sender: UserProfile, notification_to: Sequence[Text], + operator: Text) -> None: typing_notification = check_typing_notification(sender, notification_to, operator) do_send_typing_notification(typing_notification) # check_typing_notification: # Returns typing notification ready for sending with do_send_typing_notification on success # or the error message (string) on error. -def check_typing_notification(sender, notification_to, operator): - # type: (UserProfile, Sequence[Text], Text) -> Dict[str, Any] +def check_typing_notification(sender: UserProfile, notification_to: Sequence[Text], + operator: Text) -> Dict[str, Any]: if len(notification_to) == 0: raise JsonableError(_('Missing parameter: \'to\' (recipient)')) elif operator not in ('start', 'stop'): @@ -1454,8 +1427,7 @@ def check_typing_notification(sender, notification_to, operator): raise ValueError('Forbidden recipient type') return {'sender': sender, 'recipient': recipient, 'op': operator} -def stream_welcome_message(stream): - # type: (Stream) -> Text +def stream_welcome_message(stream: Stream) -> Text: content = _('Welcome to #**%s**.') % (stream.name,) if stream.description: @@ -1464,8 +1436,7 @@ def stream_welcome_message(stream): return content -def prep_stream_welcome_message(stream): - # type: (Stream) -> Optional[Dict[str, Any]] +def prep_stream_welcome_message(stream: Stream) -> Optional[Dict[str, Any]]: realm = stream.realm sender = get_system_bot(settings.WELCOME_BOT) topic = _('hello') @@ -1480,14 +1451,15 @@ def prep_stream_welcome_message(stream): return message -def send_stream_creation_event(stream, user_ids): - # type: (Stream, List[int]) -> None +def send_stream_creation_event(stream: Stream, user_ids: List[int]) -> None: event = dict(type="stream", op="create", streams=[stream.to_dict()]) send_event(event, user_ids) -def create_stream_if_needed(realm, stream_name, invite_only=False, stream_description = ""): - # type: (Realm, Text, bool, Text) -> Tuple[Stream, bool] +def create_stream_if_needed(realm: Realm, + stream_name: Text, + invite_only: bool=False, + stream_description: Text="") -> Tuple[Stream, bool]: (stream, created) = Stream.objects.get_or_create( realm=realm, name__iexact=stream_name, @@ -1505,8 +1477,8 @@ def create_stream_if_needed(realm, stream_name, invite_only=False, stream_descri send_stream_creation_event(stream, active_user_ids(stream.realm_id)) return stream, created -def create_streams_if_needed(realm, stream_dicts): - # type: (Realm, List[Mapping[str, Any]]) -> Tuple[List[Stream], List[Stream]] +def create_streams_if_needed(realm: Realm, + stream_dicts: List[Mapping[str, Any]]) -> Tuple[List[Stream], List[Stream]]: """Note that stream_dict["name"] is assumed to already be stripped of whitespace""" added_streams = [] # type: List[Stream] @@ -1525,11 +1497,10 @@ def create_streams_if_needed(realm, stream_dicts): return added_streams, existing_streams -def get_recipient_from_user_ids(recipient_profile_ids, - not_forged_mirror_message, - forwarder_user_profile, - sender): - # type: (Set[int], bool, Optional[UserProfile], UserProfile) -> Recipient +def get_recipient_from_user_ids(recipient_profile_ids: Set[int], + not_forged_mirror_message: bool, + forwarder_user_profile: Optional[UserProfile], + sender: UserProfile) -> Recipient: # Avoid mutating the passed in set of recipient_profile_ids. recipient_profile_ids = set(recipient_profile_ids) @@ -1552,8 +1523,8 @@ def get_recipient_from_user_ids(recipient_profile_ids, else: return get_personal_recipient(list(recipient_profile_ids)[0]) -def validate_recipient_user_profiles(user_profiles, sender): - # type: (List[UserProfile], UserProfile) -> Set[int] +def validate_recipient_user_profiles(user_profiles: List[UserProfile], + sender: UserProfile) -> Set[int]: recipient_profile_ids = set() # We exempt cross-realm bots from the check that all the recipients @@ -1598,8 +1569,7 @@ def recipient_for_user_profiles(user_profiles, not_forged_mirror_message, return get_recipient_from_user_ids(recipient_profile_ids, not_forged_mirror_message, forwarder_user_profile, sender) -def already_sent_mirrored_message_id(message): - # type: (Message) -> Optional[int] +def already_sent_mirrored_message_id(message: Message) -> Optional[int]: if message.recipient.type == Recipient.HUDDLE: # For huddle messages, we use a 10-second window because the # timestamps aren't guaranteed to actually match between two @@ -1621,8 +1591,7 @@ def already_sent_mirrored_message_id(message): return messages[0].id return None -def extract_recipients(s): - # type: (Union[str, Iterable[Text]]) -> List[Text] +def extract_recipients(s: Union[str, Iterable[Text]]) -> List[Text]: # We try to accept multiple incoming formats for recipients. # See test_extract_recipients() for examples of what we allow. try: @@ -1643,15 +1612,15 @@ def extract_recipients(s): recipients = [recipient.strip() for recipient in recipients] return list(set(recipient for recipient in recipients if recipient)) -def check_send_stream_message(sender, client, stream_name, topic, body): - # type: (UserProfile, Client, Text, Text, Text) -> int +def check_send_stream_message(sender: UserProfile, client: Client, stream_name: Text, + topic: Text, body: Text) -> int: addressee = Addressee.for_stream(stream_name, topic) message = check_message(sender, client, addressee, body) return do_send_messages([message])[0] -def check_send_private_message(sender, client, receiving_user, body): - # type: (UserProfile, Client, UserProfile, Text) -> int +def check_send_private_message(sender: UserProfile, client: Client, + receiving_user: UserProfile, body: Text) -> int: addressee = Addressee.for_user_profile(receiving_user) message = check_message(sender, client, addressee, body) @@ -1676,8 +1645,7 @@ def check_send_message(sender, client, message_type_name, message_to, forwarder_user_profile, local_id, sender_queue_id) return do_send_messages([message])[0] -def check_stream_name(stream_name): - # type: (Text) -> None +def check_stream_name(stream_name: Text) -> None: if stream_name.strip() == "": raise JsonableError(_("Invalid stream name '%s'" % (stream_name))) if len(stream_name) > Stream.MAX_NAME_LENGTH: @@ -1697,8 +1665,10 @@ def check_default_stream_group_name(group_name: Text) -> None: raise JsonableError(_("Default stream group name '%s' contains NULL (0x00) characters." % (group_name))) -def send_pm_if_empty_stream(sender, stream, stream_name, realm): - # type: (UserProfile, Optional[Stream], Text, Realm) -> None +def send_pm_if_empty_stream(sender: UserProfile, + stream: Optional[Stream], + stream_name: Text, + realm: Realm) -> None: """If a bot sends a message to a stream that doesn't exist or has no subscribers, sends a notification to the bot owner (if not a cross-realm bot) so that the owner can correct the issue.""" @@ -1855,8 +1825,10 @@ def check_message(sender, client, addressee, return {'message': message, 'stream': stream, 'local_id': local_id, 'sender_queue_id': sender_queue_id, 'realm': realm} -def _internal_prep_message(realm, sender, addressee, content): - # type: (Realm, UserProfile, Addressee, Text) -> Optional[Dict[str, Any]] +def _internal_prep_message(realm: Realm, + sender: UserProfile, + addressee: Addressee, + content: Text) -> Optional[Dict[str, Any]]: """ Create a message object and checks it, but doesn't send it or save it to the database. The internal function that calls this can therefore batch send a bunch of created @@ -1881,8 +1853,9 @@ def _internal_prep_message(realm, sender, addressee, content): return None -def internal_prep_stream_message(realm, sender, stream_name, topic, content): - # type: (Realm, UserProfile, Text, Text, Text) -> Optional[Dict[str, Any]] +def internal_prep_stream_message(realm: Realm, sender: UserProfile, + stream_name: Text, topic: Text, + content: Text) -> Optional[Dict[str, Any]]: """ See _internal_prep_message for details of how this works. """ @@ -1895,8 +1868,10 @@ def internal_prep_stream_message(realm, sender, stream_name, topic, content): content=content, ) -def internal_prep_private_message(realm, sender, recipient_user, content): - # type: (Realm, UserProfile, UserProfile, Text) -> Optional[Dict[str, Any]] +def internal_prep_private_message(realm: Realm, + sender: UserProfile, + recipient_user: UserProfile, + content: Text) -> Optional[Dict[str, Any]]: """ See _internal_prep_message for details of how this works. """ @@ -1940,22 +1915,24 @@ def internal_send_message(realm, sender_email, recipient_type_name, recipients, do_send_messages([msg], email_gateway=email_gateway) -def internal_send_private_message(realm, sender, recipient_user, content): - # type: (Realm, UserProfile, UserProfile, Text) -> None +def internal_send_private_message(realm: Realm, + sender: UserProfile, + recipient_user: UserProfile, + content: Text) -> None: message = internal_prep_private_message(realm, sender, recipient_user, content) if message is None: return do_send_messages([message]) -def internal_send_stream_message(realm, sender, stream_name, topic, content): - # type: (Realm, UserProfile, str, str, str) -> None +def internal_send_stream_message(realm: Realm, sender: UserProfile, stream_name: str, + topic: str, content: str) -> None: message = internal_prep_stream_message(realm, sender, stream_name, topic, content) if message is None: return do_send_messages([message]) -def internal_send_huddle_message(realm, sender, emails, content): - # type: (Realm, UserProfile, List[str], str) -> None +def internal_send_huddle_message(realm: Realm, sender: UserProfile, emails: List[str], + content: str) -> None: addressee = Addressee.for_private(emails, realm) message = _internal_prep_message( realm=realm, @@ -1967,13 +1944,11 @@ def internal_send_huddle_message(realm, sender, emails, content): return do_send_messages([message]) -def pick_color(user_profile): - # type: (UserProfile) -> Text +def pick_color(user_profile: UserProfile) -> Text: subs = get_stream_subscriptions_for_user(user_profile).filter(active=True) return pick_color_helper(user_profile, subs) -def pick_color_helper(user_profile, subs): - # type: (UserProfile, Iterable[Subscription]) -> Text +def pick_color_helper(user_profile: UserProfile, subs: Iterable[Subscription]) -> Text: # These colors are shared with the palette in subs.js. used_colors = [sub.color for sub in subs if sub.active] available_colors = [s for s in STREAM_ASSIGNMENT_COLORS if s not in used_colors] @@ -1983,8 +1958,8 @@ def pick_color_helper(user_profile, subs): else: return STREAM_ASSIGNMENT_COLORS[len(used_colors) % len(STREAM_ASSIGNMENT_COLORS)] -def validate_user_access_to_subscribers(user_profile, stream): - # type: (Optional[UserProfile], Stream) -> None +def validate_user_access_to_subscribers(user_profile: Optional[UserProfile], + stream: Stream) -> None: """ Validates whether the user can view the subscribers of a stream. Raises a JsonableError if: * The user and the stream are in different realms * The realm is MIT and the stream is not invite only. @@ -1999,8 +1974,9 @@ def validate_user_access_to_subscribers(user_profile, stream): # user is subscribed if we have to lambda: subscribed_to_stream(cast(UserProfile, user_profile), stream.id)) -def validate_user_access_to_subscribers_helper(user_profile, stream_dict, check_user_subscribed): - # type: (Optional[UserProfile], Mapping[str, Any], Callable[[], bool]) -> None +def validate_user_access_to_subscribers_helper(user_profile: Optional[UserProfile], + stream_dict: Mapping[str, Any], + check_user_subscribed: Callable[[], bool]) -> None: """ Helper for validate_user_access_to_subscribers that doesn't require a full stream object * check_user_subscribed is a function that when called with no arguments, will report whether the user is subscribed to the stream @@ -2018,8 +1994,10 @@ def validate_user_access_to_subscribers_helper(user_profile, stream_dict, check_ raise JsonableError(_("Unable to retrieve subscribers for invite-only stream")) # sub_dict is a dictionary mapping stream_id => whether the user is subscribed to that stream -def bulk_get_subscriber_user_ids(stream_dicts, user_profile, sub_dict, stream_recipient): - # type: (Iterable[Mapping[str, Any]], UserProfile, Mapping[int, bool], StreamRecipientMap) -> Dict[int, List[int]] +def bulk_get_subscriber_user_ids(stream_dicts: Iterable[Mapping[str, Any]], + user_profile: UserProfile, + sub_dict: Mapping[int, bool], + stream_recipient: StreamRecipientMap) -> Dict[int, List[int]]: target_stream_dicts = [] for stream_dict in stream_dicts: try: @@ -2083,8 +2061,7 @@ def bulk_get_subscriber_user_ids(stream_dicts, user_profile, sub_dict, stream_re return result -def get_subscribers_query(stream, requesting_user): - # type: (Stream, Optional[UserProfile]) -> QuerySet +def get_subscribers_query(stream: Stream, requesting_user: Optional[UserProfile]) -> QuerySet: # TODO: Make a generic stub for QuerySet """ Build a query to get the subscribers list for a stream, raising a JsonableError if: @@ -2103,19 +2080,18 @@ def get_subscribers_query(stream, requesting_user): ) return subscriptions -def get_subscribers(stream, requesting_user=None): - # type: (Stream, Optional[UserProfile]) -> List[UserProfile] +def get_subscribers(stream: Stream, + requesting_user: Optional[UserProfile]=None) -> List[UserProfile]: subscriptions = get_subscribers_query(stream, requesting_user).select_related() return [subscription.user_profile for subscription in subscriptions] -def get_subscriber_emails(stream, requesting_user=None): - # type: (Stream, Optional[UserProfile]) -> List[Text] +def get_subscriber_emails(stream: Stream, + requesting_user: Optional[UserProfile]=None) -> List[Text]: subscriptions_query = get_subscribers_query(stream, requesting_user) subscriptions = subscriptions_query.values('user_profile__email') return [subscription['user_profile__email'] for subscription in subscriptions] -def maybe_get_subscriber_emails(stream, user_profile): - # type: (Stream, UserProfile) -> List[Text] +def maybe_get_subscriber_emails(stream: Stream, user_profile: UserProfile) -> List[Text]: """ Alternate version of get_subscriber_emails that takes a Stream object only (not a name), and simply returns an empty list if unable to get a real subscriber list (because we're on the MIT realm). """ @@ -2125,8 +2101,10 @@ def maybe_get_subscriber_emails(stream, user_profile): subscribers = [] return subscribers -def notify_subscriptions_added(user_profile, sub_pairs, stream_user_ids, no_log=False): - # type: (UserProfile, Iterable[Tuple[Subscription, Stream]], Callable[[Stream], List[int]], bool) -> None +def notify_subscriptions_added(user_profile: UserProfile, + sub_pairs: Iterable[Tuple[Subscription, Stream]], + stream_user_ids: Callable[[Stream], List[int]], + no_log: bool=False) -> None: if not no_log: log_event({'type': 'subscription_added', 'user': user_profile.email, @@ -2151,8 +2129,9 @@ def notify_subscriptions_added(user_profile, sub_pairs, stream_user_ids, no_log= subscriptions=payload) send_event(event, [user_profile.id]) -def get_peer_user_ids_for_stream_change(stream, altered_user_ids, subscribed_user_ids): - # type: (Stream, Iterable[int], Iterable[int]) -> Set[int] +def get_peer_user_ids_for_stream_change(stream: Stream, + altered_user_ids: Iterable[int], + subscribed_user_ids: Iterable[int]) -> Set[int]: ''' altered_user_ids is a list of user_ids that we are adding/removing subscribed_user_ids is the list of already subscribed user_ids @@ -2172,8 +2151,7 @@ def get_peer_user_ids_for_stream_change(stream, altered_user_ids, subscribed_use # structure to stay up-to-date. return set(active_user_ids(stream.realm_id)) - set(altered_user_ids) -def get_user_ids_for_streams(streams): - # type: (Iterable[Stream]) -> Dict[int, List[int]] +def get_user_ids_for_streams(streams: Iterable[Stream]) -> Dict[int, List[int]]: stream_ids = [stream.id for stream in streams] all_subs = get_active_subscriptions_for_stream_ids(stream_ids).filter( @@ -2194,8 +2172,11 @@ def get_user_ids_for_streams(streams): return all_subscribers_by_stream -def bulk_add_subscriptions(streams, users, from_stream_creation=False, acting_user=None): - # type: (Iterable[Stream], Iterable[UserProfile], bool, Optional[UserProfile]) -> Tuple[List[Tuple[UserProfile, Stream]], List[Tuple[UserProfile, Stream]]] +SubT = Tuple[List[Tuple[UserProfile, Stream]], List[Tuple[UserProfile, Stream]]] +def bulk_add_subscriptions(streams: Iterable[Stream], + users: Iterable[UserProfile], + from_stream_creation: bool=False, + acting_user: Optional[UserProfile]=None) -> SubT: users = list(users) recipients_map = bulk_get_recipients(Recipient.STREAM, [stream.id for stream in streams]) # type: Mapping[int, Recipient] @@ -2289,8 +2270,7 @@ def bulk_add_subscriptions(streams, users, from_stream_creation=False, acting_us # the following code and we want to minize DB queries all_subscribers_by_stream = get_user_ids_for_streams(streams=streams) - def fetch_stream_subscriber_user_ids(stream): - # type: (Stream) -> List[int] + def fetch_stream_subscriber_user_ids(stream: Stream) -> List[int]: if stream.is_in_zephyr_realm and not stream.invite_only: return [] user_ids = all_subscribers_by_stream[stream.id] @@ -2349,8 +2329,8 @@ def bulk_add_subscriptions(streams, users, from_stream_creation=False, acting_us [(sub.user_profile, stream) for (sub, stream) in subs_to_activate], already_subscribed) -def notify_subscriptions_removed(user_profile, streams, no_log=False): - # type: (UserProfile, Iterable[Stream], bool) -> None +def notify_subscriptions_removed(user_profile: UserProfile, streams: Iterable[Stream], + no_log: bool=False) -> None: if not no_log: log_event({'type': 'subscription_removed', 'user': user_profile.email, @@ -2362,8 +2342,10 @@ def notify_subscriptions_removed(user_profile, streams, no_log=False): subscriptions=payload) send_event(event, [user_profile.id]) -def bulk_remove_subscriptions(users, streams, acting_user=None): - # type: (Iterable[UserProfile], Iterable[Stream], Optional[UserProfile]) -> Tuple[List[Tuple[UserProfile, Stream]], List[Tuple[UserProfile, Stream]]] +SubAndRemovedT = Tuple[List[Tuple[UserProfile, Stream]], List[Tuple[UserProfile, Stream]]] +def bulk_remove_subscriptions(users: Iterable[UserProfile], + streams: Iterable[Stream], + acting_user: Optional[UserProfile]=None) -> SubAndRemovedT: users = list(users) streams = list(streams) @@ -2372,8 +2354,7 @@ def bulk_remove_subscriptions(users, streams, acting_user=None): existing_subs_by_user = get_bulk_stream_subscriber_info(users, stream_dict) - def get_non_subscribed_tups(): - # type: () -> List[Tuple[UserProfile, Stream]] + def get_non_subscribed_tups() -> List[Tuple[UserProfile, Stream]]: stream_ids = {stream.id for stream in streams} not_subscribed = [] # type: List[Tuple[UserProfile, Stream]] @@ -2492,8 +2473,8 @@ def bulk_remove_subscriptions(users, streams, acting_user=None): not_subscribed, ) -def log_subscription_property_change(user_email, stream_name, property, value): - # type: (Text, Text, Text, Any) -> None +def log_subscription_property_change(user_email: Text, stream_name: Text, property: Text, + value: Any) -> None: event = {'type': 'subscription_property', 'property': property, 'user': user_email, @@ -2533,8 +2514,8 @@ def do_change_password(user_profile, password, commit=True, modified_user=user_profile, event_type='user_change_password', event_time=event_time) -def do_change_full_name(user_profile, full_name, acting_user): - # type: (UserProfile, Text, UserProfile) -> None +def do_change_full_name(user_profile: UserProfile, full_name: Text, + acting_user: UserProfile) -> None: old_name = user_profile.full_name user_profile.full_name = full_name user_profile.save(update_fields=["full_name"]) @@ -2561,8 +2542,8 @@ def check_change_full_name(user_profile: UserProfile, full_name_raw: Text, do_change_full_name(user_profile, new_full_name, acting_user) return new_full_name -def do_change_bot_owner(user_profile, bot_owner, acting_user): - # type: (UserProfile, UserProfile, UserProfile) -> None +def do_change_bot_owner(user_profile: UserProfile, bot_owner: UserProfile, + acting_user: UserProfile) -> None: user_profile.bot_owner = bot_owner user_profile.save() # Can't use update_fields because of how the foreign key works. event_time = timezone_now() @@ -2577,8 +2558,7 @@ def do_change_bot_owner(user_profile, bot_owner, acting_user): )), bot_owner_user_ids(user_profile)) -def do_change_tos_version(user_profile, tos_version): - # type: (UserProfile, Text) -> None +def do_change_tos_version(user_profile: UserProfile, tos_version: Text) -> None: user_profile.tos_version = tos_version user_profile.save(update_fields=["tos_version"]) event_time = timezone_now() @@ -2586,8 +2566,7 @@ def do_change_tos_version(user_profile, tos_version): modified_user=user_profile, event_type='user_tos_version_changed', event_time=event_time) -def do_regenerate_api_key(user_profile, acting_user): - # type: (UserProfile, UserProfile) -> None +def do_regenerate_api_key(user_profile: UserProfile, acting_user: UserProfile) -> None: user_profile.api_key = random_api_key() user_profile.save(update_fields=["api_key"]) event_time = timezone_now() @@ -2604,8 +2583,7 @@ def do_regenerate_api_key(user_profile, acting_user): )), bot_owner_user_ids(user_profile)) -def do_change_avatar_fields(user_profile, avatar_source): - # type: (UserProfile, Text) -> None +def do_change_avatar_fields(user_profile: UserProfile, avatar_source: Text) -> None: user_profile.avatar_source = avatar_source user_profile.avatar_version += 1 user_profile.save(update_fields=["avatar_source", "avatar_version"]) @@ -2638,8 +2616,7 @@ def do_change_avatar_fields(user_profile, avatar_source): active_user_ids(user_profile.realm_id)) -def do_change_icon_source(realm, icon_source, log=True): - # type: (Realm, Text, bool) -> None +def do_change_icon_source(realm: Realm, icon_source: Text, log: bool=True) -> None: realm.icon_source = icon_source realm.icon_version += 1 realm.save(update_fields=["icon_source", "icon_version"]) @@ -2656,8 +2633,7 @@ def do_change_icon_source(realm, icon_source, log=True): icon_url=realm_icon_url(realm))), active_user_ids(realm.id)) -def _default_stream_permision_check(user_profile, stream): - # type: (UserProfile, Optional[Stream]) -> None +def _default_stream_permision_check(user_profile: UserProfile, stream: Optional[Stream]) -> None: # Any user can have a None default stream if stream is not None: if user_profile.is_bot: @@ -2667,8 +2643,8 @@ def _default_stream_permision_check(user_profile, stream): if stream.invite_only and (user is None or not subscribed_to_stream(user, stream.id)): raise JsonableError(_('Insufficient permission')) -def do_change_default_sending_stream(user_profile, stream, log=True): - # type: (UserProfile, Optional[Stream], bool) -> None +def do_change_default_sending_stream(user_profile: UserProfile, stream: Optional[Stream], + log: bool=True) -> None: _default_stream_permision_check(user_profile, stream) user_profile.default_sending_stream = stream @@ -2690,8 +2666,9 @@ def do_change_default_sending_stream(user_profile, stream, log=True): )), bot_owner_user_ids(user_profile)) -def do_change_default_events_register_stream(user_profile, stream, log=True): - # type: (UserProfile, Optional[Stream], bool) -> None +def do_change_default_events_register_stream(user_profile: UserProfile, + stream: Optional[Stream], + log: bool=True) -> None: _default_stream_permision_check(user_profile, stream) user_profile.default_events_register_stream = stream @@ -2713,8 +2690,8 @@ def do_change_default_events_register_stream(user_profile, stream, log=True): )), bot_owner_user_ids(user_profile)) -def do_change_default_all_public_streams(user_profile, value, log=True): - # type: (UserProfile, bool, bool) -> None +def do_change_default_all_public_streams(user_profile: UserProfile, value: bool, + log: bool=True) -> None: user_profile.default_all_public_streams = value user_profile.save(update_fields=['default_all_public_streams']) if log: @@ -2730,8 +2707,8 @@ def do_change_default_all_public_streams(user_profile, value, log=True): )), bot_owner_user_ids(user_profile)) -def do_change_is_admin(user_profile, value, permission='administer'): - # type: (UserProfile, bool, str) -> None +def do_change_is_admin(user_profile: UserProfile, value: bool, + permission: str='administer') -> None: if permission == "administer": user_profile.is_realm_admin = value user_profile.save(update_fields=["is_realm_admin"]) @@ -2748,18 +2725,15 @@ def do_change_is_admin(user_profile, value, permission='administer'): is_admin=value)) send_event(event, active_user_ids(user_profile.realm_id)) -def do_change_bot_type(user_profile, value): - # type: (UserProfile, int) -> None +def do_change_bot_type(user_profile: UserProfile, value: int) -> None: user_profile.bot_type = value user_profile.save(update_fields=["bot_type"]) -def do_change_stream_invite_only(stream, invite_only): - # type: (Stream, bool) -> None +def do_change_stream_invite_only(stream: Stream, invite_only: bool) -> None: stream.invite_only = invite_only stream.save(update_fields=['invite_only']) -def do_rename_stream(stream, new_name, log=True): - # type: (Stream, Text, bool) -> Dict[str, Text] +def do_rename_stream(stream: Stream, new_name: Text, log: bool=True) -> Dict[str, Text]: old_name = stream.name stream.name = new_name stream.save(update_fields=["name"]) @@ -2815,8 +2789,7 @@ def do_rename_stream(stream, new_name, log=True): # email forwarding address to display the correctly-escaped new name. return {"email_address": new_email} -def do_change_stream_description(stream, new_description): - # type: (Stream, Text) -> None +def do_change_stream_description(stream: Stream, new_description: Text) -> None: stream.description = new_description stream.save(update_fields=['description']) @@ -2873,8 +2846,8 @@ def do_create_realm(string_id, name, restricted_to_domain=None, "signups", realm.display_subdomain, signup_message) return realm -def do_change_notification_settings(user_profile, name, value, log=True): - # type: (UserProfile, str, bool, bool) -> None +def do_change_notification_settings(user_profile: UserProfile, name: str, value: bool, + log: bool=True) -> None: """Takes in a UserProfile object, the name of a global notification preference to update, and the value to update to """ @@ -2899,8 +2872,8 @@ def do_change_notification_settings(user_profile, name, value, log=True): log_event(event) send_event(event, [user_profile.id]) -def do_change_autoscroll_forever(user_profile, autoscroll_forever, log=True): - # type: (UserProfile, bool, bool) -> None +def do_change_autoscroll_forever(user_profile: UserProfile, autoscroll_forever: bool, + log: bool=True) -> None: user_profile.autoscroll_forever = autoscroll_forever user_profile.save(update_fields=["autoscroll_forever"]) @@ -2909,18 +2882,18 @@ def do_change_autoscroll_forever(user_profile, autoscroll_forever, log=True): 'user': user_profile.email, 'autoscroll_forever': autoscroll_forever}) -def do_change_enter_sends(user_profile, enter_sends): - # type: (UserProfile, bool) -> None +def do_change_enter_sends(user_profile: UserProfile, enter_sends: bool) -> None: user_profile.enter_sends = enter_sends user_profile.save(update_fields=["enter_sends"]) -def do_change_default_desktop_notifications(user_profile, default_desktop_notifications): - # type: (UserProfile, bool) -> None +def do_change_default_desktop_notifications(user_profile: UserProfile, + default_desktop_notifications: bool) -> None: user_profile.default_desktop_notifications = default_desktop_notifications user_profile.save(update_fields=["default_desktop_notifications"]) -def do_set_user_display_setting(user_profile, setting_name, setting_value): - # type: (UserProfile, str, Union[bool, Text]) -> None +def do_set_user_display_setting(user_profile: UserProfile, + setting_name: str, + setting_value: Union[bool, Text]) -> None: property_type = UserProfile.property_types[setting_name] assert isinstance(setting_value, property_type) setattr(user_profile, setting_name, setting_value) @@ -2939,8 +2912,8 @@ def do_set_user_display_setting(user_profile, setting_name, setting_value): send_event(dict(type='realm_user', op='update', person=payload), active_user_ids(user_profile.realm_id)) -def lookup_default_stream_groups(default_stream_group_names, realm): - # type: (List[str], Realm) -> List[DefaultStreamGroup] +def lookup_default_stream_groups(default_stream_group_names: List[str], + realm: Realm) -> List[DefaultStreamGroup]: default_stream_groups = [] for group_name in default_stream_group_names: try: @@ -2951,8 +2924,7 @@ def lookup_default_stream_groups(default_stream_group_names, realm): default_stream_groups.append(default_stream_group) return default_stream_groups -def set_default_streams(realm, stream_dict): - # type: (Realm, Dict[Text, Dict[Text, Any]]) -> None +def set_default_streams(realm: Realm, stream_dict: Dict[Text, Dict[Text, Any]]) -> None: DefaultStream.objects.filter(realm=realm).delete() stream_names = [] for name, options in stream_dict.items(): @@ -2971,8 +2943,7 @@ def set_default_streams(realm, stream_dict): 'realm': realm.string_id, 'streams': stream_names}) -def notify_default_streams(realm_id): - # type: (int) -> None +def notify_default_streams(realm_id: int) -> None: event = dict( type="default_streams", default_streams=streams_to_dicts_sorted(get_default_streams_for_realm(realm_id)) @@ -2986,16 +2957,14 @@ def notify_default_stream_groups(realm: Realm) -> None: ) send_event(event, active_user_ids(realm.id)) -def do_add_default_stream(stream): - # type: (Stream) -> None +def do_add_default_stream(stream: Stream) -> None: realm_id = stream.realm_id stream_id = stream.id if not DefaultStream.objects.filter(realm_id=realm_id, stream_id=stream_id).exists(): DefaultStream.objects.create(realm_id=realm_id, stream_id=stream_id) notify_default_streams(realm_id) -def do_remove_default_stream(stream): - # type: (Stream) -> None +def do_remove_default_stream(stream: Stream) -> None: realm_id = stream.realm_id stream_id = stream.id DefaultStream.objects.filter(realm_id=realm_id, stream_id=stream_id).delete() @@ -3072,28 +3041,25 @@ def do_remove_default_stream_group(realm: Realm, group: DefaultStreamGroup) -> N group.delete() notify_default_stream_groups(realm) -def get_default_streams_for_realm(realm_id): - # type: (int) -> List[Stream] +def get_default_streams_for_realm(realm_id: int) -> List[Stream]: return [default.stream for default in DefaultStream.objects.select_related("stream", "stream__realm").filter( realm_id=realm_id)] -def get_default_subs(user_profile): - # type: (UserProfile) -> List[Stream] +def get_default_subs(user_profile: UserProfile) -> List[Stream]: # Right now default streams are realm-wide. This wrapper gives us flexibility # to some day further customize how we set up default streams for new users. return get_default_streams_for_realm(user_profile.realm_id) # returns default streams in json serializeable format -def streams_to_dicts_sorted(streams): - # type: (List[Stream]) -> List[Dict[str, Any]] +def streams_to_dicts_sorted(streams: List[Stream]) -> List[Dict[str, Any]]: return sorted([stream.to_dict() for stream in streams], key=lambda elt: elt["name"]) def default_stream_groups_to_dicts_sorted(groups: List[DefaultStreamGroup]) -> List[Dict[str, Any]]: return sorted([group.to_dict() for group in groups], key=lambda elt: elt["name"]) -def do_update_user_activity_interval(user_profile, log_time): - # type: (UserProfile, datetime.datetime) -> None +def do_update_user_activity_interval(user_profile: UserProfile, + log_time: datetime.datetime) -> None: effective_end = log_time + UserActivityInterval.MIN_INTERVAL_LENGTH # This code isn't perfect, because with various races we might end # up creating two overlapping intervals, but that shouldn't happen @@ -3118,8 +3084,10 @@ def do_update_user_activity_interval(user_profile, log_time): end=effective_end) @statsd_increment('user_activity') -def do_update_user_activity(user_profile, client, query, log_time): - # type: (UserProfile, Client, Text, datetime.datetime) -> None +def do_update_user_activity(user_profile: UserProfile, + client: Client, + query: Text, + log_time: datetime.datetime) -> None: (activity, created) = UserActivity.objects.get_or_create( user_profile = user_profile, client = client, @@ -3130,16 +3098,14 @@ def do_update_user_activity(user_profile, client, query, log_time): activity.last_visit = log_time activity.save(update_fields=["last_visit", "count"]) -def send_presence_changed(user_profile, presence): - # type: (UserProfile, UserPresence) -> None +def send_presence_changed(user_profile: UserProfile, presence: UserPresence) -> None: presence_dict = presence.to_dict() event = dict(type="presence", email=user_profile.email, server_timestamp=time.time(), presence={presence_dict['client']: presence_dict}) send_event(event, active_user_ids(user_profile.realm_id)) -def consolidate_client(client): - # type: (Client) -> Client +def consolidate_client(client: Client) -> Client: # The web app reports a client as 'website' # The desktop app reports a client as ZulipDesktop # due to it setting a custom user agent. We want both @@ -3152,8 +3118,10 @@ def consolidate_client(client): return client @statsd_increment('user_presence') -def do_update_user_presence(user_profile, client, log_time, status): - # type: (UserProfile, Client, datetime.datetime, int) -> None +def do_update_user_presence(user_profile: UserProfile, + client: Client, + log_time: datetime.datetime, + status: int) -> None: client = consolidate_client(client) (presence, created) = UserPresence.objects.get_or_create( user_profile = user_profile, @@ -3197,8 +3165,7 @@ def do_update_user_presence(user_profile, client, log_time, status): # realms are pretty small. send_presence_changed(user_profile, presence) -def update_user_activity_interval(user_profile, log_time): - # type: (UserProfile, datetime.datetime) -> None +def update_user_activity_interval(user_profile: UserProfile, log_time: datetime.datetime) -> None: event = {'user_profile_id': user_profile.id, 'time': datetime_to_timestamp(log_time)} queue_json_publish("user_activity_interval", event) @@ -3216,8 +3183,7 @@ def update_user_presence(user_profile, client, log_time, status, if new_user_input: update_user_activity_interval(user_profile, log_time) -def do_update_pointer(user_profile, pointer, update_flags=False): - # type: (UserProfile, int, bool) -> None +def do_update_pointer(user_profile: UserProfile, pointer: int, update_flags: bool=False) -> None: prev_pointer = user_profile.pointer user_profile.pointer = pointer user_profile.save(update_fields=["pointer"]) @@ -3235,8 +3201,7 @@ def do_update_pointer(user_profile, pointer, update_flags=False): event = dict(type='pointer', pointer=pointer) send_event(event, [user_profile.id]) -def do_mark_all_as_read(user_profile): - # type: (UserProfile) -> int +def do_mark_all_as_read(user_profile: UserProfile) -> int: log_statsd_event('bankruptcy') msgs = UserMessage.objects.filter( @@ -3261,8 +3226,9 @@ def do_mark_all_as_read(user_profile): statsd.incr("mark_all_as_read", count) return count -def do_mark_stream_messages_as_read(user_profile, stream, topic_name=None): - # type: (UserProfile, Optional[Stream], Optional[Text]) -> int +def do_mark_stream_messages_as_read(user_profile: UserProfile, + stream: Optional[Stream], + topic_name: Optional[Text]=None) -> int: log_statsd_event('mark_stream_as_read') msgs = UserMessage.objects.filter( @@ -3297,8 +3263,10 @@ def do_mark_stream_messages_as_read(user_profile, stream, topic_name=None): statsd.incr("mark_stream_as_read", count) return count -def do_update_message_flags(user_profile, operation, flag, messages): - # type: (UserProfile, Text, Text, Optional[Sequence[int]]) -> int +def do_update_message_flags(user_profile: UserProfile, + operation: Text, + flag: Text, + messages: Optional[Sequence[int]]) -> int: flagattr = getattr(UserMessage.flags, flag) assert messages is not None @@ -3338,8 +3306,7 @@ def do_update_message_flags(user_profile, operation, flag, messages): statsd.incr("flags.%s.%s" % (flag, operation), count) return count -def subscribed_to_stream(user_profile, stream_id): - # type: (UserProfile, int) -> bool +def subscribed_to_stream(user_profile: UserProfile, stream_id: int) -> bool: try: if Subscription.objects.get(user_profile=user_profile, active=True, @@ -3350,18 +3317,15 @@ def subscribed_to_stream(user_profile, stream_id): except Subscription.DoesNotExist: return False -def truncate_content(content, max_length, truncation_message): - # type: (Text, int, Text) -> Text +def truncate_content(content: Text, max_length: int, truncation_message: Text) -> Text: if len(content) > max_length: content = content[:max_length - len(truncation_message)] + truncation_message return content -def truncate_body(body): - # type: (Text) -> Text +def truncate_body(body: Text) -> Text: return truncate_content(body, MAX_MESSAGE_LENGTH, "...") -def truncate_topic(topic): - # type: (Text) -> Text +def truncate_topic(topic: Text) -> Text: return truncate_content(topic, MAX_SUBJECT_LENGTH, "...") MessageUpdateUserInfoResult = TypedDict('MessageUpdateUserInfoResult', { @@ -3369,8 +3333,7 @@ MessageUpdateUserInfoResult = TypedDict('MessageUpdateUserInfoResult', { 'mention_user_ids': Set[int], }) -def get_user_info_for_message_updates(message_id): - # type: (int) -> MessageUpdateUserInfoResult +def get_user_info_for_message_updates(message_id: int) -> MessageUpdateUserInfoResult: # We exclude UserMessage.flags.historical rows since those # users did not receive the message originally, and thus @@ -3401,15 +3364,13 @@ def get_user_info_for_message_updates(message_id): mention_user_ids=mention_user_ids, ) -def update_user_message_flags(message, ums): - # type: (Message, Iterable[UserMessage]) -> None +def update_user_message_flags(message: Message, ums: Iterable[UserMessage]) -> None: wildcard = message.mentions_wildcard mentioned_ids = message.mentions_user_ids ids_with_alert_words = message.user_ids_with_alert_words changed_ums = set() # type: Set[UserMessage] - def update_flag(um, should_set, flag): - # type: (UserMessage, bool, int) -> None + def update_flag(um: UserMessage, should_set: bool, flag: int) -> None: if should_set: if not (um.flags & flag): um.flags |= flag @@ -3431,8 +3392,7 @@ def update_user_message_flags(message, ums): for um in changed_ums: um.save(update_fields=['flags']) -def update_to_dict_cache(changed_messages): - # type: (List[Message]) -> List[int] +def update_to_dict_cache(changed_messages: List[Message]) -> List[int]: """Updates the message as stored in the to_dict cache (for serving messages).""" items_for_remote_cache = {} @@ -3448,8 +3408,10 @@ def update_to_dict_cache(changed_messages): # We use transaction.atomic to support select_for_update in the attachment codepath. @transaction.atomic -def do_update_embedded_data(user_profile, message, content, rendered_content): - # type: (UserProfile, Message, Optional[Text], Optional[Text]) -> None +def do_update_embedded_data(user_profile: UserProfile, + message: Message, + content: Optional[Text], + rendered_content: Optional[Text]) -> None: event = { 'type': 'update_message', 'sender': user_profile.email, @@ -3470,8 +3432,7 @@ def do_update_embedded_data(user_profile, message, content, rendered_content): event['message_ids'] = update_to_dict_cache(changed_messages) - def user_info(um): - # type: (UserMessage) -> Dict[str, Any] + def user_info(um: UserMessage) -> Dict[str, Any]: return { 'id': um.user_profile_id, 'flags': um.flags_list() @@ -3623,8 +3584,7 @@ def do_update_message(user_profile, message, topic_name, propagate_mode, event['message_ids'] = update_to_dict_cache(changed_messages) - def user_info(um): - # type: (UserMessage) -> Dict[str, Any] + def user_info(um: UserMessage) -> Dict[str, Any]: return { 'id': um.user_profile_id, 'flags': um.flags_list() @@ -3633,8 +3593,7 @@ def do_update_message(user_profile, message, topic_name, propagate_mode, return len(changed_messages) -def do_delete_message(user_profile, message): - # type: (UserProfile, Message) -> None +def do_delete_message(user_profile: UserProfile, message: Message) -> None: event = { 'type': 'delete_message', 'sender': user_profile.email, @@ -3645,12 +3604,10 @@ def do_delete_message(user_profile, message): send_event(event, ums) -def encode_email_address(stream): - # type: (Stream) -> Text +def encode_email_address(stream: Stream) -> Text: return encode_email_address_helper(stream.name, stream.email_token) -def encode_email_address_helper(name, email_token): - # type: (Text, Text) -> Text +def encode_email_address_helper(name: Text, email_token: Text) -> Text: # Some deployments may not use the email gateway if settings.EMAIL_GATEWAY_PATTERN == '': return '' @@ -3667,8 +3624,7 @@ def encode_email_address_helper(name, email_token): encoded_token = "%s+%s" % (encoded_name, email_token) return settings.EMAIL_GATEWAY_PATTERN % (encoded_token,) -def get_email_gateway_message_string_from_address(address): - # type: (Text) -> Optional[Text] +def get_email_gateway_message_string_from_address(address: Text) -> Optional[Text]: pattern_parts = [re.escape(part) for part in settings.EMAIL_GATEWAY_PATTERN.split('%s')] if settings.EMAIL_GATEWAY_EXTRA_PATTERN_HACK: # Accept mails delivered to any Zulip server @@ -3683,8 +3639,7 @@ def get_email_gateway_message_string_from_address(address): return msg_string -def decode_email_address(email): - # type: (Text) -> Optional[Tuple[Text, Text]] +def decode_email_address(email: Text) -> Optional[Tuple[Text, Text]]: # Perform the reverse of encode_email_address. Returns a tuple of (streamname, email_token) msg_string = get_email_gateway_message_string_from_address(email) @@ -3703,8 +3658,9 @@ def decode_email_address(email): # the code pretty ugly, but in this case, it has significant # performance impact for loading / for users with large numbers of # subscriptions, so it's worth optimizing. -def gather_subscriptions_helper(user_profile, include_subscribers=True): - # type: (UserProfile, bool) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]], List[Dict[str, Any]]] +SubHelperT = Tuple[List[Dict[str, Any]], List[Dict[str, Any]], List[Dict[str, Any]]] +def gather_subscriptions_helper(user_profile: UserProfile, + include_subscribers: bool=True) -> SubHelperT: sub_dicts = get_stream_subscriptions_for_user(user_profile).values( "recipient_id", "in_home_view", "color", "desktop_notifications", "audible_notifications", "push_notifications", "active", "pin_to_top" @@ -3817,8 +3773,7 @@ def gather_subscriptions_helper(user_profile, include_subscribers=True): sorted(unsubscribed, key=lambda x: x['name']), sorted(never_subscribed, key=lambda x: x['name'])) -def gather_subscriptions(user_profile): - # type: (UserProfile) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]] +def gather_subscriptions(user_profile: UserProfile) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: subscribed, unsubscribed, never_subscribed = gather_subscriptions_helper(user_profile) user_ids = set() for subs in [subscribed, unsubscribed, never_subscribed]: @@ -3835,8 +3790,11 @@ def gather_subscriptions(user_profile): return (subscribed, unsubscribed) -def get_active_presence_idle_user_ids(realm, sender_id, message_type, active_user_ids, user_flags): - # type: (Realm, int, str, Set[int], Dict[int, List[str]]) -> List[int] +def get_active_presence_idle_user_ids(realm: Realm, + sender_id: int, + message_type: str, + active_user_ids: Set[int], + user_flags: Dict[int, List[str]]) -> List[int]: ''' Given a list of active_user_ids, we build up a subset of those users who fit these criteria: @@ -3862,8 +3820,7 @@ def get_active_presence_idle_user_ids(realm, sender_id, message_type, active_use return filter_presence_idle_user_ids(user_ids) -def filter_presence_idle_user_ids(user_ids): - # type: (Set[int]) -> List[int] +def filter_presence_idle_user_ids(user_ids: Set[int]) -> List[int]: if not user_ids: return [] @@ -3877,16 +3834,14 @@ def filter_presence_idle_user_ids(user_ids): idle_user_ids = user_ids - active_user_ids return sorted(list(idle_user_ids)) -def get_status_dict(requesting_user_profile): - # type: (UserProfile) -> Dict[Text, Dict[Text, Dict[str, Any]]] +def get_status_dict(requesting_user_profile: UserProfile) -> Dict[Text, Dict[Text, Dict[str, Any]]]: if requesting_user_profile.realm.presence_disabled: # Return an empty dict if presence is disabled in this realm return defaultdict(dict) return UserPresence.get_status_dict_by_realm(requesting_user_profile.realm_id) -def get_cross_realm_dicts(): - # type: () -> List[Dict[str, Any]] +def get_cross_realm_dicts() -> List[Dict[str, Any]]: users = bulk_get_users(list(get_cross_realm_emails()), None, base_query=UserProfile.objects.filter( realm__string_id=settings.SYSTEM_BOT_REALM)).values() @@ -3901,8 +3856,9 @@ def get_cross_realm_dicts(): # cache with other UserProfile caches. if user.realm.string_id == settings.SYSTEM_BOT_REALM] -def do_send_confirmation_email(invitee, referrer, body): - # type: (PreregistrationUser, UserProfile, Optional[str]) -> None +def do_send_confirmation_email(invitee: PreregistrationUser, + referrer: UserProfile, + body: Optional[str]) -> None: """ Send the confirmation/welcome e-mail to an invited user. @@ -3945,8 +3901,7 @@ def validate_email_for_realm(target_realm: Realm, email: Text) -> None: # Other users should not already exist at all. raise ValidationError('%s already has an account' % (email,)) -def validate_email(user_profile, email): - # type: (UserProfile, Text) -> Tuple[Optional[str], Optional[str]] +def validate_email(user_profile: UserProfile, email: Text) -> Tuple[Optional[str], Optional[str]]: try: validators.validate_email(email) except ValidationError: @@ -3966,14 +3921,16 @@ class InvitationError(JsonableError): code = ErrorCode.INVITATION_FAILED data_fields = ['errors', 'sent_invitations'] - def __init__(self, msg, errors, sent_invitations): - # type: (Text, List[Tuple[Text, str]], bool) -> None + def __init__(self, msg: Text, errors: List[Tuple[Text, str]], sent_invitations: bool) -> None: self._msg = msg # type: Text self.errors = errors # type: List[Tuple[Text, str]] self.sent_invitations = sent_invitations # type: bool -def do_invite_users(user_profile, invitee_emails, streams, invite_as_admin=False, body=None): - # type: (UserProfile, SizedTextIterable, Iterable[Stream], Optional[bool], Optional[str]) -> None +def do_invite_users(user_profile: UserProfile, + invitee_emails: SizedTextIterable, + streams: Iterable[Stream], + invite_as_admin: Optional[bool]=False, + body: Optional[str]=None) -> None: validated_emails = [] # type: List[Text] errors = [] # type: List[Tuple[Text, str]] skipped = [] # type: List[Tuple[Text, str]] @@ -4022,8 +3979,7 @@ def do_invite_users(user_profile, invitee_emails, streams, invite_as_admin=False "invitations to everyone else!"), skipped, sent_invitations=True) -def do_get_user_invites(user_profile): - # type: (UserProfile) -> List[Dict[str, Any]] +def do_get_user_invites(user_profile: UserProfile) -> List[Dict[str, Any]]: days_to_activate = getattr(settings, 'ACCOUNT_ACTIVATION_DAYS', 7) active_value = getattr(confirmation_settings, 'STATUS_ACTIVE', 1) @@ -4042,8 +3998,7 @@ def do_get_user_invites(user_profile): return invites -def do_revoke_user_invite(invite_id, realm_id): - # type: (int, int) -> None +def do_revoke_user_invite(invite_id: int, realm_id: int) -> None: try: prereg_user = PreregistrationUser.objects.get(id=invite_id) except PreregistrationUser.DoesNotExist: @@ -4064,8 +4019,7 @@ def do_revoke_user_invite(invite_id, realm_id): prereg_user.delete() clear_scheduled_invitation_emails(email) -def do_resend_user_invite_email(invite_id, realm_id): - # type: (int, int) -> str +def do_resend_user_invite_email(invite_id: int, realm_id: int) -> str: try: prereg_user = PreregistrationUser.objects.get(id=invite_id) except PreregistrationUser.DoesNotExist: @@ -4082,7 +4036,9 @@ def do_resend_user_invite_email(invite_id, realm_id): from zerver.context_processors import common_context clear_scheduled_invitation_emails(prereg_user.email) - link = create_confirmation_link(prereg_user, prereg_user.referred_by.realm.host, Confirmation.INVITATION) + link = create_confirmation_link(prereg_user, + prereg_user.referred_by.realm.host, + Confirmation.INVITATION) context = common_context(prereg_user.referred_by) context.update({ 'activate_url': link, @@ -4098,66 +4054,58 @@ def do_resend_user_invite_email(invite_id, realm_id): return prereg_user.invited_at.strftime("%Y-%m-%d %H:%M:%S") -def notify_realm_emoji(realm): - # type: (Realm) -> None +def notify_realm_emoji(realm: Realm) -> None: event = dict(type="realm_emoji", op="update", realm_emoji=realm.get_emoji()) send_event(event, active_user_ids(realm.id)) -def check_add_realm_emoji(realm, name, file_name, author=None): - # type: (Realm, Text, Text, Optional[UserProfile]) -> None +def check_add_realm_emoji(realm: Realm, + name: Text, + file_name: Text, + author: Optional[UserProfile]=None) -> None: emoji = RealmEmoji(realm=realm, name=name, file_name=file_name, author=author) emoji.full_clean() emoji.save() notify_realm_emoji(realm) -def do_remove_realm_emoji(realm, name): - # type: (Realm, Text) -> None +def do_remove_realm_emoji(realm: Realm, name: Text) -> None: emoji = RealmEmoji.objects.get(realm=realm, name=name) emoji.deactivated = True emoji.save(update_fields=['deactivated']) notify_realm_emoji(realm) -def notify_alert_words(user_profile, words): - # type: (UserProfile, Iterable[Text]) -> None +def notify_alert_words(user_profile: UserProfile, words: Iterable[Text]) -> None: event = dict(type="alert_words", alert_words=words) send_event(event, [user_profile.id]) -def do_add_alert_words(user_profile, alert_words): - # type: (UserProfile, Iterable[Text]) -> None +def do_add_alert_words(user_profile: UserProfile, alert_words: Iterable[Text]) -> None: words = add_user_alert_words(user_profile, alert_words) notify_alert_words(user_profile, words) -def do_remove_alert_words(user_profile, alert_words): - # type: (UserProfile, Iterable[Text]) -> None +def do_remove_alert_words(user_profile: UserProfile, alert_words: Iterable[Text]) -> None: words = remove_user_alert_words(user_profile, alert_words) notify_alert_words(user_profile, words) -def do_set_alert_words(user_profile, alert_words): - # type: (UserProfile, List[Text]) -> None +def do_set_alert_words(user_profile: UserProfile, alert_words: List[Text]) -> None: set_user_alert_words(user_profile, alert_words) notify_alert_words(user_profile, alert_words) -def do_mute_topic(user_profile, stream, recipient, topic): - # type: (UserProfile, Stream, Recipient, str) -> None +def do_mute_topic(user_profile: UserProfile, stream: Stream, recipient: Recipient, topic: str) -> None: add_topic_mute(user_profile, stream.id, recipient.id, topic) event = dict(type="muted_topics", muted_topics=get_topic_mutes(user_profile)) send_event(event, [user_profile.id]) -def do_unmute_topic(user_profile, stream, topic): - # type: (UserProfile, Stream, str) -> None +def do_unmute_topic(user_profile: UserProfile, stream: Stream, topic: str) -> None: remove_topic_mute(user_profile, stream.id, topic) event = dict(type="muted_topics", muted_topics=get_topic_mutes(user_profile)) send_event(event, [user_profile.id]) -def do_mark_hotspot_as_read(user, hotspot): - # type: (UserProfile, str) -> None +def do_mark_hotspot_as_read(user: UserProfile, hotspot: str) -> None: UserHotspot.objects.get_or_create(user=user, hotspot=hotspot) event = dict(type="hotspots", hotspots=get_next_hotspots(user)) send_event(event, [user.id]) -def notify_realm_filters(realm): - # type: (Realm) -> None +def notify_realm_filters(realm: Realm) -> None: realm_filters = realm_filters_for_realm(realm.id) event = dict(type="realm_filters", realm_filters=realm_filters) send_event(event, active_user_ids(realm.id)) @@ -4166,8 +4114,7 @@ def notify_realm_filters(realm): # RegExp syntax. In addition to JS-compatible syntax, the following features are available: # * Named groups will be converted to numbered groups automatically # * Inline-regex flags will be stripped, and where possible translated to RegExp-wide flags -def do_add_realm_filter(realm, pattern, url_format_string): - # type: (Realm, Text, Text) -> int +def do_add_realm_filter(realm: Realm, pattern: Text, url_format_string: Text) -> int: pattern = pattern.strip() url_format_string = url_format_string.strip() realm_filter = RealmFilter( @@ -4179,21 +4126,19 @@ def do_add_realm_filter(realm, pattern, url_format_string): return realm_filter.id -def do_remove_realm_filter(realm, pattern=None, id=None): - # type: (Realm, Optional[Text], Optional[int]) -> None +def do_remove_realm_filter(realm: Realm, pattern: Optional[Text]=None, + id: Optional[int]=None) -> None: if pattern is not None: RealmFilter.objects.get(realm=realm, pattern=pattern).delete() else: RealmFilter.objects.get(realm=realm, pk=id).delete() notify_realm_filters(realm) -def get_emails_from_user_ids(user_ids): - # type: (Sequence[int]) -> Dict[int, Text] +def get_emails_from_user_ids(user_ids: Sequence[int]) -> Dict[int, Text]: # We may eventually use memcached to speed this up, but the DB is fast. return UserProfile.emails_from_ids(user_ids) -def do_add_realm_domain(realm, domain, allow_subdomains): - # type: (Realm, Text, bool) -> (RealmDomain) +def do_add_realm_domain(realm: Realm, domain: Text, allow_subdomains: bool) -> (RealmDomain): realm_domain = RealmDomain.objects.create(realm=realm, domain=domain, allow_subdomains=allow_subdomains) event = dict(type="realm_domains", op="add", @@ -4202,8 +4147,7 @@ def do_add_realm_domain(realm, domain, allow_subdomains): send_event(event, active_user_ids(realm.id)) return realm_domain -def do_change_realm_domain(realm_domain, allow_subdomains): - # type: (RealmDomain, bool) -> None +def do_change_realm_domain(realm_domain: RealmDomain, allow_subdomains: bool) -> None: realm_domain.allow_subdomains = allow_subdomains realm_domain.save(update_fields=['allow_subdomains']) event = dict(type="realm_domains", op="change", @@ -4211,8 +4155,7 @@ def do_change_realm_domain(realm_domain, allow_subdomains): allow_subdomains=realm_domain.allow_subdomains)) send_event(event, active_user_ids(realm_domain.realm_id)) -def do_remove_realm_domain(realm_domain): - # type: (RealmDomain) -> None +def do_remove_realm_domain(realm_domain: RealmDomain) -> None: realm = realm_domain.realm domain = realm_domain.domain realm_domain.delete() @@ -4225,8 +4168,7 @@ def do_remove_realm_domain(realm_domain): event = dict(type="realm_domains", op="remove", domain=domain) send_event(event, active_user_ids(realm.id)) -def get_occupied_streams(realm): - # type: (Realm) -> QuerySet +def get_occupied_streams(realm: Realm) -> QuerySet: # TODO: Make a generic stub for QuerySet """ Get streams with subscribers """ subs_filter = Subscription.objects.filter(active=True, user_profile__realm=realm, @@ -4279,8 +4221,7 @@ def do_get_streams(user_profile, include_public=True, include_subscribed=True, return streams -def do_claim_attachments(message): - # type: (Message) -> None +def do_claim_attachments(message: Message) -> None: attachment_url_list = attachment_url_re.findall(message.content) for url in attachment_url_list: @@ -4306,16 +4247,14 @@ def do_claim_attachments(message): claim_attachment(user_profile, path_id, message, is_message_realm_public) -def do_delete_old_unclaimed_attachments(weeks_ago): - # type: (int) -> None +def do_delete_old_unclaimed_attachments(weeks_ago: int) -> None: old_unclaimed_attachments = get_old_unclaimed_attachments(weeks_ago) for attachment in old_unclaimed_attachments: delete_message_image(attachment.path_id) attachment.delete() -def check_attachment_reference_change(prev_content, message): - # type: (Text, Message) -> None +def check_attachment_reference_change(prev_content: Text, message: Message) -> None: new_content = message.content prev_attachments = set(attachment_url_re.findall(prev_content)) new_attachments = set(attachment_url_re.findall(new_content)) @@ -4333,22 +4272,19 @@ def check_attachment_reference_change(prev_content, message): if len(to_add) > 0: do_claim_attachments(message) -def notify_realm_custom_profile_fields(realm): - # type: (Realm) -> None +def notify_realm_custom_profile_fields(realm: Realm) -> None: fields = custom_profile_fields_for_realm(realm.id) event = dict(type="custom_profile_fields", fields=[f.as_dict() for f in fields]) send_event(event, active_user_ids(realm.id)) -def try_add_realm_custom_profile_field(realm, name, field_type): - # type: (Realm, Text, int) -> CustomProfileField +def try_add_realm_custom_profile_field(realm: Realm, name: Text, field_type: int) -> CustomProfileField: field = CustomProfileField(realm=realm, name=name, field_type=field_type) field.save() notify_realm_custom_profile_fields(realm) return field -def do_remove_realm_custom_profile_field(realm, field): - # type: (Realm, CustomProfileField) -> None +def do_remove_realm_custom_profile_field(realm: Realm, field: CustomProfileField) -> None: """ Deleting a field will also delete the user profile data associated with it in CustomProfileFieldValue model. @@ -4356,14 +4292,14 @@ def do_remove_realm_custom_profile_field(realm, field): field.delete() notify_realm_custom_profile_fields(realm) -def try_update_realm_custom_profile_field(realm, field, name): - # type: (Realm, CustomProfileField, Text) -> None +def try_update_realm_custom_profile_field(realm: Realm, field: CustomProfileField, + name: Text) -> None: field.name = name field.save(update_fields=['name']) notify_realm_custom_profile_fields(realm) -def do_update_user_custom_profile_data(user_profile, data): - # type: (UserProfile, List[Dict[str, Union[int, Text]]]) -> None +def do_update_user_custom_profile_data(user_profile: UserProfile, + data: List[Dict[str, Union[int, Text]]]) -> None: with transaction.atomic(): update_or_create = CustomProfileFieldValue.objects.update_or_create for field in data: @@ -4413,7 +4349,8 @@ def do_send_user_group_members_update_event(event_name: Text, user_ids=user_ids) send_event(event, active_user_ids(user_group.realm_id)) -def bulk_add_members_to_user_group(user_group: UserGroup, user_profiles: List[UserProfile]) -> None: +def bulk_add_members_to_user_group(user_group: UserGroup, + user_profiles: List[UserProfile]) -> None: memberships = [UserGroupMembership(user_group_id=user_group.id, user_profile=user_profile) for user_profile in user_profiles] @@ -4422,7 +4359,8 @@ def bulk_add_members_to_user_group(user_group: UserGroup, user_profiles: List[Us user_ids = [up.id for up in user_profiles] do_send_user_group_members_update_event('add_members', user_group, user_ids) -def remove_members_from_user_group(user_group: UserGroup, user_profiles: List[UserProfile]) -> None: +def remove_members_from_user_group(user_group: UserGroup, + user_profiles: List[UserProfile]) -> None: UserGroupMembership.objects.filter( user_group_id=user_group.id, user_profile__in=user_profiles).delete() diff --git a/zerver/lib/bot_storage.py b/zerver/lib/bot_storage.py index afdfc8283a..26b309bd8c 100644 --- a/zerver/lib/bot_storage.py +++ b/zerver/lib/bot_storage.py @@ -9,15 +9,13 @@ from typing import Text, Optional, List, Tuple class StateError(Exception): pass -def get_bot_storage(bot_profile, key): - # type: (UserProfile, Text) -> Text +def get_bot_storage(bot_profile: UserProfile, key: Text) -> Text: try: return BotStorageData.objects.get(bot_profile=bot_profile, key=key).value except BotStorageData.DoesNotExist: raise StateError("Key does not exist.") -def get_bot_storage_size(bot_profile, key=None): - # type: (UserProfile, Optional[Text]) -> int +def get_bot_storage_size(bot_profile: UserProfile, key: Optional[Text]=None) -> int: if key is None: return BotStorageData.objects.filter(bot_profile=bot_profile) \ .annotate(key_size=Length('key'), value_size=Length('value')) \ diff --git a/zerver/lib/bugdown/__init__.py b/zerver/lib/bugdown/__init__.py index b1434c724e..45fad8846e 100644 --- a/zerver/lib/bugdown/__init__.py +++ b/zerver/lib/bugdown/__init__.py @@ -61,12 +61,7 @@ FullNameInfo = TypedDict('FullNameInfo', { version = 1 _T = TypeVar('_T') -# We need to avoid this running at runtime, but mypy will see this. -# The problem is that under python 2, Element isn't exactly a type, -# which means that at runtime Union causes this to blow up. -if False: - # mypy requires the Optional to be inside Union - ElementStringNone = Union[Element, Optional[Text]] +ElementStringNone = Union[Element, Optional[Text]] AVATAR_REGEX = r'!avatar\((?P[^)]*)\)' GRAVATAR_REGEX = r'!gravatar\((?P[^)]*)\)' @@ -82,8 +77,7 @@ STREAM_LINK_REGEX = r""" class BugdownRenderingException(Exception): pass -def url_embed_preview_enabled_for_realm(message): - # type: (Optional[Message]) -> bool +def url_embed_preview_enabled_for_realm(message: Optional[Message]) -> bool: if message is not None: realm = message.get_realm() # type: Optional[Realm] else: @@ -95,8 +89,7 @@ def url_embed_preview_enabled_for_realm(message): return True return realm.inline_url_embed_preview -def image_preview_enabled_for_realm(): - # type: () -> bool +def image_preview_enabled_for_realm() -> bool: global current_message if current_message is not None: realm = current_message.get_realm() # type: Optional[Realm] @@ -108,8 +101,7 @@ def image_preview_enabled_for_realm(): return True return realm.inline_image_preview -def list_of_tlds(): - # type: () -> List[Text] +def list_of_tlds() -> List[Text]: # HACK we manually blacklist a few domains blacklist = ['PY\n', "MD\n"] @@ -120,8 +112,9 @@ def list_of_tlds(): tlds.sort(key=len, reverse=True) return tlds -def walk_tree(root, processor, stop_after_first=False): - # type: (Element, Callable[[Element], Optional[_T]], bool) -> List[_T] +def walk_tree(root: Element, + processor: Callable[[Element], Optional[_T]], + stop_after_first: bool=False) -> List[_T]: results = [] queue = deque([root]) @@ -166,8 +159,7 @@ def add_a(root, url, link, title=None, desc=None, desc_div.set("class", "message_inline_image_desc") -def add_embed(root, link, extracted_data): - # type: (Element, Text, Dict[Text, Any]) -> None +def add_embed(root: Element, link: Text, extracted_data: Dict[Text, Any]) -> None: container = markdown.util.etree.SubElement(root, "div") container.set("class", "message_embed") @@ -206,8 +198,7 @@ def add_embed(root, link, extracted_data): @cache_with_key(lambda tweet_id: tweet_id, cache_name="database", with_statsd_key="tweet_data") -def fetch_tweet_data(tweet_id): - # type: (Text) -> Optional[Dict[Text, Any]] +def fetch_tweet_data(tweet_id: Text) -> Optional[Dict[Text, Any]]: if settings.TEST_SUITE: from . import testing_mocks res = testing_mocks.twitter(tweet_id) @@ -266,8 +257,7 @@ HEAD_END_RE = re.compile('^/head[ >]') META_START_RE = re.compile('^meta[ >]') META_END_RE = re.compile('^/meta[ >]') -def fetch_open_graph_image(url): - # type: (Text) -> Optional[Dict[str, Any]] +def fetch_open_graph_image(url: Text) -> Optional[Dict[str, Any]]: in_head = False # HTML will auto close meta tags, when we start the next tag add # a closing tag if it has not been closed yet. @@ -333,8 +323,7 @@ def fetch_open_graph_image(url): desc = og_desc.get('content') return {'image': image, 'title': title, 'desc': desc} -def get_tweet_id(url): - # type: (Text) -> Optional[Text] +def get_tweet_id(url: Text) -> Optional[Text]: parsed_url = urllib.parse.urlparse(url) if not (parsed_url.netloc == 'twitter.com' or parsed_url.netloc.endswith('.twitter.com')): return None @@ -350,8 +339,7 @@ def get_tweet_id(url): return tweet_id_match.group("tweetid") class InlineHttpsProcessor(markdown.treeprocessors.Treeprocessor): - def run(self, root): - # type: (Element) -> None + def run(self, root: Element) -> None: # Get all URLs from the blob found_imgs = walk_tree(root, lambda e: e if e.tag == "img" else None) for img in found_imgs: @@ -365,14 +353,12 @@ class InlineInterestingLinkProcessor(markdown.treeprocessors.Treeprocessor): TWITTER_MAX_IMAGE_HEIGHT = 400 TWITTER_MAX_TO_PREVIEW = 3 - def __init__(self, md, bugdown): - # type: (markdown.Markdown, Bugdown) -> None + def __init__(self, md: markdown.Markdown, bugdown: 'Bugdown') -> None: # Passing in bugdown for access to config to check if realm is zulip.com self.bugdown = bugdown markdown.treeprocessors.Treeprocessor.__init__(self, md) - def get_actual_image_url(self, url): - # type: (Text) -> Text + def get_actual_image_url(self, url: Text) -> Text: # Add specific per-site cases to convert image-preview urls to image urls. # See https://github.com/zulip/zulip/issues/4658 for more information parsed_url = urllib.parse.urlparse(url) @@ -386,8 +372,7 @@ class InlineInterestingLinkProcessor(markdown.treeprocessors.Treeprocessor): return url - def is_image(self, url): - # type: (Text) -> bool + def is_image(self, url: Text) -> bool: if not image_preview_enabled_for_realm(): return False parsed_url = urllib.parse.urlparse(url) @@ -397,8 +382,7 @@ class InlineInterestingLinkProcessor(markdown.treeprocessors.Treeprocessor): return True return False - def dropbox_image(self, url): - # type: (Text) -> Optional[Dict[str, Any]] + def dropbox_image(self, url: Text) -> Optional[Dict[str, Any]]: # TODO: The returned Dict could possibly be a TypedDict in future. parsed_url = urllib.parse.urlparse(url) if (parsed_url.netloc == 'dropbox.com' or parsed_url.netloc.endswith('.dropbox.com')): @@ -443,8 +427,7 @@ class InlineInterestingLinkProcessor(markdown.treeprocessors.Treeprocessor): return image_info return None - def youtube_id(self, url): - # type: (Text) -> Optional[Text] + def youtube_id(self, url: Text) -> Optional[Text]: if not image_preview_enabled_for_realm(): return None # Youtube video id extraction regular expression from http://pastebin.com/KyKAFv1s @@ -457,16 +440,17 @@ class InlineInterestingLinkProcessor(markdown.treeprocessors.Treeprocessor): return None return match.group(2) - def youtube_image(self, url): - # type: (Text) -> Optional[Text] + def youtube_image(self, url: Text) -> Optional[Text]: yt_id = self.youtube_id(url) if yt_id is not None: return "https://i.ytimg.com/vi/%s/default.jpg" % (yt_id,) return None - def twitter_text(self, text, urls, user_mentions, media): - # type: (Text, List[Dict[Text, Text]], List[Dict[Text, Any]], List[Dict[Text, Any]]) -> Element + def twitter_text(self, text: Text, + urls: List[Dict[Text, Text]], + user_mentions: List[Dict[Text, Any]], + media: List[Dict[Text, Any]]) -> Element: """ Use data from the twitter API to turn links, mentions and media into A tags. Also convert unicode emojis to images. @@ -542,8 +526,7 @@ class InlineInterestingLinkProcessor(markdown.treeprocessors.Treeprocessor): to_process.sort(key=lambda x: x['start']) p = current_node = markdown.util.etree.Element('p') - def set_text(text): - # type: (Text) -> None + def set_text(text: Text) -> None: """ Helper to set the text or the tail of the current_node """ @@ -571,8 +554,7 @@ class InlineInterestingLinkProcessor(markdown.treeprocessors.Treeprocessor): set_text(text[current_index:]) return p - def twitter_link(self, url): - # type: (Text) -> Optional[Element] + def twitter_link(self, url: Text) -> Optional[Element]: tweet_id = get_tweet_id(url) if tweet_id is None: @@ -641,16 +623,14 @@ class InlineInterestingLinkProcessor(markdown.treeprocessors.Treeprocessor): logging.warning(traceback.format_exc()) return None - def get_url_data(self, e): - # type: (Element) -> Optional[Tuple[Text, Text]] + def get_url_data(self, e: Element) -> Optional[Tuple[Text, Text]]: if e.tag == "a": if e.text is not None: return (e.get("href"), e.text) return (e.get("href"), e.get("href")) return None - def is_only_element(self, root, url): - # type: (Element, str) -> bool + def is_only_element(self, root: Element, url: str) -> bool: # Check if the url is the only content of the message. if not len(root) == 1: @@ -668,8 +648,7 @@ class InlineInterestingLinkProcessor(markdown.treeprocessors.Treeprocessor): return True - def run(self, root): - # type: (Element) -> None + def run(self, root: Element) -> None: # Get all URLs from the blob found_urls = walk_tree(root, self.get_url_data) @@ -735,8 +714,7 @@ class InlineInterestingLinkProcessor(markdown.treeprocessors.Treeprocessor): class Avatar(markdown.inlinepatterns.Pattern): - def handleMatch(self, match): - # type: (Match[Text]) -> Optional[Element] + def handleMatch(self, match: Match[Text]) -> Optional[Element]: img = markdown.util.etree.Element('img') email_address = match.group('email') email = email_address.strip().lower() @@ -753,8 +731,7 @@ class Avatar(markdown.inlinepatterns.Pattern): img.set('alt', email) return img -def possible_avatar_emails(content): - # type: (Text) -> Set[Text] +def possible_avatar_emails(content: Text) -> Set[Text]: emails = set() for regex in [AVATAR_REGEX, GRAVATAR_REGEX]: matches = re.findall(regex, content) @@ -819,8 +796,7 @@ unicode_emoji_regex = '(?P['\ # For more information, please refer to the following article: # http://crocodillon.com/blog/parsing-emoji-unicode-in-javascript -def make_emoji(codepoint, display_string): - # type: (Text, Text) -> Element +def make_emoji(codepoint: Text, display_string: Text) -> Element: # Replace underscore in emoji's title with space title = display_string[1:-1].replace("_", " ") span = markdown.util.etree.Element('span') @@ -829,8 +805,7 @@ def make_emoji(codepoint, display_string): span.text = display_string return span -def make_realm_emoji(src, display_string): - # type: (Text, Text) -> Element +def make_realm_emoji(src: Text, display_string: Text) -> Element: elt = markdown.util.etree.Element('img') elt.set('src', src) elt.set('class', 'emoji') @@ -838,8 +813,7 @@ def make_realm_emoji(src, display_string): elt.set("title", display_string[1:-1].replace("_", " ")) return elt -def unicode_emoji_to_codepoint(unicode_emoji): - # type: (Text) -> Text +def unicode_emoji_to_codepoint(unicode_emoji: Text) -> Text: codepoint = hex(ord(unicode_emoji))[2:] # Unicode codepoints are minimum of length 4, padded # with zeroes if the length is less than zero. @@ -848,8 +822,7 @@ def unicode_emoji_to_codepoint(unicode_emoji): return codepoint class UnicodeEmoji(markdown.inlinepatterns.Pattern): - def handleMatch(self, match): - # type: (Match[Text]) -> Optional[Element] + def handleMatch(self, match: Match[Text]) -> Optional[Element]: orig_syntax = match.group('syntax') codepoint = unicode_emoji_to_codepoint(orig_syntax) if codepoint in codepoint_to_name: @@ -859,8 +832,7 @@ class UnicodeEmoji(markdown.inlinepatterns.Pattern): return None class Emoji(markdown.inlinepatterns.Pattern): - def handleMatch(self, match): - # type: (Match[Text]) -> Optional[Element] + def handleMatch(self, match: Match[Text]) -> Optional[Element]: orig_syntax = match.group("syntax") name = orig_syntax[1:-1] @@ -877,15 +849,13 @@ class Emoji(markdown.inlinepatterns.Pattern): else: return None -def content_has_emoji_syntax(content): - # type: (Text) -> bool +def content_has_emoji_syntax(content: Text) -> bool: return re.search(EMOJI_REGEX, content) is not None class StreamSubscribeButton(markdown.inlinepatterns.Pattern): # This markdown extension has required javascript in # static/js/custom_markdown.js - def handleMatch(self, match): - # type: (Match[Text]) -> Element + def handleMatch(self, match: Match[Text]) -> Element: stream_name = match.group('stream_name') stream_name = stream_name.replace('\\)', ')').replace('\\\\', '\\') @@ -907,8 +877,7 @@ class ModalLink(markdown.inlinepatterns.Pattern): A pattern that allows including in-app modal links in messages. """ - def handleMatch(self, match): - # type: (Match[Text]) -> Element + def handleMatch(self, match: Match[Text]) -> Element: relative_url = match.group('relative_url') text = match.group('text') @@ -920,8 +889,7 @@ class ModalLink(markdown.inlinepatterns.Pattern): return a_tag class Tex(markdown.inlinepatterns.Pattern): - def handleMatch(self, match): - # type: (Match[Text]) -> Element + def handleMatch(self, match: Match[Text]) -> Element: rendered = render_tex(match.group('body'), is_inline=True) if rendered is not None: return etree.fromstring(rendered.encode('utf-8')) @@ -932,8 +900,7 @@ class Tex(markdown.inlinepatterns.Pattern): return span upload_title_re = re.compile("^(https?://[^/]*)?(/user_uploads/\\d+)(/[^/]*)?/[^/]*/(?P[^/]*)$") -def url_filename(url): - # type: (Text) -> Text +def url_filename(url: Text) -> Text: """Extract the filename if a URL is an uploaded file, or return the original URL""" match = upload_title_re.match(url) if match: @@ -941,16 +908,14 @@ def url_filename(url): else: return url -def fixup_link(link, target_blank=True): - # type: (markdown.util.etree.Element, bool) -> None +def fixup_link(link: markdown.util.etree.Element, target_blank: bool=True) -> None: """Set certain attributes we want on every link.""" if target_blank: link.set('target', '_blank') link.set('title', url_filename(link.get('href'))) -def sanitize_url(url): - # type: (Text) -> Optional[Text] +def sanitize_url(url: Text) -> Optional[Text]: """ Sanitize a url against xss attacks. See the docstring on markdown.inlinepatterns.LinkPattern.sanitize_url. @@ -1004,8 +969,7 @@ def sanitize_url(url): # Url passes all tests. Return url as-is. return urllib.parse.urlunparse((scheme, netloc, path, params, query, fragment)) -def url_to_a(url, text = None): - # type: (Text, Optional[Text]) -> Union[Element, Text] +def url_to_a(url: Text, text: Optional[Text]=None) -> Union[Element, Text]: a = markdown.util.etree.Element('a') href = sanitize_url(url) @@ -1032,8 +996,7 @@ def url_to_a(url, text = None): return a class VerbosePattern(markdown.inlinepatterns.Pattern): - def __init__(self, pattern): - # type: (Text) -> None + def __init__(self, pattern: Text) -> None: markdown.inlinepatterns.Pattern.__init__(self, ' ') # HACK: we just had python-markdown compile an empty regex. @@ -1044,8 +1007,7 @@ class VerbosePattern(markdown.inlinepatterns.Pattern): re.DOTALL | re.UNICODE | re.VERBOSE) class AutoLink(VerbosePattern): - def handleMatch(self, match): - # type: (Match[Text]) -> ElementStringNone + def handleMatch(self, match: Match[Text]) -> ElementStringNone: url = match.group('url') return url_to_a(url) @@ -1058,8 +1020,7 @@ class UListProcessor(markdown.blockprocessors.UListProcessor): TAG = 'ul' RE = re.compile('^[ ]{0,3}[*][ ]+(.*)') - def __init__(self, parser): - # type: (Any) -> None + def __init__(self, parser: Any) -> None: # HACK: Set the tab length to 2 just for the initialization of # this class, so that bulleted lists (and only bulleted lists) @@ -1074,8 +1035,7 @@ class ListIndentProcessor(markdown.blockprocessors.ListIndentProcessor): Based on markdown.blockprocessors.ListIndentProcessor, but with 2-space indent """ - def __init__(self, parser): - # type: (Any) -> None + def __init__(self, parser: Any) -> None: # HACK: Set the tab length to 2 just for the initialization of # this class, so that bulleted lists (and only bulleted lists) @@ -1095,8 +1055,7 @@ class BugdownUListPreprocessor(markdown.preprocessors.Preprocessor): LI_RE = re.compile('^[ ]{0,3}[*][ ]+(.*)', re.MULTILINE) HANGING_ULIST_RE = re.compile('^.+\\n([ ]{0,3}[*][ ]+.*)', re.MULTILINE) - def run(self, lines): - # type: (List[Text]) -> List[Text] + def run(self, lines: List[Text]) -> List[Text]: """ Insert a newline between a paragraph and ulist if missing """ inserts = 0 fence = None @@ -1123,8 +1082,7 @@ class BugdownUListPreprocessor(markdown.preprocessors.Preprocessor): class LinkPattern(markdown.inlinepatterns.Pattern): """ Return a link element from the given match. """ - def handleMatch(self, m): - # type: (Match[Text]) -> Optional[Element] + def handleMatch(self, m: Match[Text]) -> Optional[Element]: href = m.group(9) if not href: return None @@ -1141,8 +1099,7 @@ class LinkPattern(markdown.inlinepatterns.Pattern): fixup_link(el, target_blank = (href[:1] != '#')) return el -def prepare_realm_pattern(source): - # type: (Text) -> Text +def prepare_realm_pattern(source: Text) -> Text: """ Augment a realm filter so it only matches after start-of-string, whitespace, or opening delimiters, won't match if there are word characters directly after, and saves what was matched as "name". """ @@ -1153,20 +1110,19 @@ def prepare_realm_pattern(source): class RealmFilterPattern(markdown.inlinepatterns.Pattern): """ Applied a given realm filter to the input """ - def __init__(self, source_pattern, format_string, markdown_instance=None): - # type: (Text, Text, Optional[markdown.Markdown]) -> None + def __init__(self, source_pattern: Text, + format_string: Text, + markdown_instance: Optional[markdown.Markdown]=None) -> None: self.pattern = prepare_realm_pattern(source_pattern) self.format_string = format_string markdown.inlinepatterns.Pattern.__init__(self, self.pattern, markdown_instance) - def handleMatch(self, m): - # type: (Match[Text]) -> Union[Element, Text] + def handleMatch(self, m: Match[Text]) -> Union[Element, Text]: return url_to_a(self.format_string % m.groupdict(), m.group("name")) class UserMentionPattern(markdown.inlinepatterns.Pattern): - def handleMatch(self, m): - # type: (Match[Text]) -> Optional[Element] + def handleMatch(self, m: Match[Text]) -> Optional[Element]: match = m.group(2) if current_message and db_data is not None: @@ -1202,8 +1158,7 @@ class UserMentionPattern(markdown.inlinepatterns.Pattern): return None class UserGroupMentionPattern(markdown.inlinepatterns.Pattern): - def handleMatch(self, m): - # type: (Match[Text]) -> Optional[Element] + def handleMatch(self, m: Match[Text]) -> Optional[Element]: match = m.group(2) if current_message and db_data is not None: @@ -1226,15 +1181,13 @@ class UserGroupMentionPattern(markdown.inlinepatterns.Pattern): return None class StreamPattern(VerbosePattern): - def find_stream_by_name(self, name): - # type: (Match[Text]) -> Optional[Dict[str, Any]] + def find_stream_by_name(self, name: Match[Text]) -> Optional[Dict[str, Any]]: if db_data is None: return None stream = db_data['stream_names'].get(name) return stream - def handleMatch(self, m): - # type: (Match[Text]) -> Optional[Element] + def handleMatch(self, m: Match[Text]) -> Optional[Element]: name = m.group('stream_name') if current_message: @@ -1254,14 +1207,12 @@ class StreamPattern(VerbosePattern): return el return None -def possible_linked_stream_names(content): - # type: (Text) -> Set[Text] +def possible_linked_stream_names(content: Text) -> Set[Text]: matches = re.findall(STREAM_LINK_REGEX, content, re.VERBOSE) return set(matches) class AlertWordsNotificationProcessor(markdown.preprocessors.Preprocessor): - def run(self, lines): - # type: (Iterable[Text]) -> Iterable[Text] + def run(self, lines: Iterable[Text]) -> Iterable[Text]: if current_message and db_data is not None: # We check for alert words here, the set of which are # dependent on which users may see this message. @@ -1292,8 +1243,7 @@ class AlertWordsNotificationProcessor(markdown.preprocessors.Preprocessor): # Markdown link, breaking up the link. This is a monkey-patch, but it # might be worth sending a version of this change upstream. class AtomicLinkPattern(LinkPattern): - def handleMatch(self, m): - # type: (Match[Text]) -> Optional[Element] + def handleMatch(self, m: Match[Text]) -> Optional[Element]: ret = LinkPattern.handleMatch(self, m) if ret is None: return None @@ -1307,8 +1257,7 @@ DEFAULT_BUGDOWN_KEY = -1 ZEPHYR_MIRROR_BUGDOWN_KEY = -2 class Bugdown(markdown.Extension): - def __init__(self, *args, **kwargs): - # type: (*Any, **Union[bool, int, List[Any]]) -> None + def __init__(self, *args: Any, **kwargs: Union[bool, int, List[Any]]) -> None: # define default configs self.config = { "realm_filters": [kwargs['realm_filters'], @@ -1320,8 +1269,7 @@ class Bugdown(markdown.Extension): super().__init__(*args, **kwargs) - def extendMarkdown(self, md, md_globals): - # type: (markdown.Markdown, Dict[str, Any]) -> None + def extendMarkdown(self, md: markdown.Markdown, md_globals: Dict[str, Any]) -> None: del md.preprocessors['reference'] if self.getConfig('code_block_processor_disabled'): @@ -1476,13 +1424,11 @@ md_engines = {} # type: Dict[Tuple[int, bool], markdown.Markdown] realm_filter_data = {} # type: Dict[int, List[Tuple[Text, Text, int]]] class EscapeHtml(markdown.Extension): - def extendMarkdown(self, md, md_globals): - # type: (markdown.Markdown, Dict[str, Any]) -> None + def extendMarkdown(self, md: markdown.Markdown, md_globals: Dict[str, Any]) -> None: del md.preprocessors['html_block'] del md.inlinePatterns['html'] -def make_md_engine(realm_filters_key, email_gateway): - # type: (int, bool) -> None +def make_md_engine(realm_filters_key: int, email_gateway: bool) -> None: md_engine_key = (realm_filters_key, email_gateway) if md_engine_key in md_engines: del md_engines[md_engine_key] @@ -1503,8 +1449,7 @@ def make_md_engine(realm_filters_key, email_gateway): realm=realm_filters_key, code_block_processor_disabled=email_gateway)]) -def subject_links(realm_filters_key, subject): - # type: (int, Text) -> List[Text] +def subject_links(realm_filters_key: int, subject: Text) -> List[Text]: matches = [] # type: List[Text] realm_filters = realm_filters_for_realm(realm_filters_key) @@ -1515,8 +1460,7 @@ def subject_links(realm_filters_key, subject): matches += [realm_filter[1] % m.groupdict()] return matches -def maybe_update_markdown_engines(realm_filters_key, email_gateway): - # type: (Optional[int], bool) -> None +def maybe_update_markdown_engines(realm_filters_key: Optional[int], email_gateway: bool) -> None: # If realm_filters_key is None, load all filters global realm_filter_data if realm_filters_key is None: @@ -1551,8 +1495,7 @@ def maybe_update_markdown_engines(realm_filters_key, email_gateway): # We also use repr() to improve reproducibility, and to escape terminal control # codes, which can do surprisingly nasty things. _privacy_re = re.compile('\\w', flags=re.UNICODE) -def privacy_clean_markdown(content): - # type: (Text) -> Text +def privacy_clean_markdown(content: Text) -> Text: return repr(_privacy_re.sub('x', content)) @@ -1565,16 +1508,14 @@ current_message = None # type: Optional[Message] # threads themselves, as well. db_data = None # type: Optional[Dict[Text, Any]] -def log_bugdown_error(msg): - # type: (str) -> None +def log_bugdown_error(msg: str) -> None: """We use this unusual logging approach to log the bugdown error, in order to prevent AdminZulipHandler from sending the santized original markdown formatting into another Zulip message, which could cause an infinite exception loop.""" logging.getLogger('').error(msg) -def get_email_info(realm_id, emails): - # type: (int, Set[Text]) -> Dict[Text, FullNameInfo] +def get_email_info(realm_id: int, emails: Set[Text]) -> Dict[Text, FullNameInfo]: if not emails: return dict() @@ -1598,8 +1539,7 @@ def get_email_info(realm_id, emails): } return dct -def get_full_name_info(realm_id, full_names): - # type: (int, Set[Text]) -> Dict[Text, FullNameInfo] +def get_full_name_info(realm_id: int, full_names: Set[Text]) -> Dict[Text, FullNameInfo]: if not full_names: return dict() @@ -1626,8 +1566,7 @@ def get_full_name_info(realm_id, full_names): return dct class MentionData: - def __init__(self, realm_id, content): - # type: (int, Text) -> None + def __init__(self, realm_id: int, content: Text) -> None: full_names = possible_mentions(content) self.full_name_info = get_full_name_info(realm_id, full_names) self.user_ids = { @@ -1645,12 +1584,10 @@ class MentionData: user_profile_id = info['user_profile_id'] self.user_group_members[group_id].append(user_profile_id) - def get_user(self, name): - # type: (Text) -> Optional[FullNameInfo] + def get_user(self, name: Text) -> Optional[FullNameInfo]: return self.full_name_info.get(name.lower(), None) - def get_user_ids(self): - # type: () -> Set[int] + def get_user_ids(self) -> Set[int]: """ Returns the user IDs that might have been mentioned by this content. Note that because this data structure has not parsed @@ -1659,16 +1596,13 @@ class MentionData: """ return self.user_ids - def get_user_group(self, name): - # type: (Text) -> Optional[UserGroup] + def get_user_group(self, name: Text) -> Optional[UserGroup]: return self.user_group_name_info.get(name.lower(), None) - def get_group_members(self, user_group_id): - # type: (int) -> List[int] + def get_group_members(self, user_group_id: int) -> List[int]: return self.user_group_members.get(user_group_id, []) -def get_user_group_name_info(realm_id, user_group_names): - # type: (int, Set[Text]) -> Dict[Text, UserGroup] +def get_user_group_name_info(realm_id: int, user_group_names: Set[Text]) -> Dict[Text, UserGroup]: if not user_group_names: return dict() @@ -1677,8 +1611,7 @@ def get_user_group_name_info(realm_id, user_group_names): dct = {row.name.lower(): row for row in rows} return dct -def get_stream_name_info(realm, stream_names): - # type: (Realm, Set[Text]) -> Dict[Text, FullNameInfo] +def get_stream_name_info(realm: Realm, stream_names: Set[Text]) -> Dict[Text, FullNameInfo]: if not stream_names: return dict() @@ -1703,9 +1636,13 @@ def get_stream_name_info(realm, stream_names): return dct -def do_convert(content, message=None, message_realm=None, possible_words=None, sent_by_bot=False, - mention_data=None, email_gateway=False): - # type: (Text, Optional[Message], Optional[Realm], Optional[Set[Text]], Optional[bool], Optional[MentionData], Optional[bool]) -> Text +def do_convert(content: Text, + message: Optional[Message]=None, + message_realm: Optional[Realm]=None, + possible_words: Optional[Set[Text]]=None, + sent_by_bot: Optional[bool]=False, + mention_data: Optional[MentionData]=None, + email_gateway: Optional[bool]=False) -> Text: """Convert Markdown to HTML, with Zulip-specific settings and hacks.""" # This logic is a bit convoluted, but the overall goal is to support a range of use cases: # * Nothing is passed in other than content -> just run default options (e.g. for docs) @@ -1803,30 +1740,30 @@ bugdown_time_start = 0.0 bugdown_total_time = 0.0 bugdown_total_requests = 0 -def get_bugdown_time(): - # type: () -> float +def get_bugdown_time() -> float: return bugdown_total_time -def get_bugdown_requests(): - # type: () -> int +def get_bugdown_requests() -> int: return bugdown_total_requests -def bugdown_stats_start(): - # type: () -> None +def bugdown_stats_start() -> None: global bugdown_time_start bugdown_time_start = time.time() -def bugdown_stats_finish(): - # type: () -> None +def bugdown_stats_finish() -> None: global bugdown_total_time global bugdown_total_requests global bugdown_time_start bugdown_total_requests += 1 bugdown_total_time += (time.time() - bugdown_time_start) -def convert(content, message=None, message_realm=None, possible_words=None, sent_by_bot=False, - mention_data=None, email_gateway=False): - # type: (Text, Optional[Message], Optional[Realm], Optional[Set[Text]], Optional[bool], Optional[MentionData], Optional[bool]) -> Text +def convert(content: Text, + message: Optional[Message]=None, + message_realm: Optional[Realm]=None, + possible_words: Optional[Set[Text]]=None, + sent_by_bot: Optional[bool]=False, + mention_data: Optional[MentionData]=None, + email_gateway: Optional[bool]=False) -> Text: bugdown_stats_start() ret = do_convert(content, message, message_realm, possible_words, sent_by_bot, mention_data, email_gateway) diff --git a/zerver/lib/bugdown/fenced_code.py b/zerver/lib/bugdown/fenced_code.py index 26a932e28b..1f2c7ae413 100644 --- a/zerver/lib/bugdown/fenced_code.py +++ b/zerver/lib/bugdown/fenced_code.py @@ -110,8 +110,7 @@ LANG_TAG = ' class="%s"' class FencedCodeExtension(markdown.Extension): - def extendMarkdown(self, md, md_globals): - # type: (markdown.Markdown, Dict[str, Any]) -> None + def extendMarkdown(self, md: markdown.Markdown, md_globals: Dict[str, Any]) -> None: """ Add FencedBlockPreprocessor to the Markdown instance. """ md.registerExtension(self) @@ -127,41 +126,34 @@ class FencedCodeExtension(markdown.Extension): class FencedBlockPreprocessor(markdown.preprocessors.Preprocessor): - def __init__(self, md): - # type: (markdown.Markdown) -> None + def __init__(self, md: markdown.Markdown) -> None: markdown.preprocessors.Preprocessor.__init__(self, md) self.checked_for_codehilite = False self.codehilite_conf = {} # type: Dict[str, List[Any]] - def run(self, lines): - # type: (Iterable[Text]) -> List[Text] + def run(self, lines: Iterable[Text]) -> List[Text]: """ Match and store Fenced Code Blocks in the HtmlStash. """ output = [] # type: List[Text] class BaseHandler: - def handle_line(self, line): - # type: (Text) -> None + def handle_line(self, line: Text) -> None: raise NotImplementedError() - def done(self): - # type: () -> None + def done(self) -> None: raise NotImplementedError() processor = self handlers = [] # type: List[BaseHandler] - def push(handler): - # type: (BaseHandler) -> None + def push(handler: BaseHandler) -> None: handlers.append(handler) - def pop(): - # type: () -> None + def pop() -> None: handlers.pop() - def check_for_new_fence(output, line): - # type: (MutableSequence[Text], Text) -> None + def check_for_new_fence(output: MutableSequence[Text], line: Text) -> None: m = FENCE_RE.match(line) if m: fence = m.group('fence') @@ -172,20 +164,16 @@ class FencedBlockPreprocessor(markdown.preprocessors.Preprocessor): output.append(line) class OuterHandler(BaseHandler): - def __init__(self, output): - # type: (MutableSequence[Text]) -> None + def __init__(self, output: MutableSequence[Text]) -> None: self.output = output - def handle_line(self, line): - # type: (Text) -> None + def handle_line(self, line: Text) -> None: check_for_new_fence(self.output, line) - def done(self): - # type: () -> None + def done(self) -> None: pop() - def generic_handler(output, fence, lang): - # type: (MutableSequence[Text], Text, Text) -> BaseHandler + def generic_handler(output: MutableSequence[Text], fence: Text, lang: Text) -> BaseHandler: if lang in ('quote', 'quoted'): return QuoteHandler(output, fence) elif lang in ('math', 'tex', 'latex'): @@ -194,22 +182,19 @@ class FencedBlockPreprocessor(markdown.preprocessors.Preprocessor): return CodeHandler(output, fence, lang) class CodeHandler(BaseHandler): - def __init__(self, output, fence, lang): - # type: (MutableSequence[Text], Text, Text) -> None + def __init__(self, output: MutableSequence[Text], fence: Text, lang: Text) -> None: self.output = output self.fence = fence self.lang = lang self.lines = [] # type: List[Text] - def handle_line(self, line): - # type: (Text) -> None + def handle_line(self, line: Text) -> None: if line.rstrip() == self.fence: self.done() else: self.lines.append(line.rstrip()) - def done(self): - # type: () -> None + def done(self) -> None: text = '\n'.join(self.lines) text = processor.format_code(self.lang, text) text = processor.placeholder(text) @@ -220,21 +205,18 @@ class FencedBlockPreprocessor(markdown.preprocessors.Preprocessor): pop() class QuoteHandler(BaseHandler): - def __init__(self, output, fence): - # type: (MutableSequence[Text], Text) -> None + def __init__(self, output: MutableSequence[Text], fence: Text) -> None: self.output = output self.fence = fence self.lines = [] # type: List[Text] - def handle_line(self, line): - # type: (Text) -> None + def handle_line(self, line: Text) -> None: if line.rstrip() == self.fence: self.done() else: check_for_new_fence(self.lines, line) - def done(self): - # type: () -> None + def done(self) -> None: text = '\n'.join(self.lines) text = processor.format_quote(text) processed_lines = text.split('\n') @@ -244,21 +226,18 @@ class FencedBlockPreprocessor(markdown.preprocessors.Preprocessor): pop() class TexHandler(BaseHandler): - def __init__(self, output, fence): - # type: (MutableSequence[Text], Text) -> None + def __init__(self, output: MutableSequence[Text], fence: Text) -> None: self.output = output self.fence = fence self.lines = [] # type: List[Text] - def handle_line(self, line): - # type: (Text) -> None + def handle_line(self, line: Text) -> None: if line.rstrip() == self.fence: self.done() else: self.lines.append(line) - def done(self): - # type: () -> None + def done(self) -> None: text = '\n'.join(self.lines) text = processor.format_tex(text) text = processor.placeholder(text) @@ -284,8 +263,7 @@ class FencedBlockPreprocessor(markdown.preprocessors.Preprocessor): output.append('') return output - def format_code(self, lang, text): - # type: (Text, Text) -> Text + def format_code(self, lang: Text, text: Text) -> Text: if lang: langclass = LANG_TAG % (lang,) else: @@ -318,8 +296,7 @@ class FencedBlockPreprocessor(markdown.preprocessors.Preprocessor): return code - def format_quote(self, text): - # type: (Text) -> Text + def format_quote(self, text: Text) -> Text: paragraphs = text.split("\n\n") quoted_paragraphs = [] for paragraph in paragraphs: @@ -327,8 +304,7 @@ class FencedBlockPreprocessor(markdown.preprocessors.Preprocessor): quoted_paragraphs.append("\n".join("> " + line for line in lines if line != '')) return "\n\n".join(quoted_paragraphs) - def format_tex(self, text): - # type: (Text) -> Text + def format_tex(self, text: Text) -> Text: paragraphs = text.split("\n\n") tex_paragraphs = [] for paragraph in paragraphs: @@ -340,12 +316,10 @@ class FencedBlockPreprocessor(markdown.preprocessors.Preprocessor): escape(paragraph) + '') return "\n\n".join(tex_paragraphs) - def placeholder(self, code): - # type: (Text) -> Text + def placeholder(self, code: Text) -> Text: return self.markdown.htmlStash.store(code, safe=True) - def _escape(self, txt): - # type: (Text) -> Text + def _escape(self, txt: Text) -> Text: """ basic html escaping """ txt = txt.replace('&', '&') txt = txt.replace('<', '<') @@ -354,8 +328,7 @@ class FencedBlockPreprocessor(markdown.preprocessors.Preprocessor): return txt -def makeExtension(*args, **kwargs): - # type: (*Any, **Union[bool, None, Text]) -> FencedCodeExtension +def makeExtension(*args: Any, **kwargs: None) -> FencedCodeExtension: return FencedCodeExtension(*args, **kwargs) if __name__ == "__main__": diff --git a/zerver/lib/cache.py b/zerver/lib/cache.py index 330ea5db5b..7049c70aa3 100644 --- a/zerver/lib/cache.py +++ b/zerver/lib/cache.py @@ -33,29 +33,24 @@ remote_cache_time_start = 0.0 remote_cache_total_time = 0.0 remote_cache_total_requests = 0 -def get_remote_cache_time(): - # type: () -> float +def get_remote_cache_time() -> float: return remote_cache_total_time -def get_remote_cache_requests(): - # type: () -> int +def get_remote_cache_requests() -> int: return remote_cache_total_requests -def remote_cache_stats_start(): - # type: () -> None +def remote_cache_stats_start() -> None: global remote_cache_time_start remote_cache_time_start = time.time() -def remote_cache_stats_finish(): - # type: () -> None +def remote_cache_stats_finish() -> None: global remote_cache_total_time global remote_cache_total_requests global remote_cache_time_start remote_cache_total_requests += 1 remote_cache_total_time += (time.time() - remote_cache_time_start) -def get_or_create_key_prefix(): - # type: () -> Text +def get_or_create_key_prefix() -> Text: if settings.CASPER_TESTS: # This sets the prefix for the benefit of the Casper tests. # @@ -99,32 +94,27 @@ def get_or_create_key_prefix(): KEY_PREFIX = get_or_create_key_prefix() # type: Text -def bounce_key_prefix_for_testing(test_name): - # type: (Text) -> None +def bounce_key_prefix_for_testing(test_name: Text) -> None: global KEY_PREFIX KEY_PREFIX = test_name + ':' + Text(os.getpid()) + ':' # We are taking the hash of the KEY_PREFIX to decrease the size of the key. # Memcached keys should have a length of less than 256. KEY_PREFIX = hashlib.sha1(KEY_PREFIX.encode('utf-8')).hexdigest() -def get_cache_backend(cache_name): - # type: (Optional[str]) -> BaseCache +def get_cache_backend(cache_name: Optional[str]) -> BaseCache: if cache_name is None: return djcache return caches[cache_name] -def get_cache_with_key(keyfunc, cache_name=None): - # type: (Any, Optional[str]) -> Any +def get_cache_with_key(keyfunc: Any, cache_name: Optional[str]=None) -> Any: """ The main goal of this function getting value from the cache like in the "cache_with_key". A cache value can contain any data including the "None", so here used exception for case if value isn't found in the cache. """ - def decorator(func): - # type: (Callable[..., Any]) -> (Callable[..., Any]) + def decorator(func: Callable[..., Any]) -> (Callable[..., Any]): @wraps(func) - def func_with_caching(*args, **kwargs): - # type: (*Any, **Any) -> Callable[..., Any] + def func_with_caching(*args: Any, **kwargs: Any) -> Callable[..., Any]: key = keyfunc(*args, **kwargs) val = cache_get(key, cache_name=cache_name) if val is not None: @@ -144,11 +134,9 @@ def cache_with_key(keyfunc, cache_name=None, timeout=None, with_statsd_key=None) for avoiding collisions with other uses of this decorator or other uses of caching.""" - def decorator(func): - # type: (Callable[..., ReturnT]) -> Callable[..., ReturnT] + def decorator(func: Callable[..., ReturnT]) -> Callable[..., ReturnT]: @wraps(func) - def func_with_caching(*args, **kwargs): - # type: (*Any, **Any) -> ReturnT + def func_with_caching(*args: Any, **kwargs: Any) -> ReturnT: key = keyfunc(*args, **kwargs) val = cache_get(key, cache_name=cache_name) @@ -180,31 +168,28 @@ def cache_with_key(keyfunc, cache_name=None, timeout=None, with_statsd_key=None) return decorator -def cache_set(key, val, cache_name=None, timeout=None): - # type: (Text, Any, Optional[str], Optional[int]) -> None +def cache_set(key: Text, val: Any, cache_name: Optional[str]=None, timeout: Optional[int]=None) -> None: remote_cache_stats_start() cache_backend = get_cache_backend(cache_name) cache_backend.set(KEY_PREFIX + key, (val,), timeout=timeout) remote_cache_stats_finish() -def cache_get(key, cache_name=None): - # type: (Text, Optional[str]) -> Any +def cache_get(key: Text, cache_name: Optional[str]=None) -> Any: remote_cache_stats_start() cache_backend = get_cache_backend(cache_name) ret = cache_backend.get(KEY_PREFIX + key) remote_cache_stats_finish() return ret -def cache_get_many(keys, cache_name=None): - # type: (List[Text], Optional[str]) -> Dict[Text, Any] +def cache_get_many(keys: List[Text], cache_name: Optional[str]=None) -> Dict[Text, Any]: keys = [KEY_PREFIX + key for key in keys] remote_cache_stats_start() ret = get_cache_backend(cache_name).get_many(keys) remote_cache_stats_finish() return dict([(key[len(KEY_PREFIX):], value) for key, value in ret.items()]) -def cache_set_many(items, cache_name=None, timeout=None): - # type: (Dict[Text, Any], Optional[str], Optional[int]) -> None +def cache_set_many(items: Dict[Text, Any], cache_name: Optional[str]=None, + timeout: Optional[int]=None) -> None: new_items = {} for key in items: new_items[KEY_PREFIX + key] = items[key] @@ -213,14 +198,12 @@ def cache_set_many(items, cache_name=None, timeout=None): get_cache_backend(cache_name).set_many(items, timeout=timeout) remote_cache_stats_finish() -def cache_delete(key, cache_name=None): - # type: (Text, Optional[str]) -> None +def cache_delete(key: Text, cache_name: Optional[str]=None) -> None: remote_cache_stats_start() get_cache_backend(cache_name).delete(KEY_PREFIX + key) remote_cache_stats_finish() -def cache_delete_many(items, cache_name=None): - # type: (Iterable[Text], Optional[str]) -> None +def cache_delete_many(items: Iterable[Text], cache_name: Optional[str]=None) -> None: remote_cache_stats_start() get_cache_backend(cache_name).delete_many( KEY_PREFIX + item for item in items) @@ -289,8 +272,7 @@ def generic_bulk_cached_fetch( return dict((object_id, cached_objects[cache_keys[object_id]]) for object_id in object_ids if cache_keys[object_id] in cached_objects) -def cache(func): - # type: (Callable[..., ReturnT]) -> Callable[..., ReturnT] +def cache(func: Callable[..., ReturnT]) -> Callable[..., ReturnT]: """Decorator which applies Django caching to a function. Uses a key based on the function's name, filename, and @@ -299,20 +281,17 @@ def cache(func): func_uniqifier = '%s-%s' % (func.__code__.co_filename, func.__name__) @wraps(func) - def keyfunc(*args, **kwargs): - # type: (*Any, **Any) -> str + def keyfunc(*args: Any, **kwargs: Any) -> str: # Django complains about spaces because memcached rejects them key = func_uniqifier + repr((args, kwargs)) return key.replace('-', '--').replace(' ', '-s') return cache_with_key(keyfunc)(func) -def display_recipient_cache_key(recipient_id): - # type: (int) -> Text +def display_recipient_cache_key(recipient_id: int) -> Text: return u"display_recipient_dict:%d" % (recipient_id,) -def user_profile_by_email_cache_key(email): - # type: (Text) -> Text +def user_profile_by_email_cache_key(email: Text) -> Text: # See the comment in zerver/lib/avatar_hash.py:gravatar_hash for why we # are proactively encoding email addresses even though they will # with high likelihood be ASCII-only for the foreseeable future. @@ -326,16 +305,13 @@ def user_profile_cache_key(email, realm): # type: (Text, Realm) -> Text return user_profile_cache_key_id(email, realm.id) -def bot_profile_cache_key(email): - # type: (Text) -> Text +def bot_profile_cache_key(email: Text) -> Text: return u"bot_profile:%s" % (make_safe_digest(email.strip())) -def user_profile_by_id_cache_key(user_profile_id): - # type: (int) -> Text +def user_profile_by_id_cache_key(user_profile_id: int) -> Text: return u"user_profile_by_id:%s" % (user_profile_id,) -def user_profile_by_api_key_cache_key(api_key): - # type: (Text) -> Text +def user_profile_by_api_key_cache_key(api_key: Text) -> Text: return u"user_profile_by_api_key:%s" % (api_key,) # TODO: Refactor these cache helpers into another file that can import @@ -346,12 +322,10 @@ realm_user_dict_fields = [ 'avatar_source', 'avatar_version', 'is_active', 'is_realm_admin', 'is_bot', 'realm_id', 'timezone'] # type: List[str] -def realm_user_dicts_cache_key(realm_id): - # type: (int) -> Text +def realm_user_dicts_cache_key(realm_id: int) -> Text: return u"realm_user_dicts:%s" % (realm_id,) -def active_user_ids_cache_key(realm_id): - # type: (int) -> Text +def active_user_ids_cache_key(realm_id: int) -> Text: return u"active_user_ids:%s" % (realm_id,) bot_dict_fields = ['id', 'full_name', 'short_name', 'bot_type', 'email', @@ -366,8 +340,7 @@ def bot_dicts_in_realm_cache_key(realm): # type: (Realm) -> Text return u"bot_dicts_in_realm:%s" % (realm.id,) -def get_stream_cache_key(stream_name, realm_id): - # type: (Text, int) -> Text +def get_stream_cache_key(stream_name: Text, realm_id: int) -> Text: return u"stream_by_realm_and_name:%s:%s" % ( realm_id, make_safe_digest(stream_name.strip().lower())) @@ -392,13 +365,11 @@ def delete_display_recipient_cache(user_profile): # Called by models.py to flush the user_profile cache whenever we save # a user_profile object -def flush_user_profile(sender, **kwargs): - # type: (Any, **Any) -> None +def flush_user_profile(sender: Any, **kwargs: Any) -> None: user_profile = kwargs['instance'] delete_user_profile_caches([user_profile]) - def changed(fields): - # type: (List[str]) -> bool + def changed(fields: List[str]) -> bool: if kwargs.get('update_fields') is None: # adds/deletes should invalidate the cache return True @@ -434,8 +405,7 @@ def flush_user_profile(sender, **kwargs): # Called by models.py to flush various caches whenever we save # a Realm object. The main tricky thing here is that Realm info is # generally cached indirectly through user_profile objects. -def flush_realm(sender, **kwargs): - # type: (Any, **Any) -> None +def flush_realm(sender: Any, **kwargs: Any) -> None: realm = kwargs['instance'] users = realm.get_active_users() delete_user_profile_caches(users) @@ -452,8 +422,7 @@ def realm_alert_words_cache_key(realm): # Called by models.py to flush the stream cache whenever we save a stream # object. -def flush_stream(sender, **kwargs): - # type: (Any, **Any) -> None +def flush_stream(sender: Any, **kwargs: Any) -> None: from zerver.models import UserProfile stream = kwargs['instance'] items_for_remote_cache = {} @@ -466,15 +435,13 @@ def flush_stream(sender, **kwargs): Q(default_events_register_stream=stream)).exists(): cache_delete(bot_dicts_in_realm_cache_key(stream.realm)) -def to_dict_cache_key_id(message_id): - # type: (int) -> Text +def to_dict_cache_key_id(message_id: int) -> Text: return 'message_dict:%d' % (message_id,) def to_dict_cache_key(message): # type: (Message) -> Text return to_dict_cache_key_id(message.id) -def flush_message(sender, **kwargs): - # type: (Any, **Any) -> None +def flush_message(sender: Any, **kwargs: Any) -> None: message = kwargs['instance'] cache_delete(to_dict_cache_key_id(message.id)) diff --git a/zerver/lib/notifications.py b/zerver/lib/notifications.py index c38aec6399..652a1a16b3 100644 --- a/zerver/lib/notifications.py +++ b/zerver/lib/notifications.py @@ -32,8 +32,7 @@ import ujson import urllib from collections import defaultdict -def one_click_unsubscribe_link(user_profile, email_type): - # type: (UserProfile, str) -> str +def one_click_unsubscribe_link(user_profile: UserProfile, email_type: str) -> str: """ Generate a unique link that a logged-out user can visit to unsubscribe from Zulip e-mails without having to first log in. @@ -42,33 +41,28 @@ def one_click_unsubscribe_link(user_profile, email_type): Confirmation.UNSUBSCRIBE, url_args = {'email_type': email_type}) -def hash_util_encode(string): - # type: (Text) -> Text +def hash_util_encode(string: Text) -> Text: # Do the same encoding operation as hash_util.encodeHashComponent on the # frontend. # `safe` has a default value of "/", but we want those encoded, too. return urllib.parse.quote( string.encode("utf-8"), safe=b"").replace(".", "%2E").replace("%", ".") -def pm_narrow_url(realm, participants): - # type: (Realm, List[Text]) -> Text +def pm_narrow_url(realm: Realm, participants: List[Text]) -> Text: participants.sort() base_url = u"%s/#narrow/pm-with/" % (realm.uri,) return base_url + hash_util_encode(",".join(participants)) -def stream_narrow_url(realm, stream): - # type: (Realm, Text) -> Text +def stream_narrow_url(realm: Realm, stream: Text) -> Text: base_url = u"%s/#narrow/stream/" % (realm.uri,) return base_url + hash_util_encode(stream) -def topic_narrow_url(realm, stream, topic): - # type: (Realm, Text, Text) -> Text +def topic_narrow_url(realm: Realm, stream: Text, topic: Text) -> Text: base_url = u"%s/#narrow/stream/" % (realm.uri,) return u"%s%s/topic/%s" % (base_url, hash_util_encode(stream), hash_util_encode(topic)) -def relative_to_full_url(base_url, content): - # type: (Text, Text) -> Text +def relative_to_full_url(base_url: Text, content: Text) -> Text: # Convert relative URLs to absolute URLs. fragment = lxml.html.fromstring(content) @@ -101,10 +95,8 @@ def relative_to_full_url(base_url, content): return content -def fix_emojis(content, base_url, emojiset): - # type: (Text, Text, Text) -> Text - def make_emoji_img_elem(emoji_span_elem): - # type: (Any) -> Dict[str, Any] +def fix_emojis(content: Text, base_url: Text, emojiset: Text) -> Text: + def make_emoji_img_elem(emoji_span_elem: Any) -> Dict[str, Any]: # Convert the emoji spans to img tags. classes = emoji_span_elem.get('class') match = re.search('emoji-(?P\S+)', classes) @@ -138,8 +130,7 @@ def fix_emojis(content, base_url, emojiset): content = lxml.html.tostring(fragment).decode('utf-8') return content -def build_message_list(user_profile, messages): - # type: (UserProfile, List[Message]) -> List[Dict[str, Any]] +def build_message_list(user_profile: UserProfile, messages: List[Message]) -> List[Dict[str, Any]]: """ Builds the message list object for the missed message email template. The messages are collapsed into per-recipient and per-sender blocks, like @@ -147,22 +138,19 @@ def build_message_list(user_profile, messages): """ messages_to_render = [] # type: List[Dict[str, Any]] - def sender_string(message): - # type: (Message) -> Text + def sender_string(message: Message) -> Text: if message.recipient.type in (Recipient.STREAM, Recipient.HUDDLE): return message.sender.full_name else: return '' - def fix_plaintext_image_urls(content): - # type: (Text) -> Text + def fix_plaintext_image_urls(content: Text) -> Text: # Replace image URLs in plaintext content of the form # [image name](image url) # with a simple hyperlink. return re.sub(r"\[(\S*)\]\((\S*)\)", r"\2", content) - def build_message_payload(message): - # type: (Message) -> Dict[str, Text] + def build_message_payload(message: Message) -> Dict[str, Text]: plain = message.content plain = fix_plaintext_image_urls(plain) # There's a small chance of colliding with non-Zulip URLs containing @@ -181,14 +169,12 @@ def build_message_list(user_profile, messages): return {'plain': plain, 'html': html} - def build_sender_payload(message): - # type: (Message) -> Dict[str, Any] + def build_sender_payload(message: Message) -> Dict[str, Any]: sender = sender_string(message) return {'sender': sender, 'content': [build_message_payload(message)]} - def message_header(user_profile, message): - # type: (UserProfile, Message) -> Dict[str, Any] + def message_header(user_profile: UserProfile, message: Message) -> Dict[str, Any]: disp_recipient = get_display_recipient(message.recipient) if message.recipient.type == Recipient.PERSONAL: header = u"You and %s" % (message.sender.full_name,) @@ -264,8 +250,9 @@ def build_message_list(user_profile, messages): return messages_to_render @statsd_increment("missed_message_reminders") -def do_send_missedmessage_events_reply_in_zulip(user_profile, missed_messages, message_count): - # type: (UserProfile, List[Message], int) -> None +def do_send_missedmessage_events_reply_in_zulip(user_profile: UserProfile, + missed_messages: List[Message], + message_count: int) -> None: """ Send a reminder email to a user if she's missed some PMs by being offline. @@ -384,8 +371,7 @@ def do_send_missedmessage_events_reply_in_zulip(user_profile, missed_messages, m user_profile.last_reminder = timezone_now() user_profile.save(update_fields=['last_reminder']) -def handle_missedmessage_emails(user_profile_id, missed_email_events): - # type: (int, Iterable[Dict[str, Any]]) -> None +def handle_missedmessage_emails(user_profile_id: int, missed_email_events: Iterable[Dict[str, Any]]) -> None: message_ids = [event.get('message_id') for event in missed_email_events] user_profile = get_user_profile_by_id(user_profile_id) @@ -429,29 +415,25 @@ def handle_missedmessage_emails(user_profile_id, missed_email_events): message_count_by_recipient_subject[recipient_subject], ) -def clear_scheduled_invitation_emails(email): - # type: (str) -> None +def clear_scheduled_invitation_emails(email: str) -> None: """Unlike most scheduled emails, invitation emails don't have an existing user object to key off of, so we filter by address here.""" items = ScheduledEmail.objects.filter(address__iexact=email, type=ScheduledEmail.INVITATION_REMINDER) items.delete() -def clear_scheduled_emails(user_id, email_type=None): - # type: (int, Optional[int]) -> None +def clear_scheduled_emails(user_id: int, email_type: Optional[int]=None) -> None: items = ScheduledEmail.objects.filter(user_id=user_id) if email_type is not None: items = items.filter(type=email_type) items.delete() -def log_digest_event(msg): - # type: (Text) -> None +def log_digest_event(msg: Text) -> None: import logging logging.basicConfig(filename=settings.DIGEST_LOG_PATH, level=logging.INFO) logging.info(msg) -def enqueue_welcome_emails(user): - # type: (UserProfile) -> None +def enqueue_welcome_emails(user: UserProfile) -> None: from zerver.context_processors import common_context if settings.WELCOME_EMAIL_SENDER is not None: # line break to avoid triggering lint rule @@ -476,8 +458,7 @@ def enqueue_welcome_emails(user): "zerver/emails/followup_day2", to_user_id=user.id, from_name=from_name, from_address=from_address, context=context, delay=datetime.timedelta(days=1)) -def convert_html_to_markdown(html): - # type: (Text) -> Text +def convert_html_to_markdown(html: Text) -> Text: # On Linux, the tool installs as html2markdown, and there's a command called # html2text that does something totally different. On OSX, the tool installs # as html2text. diff --git a/zerver/lib/push_notifications.py b/zerver/lib/push_notifications.py index 2771e21db0..341da37b1c 100644 --- a/zerver/lib/push_notifications.py +++ b/zerver/lib/push_notifications.py @@ -44,12 +44,10 @@ else: # nocoverage -- Not convenient to add test for this. DeviceToken = Union[PushDeviceToken, RemotePushDeviceToken] # We store the token as b64, but apns-client wants hex strings -def b64_to_hex(data): - # type: (bytes) -> Text +def b64_to_hex(data: bytes) -> Text: return binascii.hexlify(base64.b64decode(data)).decode('utf-8') -def hex_to_b64(data): - # type: (Text) -> bytes +def hex_to_b64(data: Text) -> bytes: return base64.b64encode(binascii.unhexlify(data.encode('utf-8'))) # @@ -58,8 +56,7 @@ def hex_to_b64(data): _apns_client = None # type: APNsClient -def get_apns_client(): - # type: () -> APNsClient +def get_apns_client() -> APNsClient: global _apns_client if _apns_client is None: # NB if called concurrently, this will make excess connections. @@ -69,8 +66,7 @@ def get_apns_client(): use_sandbox=settings.APNS_SANDBOX) return _apns_client -def modernize_apns_payload(data): - # type: (Dict[str, Any]) -> Dict[str, Any] +def modernize_apns_payload(data: Dict[str, Any]) -> Dict[str, Any]: '''Take a payload in an unknown Zulip version's format, and return in current format.''' # TODO this isn't super robust as is -- if a buggy remote server # sends a malformed payload, we are likely to raise an exception. @@ -96,8 +92,8 @@ def modernize_apns_payload(data): APNS_MAX_RETRIES = 3 @statsd_increment("apple_push_notification") -def send_apple_push_notification(user_id, devices, payload_data): - # type: (int, List[DeviceToken], Dict[str, Any]) -> None +def send_apple_push_notification(user_id: int, devices: List[DeviceToken], + payload_data: Dict[str, Any]) -> None: logging.info("APNs: Sending notification for user %d to %d devices", user_id, len(devices)) payload = APNsPayload(**modernize_apns_payload(payload_data)) @@ -107,8 +103,7 @@ def send_apple_push_notification(user_id, devices, payload_data): for device in devices: # TODO obviously this should be made to actually use the async - def attempt_send(): - # type: () -> Optional[str] + def attempt_send() -> Optional[str]: stream_id = client.send_notification_async( device.token, payload, topic='org.zulip.Zulip', expiration=expiration) @@ -144,15 +139,14 @@ if settings.ANDROID_GCM_API_KEY: # nocoverage else: gcm = None -def send_android_push_notification_to_user(user_profile, data): - # type: (UserProfile, Dict[str, Any]) -> None +def send_android_push_notification_to_user(user_profile: UserProfile, data: Dict[str, Any]) -> None: devices = list(PushDeviceToken.objects.filter(user=user_profile, kind=PushDeviceToken.GCM)) send_android_push_notification(devices, data) @statsd_increment("android_push_notification") -def send_android_push_notification(devices, data, remote=False): - # type: (List[DeviceToken], Dict[str, Any], bool) -> None +def send_android_push_notification(devices: List[DeviceToken], data: Dict[str, Any], + remote: bool=False) -> None: if not gcm: logging.warning("Skipping sending a GCM push notification since " "PUSH_NOTIFICATION_BOUNCER_URL and ANDROID_GCM_API_KEY are both unset") @@ -218,12 +212,12 @@ def send_android_push_notification(devices, data, remote=False): # Sending to a bouncer # -def uses_notification_bouncer(): - # type: () -> bool +def uses_notification_bouncer() -> bool: return settings.PUSH_NOTIFICATION_BOUNCER_URL is not None -def send_notifications_to_bouncer(user_profile_id, apns_payload, gcm_payload): - # type: (int, Dict[str, Any], Dict[str, Any]) -> None +def send_notifications_to_bouncer(user_profile_id: int, + apns_payload: Dict[str, Any], + gcm_payload: Dict[str, Any]) -> None: post_data = { 'user_id': user_profile_id, 'apns_payload': apns_payload, @@ -231,8 +225,7 @@ def send_notifications_to_bouncer(user_profile_id, apns_payload, gcm_payload): } send_json_to_push_bouncer('POST', 'notify', post_data) -def send_json_to_push_bouncer(method, endpoint, post_data): - # type: (str, str, Dict[str, Any]) -> None +def send_json_to_push_bouncer(method: str, endpoint: str, post_data: Dict[str, Any]) -> None: send_to_push_bouncer( method, endpoint, @@ -243,8 +236,10 @@ def send_json_to_push_bouncer(method, endpoint, post_data): class PushNotificationBouncerException(Exception): pass -def send_to_push_bouncer(method, endpoint, post_data, extra_headers=None): - # type: (str, str, Union[Text, Dict[str, Any]], Optional[Dict[str, Any]]) -> None +def send_to_push_bouncer(method: str, + endpoint: str, + post_data: Union[Text, Dict[str, Any]], + extra_headers: Optional[Dict[str, Any]]=None) -> None: """While it does actually send the notice, this function has a lot of code and comments around error handling for the push notifications bouncer. There are several classes of failures, each with its own @@ -310,15 +305,16 @@ def send_to_push_bouncer(method, endpoint, post_data, extra_headers=None): # Managing device tokens # -def num_push_devices_for_user(user_profile, kind = None): - # type: (UserProfile, Optional[int]) -> PushDeviceToken +def num_push_devices_for_user(user_profile: UserProfile, kind: Optional[int]=None) -> PushDeviceToken: if kind is None: return PushDeviceToken.objects.filter(user=user_profile).count() else: return PushDeviceToken.objects.filter(user=user_profile, kind=kind).count() -def add_push_device_token(user_profile, token_str, kind, ios_app_id=None): - # type: (UserProfile, bytes, int, Optional[str]) -> None +def add_push_device_token(user_profile: UserProfile, + token_str: bytes, + kind: int, + ios_app_id: Optional[str]=None) -> None: logging.info("New push device: %d %r %d %r", user_profile.id, token_str, kind, ios_app_id) @@ -357,8 +353,7 @@ def add_push_device_token(user_profile, token_str, kind, ios_app_id=None): else: logging.info("New push device created.") -def remove_push_device_token(user_profile, token_str, kind): - # type: (UserProfile, bytes, int) -> None +def remove_push_device_token(user_profile: UserProfile, token_str: bytes, kind: int) -> None: # If we're sending things to the push notification bouncer # register this user with them here @@ -383,8 +378,7 @@ def remove_push_device_token(user_profile, token_str, kind): # Push notifications in general # -def get_alert_from_message(message): - # type: (Message) -> Text +def get_alert_from_message(message: Message) -> Text: """ Determine what alert string to display based on the missed messages. """ @@ -401,10 +395,8 @@ def get_alert_from_message(message): else: return "New Zulip mentions and private messages from %s" % (sender_str,) -def get_mobile_push_content(rendered_content): - # type: (Text) -> Text - def get_text(elem): - # type: (LH.HtmlElement) -> Text +def get_mobile_push_content(rendered_content: Text) -> Text: + def get_text(elem: LH.HtmlElement) -> Text: # Convert default emojis to their unicode equivalent. classes = elem.get("class", "") if "emoji" in classes: @@ -421,8 +413,7 @@ def get_mobile_push_content(rendered_content): return elem.text or "" - def process(elem): - # type: (LH.HtmlElement) -> Text + def process(elem: LH.HtmlElement) -> Text: plain_text = get_text(elem) for child in elem: plain_text += process(child) @@ -436,8 +427,7 @@ def get_mobile_push_content(rendered_content): plain_text = process(elem) return plain_text -def truncate_content(content): - # type: (Text) -> Text +def truncate_content(content: Text) -> Text: # We use unicode character 'HORIZONTAL ELLIPSIS' (U+2026) instead # of three dots as this saves two extra characters for textual # content. This function will need to be updated to handle unicode @@ -446,8 +436,7 @@ def truncate_content(content): return content return content[:200] + "…" -def get_apns_payload(message): - # type: (Message) -> Dict[str, Any] +def get_apns_payload(message: Message) -> Dict[str, Any]: text_content = get_mobile_push_content(message.rendered_content) truncated_content = truncate_content(text_content) return { @@ -464,8 +453,7 @@ def get_apns_payload(message): } } -def get_gcm_payload(user_profile, message): - # type: (UserProfile, Message) -> Dict[str, Any] +def get_gcm_payload(user_profile: UserProfile, message: Message) -> Dict[str, Any]: text_content = get_mobile_push_content(message.rendered_content) truncated_content = truncate_content(text_content) @@ -492,8 +480,7 @@ def get_gcm_payload(user_profile, message): return android_data @statsd_increment("push_notifications") -def handle_push_notification(user_profile_id, missed_message): - # type: (int, Dict[str, Any]) -> None +def handle_push_notification(user_profile_id: int, missed_message: Dict[str, Any]) -> None: """ missed_message is the event received by the zerver.worker.queue_processors.PushNotificationWorker.consume function. diff --git a/zerver/lib/retention.py b/zerver/lib/retention.py index ca17fa2d4d..51352b4050 100644 --- a/zerver/lib/retention.py +++ b/zerver/lib/retention.py @@ -10,8 +10,7 @@ from zerver.models import Realm, Message, UserMessage, ArchivedMessage, Archived from typing import Any, Dict, Optional, Generator -def get_realm_expired_messages(realm): - # type: (Any) -> Optional[Dict[str, Any]] +def get_realm_expired_messages(realm: Any) -> Optional[Dict[str, Any]]: expired_date = timezone_now() - timedelta(days=realm.message_retention_days) expired_messages = Message.objects.order_by('id').filter(sender__realm=realm, pub_date__lt=expired_date) @@ -20,8 +19,7 @@ def get_realm_expired_messages(realm): return {'realm_id': realm.id, 'expired_messages': expired_messages} -def get_expired_messages(): - # type: () -> Generator[Any, None, None] +def get_expired_messages() -> Generator[Any, None, None]: # Get all expired messages by Realm. realms = Realm.objects.order_by('string_id').filter( deactivated=False, message_retention_days__isnull=False) @@ -31,8 +29,7 @@ def get_expired_messages(): yield realm_expired_messages -def move_attachment_message_to_archive_by_message(message_id): - # type: (int) -> None +def move_attachment_message_to_archive_by_message(message_id: int) -> None: # Move attachments messages relation table data to archive. query = """ INSERT INTO zerver_archivedattachment_messages (id, archivedattachment_id, @@ -50,8 +47,7 @@ def move_attachment_message_to_archive_by_message(message_id): @transaction.atomic -def move_message_to_archive(message_id): - # type: (int) -> None +def move_message_to_archive(message_id: int) -> None: msg = list(Message.objects.filter(id=message_id).values()) if not msg: raise Message.DoesNotExist