Files
zulip/zerver/management/commands/deliver_scheduled_messages.py
Tim Abbott 7051d3416b scheduled_messages: Add reasonable failure handling.
Previously, it seemed possible for the scheduled messages API to try
to send infinite copies of a message if we had the very poor luck of a
persistent failure happening after a message was sent.

The failure_message field supports being able to display what happened
in the scheduled messages modal, though that's not exposed to the API
yet.
2023-05-09 13:48:28 -07:00

83 lines
3.5 KiB
Python

import logging
import time
from datetime import timedelta
from typing import Any
from django.conf import settings
from django.core.management.base import BaseCommand
from django.db import transaction
from django.utils.timezone import now as timezone_now
from django.utils.translation import gettext as _
from zerver.actions.scheduled_messages import send_scheduled_message
from zerver.lib.exceptions import JsonableError
from zerver.lib.logging_util import log_to_file
from zerver.models import ScheduledMessage
## Setup ##
logger = logging.getLogger(__name__)
log_to_file(logger, settings.DELIVER_SCHEDULED_MESSAGES_LOG_PATH)
class Command(BaseCommand):
help = """Deliver scheduled messages from the ScheduledMessage table.
Run this command under supervisor.
This management command is run via supervisor.
Usage: ./manage.py deliver_scheduled_messages
"""
def handle(self, *args: Any, **options: Any) -> None:
try:
while True:
with transaction.atomic():
scheduled_message = (
ScheduledMessage.objects.filter(
scheduled_timestamp__lte=timezone_now(),
delivered=False,
failed=False,
)
.select_for_update()
.first()
)
if scheduled_message is not None:
logger.info(
"Sending scheduled message %s with date %s (sender: %s)",
scheduled_message.id,
scheduled_message.scheduled_timestamp,
scheduled_message.sender_id,
)
try:
send_scheduled_message(scheduled_message)
except JsonableError as e:
scheduled_message.failed = True
scheduled_message.failure_message = e.msg
scheduled_message.save(update_fields=["failed", "failure_message"])
logging.info(
"Failed with message: %s", scheduled_message.failure_message
)
except Exception:
# An unexpected failure; store as a generic 500 error.
scheduled_message.refresh_from_db()
was_delivered = scheduled_message.delivered
scheduled_message.failed = True
scheduled_message.failure_message = _("Internal server error")
scheduled_message.save(update_fields=["failed", "failure_message"])
logging.exception(
"Unexpected error sending scheduled message %s (sent: %s)",
scheduled_message.id,
was_delivered,
stack_info=True,
)
continue
# If there's no overdue scheduled messages, go to sleep until the next minute.
cur_time = timezone_now()
time_next_min = (cur_time + timedelta(minutes=1)).replace(second=0, microsecond=0)
sleep_time = (time_next_min - cur_time).total_seconds()
time.sleep(sleep_time)
except KeyboardInterrupt:
pass