From 4f3976b917ff63fe5300097a490eb8ba94982fac Mon Sep 17 00:00:00 2001 From: Tim Abbott Date: Fri, 1 May 2020 11:51:18 -0700 Subject: [PATCH] process_fts_updates: Clean up logging output. This saves a couple lines of spammy output in the run-dev.py startup experience, and will be better output in production as well. --- .../files/postgresql/process_fts_updates | 32 +++++++++++++------ tools/run-dev.py | 2 +- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/puppet/zulip/files/postgresql/process_fts_updates b/puppet/zulip/files/postgresql/process_fts_updates index 8411ddf9f7..dfce2ea1f6 100755 --- a/puppet/zulip/files/postgresql/process_fts_updates +++ b/puppet/zulip/files/postgresql/process_fts_updates @@ -22,6 +22,7 @@ try: except ImportError: pass +import argparse import psycopg2 import psycopg2.extensions import select @@ -33,6 +34,19 @@ import os BATCH_SIZE = 1000 +parser = argparse.ArgumentParser() +parser.add_argument('--quiet', + action='store_true') +options = parser.parse_args() + +logging.Formatter.converter = time.gmtime +logging.basicConfig(format="%(asctime)s %(levelname)s: %(message)s") +logger = logging.getLogger("process_fts_updates") +if options.quiet: + logger.setLevel(logging.INFO) +else: + logger.setLevel(logging.DEBUG) + def update_fts_columns(cursor: psycopg2.extensions.cursor) -> int: cursor.execute("SELECT id, message_id FROM fts_update_log LIMIT %s;" % ( BATCH_SIZE,)) @@ -55,13 +69,6 @@ def am_master(cursor: psycopg2.extensions.cursor) -> bool: cursor.execute("SELECT pg_is_in_recovery()") return not cursor.fetchall()[0][0] -logging.Formatter.converter = time.gmtime -logging.basicConfig(format="%(asctime)s %(levelname)s: %(message)s") -logger = logging.getLogger("process_fts_updates") -logger.setLevel(logging.DEBUG) - -logger.info("process_fts_updates starting") - pg_args = {} # Path to the root of the Zulip codebase in production @@ -124,16 +131,21 @@ while True: while not am_master(cursor): if first_check: first_check = False - logger.info("In recovery; sleeping") + logger.warning("In recovery; sleeping") time.sleep(5) - logger.info("Not in recovery; listening for FTS updates") + logger.info("process_fts_updates: listening for search index updates") cursor.execute("LISTEN fts_update_log;") # Catch up on any historical columns while True: rows_updated = update_fts_columns(cursor) - logger.info("Processed %s rows catching up" % (rows_updated,)) + notice = "Processed %s rows catching up" % (rows_updated,) + if rows_updated > 0: + logger.info(notice) + else: + logger.debug(notice) + if rows_updated != BATCH_SIZE: # We're caught up, so proceed to the listening for updates phase. break diff --git a/tools/run-dev.py b/tools/run-dev.py index 45a1954139..627e0cbf96 100755 --- a/tools/run-dev.py +++ b/tools/run-dev.py @@ -155,7 +155,7 @@ def server_processes() -> List[List[str]]: other_cmds = [ ['./manage.py', 'process_queue', '--all'] + manage_args, ['env', 'PGHOST=127.0.0.1', # Force password authentication using .pgpass - './puppet/zulip/files/postgresql/process_fts_updates'], + './puppet/zulip/files/postgresql/process_fts_updates', '--quiet'], ['./manage.py', 'deliver_scheduled_messages'], ['/srv/zulip-thumbor-venv/bin/thumbor', '-c', './zthumbor/thumbor.conf', '-p', '%s' % (thumbor_port,)],