mirror of
https://github.com/zulip/zulip.git
synced 2025-11-06 15:03:34 +00:00
3a825ef22c
This management command creates the same indexes as migrations 82, 83, and 95, which are all indexes on the huge UserMessage table. (*) This command quickly no-ops with clear messaging when the indexes already exist, so it's idempotent in that regard. (If somebody somehow creates an index by the same name incorrectly, they can always drop it in dbshell and re-run this command.) If any of the migrations have not been run, which we detect simply by the existence of the indexes, then we create them using a `CREATE INDEX CONCURRENTLY` command. This functionality in postgres allows you to create indexes against large tables without disrupting queries against those tables. The tradeoff here is that creating indexes concurrently takes significantly longer than doing them non-concurrently. Since most tables are small, we typically just use regular Django migrations and run them during a brief interval while the app is down. For indexes on big tables, we will want to run this command as part of the upgrade process, and we will want to run it while the app is still up, otherwise it's pointless. All the code in create_indexes() is literally copy/pasted from the relevant migrations, and that scheme should work going forward. (It uses a different implementation of create_index_if_not_exist than the migrations use, but the code is identical lexically in the function.) If we ever do major restructuring of our large tables, such as UserMessage, and we end up droppping some of these indexes, then we will need to make this command migrations-aware. For now it's safe to assume that indexes are generally additive in nature, and the sooner we create them during the upgrade process, the better. (*) UserMessage is huge for large installations, of course.
84 lines
2.6 KiB
Python
84 lines
2.6 KiB
Python
from __future__ import absolute_import
|
|
from __future__ import print_function
|
|
|
|
from typing import Any, Callable, Dict, List, Set, Text
|
|
|
|
from django.db import connection
|
|
|
|
from zerver.lib.management import ZulipBaseCommand
|
|
|
|
def create_index_if_not_exist(index_name, table_name, column_string, where_clause):
|
|
# type: (Text, Text, Text, Text) -> None
|
|
#
|
|
# This function is somewhat similar to
|
|
# zerver.lib.migrate.create_index_if_not_exist.
|
|
#
|
|
# The other function gets used as part of Django migrations; this function
|
|
# uses SQL that is not supported by Django migrations.
|
|
#
|
|
# Creating concurrent indexes is kind of a pain with current versions
|
|
# of Django/postgres, because you will get this error with seemingly
|
|
# reasonable code:
|
|
#
|
|
# CREATE INDEX CONCURRENTLY cannot be executed from a function or multi-command string
|
|
#
|
|
# For a lot more detail on this process, refer to the commit message
|
|
# that added this file to the repo.
|
|
|
|
with connection.cursor() as cursor:
|
|
sql = '''
|
|
SELECT 1
|
|
FROM pg_class
|
|
where relname = %s
|
|
'''
|
|
cursor.execute(sql, [index_name])
|
|
rows = cursor.fetchall()
|
|
if len(rows) > 0:
|
|
print('Index %s already exists.' % (index_name,))
|
|
return
|
|
|
|
print("Creating index %s." % (index_name,))
|
|
sql = '''
|
|
CREATE INDEX CONCURRENTLY
|
|
%s
|
|
ON %s (%s)
|
|
%s;
|
|
''' % (index_name, table_name, column_string, where_clause)
|
|
cursor.execute(sql)
|
|
print('Finished creating %s.' % (index_name,))
|
|
|
|
|
|
def create_indexes():
|
|
# type: () -> None
|
|
|
|
# copied from 0082
|
|
create_index_if_not_exist(
|
|
index_name='zerver_usermessage_starred_message_id',
|
|
table_name='zerver_usermessage',
|
|
column_string='user_profile_id, message_id',
|
|
where_clause='WHERE (flags & 2) != 0',
|
|
)
|
|
|
|
# copied from 0083
|
|
create_index_if_not_exist(
|
|
index_name='zerver_usermessage_mentioned_message_id',
|
|
table_name='zerver_usermessage',
|
|
column_string='user_profile_id, message_id',
|
|
where_clause='WHERE (flags & 8) != 0',
|
|
)
|
|
|
|
# copied from 0095
|
|
create_index_if_not_exist(
|
|
index_name='zerver_usermessage_unread_message_id',
|
|
table_name='zerver_usermessage',
|
|
column_string='user_profile_id, message_id',
|
|
where_clause='WHERE (flags & 1) = 0',
|
|
)
|
|
|
|
class Command(ZulipBaseCommand):
|
|
help = """Create concurrent indexes for large tables."""
|
|
|
|
def handle(self, *args, **options):
|
|
# type: (*Any, **str) -> None
|
|
create_indexes()
|