mirror of
https://github.com/zulip/zulip.git
synced 2025-11-03 13:33:24 +00:00
refactor: Add Database.template_status.
This is mostly a pure code move from template_database_status().
This commit is contained in:
@@ -166,7 +166,7 @@ def main(options: argparse.Namespace) -> int:
|
|||||||
# of the development environment (it just uses the development
|
# of the development environment (it just uses the development
|
||||||
# environment to build a release tarball).
|
# environment to build a release tarball).
|
||||||
|
|
||||||
# Need to set up Django before using template_database_status
|
# Need to set up Django before using template_status
|
||||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "zproject.settings")
|
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "zproject.settings")
|
||||||
import django
|
import django
|
||||||
django.setup()
|
django.setup()
|
||||||
@@ -175,7 +175,6 @@ def main(options: argparse.Namespace) -> int:
|
|||||||
DEV_DATABASE,
|
DEV_DATABASE,
|
||||||
TEST_DATABASE,
|
TEST_DATABASE,
|
||||||
destroy_leaked_test_databases,
|
destroy_leaked_test_databases,
|
||||||
template_database_status,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -190,7 +189,7 @@ def main(options: argparse.Namespace) -> int:
|
|||||||
else:
|
else:
|
||||||
print("No need to run `scripts/setup/configure-rabbitmq.")
|
print("No need to run `scripts/setup/configure-rabbitmq.")
|
||||||
|
|
||||||
dev_template_db_status = template_database_status(DEV_DATABASE)
|
dev_template_db_status = DEV_DATABASE.template_status()
|
||||||
if options.is_force or dev_template_db_status == 'needs_rebuild':
|
if options.is_force or dev_template_db_status == 'needs_rebuild':
|
||||||
run(["tools/setup/postgres-init-dev-db"])
|
run(["tools/setup/postgres-init-dev-db"])
|
||||||
run(["tools/do-destroy-rebuild-database"])
|
run(["tools/do-destroy-rebuild-database"])
|
||||||
@@ -199,7 +198,7 @@ def main(options: argparse.Namespace) -> int:
|
|||||||
elif dev_template_db_status == 'current':
|
elif dev_template_db_status == 'current':
|
||||||
print("No need to regenerate the dev DB.")
|
print("No need to regenerate the dev DB.")
|
||||||
|
|
||||||
test_template_db_status = template_database_status(TEST_DATABASE)
|
test_template_db_status = TEST_DATABASE.template_status()
|
||||||
if options.is_force or test_template_db_status == 'needs_rebuild':
|
if options.is_force or test_template_db_status == 'needs_rebuild':
|
||||||
run(["tools/setup/postgres-init-test-db"])
|
run(["tools/setup/postgres-init-test-db"])
|
||||||
run(["tools/do-destroy-rebuild-test-database"])
|
run(["tools/do-destroy-rebuild-test-database"])
|
||||||
|
|||||||
@@ -83,6 +83,72 @@ class Database:
|
|||||||
|
|
||||||
return 'migrate'
|
return 'migrate'
|
||||||
|
|
||||||
|
def template_status(self) -> str:
|
||||||
|
# This function returns a status string specifying the type of
|
||||||
|
# state the template db is in and thus the kind of action required.
|
||||||
|
database_name = self.database_name
|
||||||
|
|
||||||
|
check_files = [
|
||||||
|
'zilencer/management/commands/populate_db.py',
|
||||||
|
'zerver/lib/bulk_create.py',
|
||||||
|
'zerver/lib/generate_test_data.py',
|
||||||
|
'zerver/lib/server_initialization.py',
|
||||||
|
'tools/setup/postgres-init-test-db',
|
||||||
|
'tools/setup/postgres-init-dev-db',
|
||||||
|
'zerver/migrations/0258_enable_online_push_notifications_default.py',
|
||||||
|
]
|
||||||
|
check_settings = [
|
||||||
|
'REALM_INTERNAL_BOTS',
|
||||||
|
]
|
||||||
|
|
||||||
|
# Construct a directory to store hashes named after the target database.
|
||||||
|
status_dir = os.path.join(UUID_VAR_DIR, database_name + '_db_status')
|
||||||
|
if not os.path.exists(status_dir):
|
||||||
|
os.mkdir(status_dir)
|
||||||
|
|
||||||
|
if not database_exists(database_name):
|
||||||
|
# TODO: It's possible that `database_exists` will
|
||||||
|
# return `False` even though the database
|
||||||
|
# exists, but we just have the wrong password,
|
||||||
|
# probably due to changing the secrets file.
|
||||||
|
#
|
||||||
|
# The only problem this causes is that we waste
|
||||||
|
# some time rebuilding the whole database, but
|
||||||
|
# it's better to err on that side, generally.
|
||||||
|
return 'needs_rebuild'
|
||||||
|
|
||||||
|
# To ensure Python evaluates all the hash tests (and thus creates the
|
||||||
|
# hash files about the current state), we evaluate them in a
|
||||||
|
# list and then process the result
|
||||||
|
files_hash_status = all([check_file_hash(fn, status_dir) for fn in check_files])
|
||||||
|
settings_hash_status = all([check_setting_hash(setting_name, status_dir)
|
||||||
|
for setting_name in check_settings])
|
||||||
|
hash_status = files_hash_status and settings_hash_status
|
||||||
|
if not hash_status:
|
||||||
|
return 'needs_rebuild'
|
||||||
|
|
||||||
|
# Here we hash and compare our migration files before doing
|
||||||
|
# the work of seeing what to do with them; if there are no
|
||||||
|
# changes, we can safely assume we don't need to run
|
||||||
|
# migrations without spending a few 100ms parsing all the
|
||||||
|
# Python migration code.
|
||||||
|
paths = [
|
||||||
|
*glob.glob('*/migrations/*.py'),
|
||||||
|
'requirements/dev.txt',
|
||||||
|
]
|
||||||
|
check_migrations = file_or_package_hash_updated(paths, "migrations_hash_" + database_name)
|
||||||
|
if not check_migrations:
|
||||||
|
return 'current'
|
||||||
|
|
||||||
|
migration_op = self.what_to_do_with_migrations()
|
||||||
|
if migration_op == 'scrap':
|
||||||
|
return 'needs_rebuild'
|
||||||
|
|
||||||
|
if migration_op == 'migrate':
|
||||||
|
return 'run_migrations'
|
||||||
|
|
||||||
|
return 'current'
|
||||||
|
|
||||||
DEV_DATABASE = Database(
|
DEV_DATABASE = Database(
|
||||||
platform='dev',
|
platform='dev',
|
||||||
database_name='zulip',
|
database_name='zulip',
|
||||||
@@ -116,7 +182,7 @@ def update_test_databases_if_required(use_force: bool=False,
|
|||||||
If use_force is specified, it will always do a full rebuild.
|
If use_force is specified, it will always do a full rebuild.
|
||||||
"""
|
"""
|
||||||
generate_fixtures_command = ['tools/setup/generate-fixtures']
|
generate_fixtures_command = ['tools/setup/generate-fixtures']
|
||||||
test_template_db_status = template_database_status(TEST_DATABASE)
|
test_template_db_status = TEST_DATABASE.template_status()
|
||||||
if use_force or test_template_db_status == 'needs_rebuild':
|
if use_force or test_template_db_status == 'needs_rebuild':
|
||||||
generate_fixtures_command.append('--force')
|
generate_fixtures_command.append('--force')
|
||||||
elif test_template_db_status == 'run_migrations':
|
elif test_template_db_status == 'run_migrations':
|
||||||
@@ -209,72 +275,6 @@ def check_setting_hash(setting_name: str, status_dir: str) -> bool:
|
|||||||
|
|
||||||
return _check_hash(source_hash_file, target_content)
|
return _check_hash(source_hash_file, target_content)
|
||||||
|
|
||||||
def template_database_status(database: Database) -> str:
|
|
||||||
# This function returns a status string specifying the type of
|
|
||||||
# state the template db is in and thus the kind of action required.
|
|
||||||
database_name = database.database_name
|
|
||||||
|
|
||||||
check_files = [
|
|
||||||
'zilencer/management/commands/populate_db.py',
|
|
||||||
'zerver/lib/bulk_create.py',
|
|
||||||
'zerver/lib/generate_test_data.py',
|
|
||||||
'zerver/lib/server_initialization.py',
|
|
||||||
'tools/setup/postgres-init-test-db',
|
|
||||||
'tools/setup/postgres-init-dev-db',
|
|
||||||
'zerver/migrations/0258_enable_online_push_notifications_default.py',
|
|
||||||
]
|
|
||||||
check_settings = [
|
|
||||||
'REALM_INTERNAL_BOTS',
|
|
||||||
]
|
|
||||||
|
|
||||||
# Construct a directory to store hashes named after the target database.
|
|
||||||
status_dir = os.path.join(UUID_VAR_DIR, database_name + '_db_status')
|
|
||||||
if not os.path.exists(status_dir):
|
|
||||||
os.mkdir(status_dir)
|
|
||||||
|
|
||||||
if not database_exists(database_name):
|
|
||||||
# TODO: It's possible that `database_exists` will
|
|
||||||
# return `False` even though the database
|
|
||||||
# exists, but we just have the wrong password,
|
|
||||||
# probably due to changing the secrets file.
|
|
||||||
#
|
|
||||||
# The only problem this causes is that we waste
|
|
||||||
# some time rebuilding the whole database, but
|
|
||||||
# it's better to err on that side, generally.
|
|
||||||
return 'needs_rebuild'
|
|
||||||
|
|
||||||
# To ensure Python evaluates all the hash tests (and thus creates the
|
|
||||||
# hash files about the current state), we evaluate them in a
|
|
||||||
# list and then process the result
|
|
||||||
files_hash_status = all([check_file_hash(fn, status_dir) for fn in check_files])
|
|
||||||
settings_hash_status = all([check_setting_hash(setting_name, status_dir)
|
|
||||||
for setting_name in check_settings])
|
|
||||||
hash_status = files_hash_status and settings_hash_status
|
|
||||||
if not hash_status:
|
|
||||||
return 'needs_rebuild'
|
|
||||||
|
|
||||||
# Here we hash and compare our migration files before doing
|
|
||||||
# the work of seeing what to do with them; if there are no
|
|
||||||
# changes, we can safely assume we don't need to run
|
|
||||||
# migrations without spending a few 100ms parsing all the
|
|
||||||
# Python migration code.
|
|
||||||
paths = [
|
|
||||||
*glob.glob('*/migrations/*.py'),
|
|
||||||
'requirements/dev.txt',
|
|
||||||
]
|
|
||||||
check_migrations = file_or_package_hash_updated(paths, "migrations_hash_" + database_name)
|
|
||||||
if not check_migrations:
|
|
||||||
return 'current'
|
|
||||||
|
|
||||||
migration_op = database.what_to_do_with_migrations()
|
|
||||||
if migration_op == 'scrap':
|
|
||||||
return 'needs_rebuild'
|
|
||||||
|
|
||||||
if migration_op == 'migrate':
|
|
||||||
return 'run_migrations'
|
|
||||||
|
|
||||||
return 'current'
|
|
||||||
|
|
||||||
def destroy_leaked_test_databases(expiry_time: int = 60 * 60) -> int:
|
def destroy_leaked_test_databases(expiry_time: int = 60 * 60) -> int:
|
||||||
"""The logic in zerver/lib/test_runner.py tries to delete all the
|
"""The logic in zerver/lib/test_runner.py tries to delete all the
|
||||||
temporary test databases generated by test-backend threads, but it
|
temporary test databases generated by test-backend threads, but it
|
||||||
|
|||||||
Reference in New Issue
Block a user