mirror of
https://github.com/zulip/zulip.git
synced 2025-11-09 08:26:11 +00:00
clean-unused-caches: Optimize performance.
This saves about 400ms when running clean-unused-caches, basically by calling its sub-rountines by import (rather than `subprocess.check_call()`). The performance optimization seems well worth it. Fixes #9766.
This commit is contained in:
@@ -164,7 +164,7 @@ highlighting. The system is largely managed by the code in
|
||||
finally, we use `pip`'s built-in caching to ensure that a specific
|
||||
version of a specific package is only downloaded once.
|
||||
* **Garbage-collecting caches**. We have a tool,
|
||||
`scripts/lib/clean-venv-cache`, which will clean old cached
|
||||
`scripts/lib/clean_venv_cache.py`, which will clean old cached
|
||||
virtualenvs that are no longer in use. In production, the algorithm
|
||||
preserves recent virtualenvs as well as those in use by any current
|
||||
production deployment directory under `/home/zulip/deployments/`.
|
||||
@@ -188,7 +188,7 @@ reasoning here.
|
||||
`scripts/lib/node_cache.py` manages cached `node_modules`
|
||||
directories in `/srv/zulip-npm-cache`. Each is named by its hash,
|
||||
computed by the `generate_sha1sum_node_modules` function.
|
||||
`scripts/lib/clean-npm-cache` handles garbage-collection.
|
||||
`scripts/lib/clean_node_cache.py` handles garbage-collection.
|
||||
* We use [yarn][], a `pip`-like tool for JavaScript, to download most
|
||||
JavaScript dependencies. Yarn talks to standard the [npm][]
|
||||
repository. We use the standard `package.json` file to declare our
|
||||
@@ -248,7 +248,7 @@ environments where they need to be displayed.
|
||||
Since processing emoji is a relatively expensive operation, as part of
|
||||
optimizing provisioning, we use the same caching strategy for the
|
||||
compiled emoji data as we use for virtualenvs and `node_modules`
|
||||
directories, with `scripts/lib/clean-emoji-cache` responsible for
|
||||
directories, with `scripts/lib/clean_emoji_cache.py` responsible for
|
||||
garbage-collection. This caching and garbage-collection is required
|
||||
because a correct emoji implementation involves over 1000 small image
|
||||
files and a few large ones. There is a more extended article on our
|
||||
|
||||
@@ -7,14 +7,15 @@ import sys
|
||||
ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
sys.path.append(ZULIP_PATH)
|
||||
from scripts.lib.zulip_tools import get_environment, parse_cache_script_args
|
||||
from scripts.lib import clean_venv_cache, clean_node_cache, clean_emoji_cache
|
||||
|
||||
def main():
|
||||
# type: () -> None
|
||||
parse_cache_script_args("This script cleans unused zulip caches.")
|
||||
args = parse_cache_script_args("This script cleans unused zulip caches.")
|
||||
os.chdir(ZULIP_PATH)
|
||||
subprocess.check_call(["./scripts/lib/clean-venv-cache"] + sys.argv[1:])
|
||||
subprocess.check_call(["./scripts/lib/clean-npm-cache"] + sys.argv[1:])
|
||||
subprocess.check_call(["./scripts/lib/clean-emoji-cache"] + sys.argv[1:])
|
||||
clean_venv_cache.main(args)
|
||||
clean_node_cache.main(args)
|
||||
clean_emoji_cache.main(args)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -38,12 +38,11 @@ def get_caches_in_use(threshold_days):
|
||||
caches_in_use.add(os.readlink(emoji_link_path))
|
||||
return caches_in_use
|
||||
|
||||
def main():
|
||||
# type: () -> None
|
||||
args = parse_cache_script_args("This script cleans unused zulip emoji caches.")
|
||||
def main(args: argparse.Namespace) -> None:
|
||||
caches_in_use = get_caches_in_use(args.threshold_days)
|
||||
purge_unused_caches(
|
||||
EMOJI_CACHE_PATH, caches_in_use, "emoji cache", args)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
args = parse_cache_script_args("This script cleans unused zulip emoji caches.")
|
||||
main(args)
|
||||
@@ -49,12 +49,11 @@ def get_caches_in_use(threshold_days):
|
||||
|
||||
return caches_in_use
|
||||
|
||||
def main():
|
||||
# type: () -> None
|
||||
args = parse_cache_script_args("This script cleans unused zulip npm caches.")
|
||||
def main(args: argparse.Namespace) -> None:
|
||||
caches_in_use = get_caches_in_use(args.threshold_days)
|
||||
purge_unused_caches(
|
||||
NODE_MODULES_CACHE_PATH, caches_in_use, "node modules cache", args)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
args = parse_cache_script_args("This script cleans unused zulip npm caches.")
|
||||
main(args)
|
||||
@@ -51,12 +51,11 @@ def get_caches_in_use(threshold_days):
|
||||
|
||||
return caches_in_use
|
||||
|
||||
def main():
|
||||
# type: () -> None
|
||||
args = parse_cache_script_args("This script cleans unused zulip venv caches.")
|
||||
def main(args: argparse.Namespace) -> None:
|
||||
caches_in_use = get_caches_in_use(args.threshold_days)
|
||||
purge_unused_caches(
|
||||
VENV_CACHE_DIR, caches_in_use, "venv cache", args)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
args = parse_cache_script_args("This script cleans unused zulip venv caches.")
|
||||
main(args)
|
||||
Reference in New Issue
Block a user