mirror of
https://github.com/zulip/zulip.git
synced 2025-11-10 00:46:03 +00:00
clean-unused-caches: Optimize performance.
This saves about 400ms when running clean-unused-caches, basically by calling its sub-rountines by import (rather than `subprocess.check_call()`). The performance optimization seems well worth it. Fixes #9766.
This commit is contained in:
@@ -164,7 +164,7 @@ highlighting. The system is largely managed by the code in
|
|||||||
finally, we use `pip`'s built-in caching to ensure that a specific
|
finally, we use `pip`'s built-in caching to ensure that a specific
|
||||||
version of a specific package is only downloaded once.
|
version of a specific package is only downloaded once.
|
||||||
* **Garbage-collecting caches**. We have a tool,
|
* **Garbage-collecting caches**. We have a tool,
|
||||||
`scripts/lib/clean-venv-cache`, which will clean old cached
|
`scripts/lib/clean_venv_cache.py`, which will clean old cached
|
||||||
virtualenvs that are no longer in use. In production, the algorithm
|
virtualenvs that are no longer in use. In production, the algorithm
|
||||||
preserves recent virtualenvs as well as those in use by any current
|
preserves recent virtualenvs as well as those in use by any current
|
||||||
production deployment directory under `/home/zulip/deployments/`.
|
production deployment directory under `/home/zulip/deployments/`.
|
||||||
@@ -188,7 +188,7 @@ reasoning here.
|
|||||||
`scripts/lib/node_cache.py` manages cached `node_modules`
|
`scripts/lib/node_cache.py` manages cached `node_modules`
|
||||||
directories in `/srv/zulip-npm-cache`. Each is named by its hash,
|
directories in `/srv/zulip-npm-cache`. Each is named by its hash,
|
||||||
computed by the `generate_sha1sum_node_modules` function.
|
computed by the `generate_sha1sum_node_modules` function.
|
||||||
`scripts/lib/clean-npm-cache` handles garbage-collection.
|
`scripts/lib/clean_node_cache.py` handles garbage-collection.
|
||||||
* We use [yarn][], a `pip`-like tool for JavaScript, to download most
|
* We use [yarn][], a `pip`-like tool for JavaScript, to download most
|
||||||
JavaScript dependencies. Yarn talks to standard the [npm][]
|
JavaScript dependencies. Yarn talks to standard the [npm][]
|
||||||
repository. We use the standard `package.json` file to declare our
|
repository. We use the standard `package.json` file to declare our
|
||||||
@@ -248,7 +248,7 @@ environments where they need to be displayed.
|
|||||||
Since processing emoji is a relatively expensive operation, as part of
|
Since processing emoji is a relatively expensive operation, as part of
|
||||||
optimizing provisioning, we use the same caching strategy for the
|
optimizing provisioning, we use the same caching strategy for the
|
||||||
compiled emoji data as we use for virtualenvs and `node_modules`
|
compiled emoji data as we use for virtualenvs and `node_modules`
|
||||||
directories, with `scripts/lib/clean-emoji-cache` responsible for
|
directories, with `scripts/lib/clean_emoji_cache.py` responsible for
|
||||||
garbage-collection. This caching and garbage-collection is required
|
garbage-collection. This caching and garbage-collection is required
|
||||||
because a correct emoji implementation involves over 1000 small image
|
because a correct emoji implementation involves over 1000 small image
|
||||||
files and a few large ones. There is a more extended article on our
|
files and a few large ones. There is a more extended article on our
|
||||||
|
|||||||
@@ -7,14 +7,15 @@ import sys
|
|||||||
ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
sys.path.append(ZULIP_PATH)
|
sys.path.append(ZULIP_PATH)
|
||||||
from scripts.lib.zulip_tools import get_environment, parse_cache_script_args
|
from scripts.lib.zulip_tools import get_environment, parse_cache_script_args
|
||||||
|
from scripts.lib import clean_venv_cache, clean_node_cache, clean_emoji_cache
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
# type: () -> None
|
# type: () -> None
|
||||||
parse_cache_script_args("This script cleans unused zulip caches.")
|
args = parse_cache_script_args("This script cleans unused zulip caches.")
|
||||||
os.chdir(ZULIP_PATH)
|
os.chdir(ZULIP_PATH)
|
||||||
subprocess.check_call(["./scripts/lib/clean-venv-cache"] + sys.argv[1:])
|
clean_venv_cache.main(args)
|
||||||
subprocess.check_call(["./scripts/lib/clean-npm-cache"] + sys.argv[1:])
|
clean_node_cache.main(args)
|
||||||
subprocess.check_call(["./scripts/lib/clean-emoji-cache"] + sys.argv[1:])
|
clean_emoji_cache.main(args)
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
|
|||||||
@@ -38,12 +38,11 @@ def get_caches_in_use(threshold_days):
|
|||||||
caches_in_use.add(os.readlink(emoji_link_path))
|
caches_in_use.add(os.readlink(emoji_link_path))
|
||||||
return caches_in_use
|
return caches_in_use
|
||||||
|
|
||||||
def main():
|
def main(args: argparse.Namespace) -> None:
|
||||||
# type: () -> None
|
|
||||||
args = parse_cache_script_args("This script cleans unused zulip emoji caches.")
|
|
||||||
caches_in_use = get_caches_in_use(args.threshold_days)
|
caches_in_use = get_caches_in_use(args.threshold_days)
|
||||||
purge_unused_caches(
|
purge_unused_caches(
|
||||||
EMOJI_CACHE_PATH, caches_in_use, "emoji cache", args)
|
EMOJI_CACHE_PATH, caches_in_use, "emoji cache", args)
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
args = parse_cache_script_args("This script cleans unused zulip emoji caches.")
|
||||||
|
main(args)
|
||||||
@@ -49,12 +49,11 @@ def get_caches_in_use(threshold_days):
|
|||||||
|
|
||||||
return caches_in_use
|
return caches_in_use
|
||||||
|
|
||||||
def main():
|
def main(args: argparse.Namespace) -> None:
|
||||||
# type: () -> None
|
|
||||||
args = parse_cache_script_args("This script cleans unused zulip npm caches.")
|
|
||||||
caches_in_use = get_caches_in_use(args.threshold_days)
|
caches_in_use = get_caches_in_use(args.threshold_days)
|
||||||
purge_unused_caches(
|
purge_unused_caches(
|
||||||
NODE_MODULES_CACHE_PATH, caches_in_use, "node modules cache", args)
|
NODE_MODULES_CACHE_PATH, caches_in_use, "node modules cache", args)
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
args = parse_cache_script_args("This script cleans unused zulip npm caches.")
|
||||||
|
main(args)
|
||||||
@@ -51,12 +51,11 @@ def get_caches_in_use(threshold_days):
|
|||||||
|
|
||||||
return caches_in_use
|
return caches_in_use
|
||||||
|
|
||||||
def main():
|
def main(args: argparse.Namespace) -> None:
|
||||||
# type: () -> None
|
|
||||||
args = parse_cache_script_args("This script cleans unused zulip venv caches.")
|
|
||||||
caches_in_use = get_caches_in_use(args.threshold_days)
|
caches_in_use = get_caches_in_use(args.threshold_days)
|
||||||
purge_unused_caches(
|
purge_unused_caches(
|
||||||
VENV_CACHE_DIR, caches_in_use, "venv cache", args)
|
VENV_CACHE_DIR, caches_in_use, "venv cache", args)
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
args = parse_cache_script_args("This script cleans unused zulip venv caches.")
|
||||||
|
main(args)
|
||||||
Reference in New Issue
Block a user