From c0afa041c83af699c87d9915ef73c5749abbdb94 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 11 Apr 2020 21:17:13 +0530 Subject: [PATCH 1/5] feat: bench apps commands caching Port of https://github.com/frappe/bench/pull/888 --- bench/commands/__init__.py | 6 ++++-- bench/commands/utils.py | 12 ++++++++++++ bench/utils.py | 39 +++++++++++++++++++++++++++++++++++++- 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/bench/commands/__init__.py b/bench/commands/__init__.py index 090013c2..da4eb8c8 100755 --- a/bench/commands/__init__.py +++ b/bench/commands/__init__.py @@ -41,7 +41,8 @@ bench_command.add_command(switch_to_develop) from bench.commands.utils import (start, restart, set_nginx_port, set_ssl_certificate, set_ssl_certificate_key, set_url_root, set_mariadb_host, set_default_site, download_translations, backup_site, backup_all_sites, release, renew_lets_encrypt, - disable_production, bench_src, prepare_beta_release, set_redis_cache_host, set_redis_queue_host, set_redis_socketio_host, find_benches, migrate_env) + disable_production, bench_src, prepare_beta_release, set_redis_cache_host, set_redis_queue_host, set_redis_socketio_host, find_benches, migrate_env, + generate_command_cache, clear_command_cache) bench_command.add_command(start) bench_command.add_command(restart) bench_command.add_command(set_nginx_port) @@ -63,7 +64,8 @@ bench_command.add_command(bench_src) bench_command.add_command(prepare_beta_release) bench_command.add_command(find_benches) bench_command.add_command(migrate_env) - +bench_command.add_command(generate_command_cache) +bench_command.add_command(clear_command_cache) from bench.commands.setup import setup bench_command.add_command(setup) diff --git a/bench/commands/utils.py b/bench/commands/utils.py index 822086a7..fcb11512 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -180,3 +180,15 @@ def find_benches(location): def migrate_env(python, backup=True): from bench.utils import migrate_env migrate_env(python=python, backup=backup) + + +@click.command('generate-command-cache', help="Caches Frappe Framework commands") +def generate_command_cache(bench_path='.'): + from bench.utils import generate_command_cache + return generate_command_cache(bench_path=bench_path) + + +@click.command('clear-command-cache', help="Clears Frappe Framework cached commands") +def clear_command_cache(bench_path='.'): + from bench.utils import clear_command_cache + return clear_command_cache(bench_path=bench_path) diff --git a/bench/utils.py b/bench/utils.py index 897acb64..33a9a6cb 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -39,7 +39,7 @@ class CommandFailedError(Exception): pass logger = logging.getLogger(__name__) - +bench_cache_file = '.bench.cmd' folders_in_bench = ('apps', 'sites', 'config', 'logs', 'config/pids') sudoers_file = '/etc/sudoers.d/frappe' @@ -187,6 +187,8 @@ def update(pull=False, patch=False, build=False, requirements=False, backup=True patches.run(bench_path=bench_path) conf = get_config(bench_path) + clear_command_cache(bench_path='.') + if conf.get('release_bench'): print('Release bench detected, cannot update!') sys.exit(1) @@ -1137,3 +1139,38 @@ def find_parent_bench(path): # NOTE: the os.path.split assumes that given path is absolute parent_dir = os.path.split(path)[0] return find_parent_bench(parent_dir) + + +def generate_command_cache(bench_path='.'): + """ + Caches all available commands (even custom apps) via Frappe + Default caching behaviour: generated the first time any command (for a specific bench directory) + """ + + python = get_env_cmd('python', bench_path=bench_path) + sites_path = os.path.join(bench_path, 'sites') + + if os.path.exists(bench_cache_file): + os.remove(bench_cache_file) + + try: + output = get_cmd_output("{0} -m frappe.utils.bench_helper get-frappe-commands".format(python), cwd=sites_path) + with open(bench_cache_file, 'w') as f: + json.dump(eval(output), f) + return json.loads(output) + + except subprocess.CalledProcessError as e: + if hasattr(e, "stderr"): + print(e.stderr.decode('utf-8')) + + +def clear_command_cache(bench_path='.'): + """ + Clears commands cached + Default invalidation behaviour: destroyed on each run of `bench update` + """ + + if os.path.exists(bench_cache_file): + os.remove(bench_cache_file) + else: + print("Bench command cache doesn't exist in this folder!") \ No newline at end of file From e39ae1e0b5699e41dd552f48707299ccda4e86e3 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 11 Apr 2020 21:20:24 +0530 Subject: [PATCH 2/5] fix: better warning and cache handling --- bench/cli.py | 89 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 54 insertions(+), 35 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index c1c75bc5..723c596e 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -1,16 +1,27 @@ +# imports - standard imports import atexit +import json +import logging +import os +import pwd +import subprocess +import sys + +# imports - third party imports import click -import os, sys, logging, json, pwd, subprocess -from bench.utils import is_root, PatchError, drop_privileges, get_env_cmd, get_cmd_output, get_frappe, log, is_dist_editable, find_parent_bench, check_latest_version + +# imports - module imports from bench.app import get_apps -from bench.config.common_site_config import get_config from bench.commands import bench_command +from bench.config.common_site_config import get_config +from bench.utils import PatchError, bench_cache_file, check_latest_version, drop_privileges, find_parent_bench, generate_command_cache, get_cmd_output, get_env_cmd, get_frappe, is_bench_directory, is_dist_editable, is_root, log logger = logging.getLogger('bench') from_command_line = False change_uid_msg = "You should not run this command as root" + def cli(): global from_command_line from_command_line = True @@ -20,46 +31,50 @@ def cli(): change_dir() change_uid() - if is_dist_editable("bench"): + if is_dist_editable("bench") and len(sys.argv) > 1 and sys.argv[1] != "src": log("bench is installed in editable mode!\n\nThis is not the recommended mode of installation for production. Instead, install the package from PyPI with: `pip install frappe-bench`\n", level=3) + if not is_bench_directory() and not cmd_requires_root() and len(sys.argv) > 1 and sys.argv[1] not in ("init", "find", "src"): + log("Command not being executed in bench directory", level=3) + if len(sys.argv) > 2 and sys.argv[1] == "frappe": return old_frappe_cli() - elif len(sys.argv) > 1 and sys.argv[1] in get_frappe_commands(): - return frappe_cmd() + elif len(sys.argv) > 1: + if sys.argv[1] in get_frappe_commands() + ["--site", "--verbose", "--force", "--profile"]: + return frappe_cmd() - elif len(sys.argv) > 1 and sys.argv[1] in ("--site", "--verbose", "--force", "--profile"): - return frappe_cmd() + elif sys.argv[1] == "--help": + print(click.Context(bench_command).get_help()) + print(get_frappe_help()) + return - elif len(sys.argv) > 1 and sys.argv[1]=="--help": - print(click.Context(bench_command).get_help()) - print(get_frappe_help()) - return + elif sys.argv[1] in get_apps(): + return app_cmd() - elif len(sys.argv) > 1 and sys.argv[1] in get_apps(): - return app_cmd() + if not (len(sys.argv) > 1 and sys.argv[1] == "src"): + atexit.register(check_latest_version) + + try: + bench_command() + except PatchError: + sys.exit(1) - else: - try: - atexit.register(check_latest_version) - bench_command() - except PatchError: - sys.exit(1) def check_uid(): if cmd_requires_root() and not is_root(): log('superuser privileges required for this command', level=3) sys.exit(1) + def cmd_requires_root(): if len(sys.argv) > 2 and sys.argv[2] in ('production', 'sudoers', 'supervisor', 'lets-encrypt', 'fonts', - 'print', 'firewall', 'ssh-port', 'role', 'fail2ban', 'wildcard-ssl'): + 'print', 'firewall', 'ssh-port', 'role', 'fail2ban', 'wildcard-ssl', 'install'): return True - if len(sys.argv) >= 2 and sys.argv[1] in ('patch', 'renew-lets-encrypt', 'disable-production', - 'install'): + if len(sys.argv) >= 2 and sys.argv[1] in ('patch', 'renew-lets-encrypt', 'disable-production'): return True + def change_dir(): if os.path.exists('config.json') or "init" in sys.argv: return @@ -70,6 +85,7 @@ def change_dir(): if os.path.exists(dir_path): os.chdir(dir_path) + def change_uid(): if is_root() and not cmd_requires_root(): frappe_user = get_config(".").get('frappe_user') @@ -80,35 +96,37 @@ def change_uid(): log(change_uid_msg, level=3) sys.exit(1) + def old_frappe_cli(bench_path='.'): f = get_frappe(bench_path=bench_path) os.chdir(os.path.join(bench_path, 'sites')) os.execv(f, [f] + sys.argv[2:]) + def app_cmd(bench_path='.'): f = get_env_cmd('python', bench_path=bench_path) os.chdir(os.path.join(bench_path, 'sites')) os.execv(f, [f] + ['-m', 'frappe.utils.bench_helper'] + sys.argv[1:]) + def frappe_cmd(bench_path='.'): f = get_env_cmd('python', bench_path=bench_path) os.chdir(os.path.join(bench_path, 'sites')) os.execv(f, [f] + ['-m', 'frappe.utils.bench_helper', 'frappe'] + sys.argv[1:]) -def get_frappe_commands(bench_path='.'): - python = get_env_cmd('python', bench_path=bench_path) - sites_path = os.path.join(bench_path, 'sites') - if not os.path.exists(sites_path): - log("Command not being executed in bench directory", level=3) - return [] - try: - output = get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-commands".format(python=python), cwd=sites_path) - return json.loads(output) - except subprocess.CalledProcessError as e: - if hasattr(e, "stderr"): - print(e.stderr.decode('utf-8')) + +def get_frappe_commands(): + if not is_bench_directory(): return [] + if os.path.exists(bench_cache_file): + command_dump = open(bench_cache_file, 'r').read() or '[]' + return json.loads(command_dump) + + else: + return generate_command_cache() + + def get_frappe_help(bench_path='.'): python = get_env_cmd('python', bench_path=bench_path) sites_path = os.path.join(bench_path, 'sites') @@ -118,6 +136,7 @@ def get_frappe_help(bench_path='.'): except: return "" + def change_working_directory(): """Allows bench commands to be run from anywhere inside a bench directory""" cur_dir = os.path.abspath(".") From 7e7c3969a764faffe814e72a0940103c4bb69d67 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 11 Apr 2020 21:21:07 +0530 Subject: [PATCH 3/5] fix: set translations download limit based on cpu cores --- bench/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/utils.py b/bench/utils.py index 33a9a6cb..332bda2a 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -844,7 +844,7 @@ def update_translations_p(args): def download_translations_p(): - pool = multiprocessing.Pool(4) + pool = multiprocessing.Pool(multiprocessing.cpu_count()) langs = get_langs() apps = ('frappe', 'erpnext') From 2d436126fd4c1e187bbb9b2685f2b7575dfe19d4 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 11 Apr 2020 21:36:34 +0530 Subject: [PATCH 4/5] fix: help for bench install without sudo --- bench/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bench/cli.py b/bench/cli.py index 723c596e..a3235382 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -69,10 +69,12 @@ def check_uid(): def cmd_requires_root(): if len(sys.argv) > 2 and sys.argv[2] in ('production', 'sudoers', 'supervisor', 'lets-encrypt', 'fonts', - 'print', 'firewall', 'ssh-port', 'role', 'fail2ban', 'wildcard-ssl', 'install'): + 'print', 'firewall', 'ssh-port', 'role', 'fail2ban', 'wildcard-ssl'): return True if len(sys.argv) >= 2 and sys.argv[1] in ('patch', 'renew-lets-encrypt', 'disable-production'): return True + if len(sys.argv) > 2 and sys.argv[1] in ('install'): + return True def change_dir(): From 279073f1c0e3d1b4ee30ad81572b2b5c86ffce01 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 13 Apr 2020 13:02:51 +0530 Subject: [PATCH 5/5] style: fixed docstring and unused imports --- bench/cli.py | 1 - bench/utils.py | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index a3235382..628f2413 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -4,7 +4,6 @@ import json import logging import os import pwd -import subprocess import sys # imports - third party imports diff --git a/bench/utils.py b/bench/utils.py index 332bda2a..38786545 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -1142,8 +1142,7 @@ def find_parent_bench(path): def generate_command_cache(bench_path='.'): - """ - Caches all available commands (even custom apps) via Frappe + """Caches all available commands (even custom apps) via Frappe Default caching behaviour: generated the first time any command (for a specific bench directory) """ @@ -1165,8 +1164,7 @@ def generate_command_cache(bench_path='.'): def clear_command_cache(bench_path='.'): - """ - Clears commands cached + """Clears commands cached Default invalidation behaviour: destroyed on each run of `bench update` """