From bb677d17df08fc50d590aa6d0c598abdc5258c9b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 31 Dec 2019 14:06:00 +0530 Subject: [PATCH] perf: Faster Commands execution by commands caching --- bench/cli.py | 36 ++++++++++++++++++++++----------- bench/commands/__init__.py | 5 ++++- bench/commands/update.py | 4 +++- bench/commands/utils.py | 20 +++++++++++++++++++ bench/utils.py | 41 ++++++++++++++++++++++++++++++++------ 5 files changed, 86 insertions(+), 20 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index fd419d00..a51bf305 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -1,13 +1,24 @@ +import json +import logging +import os +import pwd +import subprocess +import sys + import click -import os, sys, logging, json, pwd, subprocess -from bench.utils import is_root, PatchError, drop_privileges, get_env_cmd, get_cmd_output, get_frappe + from bench.app import get_apps -from bench.config.common_site_config import get_config from bench.commands import bench_command +from bench.config.common_site_config import get_config +from bench.utils import (PatchError, clear_command_cache, drop_privileges, + frappe_commands_file, generate_command_cache, + get_cmd_output, get_env_cmd, get_frappe, is_root) + logger = logging.getLogger('bench') from_command_line = False + def cli(): global from_command_line from_command_line = True @@ -90,18 +101,19 @@ def frappe_cmd(bench_path='.'): os.execv(f, [f] + ['-m', 'frappe.utils.bench_helper', 'frappe'] + sys.argv[1:]) def get_frappe_commands(bench_path='.'): - python = get_env_cmd('python', bench_path=bench_path) sites_path = os.path.join(bench_path, 'sites') + if not os.path.exists(sites_path): return [] - try: - output = get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-commands".format(python=python), cwd=sites_path) - # output = output.decode('utf-8') - return json.loads(output) - except subprocess.CalledProcessError as e: - if hasattr(e, "stderr"): - print(e.stderr.decode('utf-8')) - return [] + + if os.path.exists(frappe_commands_file): + command_dump = open(frappe_commands_file, 'r').read() + output = json.loads(command_dump) if command_dump else [] + + else: + output = generate_command_cache() + + return output or [] def get_frappe_help(bench_path='.'): python = get_env_cmd('python', bench_path=bench_path) diff --git a/bench/commands/__init__.py b/bench/commands/__init__.py index 8772a0a2..184d6cfd 100755 --- a/bench/commands/__init__.py +++ b/bench/commands/__init__.py @@ -47,7 +47,8 @@ bench_command.add_command(switch_to_develop) from bench.commands.utils import (start, restart, set_nginx_port, set_ssl_certificate, set_ssl_certificate_key, set_url_root, set_mariadb_host, set_default_site, download_translations, shell, backup_site, backup_all_sites, release, renew_lets_encrypt, - disable_production, bench_src, prepare_beta_release, set_redis_cache_host, set_redis_queue_host, set_redis_socketio_host) + disable_production, bench_src, prepare_beta_release, set_redis_cache_host, set_redis_queue_host, set_redis_socketio_host, + generate_command_cache, clear_command_cache) bench_command.add_command(start) bench_command.add_command(restart) bench_command.add_command(set_nginx_port) @@ -68,6 +69,8 @@ bench_command.add_command(renew_lets_encrypt) bench_command.add_command(disable_production) bench_command.add_command(bench_src) bench_command.add_command(prepare_beta_release) +bench_command.add_command(generate_command_cache) +bench_command.add_command(clear_command_cache) from bench.commands.setup import setup bench_command.add_command(setup) diff --git a/bench/commands/update.py b/bench/commands/update.py index 82f857ba..68653dcf 100755 --- a/bench/commands/update.py +++ b/bench/commands/update.py @@ -5,7 +5,7 @@ from bench.config.common_site_config import get_config, update_config from bench.app import pull_all_apps, is_version_upgrade, validate_branch from bench.utils import (update_bench, validate_upgrade, pre_upgrade, post_upgrade, before_update, update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets, - restart_supervisor_processes, restart_systemd_processes, is_bench_directory) + restart_supervisor_processes, restart_systemd_processes, is_bench_directory, clear_command_cache) from bench import patches from six.moves import reload_module @@ -30,6 +30,8 @@ def update(pull=False, patch=False, build=False, bench=False, auto=False, restar update_bench(bench_repo=True, requirements=True) sys.exit() + clear_command_cache(bench_path='.') + if not (pull or patch or build or bench or requirements): pull, patch, build, bench, requirements = True, True, True, True, True diff --git a/bench/commands/utils.py b/bench/commands/utils.py index 9ba55bb8..0c289e8f 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -190,3 +190,23 @@ def bench_src(): """Prints bench source folder path, which can be used as: cd `bench src` """ import bench print(os.path.dirname(bench.__path__[0])) + + +@click.command('generate-command-cache') +def generate_command_cache(bench_path='.'): + """ + Caches Frappe commands (speeds up bench commands execution) + Default caching behaviour: generated the first time any command is run + """ + from bench.utils import generate_command_cache + return generate_command_cache(bench_path=bench_path) + + +@click.command('clear-command-cache') +def clear_command_cache(bench_path='.'): + """ + Clears commands cached + Default invalidation behaviour: destroyed on each run of `bench update` + """ + from bench.utils import clear_command_cache + return clear_command_cache(bench_path=bench_path) diff --git a/bench/utils.py b/bench/utils.py index 82fe900f..5ba3b7ba 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -1,7 +1,7 @@ -import os, sys, shutil, subprocess, logging, itertools, requests, json, platform, select, pwd, grp, multiprocessing, hashlib, glob +import os, sys, shutil, subprocess, logging, itertools, requests, json, platform, select, pwd, grp, multiprocessing, hashlib, glob, errno +import semantic_version from distutils.spawn import find_executable import bench -import semantic_version from bench import env from six import iteritems, PY2 @@ -13,7 +13,7 @@ class CommandFailedError(Exception): pass logger = logging.getLogger(__name__) - +frappe_commands_file = '.frappe-cmd' folders_in_bench = ('apps', 'sites', 'config', 'logs', 'config/pids') @@ -188,9 +188,7 @@ def setup_env(bench_path='.', python = 'python3'): pip = os.path.join('env', 'bin', 'pip') exec_cmd('virtualenv -q {} -p {}'.format('env', python), cwd=bench_path) - exec_cmd('{} -q install --upgrade pip'.format(pip), cwd=bench_path) - exec_cmd('{} -q install wheel'.format(pip), cwd=bench_path) - exec_cmd('{} -q install six'.format(pip), cwd=bench_path) + exec_cmd('{} -q install --upgrade pip wheel six'.format(pip), cwd=bench_path) exec_cmd('{} -q install -e git+https://github.com/frappe/python-pdfkit.git#egg=pdfkit'.format(pip), cwd=bench_path) def setup_socketio(bench_path='.'): @@ -860,3 +858,34 @@ def run_playbook(playbook_name, extra_vars=None, tag=None): args.extend(['-t', tag]) subprocess.check_call(args, cwd=os.path.join(os.path.dirname(bench.__path__[0]), 'playbooks')) + +def generate_command_cache(bench_path='.'): + """ + Caches Frappe commands (speeds up bench commands execution) + Default caching behaviour: generated the first time any command is run + """ + + python = get_env_cmd('python', bench_path=bench_path) + sites_path = os.path.join(bench_path, 'sites') + + if os.path.exists(frappe_commands_file): + os.remove(frappe_commands_file) + + try: + output = get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-commands".format(python=python), cwd=sites_path) + json.dump(eval(output), open(frappe_commands_file, 'w')) + return output + except subprocess.CalledProcessError as e: + if hasattr(e, "stderr"): + print(e.stderr.decode('utf-8')) + +def clear_command_cache(bench_path='.'): + """ + Clears commands cached + Default invalidation behaviour: destroyed on each run of `bench update` + """ + + if os.path.exists(frappe_commands_file): + os.remove(frappe_commands_file) + else: + print("Bench command cache doesn't exist in this folder!") \ No newline at end of file