From 76d8b3d9f6285c24b8de5947f8b41438aa28b567 Mon Sep 17 00:00:00 2001 From: gavin Date: Tue, 7 Jan 2020 16:06:15 +0530 Subject: [PATCH] Revert "perf: faster bench" --- bench/cli.py | 36 ++++++++++-------------------- bench/commands/__init__.py | 5 +---- bench/commands/update.py | 4 +--- bench/commands/utils.py | 20 ----------------- bench/utils.py | 45 +++++++------------------------------- 5 files changed, 22 insertions(+), 88 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index a51bf305..fd419d00 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -1,24 +1,13 @@ -import json -import logging -import os -import pwd -import subprocess -import sys - import click - +import os, sys, logging, json, pwd, subprocess +from bench.utils import is_root, PatchError, drop_privileges, get_env_cmd, get_cmd_output, get_frappe from bench.app import get_apps -from bench.commands import bench_command from bench.config.common_site_config import get_config -from bench.utils import (PatchError, clear_command_cache, drop_privileges, - frappe_commands_file, generate_command_cache, - get_cmd_output, get_env_cmd, get_frappe, is_root) - +from bench.commands import bench_command logger = logging.getLogger('bench') from_command_line = False - def cli(): global from_command_line from_command_line = True @@ -101,19 +90,18 @@ def frappe_cmd(bench_path='.'): os.execv(f, [f] + ['-m', 'frappe.utils.bench_helper', 'frappe'] + sys.argv[1:]) def get_frappe_commands(bench_path='.'): + python = get_env_cmd('python', bench_path=bench_path) sites_path = os.path.join(bench_path, 'sites') - if not os.path.exists(sites_path): return [] - - if os.path.exists(frappe_commands_file): - command_dump = open(frappe_commands_file, 'r').read() - output = json.loads(command_dump) if command_dump else [] - - else: - output = generate_command_cache() - - return output or [] + try: + output = get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-commands".format(python=python), cwd=sites_path) + # output = output.decode('utf-8') + return json.loads(output) + except subprocess.CalledProcessError as e: + if hasattr(e, "stderr"): + print(e.stderr.decode('utf-8')) + return [] def get_frappe_help(bench_path='.'): python = get_env_cmd('python', bench_path=bench_path) diff --git a/bench/commands/__init__.py b/bench/commands/__init__.py index 184d6cfd..8772a0a2 100755 --- a/bench/commands/__init__.py +++ b/bench/commands/__init__.py @@ -47,8 +47,7 @@ bench_command.add_command(switch_to_develop) from bench.commands.utils import (start, restart, set_nginx_port, set_ssl_certificate, set_ssl_certificate_key, set_url_root, set_mariadb_host, set_default_site, download_translations, shell, backup_site, backup_all_sites, release, renew_lets_encrypt, - disable_production, bench_src, prepare_beta_release, set_redis_cache_host, set_redis_queue_host, set_redis_socketio_host, - generate_command_cache, clear_command_cache) + disable_production, bench_src, prepare_beta_release, set_redis_cache_host, set_redis_queue_host, set_redis_socketio_host) bench_command.add_command(start) bench_command.add_command(restart) bench_command.add_command(set_nginx_port) @@ -69,8 +68,6 @@ bench_command.add_command(renew_lets_encrypt) bench_command.add_command(disable_production) bench_command.add_command(bench_src) bench_command.add_command(prepare_beta_release) -bench_command.add_command(generate_command_cache) -bench_command.add_command(clear_command_cache) from bench.commands.setup import setup bench_command.add_command(setup) diff --git a/bench/commands/update.py b/bench/commands/update.py index 68653dcf..82f857ba 100755 --- a/bench/commands/update.py +++ b/bench/commands/update.py @@ -5,7 +5,7 @@ from bench.config.common_site_config import get_config, update_config from bench.app import pull_all_apps, is_version_upgrade, validate_branch from bench.utils import (update_bench, validate_upgrade, pre_upgrade, post_upgrade, before_update, update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets, - restart_supervisor_processes, restart_systemd_processes, is_bench_directory, clear_command_cache) + restart_supervisor_processes, restart_systemd_processes, is_bench_directory) from bench import patches from six.moves import reload_module @@ -30,8 +30,6 @@ def update(pull=False, patch=False, build=False, bench=False, auto=False, restar update_bench(bench_repo=True, requirements=True) sys.exit() - clear_command_cache(bench_path='.') - if not (pull or patch or build or bench or requirements): pull, patch, build, bench, requirements = True, True, True, True, True diff --git a/bench/commands/utils.py b/bench/commands/utils.py index 13141c8e..b93d2f59 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -190,23 +190,3 @@ def bench_src(): """Prints bench source folder path, which can be used as: cd `bench src` """ import bench print(os.path.dirname(bench.__path__[0])) - - -@click.command('generate-command-cache') -def generate_command_cache(bench_path='.'): - """ - Caches Frappe commands (speeds up bench commands execution) - Default caching behaviour: generated the first time any command is run - """ - from bench.utils import generate_command_cache - return generate_command_cache(bench_path=bench_path) - - -@click.command('clear-command-cache') -def clear_command_cache(bench_path='.'): - """ - Clears commands cached - Default invalidation behaviour: destroyed on each run of `bench update` - """ - from bench.utils import clear_command_cache - return clear_command_cache(bench_path=bench_path) diff --git a/bench/utils.py b/bench/utils.py index ba9d2ea5..084e4e50 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -1,7 +1,7 @@ -import os, sys, shutil, subprocess, logging, itertools, requests, json, platform, select, pwd, grp, multiprocessing, hashlib, glob, errno -import semantic_version +import os, sys, shutil, subprocess, logging, itertools, requests, json, platform, select, pwd, grp, multiprocessing, hashlib, glob from distutils.spawn import find_executable import bench +import semantic_version from bench import env from six import iteritems, PY2 @@ -13,7 +13,7 @@ class CommandFailedError(Exception): pass logger = logging.getLogger(__name__) -frappe_commands_file = '.frappe-cmd' + folders_in_bench = ('apps', 'sites', 'config', 'logs', 'config/pids') @@ -69,7 +69,7 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, try: os.makedirs(os.path.join(path, dirname)) except OSError as e: - if e.errno == errno.EEXIST: + if e.errno == os.errno.EEXIST: pass setup_logging() @@ -188,7 +188,9 @@ def setup_env(bench_path='.', python = 'python3'): pip = os.path.join('env', 'bin', 'pip') exec_cmd('virtualenv -q {} -p {}'.format('env', python), cwd=bench_path) - exec_cmd('{} -q install --upgrade pip wheel six'.format(pip), cwd=bench_path) + exec_cmd('{} -q install --upgrade pip'.format(pip), cwd=bench_path) + exec_cmd('{} -q install wheel'.format(pip), cwd=bench_path) + exec_cmd('{} -q install six'.format(pip), cwd=bench_path) exec_cmd('{} -q install -e git+https://github.com/frappe/python-pdfkit.git#egg=pdfkit'.format(pip), cwd=bench_path) def setup_socketio(bench_path='.'): @@ -712,7 +714,7 @@ def update_translations_p(args): print('Download failed for', args[0], args[1]) def download_translations_p(): - pool = multiprocessing.Pool(multiprocessing.cpu_count()) + pool = multiprocessing.Pool(4) langs = get_langs() apps = ('frappe', 'erpnext') @@ -854,34 +856,3 @@ def run_playbook(playbook_name, extra_vars=None, tag=None): args.extend(['-t', tag]) subprocess.check_call(args, cwd=os.path.join(os.path.dirname(bench.__path__[0]), 'playbooks')) - -def generate_command_cache(bench_path='.'): - """ - Caches Frappe commands (speeds up bench commands execution) - Default caching behaviour: generated the first time any command is run - """ - - python = get_env_cmd('python', bench_path=bench_path) - sites_path = os.path.join(bench_path, 'sites') - - if os.path.exists(frappe_commands_file): - os.remove(frappe_commands_file) - - try: - output = get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-commands".format(python=python), cwd=sites_path) - json.dump(eval(output), open(frappe_commands_file, 'w')) - return output - except subprocess.CalledProcessError as e: - if hasattr(e, "stderr"): - print(e.stderr.decode('utf-8')) - -def clear_command_cache(bench_path='.'): - """ - Clears commands cached - Default invalidation behaviour: destroyed on each run of `bench update` - """ - - if os.path.exists(frappe_commands_file): - os.remove(frappe_commands_file) - else: - print("Bench command cache doesn't exist in this folder!") \ No newline at end of file