Merge pull request #888 from gavindsouza/faster-commands

perf: faster bench

bench/cli.py

@@ -1,13 +1,24 @@
+import json
+import logging
+import os
+import pwd
+import subprocess
+import sys
+
 import click
-import os, sys, logging, json, pwd, subprocess
-from bench.utils import is_root, PatchError, drop_privileges, get_env_cmd, get_cmd_output, get_frappe
+
 from bench.app import get_apps
-from bench.config.common_site_config import get_config
 from bench.commands import bench_command
+from bench.config.common_site_config import get_config
+from bench.utils import (PatchError, clear_command_cache, drop_privileges,
+						 frappe_commands_file, generate_command_cache,
+						 get_cmd_output, get_env_cmd, get_frappe, is_root)
+
 
 logger = logging.getLogger('bench')
 from_command_line = False
 
+
 def cli():
 	global from_command_line
 	from_command_line = True
@@ -90,18 +101,19 @@ def frappe_cmd(bench_path='.'):
 	os.execv(f, [f] + ['-m', 'frappe.utils.bench_helper', 'frappe'] + sys.argv[1:])
 
 def get_frappe_commands(bench_path='.'):
-	python = get_env_cmd('python', bench_path=bench_path)
 	sites_path = os.path.join(bench_path, 'sites')
+
 	if not os.path.exists(sites_path):
 		return []
-	try:
-		output = get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-commands".format(python=python), cwd=sites_path)
-		# output = output.decode('utf-8')
-		return json.loads(output)
-	except subprocess.CalledProcessError as e:
-		if hasattr(e, "stderr"):
-			print(e.stderr.decode('utf-8'))
-		return []
+
+	if os.path.exists(frappe_commands_file):
+		command_dump = open(frappe_commands_file, 'r').read()
+		output = json.loads(command_dump) if command_dump else []
+
+	else:
+		output = generate_command_cache()
+
+	return output or []
 
 def get_frappe_help(bench_path='.'):
 	python = get_env_cmd('python', bench_path=bench_path)

bench/commands/__init__.py

@@ -47,7 +47,8 @@ bench_command.add_command(switch_to_develop)
 
 from bench.commands.utils import (start, restart, set_nginx_port, set_ssl_certificate, set_ssl_certificate_key, set_url_root,
 	set_mariadb_host, set_default_site, download_translations, shell, backup_site, backup_all_sites, release, renew_lets_encrypt,
-	disable_production, bench_src, prepare_beta_release, set_redis_cache_host, set_redis_queue_host, set_redis_socketio_host)
+	disable_production, bench_src, prepare_beta_release, set_redis_cache_host, set_redis_queue_host, set_redis_socketio_host,
+	generate_command_cache, clear_command_cache)
 bench_command.add_command(start)
 bench_command.add_command(restart)
 bench_command.add_command(set_nginx_port)
@@ -68,6 +69,8 @@ bench_command.add_command(renew_lets_encrypt)
 bench_command.add_command(disable_production)
 bench_command.add_command(bench_src)
 bench_command.add_command(prepare_beta_release)
+bench_command.add_command(generate_command_cache)
+bench_command.add_command(clear_command_cache)
 
 from bench.commands.setup import setup
 bench_command.add_command(setup)

bench/commands/update.py

@@ -5,7 +5,7 @@ from bench.config.common_site_config import get_config, update_config
 from bench.app import pull_all_apps, is_version_upgrade, validate_branch
 from bench.utils import (update_bench, validate_upgrade, pre_upgrade, post_upgrade, before_update,
 	update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets,
-	restart_supervisor_processes, restart_systemd_processes, is_bench_directory)
+	restart_supervisor_processes, restart_systemd_processes, is_bench_directory, clear_command_cache)
 from bench import patches
 from six.moves import reload_module
 
@@ -30,6 +30,8 @@ def update(pull=False, patch=False, build=False, bench=False, auto=False, restar
 		update_bench(bench_repo=True, requirements=True)
 		sys.exit()
 
+	clear_command_cache(bench_path='.')
+
 	if not (pull or patch or build or bench or requirements):
 		pull, patch, build, bench, requirements = True, True, True, True, True
 

bench/commands/utils.py

@@ -190,3 +190,23 @@ def bench_src():
 	"""Prints bench source folder path, which can be used as: cd `bench src` """
 	import bench
 	print(os.path.dirname(bench.__path__[0]))
+
+
+@click.command('generate-command-cache')
+def generate_command_cache(bench_path='.'):
+	"""
+	Caches Frappe commands (speeds up bench commands execution)
+	Default caching behaviour: generated the first time any command is run
+	"""
+	from bench.utils import generate_command_cache
+	return generate_command_cache(bench_path=bench_path)
+
+
+@click.command('clear-command-cache')
+def clear_command_cache(bench_path='.'):
+	"""
+	Clears commands cached
+	Default invalidation behaviour: destroyed on each run of `bench update`
+	"""
+	from bench.utils import clear_command_cache
+	return clear_command_cache(bench_path=bench_path)

bench/utils.py

@@ -1,7 +1,7 @@
-import os, sys, shutil, subprocess, logging, itertools, requests, json, platform, select, pwd, grp, multiprocessing, hashlib, glob
+import os, sys, shutil, subprocess, logging, itertools, requests, json, platform, select, pwd, grp, multiprocessing, hashlib, glob, errno
+import semantic_version
 from distutils.spawn import find_executable
 import bench
-import semantic_version
 from bench import env
 from six import iteritems, PY2
 
@@ -13,7 +13,7 @@ class CommandFailedError(Exception):
 	pass
 
 logger = logging.getLogger(__name__)
-
+frappe_commands_file = '.frappe-cmd'
 folders_in_bench = ('apps', 'sites', 'config', 'logs', 'config/pids')
 
 
@@ -69,7 +69,7 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False,
 		try:
 			os.makedirs(os.path.join(path, dirname))
 		except OSError as e:
-			if e.errno == os.errno.EEXIST:
+			if e.errno == errno.EEXIST:
 				pass
 
 	setup_logging()
@@ -188,9 +188,7 @@ def setup_env(bench_path='.', python = 'python3'):
 	pip    = os.path.join('env', 'bin', 'pip')
 
 	exec_cmd('virtualenv -q {} -p {}'.format('env', python), cwd=bench_path)
-	exec_cmd('{} -q install --upgrade pip'.format(pip), cwd=bench_path)
-	exec_cmd('{} -q install wheel'.format(pip), cwd=bench_path)
-	exec_cmd('{} -q install six'.format(pip), cwd=bench_path)
+	exec_cmd('{} -q install --upgrade pip wheel six'.format(pip), cwd=bench_path)
 	exec_cmd('{} -q install -e git+https://github.com/frappe/python-pdfkit.git#egg=pdfkit'.format(pip), cwd=bench_path)
 
 def setup_socketio(bench_path='.'):
@@ -714,7 +712,7 @@ def update_translations_p(args):
 		print('Download failed for', args[0], args[1])
 
 def download_translations_p():
-	pool = multiprocessing.Pool(4)
+	pool = multiprocessing.Pool(multiprocessing.cpu_count())
 
 	langs = get_langs()
 	apps = ('frappe', 'erpnext')
@@ -856,3 +854,34 @@ def run_playbook(playbook_name, extra_vars=None, tag=None):
 		args.extend(['-t', tag])
 
 	subprocess.check_call(args, cwd=os.path.join(os.path.dirname(bench.__path__[0]), 'playbooks'))
+
+def generate_command_cache(bench_path='.'):
+	"""
+	Caches Frappe commands (speeds up bench commands execution)
+	Default caching behaviour: generated the first time any command is run
+	"""
+
+	python = get_env_cmd('python', bench_path=bench_path)
+	sites_path = os.path.join(bench_path, 'sites')
+
+	if os.path.exists(frappe_commands_file):
+		os.remove(frappe_commands_file)
+
+	try:
+		output = get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-commands".format(python=python), cwd=sites_path)
+		json.dump(eval(output), open(frappe_commands_file, 'w'))
+		return output
+	except subprocess.CalledProcessError as e:
+		if hasattr(e, "stderr"):
+			print(e.stderr.decode('utf-8'))
+
+def clear_command_cache(bench_path='.'):
+	"""
+	Clears commands cached
+	Default invalidation behaviour: destroyed on each run of `bench update`
+	"""
+
+	if os.path.exists(frappe_commands_file):
+		os.remove(frappe_commands_file)
+	else:
+		print("Bench command cache doesn't exist in this folder!")