Merge pull request #967 from gavindsouza/cli-cache

feat: faster bench

bench/cli.py

@@ -1,16 +1,26 @@
+# imports - standard imports
 import atexit
+import json
+import logging
+import os
+import pwd
+import sys
+
+# imports - third party imports
 import click
-import os, sys, logging, json, pwd, subprocess
-from bench.utils import is_root, PatchError, drop_privileges, get_env_cmd, get_cmd_output, get_frappe, log, is_dist_editable, find_parent_bench, check_latest_version
+
+# imports - module imports
 from bench.app import get_apps
-from bench.config.common_site_config import get_config
 from bench.commands import bench_command
+from bench.config.common_site_config import get_config
+from bench.utils import PatchError, bench_cache_file, check_latest_version, drop_privileges, find_parent_bench, generate_command_cache, get_cmd_output, get_env_cmd, get_frappe, is_bench_directory, is_dist_editable, is_root, log
 
 
 logger = logging.getLogger('bench')
 from_command_line = False
 change_uid_msg = "You should not run this command as root"
 
+
 def cli():
 	global from_command_line
 	from_command_line = True
@@ -20,45 +30,51 @@ def cli():
 	change_dir()
 	change_uid()
 
-	if is_dist_editable("bench"):
+	if is_dist_editable("bench") and len(sys.argv) > 1 and sys.argv[1] != "src":
 		log("bench is installed in editable mode!\n\nThis is not the recommended mode of installation for production. Instead, install the package from PyPI with: `pip install frappe-bench`\n", level=3)
 
+	if not is_bench_directory() and not cmd_requires_root() and len(sys.argv) > 1 and sys.argv[1] not in ("init", "find", "src"):
+		log("Command not being executed in bench directory", level=3)
+
 	if len(sys.argv) > 2 and sys.argv[1] == "frappe":
 		return old_frappe_cli()
 
-	elif len(sys.argv) > 1 and sys.argv[1] in get_frappe_commands():
+	elif len(sys.argv) > 1:
+		if sys.argv[1] in get_frappe_commands() + ["--site", "--verbose", "--force", "--profile"]:
 			return frappe_cmd()
 
-	elif len(sys.argv) > 1 and sys.argv[1] in ("--site", "--verbose", "--force", "--profile"):
-		return frappe_cmd()
-
-	elif len(sys.argv) > 1 and sys.argv[1]=="--help":
+		elif sys.argv[1] == "--help":
 			print(click.Context(bench_command).get_help())
 			print(get_frappe_help())
 			return
 
-	elif len(sys.argv) > 1 and sys.argv[1] in get_apps():
+		elif sys.argv[1] in get_apps():
 			return app_cmd()
 
-	else:
-		try:
+	if not (len(sys.argv) > 1 and sys.argv[1] == "src"):
 		atexit.register(check_latest_version)
+
+	try:
 		bench_command()
 	except PatchError:
 		sys.exit(1)
 
+
 def check_uid():
 	if cmd_requires_root() and not is_root():
 		log('superuser privileges required for this command', level=3)
 		sys.exit(1)
 
+
 def cmd_requires_root():
 	if len(sys.argv) > 2 and sys.argv[2] in ('production', 'sudoers', 'supervisor', 'lets-encrypt', 'fonts',
 		'print', 'firewall', 'ssh-port', 'role', 'fail2ban', 'wildcard-ssl'):
 		return True
-	if len(sys.argv) >= 2 and sys.argv[1] in ('patch', 'renew-lets-encrypt', 'disable-production',
-		'install'):
+	if len(sys.argv) >= 2 and sys.argv[1] in ('patch', 'renew-lets-encrypt', 'disable-production'):
 		return True
+	if len(sys.argv) > 2 and sys.argv[1] in ('install'):
+		return True
+
 
 def change_dir():
 	if os.path.exists('config.json') or "init" in sys.argv:
@@ -70,6 +86,7 @@ def change_dir():
 		if os.path.exists(dir_path):
 			os.chdir(dir_path)
 
+
 def change_uid():
 	if is_root() and not cmd_requires_root():
 		frappe_user = get_config(".").get('frappe_user')
@@ -80,35 +97,37 @@ def change_uid():
 			log(change_uid_msg, level=3)
 			sys.exit(1)
 
+
 def old_frappe_cli(bench_path='.'):
 	f = get_frappe(bench_path=bench_path)
 	os.chdir(os.path.join(bench_path, 'sites'))
 	os.execv(f, [f] + sys.argv[2:])
 
+
 def app_cmd(bench_path='.'):
 	f = get_env_cmd('python', bench_path=bench_path)
 	os.chdir(os.path.join(bench_path, 'sites'))
 	os.execv(f, [f] + ['-m', 'frappe.utils.bench_helper'] + sys.argv[1:])
 
+
 def frappe_cmd(bench_path='.'):
 	f = get_env_cmd('python', bench_path=bench_path)
 	os.chdir(os.path.join(bench_path, 'sites'))
 	os.execv(f, [f] + ['-m', 'frappe.utils.bench_helper', 'frappe'] + sys.argv[1:])
 
-def get_frappe_commands(bench_path='.'):
-	python = get_env_cmd('python', bench_path=bench_path)
-	sites_path = os.path.join(bench_path, 'sites')
-	if not os.path.exists(sites_path):
-		log("Command not being executed in bench directory", level=3)
-		return []
-	try:
-		output = get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-commands".format(python=python), cwd=sites_path)
-		return json.loads(output)
-	except subprocess.CalledProcessError as e:
-		if hasattr(e, "stderr"):
-			print(e.stderr.decode('utf-8'))
+
+def get_frappe_commands():
+	if not is_bench_directory():
 		return []
 
+	if os.path.exists(bench_cache_file):
+		command_dump = open(bench_cache_file, 'r').read() or '[]'
+		return json.loads(command_dump)
+
+	else:
+		return generate_command_cache()
+
+
 def get_frappe_help(bench_path='.'):
 	python = get_env_cmd('python', bench_path=bench_path)
 	sites_path = os.path.join(bench_path, 'sites')
@@ -118,6 +137,7 @@ def get_frappe_help(bench_path='.'):
 	except:
 		return ""
 
+
 def change_working_directory():
 	"""Allows bench commands to be run from anywhere inside a bench directory"""
 	cur_dir = os.path.abspath(".")

bench/commands/__init__.py

@@ -41,7 +41,8 @@ bench_command.add_command(switch_to_develop)
 
 from bench.commands.utils import (start, restart, set_nginx_port, set_ssl_certificate, set_ssl_certificate_key, set_url_root,
 	set_mariadb_host, set_default_site, download_translations, backup_site, backup_all_sites, release, renew_lets_encrypt,
-	disable_production, bench_src, prepare_beta_release, set_redis_cache_host, set_redis_queue_host, set_redis_socketio_host, find_benches, migrate_env)
+	disable_production, bench_src, prepare_beta_release, set_redis_cache_host, set_redis_queue_host, set_redis_socketio_host, find_benches, migrate_env,
+	generate_command_cache, clear_command_cache)
 bench_command.add_command(start)
 bench_command.add_command(restart)
 bench_command.add_command(set_nginx_port)
@@ -63,7 +64,8 @@ bench_command.add_command(bench_src)
 bench_command.add_command(prepare_beta_release)
 bench_command.add_command(find_benches)
 bench_command.add_command(migrate_env)
-
+bench_command.add_command(generate_command_cache)
+bench_command.add_command(clear_command_cache)
 
 from bench.commands.setup import setup
 bench_command.add_command(setup)

bench/commands/utils.py

@@ -180,3 +180,15 @@ def find_benches(location):
 def migrate_env(python, backup=True):
 	from bench.utils import migrate_env
 	migrate_env(python=python, backup=backup)
+
+
+@click.command('generate-command-cache', help="Caches Frappe Framework commands")
+def generate_command_cache(bench_path='.'):
+	from bench.utils import generate_command_cache
+	return generate_command_cache(bench_path=bench_path)
+
+
+@click.command('clear-command-cache', help="Clears Frappe Framework cached commands")
+def clear_command_cache(bench_path='.'):
+	from bench.utils import clear_command_cache
+	return clear_command_cache(bench_path=bench_path)

bench/utils.py

@@ -39,7 +39,7 @@ class CommandFailedError(Exception):
 	pass
 
 logger = logging.getLogger(__name__)
-
+bench_cache_file = '.bench.cmd'
 folders_in_bench = ('apps', 'sites', 'config', 'logs', 'config/pids')
 sudoers_file = '/etc/sudoers.d/frappe'
 
@@ -187,6 +187,8 @@ def update(pull=False, patch=False, build=False, requirements=False, backup=True
 	patches.run(bench_path=bench_path)
 	conf = get_config(bench_path)
 
+	clear_command_cache(bench_path='.')
+
 	if conf.get('release_bench'):
 		print('Release bench detected, cannot update!')
 		sys.exit(1)
@@ -842,7 +844,7 @@ def update_translations_p(args):
 
 
 def download_translations_p():
-	pool = multiprocessing.Pool(4)
+	pool = multiprocessing.Pool(multiprocessing.cpu_count())
 
 	langs = get_langs()
 	apps = ('frappe', 'erpnext')
@@ -1137,3 +1139,36 @@ def find_parent_bench(path):
 		# NOTE: the os.path.split assumes that given path is absolute
 		parent_dir = os.path.split(path)[0]
 		return find_parent_bench(parent_dir)
+
+
+def generate_command_cache(bench_path='.'):
+	"""Caches all available commands (even custom apps) via Frappe
+	Default caching behaviour: generated the first time any command (for a specific bench directory)
+	"""
+
+	python = get_env_cmd('python', bench_path=bench_path)
+	sites_path = os.path.join(bench_path, 'sites')
+
+	if os.path.exists(bench_cache_file):
+		os.remove(bench_cache_file)
+
+	try:
+		output = get_cmd_output("{0} -m frappe.utils.bench_helper get-frappe-commands".format(python), cwd=sites_path)
+		with open(bench_cache_file, 'w') as f:
+			json.dump(eval(output), f)
+		return json.loads(output)
+
+	except subprocess.CalledProcessError as e:
+		if hasattr(e, "stderr"):
+			print(e.stderr.decode('utf-8'))
+
+
+def clear_command_cache(bench_path='.'):
+	"""Clears commands cached
+	Default invalidation behaviour: destroyed on each run of `bench update`
+	"""
+
+	if os.path.exists(bench_cache_file):
+		os.remove(bench_cache_file)
+	else:
+		print("Bench command cache doesn't exist in this folder!")