Merge pull request #1124 from adityahase/perf-cli

perf: Faster CLI
2024-09-28 06:49:06 +00:00 · 2021-02-26 16:39:01 +05:30 · 2021-02-26 16:39:01 +05:30 · 034e0085e9
commit 034e0085e9
parent 6b08292d91 b70aa21a73
14 changed files with 89 additions and 55 deletions
--- a/bench/app.py
+++ b/bench/app.py
@ -6,20 +6,14 @@ import json
 import logging
 import os
 import re
-import shutil
 import subprocess
 import sys

 # imports - third party imports
 import click
-import git
-import requests
-import semantic_version
-from six.moves import reload_module

 # imports - module imports
 import bench
-from bench.config.common_site_config import get_config
 from bench.utils import color, CommandFailedError, build_assets, check_git_for_shallow_clone, exec_cmd, get_cmd_output, get_frappe, restart_supervisor_processes, restart_systemd_processes, run_frappe_cmd


@ -91,6 +85,9 @@ def remove_from_excluded_apps_txt(app, bench_path='.'):
 		return write_excluded_apps_txt(apps, bench_path=bench_path)

 def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=False, restart_bench=True, overwrite=False):
+	import requests
+	import shutil
+
 	if not os.path.exists(git_url):
 		if not is_git_url(git_url):
 			orgs = ['frappe', 'erpnext']
@ -166,6 +163,8 @@ def new_app(app, bench_path='.'):


 def install_app(app, bench_path=".", verbose=False, no_cache=False, restart_bench=True, skip_assets=False):
+	from bench.config.common_site_config import get_config
+
 	print('\n{0}Installing {1}{2}'.format(color.yellow, app, color.nc))
 	logger.log("installing {}".format(app))

@ -194,6 +193,9 @@ def install_app(app, bench_path=".", verbose=False, no_cache=False, restart_benc


 def remove_app(app, bench_path='.'):
+	import shutil
+	from bench.config.common_site_config import get_config
+
 	if app not in get_apps(bench_path):
 		print("No app named {0}".format(app))
 		sys.exit(1)
@ -221,6 +223,8 @@ def remove_app(app, bench_path='.'):

 def pull_apps(apps=None, bench_path='.', reset=False):
 	'''Check all apps if there no local changes, pull'''
+	from bench.config.common_site_config import get_config
+
 	rebase = '--rebase' if get_config(bench_path).get('rebase_on_pull') else ''

 	apps = apps or get_apps(bench_path=bench_path)
@ -358,6 +362,8 @@ def get_repo_dir(app, bench_path='.'):
 	return os.path.join(bench_path, 'apps', app)

 def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrade=True):
+	import git
+	from six.moves import reload_module
 	from bench.utils import update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets, post_upgrade
 	apps_dir = os.path.join(bench_path, 'apps')
 	version_upgrade = (False,)
@ -425,6 +431,8 @@ def get_version_from_string(contents, field='__version__'):
 	return match.group(2)

 def get_major_version(version):
+	import semantic_version
+
 	return semantic_version.Version(version).major

 def install_apps_from_path(path, bench_path='.'):
@ -433,6 +441,8 @@ def install_apps_from_path(path, bench_path='.'):
 		get_app(app['url'], branch=app.get('branch'), bench_path=bench_path, skip_assets=True)

 def get_apps_json(path):
+	import requests
+
 	if path.startswith('http'):
 		r = requests.get(path)
 		return r.json()
--- a/bench/commands/config.py
+++ b/bench/commands/config.py
@ -1,6 +1,3 @@
-# imports - standard imports
-import ast
-
 # imports - module imports
 from bench.config.common_site_config import update_config, get_config, put_config

@ -52,6 +49,8 @@ def config_http_timeout(seconds):
@click.command('set-common-config', help='Set value in common config')
@click.option('configs', '-c', '--config', multiple=True, type=(str, str))
 def set_common_config(configs):
+	import ast
+
 	common_site_config = {}
 	for key, value in configs:
 		if value in ('true', 'false'):
--- a/bench/commands/setup.py
+++ b/bench/commands/setup.py
@ -6,10 +6,7 @@ import sys
 import click

 # imports - module imports
-import bench.config.lets_encrypt
-import bench.config.nginx
 import bench.config.procfile
-import bench.config.production_setup
 import bench.config.redis
 import bench.config.site_config
 import bench.config.supervisor
@ -31,11 +28,15 @@ def setup_sudoers(user):
@click.command("nginx", help="Generate configuration files for NGINX")
@click.option("--yes", help="Yes to regeneration of nginx config file", default=False, is_flag=True)
 def setup_nginx(yes=False):
+	import bench.config.nginx
+
 	bench.config.nginx.make_nginx_conf(bench_path=".", yes=yes)


@click.command("reload-nginx", help="Checks NGINX config file and reloads service")
 def reload_nginx():
+	import bench.config.production_setup
+
 	bench.config.production_setup.reload_nginx()


@ -61,6 +62,8 @@ def setup_fonts():
@click.argument("user")
@click.option("--yes", help="Yes to regeneration config", is_flag=True, default=False)
 def setup_production(user, yes=False):
+	import bench.config.production_setup
+
 	bench.config.production_setup.setup_production(user=user, yes=yes)


@ -103,6 +106,8 @@ def set_ssh_port(port, force=False):
@click.option("--custom-domain")
@click.option('-n', '--non-interactive', default=False, is_flag=True, help="Run command non-interactively. This flag restarts nginx and runs certbot non interactively. Shouldn't be used on 1'st attempt")
 def setup_letsencrypt(site, custom_domain, non_interactive):
+	import bench.config.lets_encrypt
+
 	bench.config.lets_encrypt.setup_letsencrypt(site, custom_domain, bench_path=".", interactive=not non_interactive)


@ -111,6 +116,8 @@ def setup_letsencrypt(site, custom_domain, non_interactive):
@click.option("--email")
@click.option("--exclude-base-domain", default=False, is_flag=True, help="SSL Certificate not applicable for base domain")
 def setup_wildcard_ssl(domain, email, exclude_base_domain):
+	import bench.config.lets_encrypt
+
 	bench.config.lets_encrypt.setup_wildcard_ssl(domain, email, bench_path=".", exclude_base_domain=exclude_base_domain)


--- a/bench/config/init.py
+++ b/bench/config/init.py
@ -1,6 +1,6 @@
 """Module for setting up system and respective bench configurations"""

-# imports - third party imports
-from jinja2 import Environment, PackageLoader

-env = Environment(loader=PackageLoader('bench.config'))
+def env():
+	from jinja2 import Environment, PackageLoader
+	return Environment(loader=PackageLoader('bench.config'))
--- a/bench/config/common_site_config.py
+++ b/bench/config/common_site_config.py
@ -1,11 +1,8 @@
 # imports - standard imports
 import getpass
 import json
-import multiprocessing
 import os

-# imports - third party imports
-from six.moves.urllib.parse import urlparse


 default_config = {
@ -54,6 +51,8 @@ def get_config_path(bench_path):
 def get_gunicorn_workers():
 	'''This function will return the maximum workers that can be started depending upon
 	number of cpu's present on the machine'''
+	import multiprocessing
+
 	return {
 		"gunicorn_workers": multiprocessing.cpu_count() * 2 + 1
 	}
@ -73,6 +72,8 @@ def update_config_for_frappe(config, bench_path):
 	# TODO Optionally we need to add the host or domain name in case dns_multitenant is false

 def make_ports(bench_path):
+	from six.moves.urllib.parse import urlparse
+
 	benches_path = os.path.dirname(os.path.abspath(bench_path))

 	default_ports = {
--- a/bench/config/lets_encrypt.py
+++ b/bench/config/lets_encrypt.py
@ -3,8 +3,6 @@ import os

 # imports - third party imports
 import click
-from crontab import CronTab
-from six.moves.urllib.request import urlretrieve

 # imports - module imports
 import bench
@ -48,7 +46,7 @@ def setup_letsencrypt(site, custom_domain, bench_path, interactive):


 def create_config(site, custom_domain):
-	config = bench.config.env.get_template('letsencrypt.cfg').render(domain=custom_domain or site)
+	config = bench.config.env().get_template('letsencrypt.cfg').render(domain=custom_domain or site)
 	config_path = '/etc/letsencrypt/configs/{site}.cfg'.format(site=custom_domain or site)
 	create_dir_if_missing(config_path)

@ -86,6 +84,8 @@ def run_certbot_and_setup_ssl(site, custom_domain, bench_path, interactive=True)


 def setup_crontab():
+	from crontab import CronTab
+
 	job_command = '/opt/certbot-auto renew -a nginx --post-hook "systemctl reload nginx"'
 	job_comment = 'Renew lets-encrypt every month'
 	print("Setting Up cron job to {0}".format(job_comment))
@ -106,6 +106,8 @@ def create_dir_if_missing(path):


 def get_certbot():
+	from six.moves.urllib.request import urlretrieve
+
 	certbot_path = get_certbot_path()
 	create_dir_if_missing(certbot_path)

--- a/bench/config/nginx.py
+++ b/bench/config/nginx.py
@ -20,7 +20,7 @@ def make_nginx_conf(bench_path, yes=False):
 		if not click.confirm('nginx.conf already exists and this will overwrite it. Do you want to continue?'):
 			return

-	template = bench.config.env.get_template('nginx.conf')
+	template = bench.config.env().get_template('nginx.conf')
 	bench_path = os.path.abspath(bench_path)
 	sites_path = os.path.join(bench_path, "sites")

@ -59,7 +59,7 @@ def make_bench_manager_nginx_conf(bench_path, yes=False, port=23624, domain=None
 	from bench.config.site_config import get_site_config
 	from bench.config.common_site_config import get_config

-	template = bench.config.env.get_template('bench_manager_nginx.conf')
+	template = bench.config.env().get_template('bench_manager_nginx.conf')
 	bench_path = os.path.abspath(bench_path)
 	sites_path = os.path.join(bench_path, "sites")

--- a/bench/config/procfile.py
+++ b/bench/config/procfile.py
@ -18,7 +18,7 @@ def setup_procfile(bench_path, yes=False, skip_redis=False):
 		click.confirm('A Procfile already exists and this will overwrite it. Do you want to continue?',
 			abort=True)

-	procfile = bench.config.env.get_template('Procfile').render(
+	procfile = bench.config.env().get_template('Procfile').render(
 		node=find_executable("node") or find_executable("nodejs"),
 		use_rq=use_rq(bench_path),
 		webserver_port=config.get('webserver_port'),
--- a/bench/config/redis.py
+++ b/bench/config/redis.py
@ -3,16 +3,14 @@ import os
 import re
 import subprocess

-# imports - third party imports
-import semantic_version
-from six.moves.urllib.parse import urlparse
-
 # imports - module imports
 import bench
 from bench.config.common_site_config import get_config


 def generate_config(bench_path):
+	from six.moves.urllib.parse import urlparse
+
 	config = get_config(bench_path)

 	ports = {}
@ -52,7 +50,7 @@ def generate_config(bench_path):
 		os.makedirs(pid_path)

 def write_redis_config(template_name, context, bench_path):
-	template = bench.config.env.get_template(template_name)
+	template = bench.config.env().get_template(template_name)

 	if "pid_path" not in context:
 		context["pid_path"] = os.path.abspath(os.path.join(bench_path, "config", "pids"))
@ -61,6 +59,8 @@ def write_redis_config(template_name, context, bench_path):
 		f.write(template.render(**context))

 def get_redis_version():
+	import semantic_version
+
 	version_string = subprocess.check_output('redis-server --version', shell=True)
 	version_string = version_string.decode('utf-8').strip()
 	# extract version number from string
--- a/bench/config/site_config.py
+++ b/bench/config/site_config.py
@ -4,7 +4,6 @@ import os
 from collections import defaultdict

 # imports - module imports
-from bench.config.nginx import make_nginx_conf
 from bench.utils import get_sites


@ -35,6 +34,8 @@ def set_ssl_certificate_key(site, ssl_certificate_key, bench_path='.', gen_confi
 	set_site_config_nginx_property(site, {"ssl_certificate_key": ssl_certificate_key}, bench_path=bench_path, gen_config=gen_config)

 def set_site_config_nginx_property(site, config, bench_path='.', gen_config=True):
+	from bench.config.nginx import make_nginx_conf
+
 	if site not in get_sites(bench_path=bench_path):
 		raise Exception("No such site")
 	update_site_config(site, config, bench_path=bench_path)
--- a/bench/config/supervisor.py
+++ b/bench/config/supervisor.py
@ -11,7 +11,6 @@ from bench.config.common_site_config import get_config, update_config, get_gunic

 # imports - third party imports
 import click
-from six.moves import configparser


 logger = logging.getLogger(bench.PROJECT_NAME)
@ -22,7 +21,7 @@ def generate_supervisor_config(bench_path, user=None, yes=False):
 	if not user:
 		user = getpass.getuser()

-	template = bench.config.env.get_template('supervisor.conf')
+	template = bench.config.env().get_template('supervisor.conf')
 	config = get_config(bench_path=bench_path)
 	bench_dir = os.path.abspath(bench_path)

@ -68,6 +67,8 @@ def get_supervisord_conf():

 def update_supervisord_config(user=None, yes=False):
 	"""From bench v5.x, we're moving to supervisor running as user"""
+	from six.moves import configparser
+
 	from bench.config.production_setup import service

 	if not user:
--- a/bench/config/systemd.py
+++ b/bench/config/systemd.py
@ -85,7 +85,7 @@ def setup_systemd_directory(bench_path):

 def setup_main_config(bench_info, bench_path):
 	# Main config
-	bench_template = bench.config.env.get_template('systemd/frappe-bench.target')
+	bench_template = bench.config.env().get_template('systemd/frappe-bench.target')
 	bench_config = bench_template.render(**bench_info)
 	bench_config_path = os.path.join(bench_path, 'config', 'systemd' , bench_info.get("bench_name") + '.target')

@ -94,11 +94,11 @@ def setup_main_config(bench_info, bench_path):

 def setup_workers_config(bench_info, bench_path):
 	# Worker Group
-	bench_workers_target_template = bench.config.env.get_template('systemd/frappe-bench-workers.target')
-	bench_default_worker_template = bench.config.env.get_template('systemd/frappe-bench-frappe-default-worker.service')
-	bench_short_worker_template = bench.config.env.get_template('systemd/frappe-bench-frappe-short-worker.service')
-	bench_long_worker_template = bench.config.env.get_template('systemd/frappe-bench-frappe-long-worker.service')
-	bench_schedule_worker_template = bench.config.env.get_template('systemd/frappe-bench-frappe-schedule.service')
+	bench_workers_target_template = bench.config.env().get_template('systemd/frappe-bench-workers.target')
+	bench_default_worker_template = bench.config.env().get_template('systemd/frappe-bench-frappe-default-worker.service')
+	bench_short_worker_template = bench.config.env().get_template('systemd/frappe-bench-frappe-short-worker.service')
+	bench_long_worker_template = bench.config.env().get_template('systemd/frappe-bench-frappe-long-worker.service')
+	bench_schedule_worker_template = bench.config.env().get_template('systemd/frappe-bench-frappe-schedule.service')

 	bench_workers_target_config = bench_workers_target_template.render(**bench_info)
 	bench_default_worker_config = bench_default_worker_template.render(**bench_info)
@ -129,9 +129,9 @@ def setup_workers_config(bench_info, bench_path):

 def setup_web_config(bench_info, bench_path):
 	# Web Group
-	bench_web_target_template = bench.config.env.get_template('systemd/frappe-bench-web.target')
-	bench_web_service_template = bench.config.env.get_template('systemd/frappe-bench-frappe-web.service')
-	bench_node_socketio_template = bench.config.env.get_template('systemd/frappe-bench-node-socketio.service')
+	bench_web_target_template = bench.config.env().get_template('systemd/frappe-bench-web.target')
+	bench_web_service_template = bench.config.env().get_template('systemd/frappe-bench-frappe-web.service')
+	bench_node_socketio_template = bench.config.env().get_template('systemd/frappe-bench-node-socketio.service')

 	bench_web_target_config = bench_web_target_template.render(**bench_info)
 	bench_web_service_config = bench_web_service_template.render(**bench_info)
@ -152,10 +152,10 @@ def setup_web_config(bench_info, bench_path):

 def setup_redis_config(bench_info, bench_path):
 	# Redis Group
-	bench_redis_target_template = bench.config.env.get_template('systemd/frappe-bench-redis.target')
-	bench_redis_cache_template = bench.config.env.get_template('systemd/frappe-bench-redis-cache.service')
-	bench_redis_queue_template = bench.config.env.get_template('systemd/frappe-bench-redis-queue.service')
-	bench_redis_socketio_template = bench.config.env.get_template('systemd/frappe-bench-redis-socketio.service')
+	bench_redis_target_template = bench.config.env().get_template('systemd/frappe-bench-redis.target')
+	bench_redis_cache_template = bench.config.env().get_template('systemd/frappe-bench-redis-cache.service')
+	bench_redis_queue_template = bench.config.env().get_template('systemd/frappe-bench-redis-queue.service')
+	bench_redis_socketio_template = bench.config.env().get_template('systemd/frappe-bench-redis-socketio.service')

 	bench_redis_target_config = bench_redis_target_template.render(**bench_info)
 	bench_redis_cache_config = bench_redis_cache_template.render(**bench_info)
--- a/bench/release.py
+++ b/bench/release.py
@ -4,11 +4,8 @@ import os
 import sys
 import semantic_version
 import git
-import requests
 import getpass
 import re
-from requests.auth import HTTPBasicAuth
-import requests.exceptions
 from time import sleep
 from .config.common_site_config import get_config
 import click
@ -47,6 +44,9 @@ def release(bench_path, app, bump_type, from_branch, to_branch,
 		repo_name=repo_name, remote=remote, frontport=frontport)

 def validate(bench_path, config):
+	import requests
+	from requests.auth import HTTPBasicAuth
+
 	global github_username, github_password

 	github_username = config.get('github_username')
@ -306,6 +306,9 @@ def push_release(repo_path, from_branch, to_branch, remote='upstream'):

 def create_github_release(repo_path, tag_name, message, remote='upstream', owner='frappe', repo_name=None,
 		gh_username=None, gh_password=None, prerelease=False):
+	import requests
+	import requests.exceptions
+	from requests.auth import HTTPBasicAuth

 	print('creating release on github')

--- a/bench/utils.py
+++ b/bench/utils.py
@ -9,12 +9,10 @@ import grp
 import itertools
 import json
 import logging
-import multiprocessing
 import os
 import pwd
 import re
 import select
-import shutil
 import site
 import subprocess
 import sys
@ -23,11 +21,7 @@ from distutils.spawn import find_executable

 # imports - third party imports
 import click
-from crontab import CronTab
-import requests
-from semantic_version import Version
 from six import iteritems
-from six.moves.urllib.parse import urlparse

 # imports - module imports
 import bench
@ -93,6 +87,9 @@ def safe_decode(string, encoding = 'utf-8'):


 def check_latest_version():
+	import requests
+	from semantic_version import Version
+
 	try:
 		pypi_request = requests.get("https://pypi.org/pypi/frappe-bench/json")
 	except Exception:
@ -263,6 +260,8 @@ def update(pull=False, apps=None, patch=False, build=False, requirements=False,


 def copy_patches_txt(bench_path):
+	import shutil
+
 	shutil.copy(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'patches', 'patches.txt'),
 		os.path.join(bench_path, 'patches.txt'))

@ -386,6 +385,7 @@ def get_sites(bench_path='.'):


 def setup_backups(bench_path='.'):
+	from crontab import CronTab
 	from bench.config.common_site_config import get_config
 	logger.log('setting up backups')

@ -422,7 +422,7 @@ def setup_sudoers(user):
 		if set_permissions:
 			os.chmod('/etc/sudoers', 0o440)

-	template = bench.config.env.get_template('frappe_sudoers')
+	template = bench.config.env().get_template('frappe_sudoers')
 	frappe_sudoers = template.render(**{
 		'user': user,
 		'service': find_executable('service'),
@ -815,6 +815,8 @@ sudo supervisorctl reload


 def update_translations_p(args):
+	import requests
+
 	try:
 		update_translations(*args)
 	except requests.exceptions.HTTPError:
@ -822,6 +824,8 @@ def update_translations_p(args):


 def download_translations_p():
+	import multiprocessing
+
 	pool = multiprocessing.Pool(multiprocessing.cpu_count())

 	langs = get_langs()
@ -846,6 +850,8 @@ def get_langs():


 def update_translations(app, lang):
+	import requests
+
 	translations_dir = os.path.join('apps', app, app, 'translations')
 	csv_file = os.path.join(translations_dir, lang + '.csv')
 	url = "https://translate.erpnext.com/files/{}-{}.csv".format(app, lang)
@ -895,6 +901,8 @@ def get_bench_name(bench_path):


 def setup_fonts():
+	import shutil
+
 	fonts_path = os.path.join('/tmp', 'fonts')

 	if os.path.exists('/etc/fonts_backup'):
@ -967,6 +975,8 @@ def find_benches(directory=None):


 def migrate_env(python, backup=False):
+	import shutil
+	from six.moves.urllib.parse import urlparse
 	from bench.config.common_site_config import get_config
 	from bench.app import get_apps