From da85e0ccdd83e40956c30a1bd00e3a1543faa682 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 23 Apr 2021 10:58:21 +0530 Subject: [PATCH] refactor: Optimize and simplify imports * make relative imports absolute * minor: Simplify logic --- bench/config/procfile.py | 4 +- bench/config/production_setup.py | 18 ++++----- bench/config/supervisor.py | 8 ++-- bench/config/systemd.py | 8 ++-- bench/utils.py | 67 +++++++++++++++++--------------- 5 files changed, 54 insertions(+), 51 deletions(-) diff --git a/bench/config/procfile.py b/bench/config/procfile.py index e57e579d..50a45560 100755 --- a/bench/config/procfile.py +++ b/bench/config/procfile.py @@ -8,7 +8,7 @@ import click import bench from bench.app import use_rq from bench.config.common_site_config import get_config -from bench.utils import find_executable +from bench.utils import which def setup_procfile(bench_path, yes=False, skip_redis=False): @@ -19,7 +19,7 @@ def setup_procfile(bench_path, yes=False, skip_redis=False): abort=True) procfile = bench.config.env().get_template('Procfile').render( - node=find_executable("node") or find_executable("nodejs"), + node=which("node") or which("nodejs"), use_rq=use_rq(bench_path), webserver_port=config.get('webserver_port'), CI=os.environ.get('CI'), diff --git a/bench/config/production_setup.py b/bench/config/production_setup.py index 6809b733..13640e6f 100755 --- a/bench/config/production_setup.py +++ b/bench/config/production_setup.py @@ -9,7 +9,7 @@ from bench.config.common_site_config import get_config from bench.config.nginx import make_nginx_conf from bench.config.supervisor import generate_supervisor_config, update_supervisord_config from bench.config.systemd import generate_systemd_config -from bench.utils import CommandFailedError, exec_cmd, find_executable, fix_prod_setup_perms, get_bench_name, get_cmd_output, log +from bench.utils import CommandFailedError, exec_cmd, which, fix_prod_setup_perms, get_bench_name, get_cmd_output, log logger = logging.getLogger(bench.PROJECT_NAME) @@ -17,13 +17,13 @@ logger = logging.getLogger(bench.PROJECT_NAME) def setup_production_prerequisites(): """Installs ansible, fail2banc, NGINX and supervisor""" - if not find_executable("ansible"): + if not which("ansible"): exec_cmd("sudo {0} -m pip install ansible".format(sys.executable)) - if not find_executable("fail2ban-client"): + if not which("fail2ban-client"): exec_cmd("bench setup role fail2ban") - if not find_executable("nginx"): + if not which("nginx"): exec_cmd("bench setup role nginx") - if not find_executable("supervisord"): + if not which("supervisord"): exec_cmd("bench setup role supervisor") @@ -95,11 +95,11 @@ def disable_production(bench_path='.'): def service(service_name, service_option): - if os.path.basename(find_executable('systemctl') or '') == 'systemctl' and is_running_systemd(): + if os.path.basename(which('systemctl') or '') == 'systemctl' and is_running_systemd(): systemctl_cmd = "sudo {service_manager} {service_option} {service_name}" exec_cmd(systemctl_cmd.format(service_manager='systemctl', service_option=service_option, service_name=service_name)) - elif os.path.basename(find_executable('service') or '') == 'service': + elif os.path.basename(which('service') or '') == 'service': service_cmd = "sudo {service_manager} {service_name} {service_option}" exec_cmd(service_cmd.format(service_manager='service', service_name=service_name, service_option=service_option)) @@ -145,7 +145,7 @@ def is_running_systemd(): def reload_supervisor(): - supervisorctl = find_executable('supervisorctl') + supervisorctl = which('supervisorctl') try: # first try reread/update @@ -178,7 +178,7 @@ def reload_supervisor(): def reload_nginx(): try: - exec_cmd('sudo {0} -t'.format(find_executable('nginx'))) + exec_cmd('sudo {0} -t'.format(which('nginx'))) except: raise diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 76ffd877..0cea009e 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -6,7 +6,7 @@ import os # imports - module imports import bench from bench.app import use_rq -from bench.utils import get_bench_name, find_executable +from bench.utils import get_bench_name, which from bench.config.common_site_config import get_config, update_config, get_gunicorn_workers # imports - third party imports @@ -31,8 +31,8 @@ def generate_supervisor_config(bench_path, user=None, yes=False, skip_redis=Fals "user": user, "use_rq": use_rq(bench_path), "http_timeout": config.get("http_timeout", 120), - "redis_server": find_executable('redis-server'), - "node": find_executable('node') or find_executable('nodejs'), + "redis_server": which('redis-server'), + "node": which('node') or which('nodejs'), "redis_cache_config": os.path.join(bench_dir, 'config', 'redis_cache.conf'), "redis_socketio_config": os.path.join(bench_dir, 'config', 'redis_socketio.conf'), "redis_queue_config": os.path.join(bench_dir, 'config', 'redis_queue.conf'), @@ -40,7 +40,7 @@ def generate_supervisor_config(bench_path, user=None, yes=False, skip_redis=Fals "gunicorn_workers": config.get('gunicorn_workers', get_gunicorn_workers()["gunicorn_workers"]), "bench_name": get_bench_name(bench_path), "background_workers": config.get('background_workers') or 1, - "bench_cmd": find_executable('bench'), + "bench_cmd": which('bench'), "skip_redis": skip_redis, }) diff --git a/bench/config/systemd.py b/bench/config/systemd.py index c0a0053c..1501d455 100644 --- a/bench/config/systemd.py +++ b/bench/config/systemd.py @@ -9,7 +9,7 @@ import click import bench from bench.app import use_rq from bench.config.common_site_config import get_config, get_gunicorn_workers, update_config -from bench.utils import exec_cmd, find_executable, get_bench_name +from bench.utils import exec_cmd, which, get_bench_name def generate_systemd_config(bench_path, user=None, yes=False, @@ -53,8 +53,8 @@ def generate_systemd_config(bench_path, user=None, yes=False, "user": user, "use_rq": use_rq(bench_path), "http_timeout": config.get("http_timeout", 120), - "redis_server": find_executable('redis-server'), - "node": find_executable('node') or find_executable('nodejs'), + "redis_server": which('redis-server'), + "node": which('node') or which('nodejs'), "redis_cache_config": os.path.join(bench_dir, 'config', 'redis_cache.conf'), "redis_socketio_config": os.path.join(bench_dir, 'config', 'redis_socketio.conf'), "redis_queue_config": os.path.join(bench_dir, 'config', 'redis_queue.conf'), @@ -62,7 +62,7 @@ def generate_systemd_config(bench_path, user=None, yes=False, "gunicorn_workers": config.get('gunicorn_workers', get_gunicorn_workers()["gunicorn_workers"]), "bench_name": get_bench_name(bench_path), "worker_target_wants": " ".join(background_workers), - "bench_cmd": find_executable('bench') + "bench_cmd": which('bench') } if not yes: diff --git a/bench/utils.py b/bench/utils.py index 43525bb5..ede9b169 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -2,26 +2,17 @@ # -*- coding: utf-8 -*- # imports - standard imports -import compileall -import errno -import glob import grp import itertools import json import logging import os import pwd -import re -import select -import site import subprocess import sys -from datetime import datetime -from distutils.spawn import find_executable # imports - third party imports import click -from six import iteritems # imports - module imports import bench @@ -143,6 +134,8 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, try: os.makedirs(os.path.join(path, dirname)) except OSError as e: + import errno + if e.errno == errno.EEXIST: pass @@ -184,6 +177,7 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, def update(pull=False, apps=None, patch=False, build=False, requirements=False, backup=True, compile=True, force=False, reset=False, restart_supervisor=False, restart_systemd=False): """command: bench update""" + import re from bench import patches from bench.app import is_version_upgrade, pull_apps, validate_branch from bench.config.common_site_config import get_config, update_config @@ -244,9 +238,11 @@ def update(pull=False, apps=None, patch=False, build=False, requirements=False, post_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) if pull and compile: + from compileall import compile_dir + print('Compiling Python files...') apps_dir = os.path.join(bench_path, 'apps') - compileall.compile_dir(apps_dir, quiet=1, rx=re.compile('.*node_modules.*')) + compile_dir(apps_dir, quiet=1, rx=re.compile('.*node_modules.*')) if restart_supervisor or conf.get('restart_supervisor_on_update'): restart_supervisor_processes(bench_path=bench_path) @@ -268,7 +264,7 @@ def copy_patches_txt(bench_path): def clone_apps_from(bench_path, clone_from, update_app=True): - from .app import install_app + from bench.app import install_app print('Copying apps from {0}...'.format(clone_from)) subprocess.check_output(['cp', '-R', os.path.join(clone_from, 'apps'), bench_path]) @@ -316,7 +312,9 @@ def exec_cmd(cmd, cwd='.'): logger.warning("{0} executed with exit code {1}".format(cmd_log, return_code)) -def which(executable, raise_err = False): +def which(executable, raise_err=False): + from distutils.spawn import find_executable + exec_ = find_executable(executable) if not exec_ and raise_err: @@ -411,9 +409,9 @@ def setup_sudoers(user): template = bench.config.env().get_template('frappe_sudoers') frappe_sudoers = template.render(**{ 'user': user, - 'service': find_executable('service'), - 'systemctl': find_executable('systemctl'), - 'nginx': find_executable('nginx'), + 'service': which('service'), + 'systemctl': which('systemctl'), + 'nginx': which('nginx'), }) frappe_sudoers = safe_decode(frappe_sudoers) @@ -449,7 +447,7 @@ def setup_logging(bench_path='.'): def get_process_manager(): for proc_man in ['honcho', 'foreman', 'forego']: - proc_man_path = find_executable(proc_man) + proc_man_path = which(proc_man) if proc_man_path: return proc_man_path @@ -486,7 +484,7 @@ def get_git_version(): def check_git_for_shallow_clone(): - from .config.common_site_config import get_config + from bench.config.common_site_config import get_config config = get_config('.') if config: @@ -514,7 +512,7 @@ def get_cmd_output(cmd, cwd='.', _raise=True): def restart_supervisor_processes(bench_path='.', web_workers=False): - from .config.common_site_config import get_config + from bench.config.common_site_config import get_config conf = get_config(bench_path=bench_path) bench_name = get_bench_name(bench_path) @@ -600,7 +598,7 @@ def update_node_packages(bench_path='.'): def update_yarn_packages(bench_path='.'): apps_dir = os.path.join(bench_path, 'apps') - if not find_executable('yarn'): + if not which('yarn'): print("Please install yarn using below command and try again.") print("`npm install -g yarn`") return @@ -621,6 +619,8 @@ def update_npm_packages(bench_path='.'): if os.path.exists(package_json_path): with open(package_json_path, "r") as f: + from six import iteritems + app_package_json = json.loads(f.read()) # package.json is usually a dict in a dict for key, value in iteritems(app_package_json): @@ -658,9 +658,7 @@ def backup_all_sites(bench_path='.'): def is_root(): - if os.getuid() == 0: - return True - return False + return os.getuid() == 0 def set_mariadb_host(host, bench_path='.'): @@ -718,7 +716,8 @@ def drop_privileges(uid_name='nobody', gid_name='nogroup'): def fix_prod_setup_perms(bench_path='.', frappe_user=None): - from .config.common_site_config import get_config + from glob import glob + from bench.config.common_site_config import get_config if not frappe_user: frappe_user = get_config(bench_path).get('frappe_user') @@ -729,14 +728,14 @@ def fix_prod_setup_perms(bench_path='.', frappe_user=None): globs = ["logs/*", "config/*"] for glob_name in globs: - for path in glob.glob(glob_name): + for path in glob(glob_name): uid = pwd.getpwnam(frappe_user).pw_uid gid = grp.getgrnam(frappe_user).gr_gid os.chown(path, uid, gid) def run_frappe_cmd(*args, **kwargs): - from .cli import from_command_line + from bench.cli import from_command_line bench_path = kwargs.get('bench_path', '.') f = get_env_cmd('python', bench_path=bench_path) @@ -762,15 +761,15 @@ def run_frappe_cmd(*args, **kwargs): def validate_upgrade(from_ver, to_ver, bench_path='.'): if to_ver >= 6: - if not find_executable('npm') and not (find_executable('node') or find_executable('nodejs')): + if not which('npm') and not (which('node') or which('nodejs')): raise Exception("Please install nodejs and npm") def post_upgrade(from_ver, to_ver, bench_path='.'): - from .config.common_site_config import get_config - from .config import redis - from .config.supervisor import generate_supervisor_config - from .config.nginx import make_nginx_conf + from bench.config.common_site_config import get_config + from bench.config import redis + from bench.config.supervisor import generate_supervisor_config + from bench.config.nginx import make_nginx_conf conf = get_config(bench_path=bench_path) print("-" * 80 + "Your bench was upgraded to version {0}".format(to_ver)) @@ -848,8 +847,10 @@ def update_translations(app, lang): def print_output(p): + from select import select + while p.poll() is None: - readx = select.select([p.stdout.fileno(), p.stderr.fileno()], [], [])[0] + readx = select([p.stdout.fileno(), p.stderr.fileno()], [], [])[0] send_buffer = [] for fd in readx: if fd == p.stdout.fileno(): @@ -910,7 +911,7 @@ def set_git_remote_url(git_url, bench_path='.'): def run_playbook(playbook_name, extra_vars=None, tag=None): - if not find_executable('ansible'): + if not which('ansible'): print("Ansible is needed to run this command, please install it using 'pip install ansible'") sys.exit(1) args = ['ansible-playbook', '-c', 'local', playbook_name, '-vvvv'] @@ -982,6 +983,8 @@ def migrate_env(python, backup=False): # Backup venv: restore using `virtualenv --relocatable` if needed if backup: + from datetime import datetime + parch = os.path.join(path, 'archived_envs') if not os.path.exists(parch): os.mkdir(parch)