From b4f4e717197b08705f1f4b40c876172cbae3f471 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 13 Feb 2020 04:15:30 +0530 Subject: [PATCH] chore: move 'find_parent_bench' to utils style: consistent spaces and formatting, code cleanup --- bench/cli.py | 23 ++----------- bench/prepare_staging.py | 6 ++-- bench/release.py | 2 -- bench/utils.py | 74 +++++++++++++++++++++++++++++++++++++--- 4 files changed, 75 insertions(+), 30 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index 733a0dd6..243e2640 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -1,6 +1,6 @@ import click import os, sys, logging, json, pwd, subprocess -from bench.utils import is_root, PatchError, drop_privileges, get_env_cmd, get_cmd_output, get_frappe, log, is_bench_directory +from bench.utils import is_root, PatchError, drop_privileges, get_env_cmd, get_cmd_output, get_frappe, log, is_bench_directory, find_parent_bench from bench.app import get_apps from bench.config.common_site_config import get_config from bench.commands import bench_command @@ -29,7 +29,6 @@ def cli(): elif len(sys.argv) > 1 and sys.argv[1]=="--help": print(click.Context(bench_command).get_help()) - print() print(get_frappe_help()) return @@ -99,7 +98,6 @@ def get_frappe_commands(bench_path='.'): return [] try: output = get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-commands".format(python=python), cwd=sites_path) - # output = output.decode('utf-8') return json.loads(output) except subprocess.CalledProcessError as e: if hasattr(e, "stderr"): @@ -109,27 +107,12 @@ def get_frappe_commands(bench_path='.'): def get_frappe_help(bench_path='.'): python = get_env_cmd('python', bench_path=bench_path) sites_path = os.path.join(bench_path, 'sites') - if not os.path.exists(sites_path): - return [] try: out = get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-help".format(python=python), cwd=sites_path) - return "Framework commands:\n" + out.split('Commands:')[1] - except subprocess.CalledProcessError: + return "\n\nFramework commands:\n" + out.split('Commands:')[1] + except: return "" -def find_parent_bench(path): - """Checks if parent directories are benches""" - if is_bench_directory(directory=path): - return path - - home_path = os.path.expanduser("~") - root_path = os.path.abspath(os.sep) - - if path not in {home_path, root_path}: - # NOTE: the os.path.split assumes that given path is absolute - parent_dir = os.path.split(path)[0] - return find_parent_bench(parent_dir) - def change_working_directory(): """Allows bench commands to be run from anywhere inside a bench directory""" cur_dir = os.path.abspath(".") diff --git a/bench/prepare_staging.py b/bench/prepare_staging.py index baa2a7c3..9d004684 100755 --- a/bench/prepare_staging.py +++ b/bench/prepare_staging.py @@ -19,9 +19,7 @@ def prepare_staging(bench_path, app, remote='upstream'): print('No commits to release') return - print() print(message) - print() click.confirm('Do you want to continue?', abort=True) @@ -52,13 +50,13 @@ def create_staging(repo_path, from_branch='develop'): g.merge(from_branch, '--no-ff') except git.exc.GitCommandError as e: handle_merge_error(e, source=from_branch, target='staging') - + g.checkout(from_branch) try: g.merge('staging') except git.exc.GitCommandError as e: handle_merge_error(e, source='staging', target=from_branch) - + def push_commits(repo_path, remote='upstream'): print('pushing staging branch of', repo_path) diff --git a/bench/release.py b/bench/release.py index ebf60f6a..f026b379 100755 --- a/bench/release.py +++ b/bench/release.py @@ -83,9 +83,7 @@ def bump(bench_path, app, bump_type, from_branch, to_branch, remote, owner, repo print('No commits to release') return - print() print(message) - print() click.confirm('Do you want to continue?', abort=True) diff --git a/bench/utils.py b/bench/utils.py index c00d9488..35c1a319 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -80,6 +80,7 @@ def safe_decode(string, encoding = 'utf-8'): pass return string + def get_frappe(bench_path='.'): frappe = get_env_cmd('frappe', bench_path=bench_path) if not os.path.exists(frappe): @@ -87,9 +88,11 @@ def get_frappe(bench_path='.'): print('bench get-app https://github.com/frappe/frappe.git') return frappe + def get_env_cmd(cmd, bench_path='.'): return os.path.abspath(os.path.join(bench_path, 'env', 'bin', cmd)) + def init(path, apps_path=None, no_procfile=False, no_backups=False, frappe_path=None, frappe_branch=None, verbose=False, clone_from=None, skip_redis_config_generation=False, clone_without_update=False, ignore_exist=False, skip_assets=False, @@ -152,10 +155,12 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, copy_patches_txt(path) + def restart_update(kwargs): args = ['--'+k for k, v in list(kwargs.items()) if v] os.execv(sys.argv[0], sys.argv[:2] + args) + def update(pull=False, patch=False, build=False, bench=False, restart_supervisor=False, restart_systemd=False, requirements=False, backup=True, force=False, reset=False): """command: bench update""" @@ -242,10 +247,12 @@ def update(pull=False, patch=False, build=False, bench=False, restart_supervisor print("_" * 80 + "\nBench: Deployment tool for Frappe and Frappe Applications (https://frappe.io/bench).\nOpen source depends on your contributions, so please contribute bug reports, patches, fixes or cash and be a part of the community") + def copy_patches_txt(bench_path): shutil.copy(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'patches', 'patches.txt'), os.path.join(bench_path, 'patches.txt')) + def clone_apps_from(bench_path, clone_from, update_app=True): from .app import install_app print('Copying apps from {0}...'.format(clone_from)) @@ -282,6 +289,7 @@ def clone_apps_from(bench_path, clone_from, update_app=True): for app in apps: setup_app(app) + def exec_cmd(cmd, cwd='.'): from .cli import from_command_line @@ -304,6 +312,7 @@ def exec_cmd(cmd, cwd='.'): if return_code > 0: raise CommandFailedError(cmd) + def which(executable, raise_err = False): from distutils.spawn import find_executable exec_ = find_executable(executable) @@ -315,6 +324,7 @@ def which(executable, raise_err = False): return exec_ + def setup_env(bench_path='.', python = 'python3'): python = which(python, raise_err = True) pip = os.path.join('env', 'bin', 'pip') @@ -323,10 +333,12 @@ def setup_env(bench_path='.', python = 'python3'): exec_cmd('{} -q install -U pip wheel six'.format(pip), cwd=bench_path) exec_cmd('{} -q install -e git+https://github.com/frappe/python-pdfkit.git#egg=pdfkit'.format(pip), cwd=bench_path) + def setup_socketio(bench_path='.'): exec_cmd("npm install socket.io redis express superagent cookie babel-core less chokidar \ babel-cli babel-preset-es2015 babel-preset-es2016 babel-preset-es2017 babel-preset-babili", cwd=bench_path) + def patch_sites(bench_path='.'): bench.set_frappe_version(bench_path=bench_path) @@ -338,6 +350,7 @@ def patch_sites(bench_path='.'): except subprocess.CalledProcessError: raise PatchError + def build_assets(bench_path='.', app=None): bench.set_frappe_version(bench_path=bench_path) @@ -349,14 +362,17 @@ def build_assets(bench_path='.', app=None): command += ' --app {}'.format(app) exec_cmd(command, cwd=bench_path) + def get_sites(bench_path='.'): sites_path = os.path.join(bench_path, 'sites') sites = (site for site in os.listdir(sites_path) if os.path.exists(os.path.join(sites_path, site, 'site_config.json'))) return sites + def get_bench_dir(bench_path='.'): return os.path.abspath(bench_path) + def setup_backups(bench_path='.'): logger.info('setting up backups') bench_dir = get_bench_dir(bench_path=bench_path) @@ -370,6 +386,7 @@ def setup_backups(bench_path='.'): add_to_crontab('0 */6 * * * {backup_command} >> {logfile} 2>&1'.format(backup_command=backup_command, logfile=os.path.join(get_bench_dir(bench_path=bench_path), 'logs', 'backup.log'))) + def add_to_crontab(line): current_crontab = read_crontab() line = str.encode(line) @@ -382,12 +399,14 @@ def add_to_crontab(line): s.stdin.write(line + b'\n') s.stdin.close() + def read_crontab(): s = subprocess.Popen(["crontab", "-l"], stdin=subprocess.PIPE, stdout=subprocess.PIPE) out = s.stdout.read() s.stdout.close() return out + def update_bench(bench_repo=True, requirements=True): logger.info("Updating bench") @@ -406,6 +425,7 @@ def update_bench(bench_repo=True, requirements=True): logger.info("Bench Updated!") + def setup_sudoers(user): from bench import env @@ -440,6 +460,7 @@ def setup_sudoers(user): os.chmod(sudoers_file, 0o440) + def setup_logging(bench_path='.'): if os.path.exists(os.path.join(bench_path, 'logs')): logger = logging.getLogger('bench') @@ -450,6 +471,7 @@ def setup_logging(bench_path='.'): logger.addHandler(hdlr) logger.setLevel(logging.DEBUG) + def get_program(programs): program = None for p in programs: @@ -458,9 +480,11 @@ def get_program(programs): break return program + def get_process_manager(): return get_program(['foreman', 'forego', 'honcho']) + def start(no_dev=False, concurrency=None, procfile=None): program = get_process_manager() if not program: @@ -478,6 +502,7 @@ def start(no_dev=False, concurrency=None, procfile=None): os.execv(program, command) + def check_cmd(cmd, cwd='.'): try: subprocess.check_call(cmd, cwd=cwd, shell=True) @@ -485,6 +510,7 @@ def check_cmd(cmd, cwd='.'): except subprocess.CalledProcessError: return False + def get_git_version(): '''returns git version from `git --version` extracts version number from string `get version 1.9.1` etc''' @@ -494,6 +520,7 @@ def get_git_version(): version = '.'.join(version.split('.')[0:2]) return float(version) + def check_git_for_shallow_clone(): from .config.common_site_config import get_config config = get_config('.') @@ -509,6 +536,7 @@ def check_git_for_shallow_clone(): if git_version > 1.9: return True + def get_cmd_output(cmd, cwd='.'): try: output = subprocess.check_output(cmd, cwd=cwd, shell=True, stderr=subprocess.PIPE).strip() @@ -519,6 +547,7 @@ def get_cmd_output(cmd, cwd='.'): print(e.output) raise + def safe_encode(what, encoding = 'utf-8'): try: what = what.encode(encoding) @@ -527,6 +556,7 @@ def safe_encode(what, encoding = 'utf-8'): return what + def restart_supervisor_processes(bench_path='.', web_workers=False): from .config.common_site_config import get_config conf = get_config(bench_path=bench_path) @@ -556,26 +586,31 @@ def restart_supervisor_processes(bench_path='.', web_workers=False): exec_cmd('sudo supervisorctl restart {group}'.format(group=group), cwd=bench_path) + def restart_systemd_processes(bench_path='.', web_workers=False): from .config.common_site_config import get_config bench_name = get_bench_name(bench_path) exec_cmd('sudo systemctl stop -- $(systemctl show -p Requires {bench_name}.target | cut -d= -f2)'.format(bench_name=bench_name)) exec_cmd('sudo systemctl start -- $(systemctl show -p Requires {bench_name}.target | cut -d= -f2)'.format(bench_name=bench_name)) + def set_default_site(site, bench_path='.'): if site not in get_sites(bench_path=bench_path): raise Exception("Site not in bench") exec_cmd("{frappe} --use {site}".format(frappe=get_frappe(bench_path=bench_path), site=site), cwd=os.path.join(bench_path, 'sites')) + def update_bench_requirements(): bench_req_file = os.path.join(os.path.dirname(bench.__path__[0]), 'requirements.txt') install_requirements(bench_req_file, user=True) + def update_env_pip(bench_path): env_pip = os.path.join(bench_path, 'env', 'bin', 'pip') exec_cmd("{pip} install -q -U pip".format(pip=env_pip)) + def update_requirements(bench_path='.'): from bench.app import get_apps, install_app print('Updating Python libraries...') @@ -589,13 +624,13 @@ def update_requirements(bench_path='.'): for app in get_apps(): install_app(app, bench_path=bench_path) + def update_node_packages(bench_path='.'): print('Updating node packages...') from bench.app import get_develop_version from distutils.version import LooseVersion v = LooseVersion(get_develop_version('frappe', bench_path = bench_path)) - # After rollup was merged, frappe_version = 10.1 # if develop_verion is 11 and up, only then install yarn if v < LooseVersion('11.x.x-develop'): @@ -603,6 +638,7 @@ def update_node_packages(bench_path='.'): else: update_yarn_packages(bench_path) + def update_yarn_packages(bench_path='.'): apps_dir = os.path.join(bench_path, 'apps') @@ -663,6 +699,7 @@ def install_requirements(req_file, user=False): exec_cmd("{python} -m pip install {user_flag} -q -U -r {req_file}".format(python=python, user_flag=user_flag, req_file=req_file)) + def backup_site(site, bench_path='.'): bench.set_frappe_version(bench_path=bench_path) @@ -672,30 +709,38 @@ def backup_site(site, bench_path='.'): else: run_frappe_cmd('--site', site, 'backup', bench_path=bench_path) + def backup_all_sites(bench_path='.'): for site in get_sites(bench_path=bench_path): backup_site(site, bench_path=bench_path) + def is_root(): if os.getuid() == 0: return True return False + def set_mariadb_host(host, bench_path='.'): update_common_site_config({'db_host': host}, bench_path=bench_path) + def set_redis_cache_host(host, bench_path='.'): update_common_site_config({'redis_cache': "redis://{}".format(host)}, bench_path=bench_path) + def set_redis_queue_host(host, bench_path='.'): update_common_site_config({'redis_queue': "redis://{}".format(host)}, bench_path=bench_path) + def set_redis_socketio_host(host, bench_path='.'): update_common_site_config({'redis_socketio': "redis://{}".format(host)}, bench_path=bench_path) + def update_common_site_config(ddict, bench_path='.'): update_json_file(os.path.join(bench_path, 'sites', 'common_site_config.json'), ddict) + def update_json_file(filename, ddict): if os.path.exists(filename): with open(filename, 'r') as f: @@ -708,6 +753,7 @@ def update_json_file(filename, ddict): with open(filename, 'w') as f: json.dump(content, f, indent=1, sort_keys=True) + def drop_privileges(uid_name='nobody', gid_name='nogroup'): # from http://stackoverflow.com/a/2699996 if os.getuid() != 0: @@ -728,6 +774,7 @@ def drop_privileges(uid_name='nobody', gid_name='nogroup'): # Ensure a very conservative umask os.umask(0o22) + def fix_prod_setup_perms(bench_path='.', frappe_user=None): from .config.common_site_config import get_config @@ -745,6 +792,7 @@ def fix_prod_setup_perms(bench_path='.', frappe_user=None): gid = grp.getgrnam(frappe_user).gr_gid os.chown(path, uid, gid) + def fix_file_perms(): for dir_path, dirs, files in os.walk('.'): for _dir in dirs: @@ -757,10 +805,12 @@ def fix_file_perms(): if not _file.startswith('activate'): os.chmod(os.path.join(bin_dir, _file), 0o755) + def get_current_frappe_version(bench_path='.'): from .app import get_current_frappe_version as fv return fv(bench_path=bench_path) + def run_frappe_cmd(*args, **kwargs): from .cli import from_command_line @@ -784,7 +834,7 @@ def run_frappe_cmd(*args, **kwargs): if return_code > 0: sys.exit(return_code) - #raise CommandFailedError(args) + def get_frappe_cmd_output(*args, **kwargs): bench_path = kwargs.get('bench_path', '.') @@ -792,19 +842,20 @@ def get_frappe_cmd_output(*args, **kwargs): sites_dir = os.path.join(bench_path, 'sites') return subprocess.check_output((f, '-m', 'frappe.utils.bench_helper', 'frappe') + args, cwd=sites_dir) + def validate_upgrade(from_ver, to_ver, bench_path='.'): if to_ver >= 6: if not find_executable('npm') and not (find_executable('node') or find_executable('nodejs')): raise Exception("Please install nodejs and npm") + def post_upgrade(from_ver, to_ver, bench_path='.'): from .config.common_site_config import get_config from .config import redis from .config.supervisor import generate_supervisor_config from .config.nginx import make_nginx_conf conf = get_config(bench_path=bench_path) - print("-"*80) - print("Your bench was upgraded to version {0}".format(to_ver)) + print("-" * 80 + "Your bench was upgraded to version {0}".format(to_ver)) if conf.get('restart_supervisor_on_update'): redis.generate_config(bench_path=bench_path) @@ -1080,6 +1131,7 @@ def in_virtual_env(): return False + def migrate_env(python, backup=False): from bench.config.common_site_config import get_config from bench.app import get_apps @@ -1136,3 +1188,17 @@ def migrate_env(python, backup=False): except: log.debug('Migration Error') raise + + +def find_parent_bench(path): + """Checks if parent directories are benches""" + if is_bench_directory(directory=path): + return path + + home_path = os.path.expanduser("~") + root_path = os.path.abspath(os.sep) + + if path not in {home_path, root_path}: + # NOTE: the os.path.split assumes that given path is absolute + parent_dir = os.path.split(path)[0] + return find_parent_bench(parent_dir)