From d1a3017172a415919f3335c4e43edd7488ea69b2 Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Sun, 15 Dec 2019 12:06:12 +0530 Subject: [PATCH 01/44] fix(get-app): handle existing directory and other formatting changes * if a directory for the application already exists, provide user with 2 options: * ask to remove the directory and reclone (overwrite) * ask if the user wants to reinstall the application * separate get_app_name from get_app * move building assets and postprocessing to install_app method * make formatting changes to the code for flake8 * use reload_module from six instead of builtins Signed-off-by: Chinmay D. Pai --- bench/app.py | 102 ++++++++++++++++++++++++----------------- bench/commands/make.py | 3 +- 2 files changed, 61 insertions(+), 44 deletions(-) diff --git a/bench/app.py b/bench/app.py index 5becd1eb..ed6cee3d 100755 --- a/bench/app.py +++ b/bench/app.py @@ -4,6 +4,7 @@ from .utils import (exec_cmd, get_frappe, check_git_for_shallow_clone, build_ass restart_supervisor_processes, get_cmd_output, run_frappe_cmd, CommandFailedError, restart_systemd_processes) from .config.common_site_config import get_config +from six import reload_module import logging import requests @@ -14,6 +15,7 @@ import subprocess import bench import sys import shutil +import click logging.basicConfig(level="DEBUG") logger = logging.getLogger(__name__) @@ -50,7 +52,7 @@ def write_appstxt(apps, bench_path='.'): with open(os.path.join(bench_path, 'sites', 'apps.txt'), 'w') as f: return f.write('\n'.join(apps)) -def check_url(url, raise_err = True): +def check_url(url, raise_err=True): try: from urlparse import urlparse except ImportError: @@ -59,7 +61,7 @@ def check_url(url, raise_err = True): parsed = urlparse(url) if not parsed.scheme: if raise_err: - raise TypeError('{url} Not a valid URL'.format(url = url)) + raise TypeError('{url} Not a valid URL'.format(url=url)) else: return False @@ -93,58 +95,64 @@ def remove_from_excluded_apps_txt(app, bench_path='.'): return write_excluded_apps_txt(apps, bench_path=bench_path) def get_app(git_url, branch=None, bench_path='.', build_asset_files=True, verbose=False, - postprocess = True): + postprocess=True, overwrite=False): # from bench.utils import check_url try: from urlparse import urljoin except ImportError: from urllib.parse import urljoin - if not check_url(git_url, raise_err = False): + if not check_url(git_url, raise_err=False): orgs = ['frappe', 'erpnext'] for org in orgs: - url = 'https://api.github.com/repos/{org}/{app}'.format(org = org, app = git_url) + url = 'https://api.github.com/repos/{org}/{app}'.format(org=org, app=git_url) res = requests.get(url) if res.ok: - data = res.json() + data = res.json() if 'name' in data: if git_url == data['name']: - git_url = 'https://github.com/{org}/{app}'.format(org = org, app = git_url) + git_url = 'https://github.com/{org}/{app}'.format(org=org, app=git_url) break - #Gets repo name from URL + # Gets repo name from URL repo_name = git_url.rsplit('/', 1)[1].rsplit('.', 1)[0] - logger.info('getting app {}'.format(repo_name)) shallow_clone = '--depth 1' if check_git_for_shallow_clone() else '' branch = '--branch {branch}'.format(branch=branch) if branch else '' - exec_cmd("git clone {git_url} {branch} {shallow_clone} --origin upstream".format( - git_url=git_url, - shallow_clone=shallow_clone, - branch=branch), - cwd=os.path.join(bench_path, 'apps')) + if os.path.isdir(os.path.join(bench_path, 'apps', repo_name)): + # application directory already exists + # prompt user to overwrite it + if overwrite or click.confirm('''A directory for the application "{0}" already exists. +Do you want to continue and overwrite it?'''): + shutil.rmtree(os.path.join(bench_path, 'apps', repo_name)) + elif click.confirm('''Do you want to reinstall the existing application?''', abort=True): + app_name = get_app_name(bench_path, repo_name) + print("Reinstalling {0}".format(app_name)) + install_app(app=app_name, bench_path=bench_path, verbose=verbose, build_asset_files=build_asset_files) + sys.exit(1) - #Retrieves app name from setup.py + logger.info('Getting app {0}'.format(repo_name)) + exec_cmd("git clone {git_url} {branch} {shallow_clone} --origin upstream".format( + git_url=git_url, + shallow_clone=shallow_clone, + branch=branch), + cwd=os.path.join(bench_path, 'apps')) + + app_name = get_app_name(bench_path, repo_name) + print("Installing {0}".format(app_name)) + install_app(app=app_name, bench_path=bench_path, verbose=verbose, build_asset_files=build_asset_files) + + +def get_app_name(bench_path, repo_name): + # retrieves app name from setup.py app_path = os.path.join(bench_path, 'apps', repo_name, 'setup.py') with open(app_path, 'rb') as f: app_name = re.search(r'name\s*=\s*[\'"](.*)[\'"]', f.read().decode('utf-8')).group(1) if repo_name != app_name: apps_path = os.path.join(os.path.abspath(bench_path), 'apps') os.rename(os.path.join(apps_path, repo_name), os.path.join(apps_path, app_name)) + return app_name - print('installing', app_name) - install_app(app=app_name, bench_path=bench_path, verbose=verbose) - - if postprocess: - - if build_asset_files: - build_assets(bench_path=bench_path, app=app_name) - conf = get_config(bench_path=bench_path) - - if conf.get('restart_supervisor_on_update'): - restart_supervisor_processes(bench_path=bench_path) - if conf.get('restart_systemd_on_update'): - restart_systemd_processes(bench_path=bench_path) def new_app(app, bench_path='.'): # For backwards compatibility @@ -160,7 +168,8 @@ def new_app(app, bench_path='.'): run_frappe_cmd('make-app', apps, app, bench_path=bench_path) install_app(app, bench_path=bench_path) -def install_app(app, bench_path=".", verbose=False, no_cache=False): + +def install_app(app, bench_path=".", verbose=False, no_cache=False, postprocess=True, build_asset_files=True): logger.info("installing {}".format(app)) pip_path = os.path.join(bench_path, "env", "bin", "pip") @@ -168,11 +177,23 @@ def install_app(app, bench_path=".", verbose=False, no_cache=False): app_path = os.path.join(bench_path, "apps", app) cache_flag = "--no-cache-dir" if no_cache else "" - exec_cmd("{pip} install {quiet} -U -e {app} {no_cache}".format(pip=pip_path, quiet=quiet_flag, app=app_path, no_cache=cache_flag)) + exec_cmd("{pip} install {quiet} -U -e {app} {no_cache}".format(pip=pip_path, + quiet=quiet_flag, app=app_path, no_cache=cache_flag)) add_to_appstxt(app, bench_path=bench_path) + if postprocess: + if build_asset_files: + build_assets(bench_path=bench_path, app=app) + conf = get_config(bench_path=bench_path) + + if conf.get('restart_supervisor_on_update'): + restart_supervisor_processes(bench_path=bench_path) + if conf.get('restart_systemd_on_update'): + restart_systemd_processes(bench_path=bench_path) + + def remove_app(app, bench_path='.'): - if not app in get_apps(bench_path): + if app not in get_apps(bench_path): print("No app named {0}".format(app)) sys.exit(1) @@ -281,8 +302,7 @@ def get_current_branch(app, bench_path='.'): def get_remote(app, bench_path='.'): repo_dir = get_repo_dir(app, bench_path=bench_path) - contents = subprocess.check_output(['git', 'remote', '-v'], cwd=repo_dir, - stderr=subprocess.STDOUT) + contents = subprocess.check_output(['git', 'remote', '-v'], cwd=repo_dir, stderr=subprocess.STDOUT) contents = contents.decode('utf-8') if re.findall('upstream[\s]+', contents): return 'upstream' @@ -371,7 +391,7 @@ def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrad except CommandFailedError: print("Error switching to branch {0} for {1}".format(branch, app)) except InvalidRemoteException: - print("Remote does not exist for app "+app) + print("Remote does not exist for app {0}".format(app)) except InvalidBranchException: print("Branch {0} does not exist in Upstream for {1}".format(branch, app)) @@ -382,11 +402,7 @@ def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrad update_requirements() update_node_packages() pre_upgrade(version_upgrade[1], version_upgrade[2]) - if sys.version_info >= (3, 4): - import importlib - importlib.reload(utils) - else: - reload(utils) + reload_module(utils) backup_all_sites() patch_sites() build_assets() @@ -402,8 +418,7 @@ def switch_to_develop(apps=None, bench_path='.', upgrade=True): switch_branch('develop', apps=apps, bench_path=bench_path, upgrade=upgrade) def get_version_from_string(contents, field='__version__'): - match = re.search(r"^(\s*%s\s*=\s*['\\\"])(.+?)(['\"])(?sm)" % field, - contents) + match = re.search(r"^(\s*%s\s*=\s*['\\\"])(.+?)(['\"])(?sm)" % field, contents) return match.group(2) def get_major_version(version): @@ -427,7 +442,8 @@ def validate_branch(): branch = get_current_branch(app) if branch == "master": - print(''' master branch is renamed to version-11 and develop to version-12. Please switch to new branches to get future updates. + print('''master branch is renamed to version-11 and develop to version-12. +Please switch to new branches to get future updates. To switch to version 11, run the following commands: bench switch-to-branch version-11''') - sys.exit(1) \ No newline at end of file + sys.exit(1) diff --git a/bench/commands/make.py b/bench/commands/make.py index b3ca8341..dc10515a 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -35,10 +35,11 @@ def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, @click.argument('name', nargs=-1) # Dummy argument for backward compatibility @click.argument('git-url') @click.option('--branch', default=None, help="branch to checkout") +@click.option('--overwrite' is_flag=True) def get_app(git_url, branch, name=None): "clone an app from the internet and set it up in your bench" from bench.app import get_app - get_app(git_url, branch=branch) + get_app(git_url, branch=branch, overwrite=overwrite) @click.command('new-app') From 699705f64b8e549e0a8322a0ea429a5d5361da07 Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Sun, 15 Dec 2019 13:46:14 +0530 Subject: [PATCH 02/44] fix: incorrect import statement use six.moves instead of six Signed-off-by: Chinmay D. Pai --- bench/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/app.py b/bench/app.py index ed6cee3d..b590fbf3 100755 --- a/bench/app.py +++ b/bench/app.py @@ -4,7 +4,7 @@ from .utils import (exec_cmd, get_frappe, check_git_for_shallow_clone, build_ass restart_supervisor_processes, get_cmd_output, run_frappe_cmd, CommandFailedError, restart_systemd_processes) from .config.common_site_config import get_config -from six import reload_module +from six.moves import reload_module import logging import requests From cac66a6b8811cb55049134ade75a9c8e86f56809 Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Sun, 15 Dec 2019 13:59:12 +0530 Subject: [PATCH 03/44] fix: invalid syntax for click option ughhhhhhhhhh Signed-off-by: Chinmay D. Pai --- bench/commands/make.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/commands/make.py b/bench/commands/make.py index dc10515a..c2d27431 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -35,7 +35,7 @@ def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, @click.argument('name', nargs=-1) # Dummy argument for backward compatibility @click.argument('git-url') @click.option('--branch', default=None, help="branch to checkout") -@click.option('--overwrite' is_flag=True) +@click.option('--overwrite', is_flag=True) def get_app(git_url, branch, name=None): "clone an app from the internet and set it up in your bench" from bench.app import get_app From 1442cf6980eb62511d51b7936c750fb907830922 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 17 Feb 2020 16:35:52 +0530 Subject: [PATCH 04/44] refactor: bench.commands.setup * updated help for all commands * sorted imports * code formatting and consistent quotes --- bench/commands/setup.py | 270 +++++++++++++++++++--------------------- 1 file changed, 131 insertions(+), 139 deletions(-) diff --git a/bench/commands/setup.py b/bench/commands/setup.py index c027c0c4..920c1435 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -1,206 +1,196 @@ -from bench.utils import exec_cmd -from six import PY3 -import click, sys, json +# imports - standard imports import os +import sys +import json -@click.group() +# imports - module imports +from bench.utils import exec_cmd + +# imports - third party imports +from six import PY3 +import click + + +@click.group(help="Setup command group for enabling setting up a Frappe environment") def setup(): - "Setup bench" pass -@click.command('sudoers') -@click.argument('user') + +@click.command("sudoers", help="Add commands to sudoers list for execution without password") +@click.argument("user") def setup_sudoers(user): - "Add commands to sudoers list for execution without password" from bench.utils import setup_sudoers setup_sudoers(user) -@click.command('nginx') -@click.option('--yes', help='Yes to regeneration of nginx config file', default=False, is_flag=True) + +@click.command("nginx", help="Generate configuration files for NGINX") +@click.option("--yes", help="Yes to regeneration of nginx config file", default=False, is_flag=True) def setup_nginx(yes=False): - "generate config for nginx" from bench.config.nginx import make_nginx_conf make_nginx_conf(bench_path=".", yes=yes) -@click.command('reload-nginx') + +@click.command("reload-nginx", help="Checks NGINX config file and reloads service") def reload_nginx(): from bench.config.production_setup import reload_nginx reload_nginx() -@click.command('supervisor') -@click.option('--user') -@click.option('--yes', help='Yes to regeneration of supervisor config', is_flag=True, default=False) + +@click.command("supervisor", help="Generate configuration for supervisor") +@click.option("--user", help="optional user argument") +@click.option("--yes", help="Yes to regeneration of supervisor config", is_flag=True, default=False) def setup_supervisor(user=None, yes=False): - "generate config for supervisor with an optional user argument" from bench.config.supervisor import generate_supervisor_config generate_supervisor_config(bench_path=".", user=user, yes=yes) -@click.command('redis') + +@click.command("redis", help="Generates configuration for Redis") def setup_redis(): - "generate config for redis cache" from bench.config.redis import generate_config - generate_config('.') + generate_config(".") -@click.command('fonts') +@click.command("fonts", help="Add Frappe fonts to system") def setup_fonts(): - "Add frappe fonts to system" from bench.utils import setup_fonts setup_fonts() -@click.command('production') -@click.argument('user') -@click.option('--yes', help='Yes to regeneration config', is_flag=True, default=False) + +@click.command("production", help="Setup Frappe production environment for specific user") +@click.argument("user") +@click.option("--yes", help="Yes to regeneration config", is_flag=True, default=False) def setup_production(user, yes=False): - "setup bench for production" from bench.config.production_setup import setup_production from bench.utils import run_playbook # Install prereqs for production from distutils.spawn import find_executable - if not find_executable('ansible'): + if not find_executable("ansible"): exec_cmd("sudo {0} install ansible".format("pip3" if PY3 else "pip2")) - if not find_executable('fail2ban-client'): + if not find_executable("fail2ban-client"): exec_cmd("bench setup role fail2ban") - if not find_executable('nginx'): + if not find_executable("nginx"): exec_cmd("bench setup role nginx") - if not find_executable('supervisord'): + if not find_executable("supervisord"): exec_cmd("bench setup role supervisor") setup_production(user=user, yes=yes) -@click.command('auto-update') +@click.command("auto-update", help="Add cronjob for bench auto update") def setup_auto_update(): - "Add cronjob for bench auto update" from bench.utils import setup_auto_update setup_auto_update() -@click.command('backups') +@click.command("backups", help="Add cronjob for bench backups") def setup_backups(): - "Add cronjob for bench backups" from bench.utils import setup_backups setup_backups() -@click.command('env') -@click.option('--python', type = str, default = 'python3', help = 'Path to Python Executable.') -def setup_env(python='python3'): - "Setup virtualenv for bench" + +@click.command("env", help="Setup virtualenv for bench") +@click.option("--python", type = str, default = "python3", help = "Path to Python Executable.") +def setup_env(python="python3"): from bench.utils import setup_env setup_env(python=python) -@click.command('firewall') -@click.option('--ssh_port') -@click.option('--force') + +@click.command("firewall", help="Setup firewall for system") +@click.option("--ssh_port") +@click.option("--force") def setup_firewall(ssh_port=None, force=False): - "Setup firewall" from bench.utils import run_playbook if not force: - click.confirm('Setting up the firewall will block all ports except 80, 443 and 22\n' - 'Do you want to continue?', - abort=True) + click.confirm("Setting up the firewall will block all ports except 80, 443 and {0}\nDo you want to continue?".format(ssh_port), abort=True) if not ssh_port: ssh_port = 22 - run_playbook('roles/bench/tasks/setup_firewall.yml', {"ssh_port": ssh_port}) + run_playbook("roles/bench/tasks/setup_firewall.yml", {"ssh_port": ssh_port}) -@click.command('ssh-port') -@click.argument('port') -@click.option('--force') + +@click.command("ssh-port", help="Set SSH Port for system") +@click.argument("port") +@click.option("--force") def set_ssh_port(port, force=False): - "Set SSH Port" from bench.utils import run_playbook if not force: - click.confirm('This will change your SSH Port to {}\n' - 'Do you want to continue?'.format(port), - abort=True) + click.confirm("This will change your SSH Port to {}\nDo you want to continue?".format(port), abort=True) - run_playbook('roles/bench/tasks/change_ssh_port.yml', {"ssh_port": port}) + run_playbook("roles/bench/tasks/change_ssh_port.yml", {"ssh_port": port}) -@click.command('lets-encrypt') -@click.argument('site') -@click.option('--custom-domain') -@click.option('-n', '--non-interactive', default=False, is_flag=True, help="Run certbot non-interactively. Shouldn't be used on 1'st attempt") + +@click.command("lets-encrypt", help="Setup lets-encrypt SSL for site") +@click.argument("site") +@click.option("--custom-domain") +@click.option("-n", "--non-interactive", default=False, is_flag=True, help="Run certbot non-interactively. Shouldn't be used on 1'st attempt") def setup_letsencrypt(site, custom_domain, non_interactive): - "Setup lets-encrypt for site" from bench.config.lets_encrypt import setup_letsencrypt - setup_letsencrypt(site, custom_domain, bench_path='.', interactive=not non_interactive) + setup_letsencrypt(site, custom_domain, bench_path=".", interactive=not non_interactive) -@click.command('wildcard-ssl') -@click.argument('domain') -@click.option('--email') -@click.option('--exclude-base-domain', default=False, is_flag=True, help="SSL Certificate not applicable for base domain") +@click.command("wildcard-ssl", help="Setup wildcard SSL certificate for multi-tenant bench") +@click.argument("domain") +@click.option("--email") +@click.option("--exclude-base-domain", default=False, is_flag=True, help="SSL Certificate not applicable for base domain") def setup_wildcard_ssl(domain, email, exclude_base_domain): - ''' Setup wildcard ssl certificate ''' from bench.config.lets_encrypt import setup_wildcard_ssl - setup_wildcard_ssl(domain, email, bench_path='.', exclude_base_domain=exclude_base_domain) + setup_wildcard_ssl(domain, email, bench_path=".", exclude_base_domain=exclude_base_domain) -@click.command('procfile') +@click.command("procfile", help="Generate Procfile for bench start") def setup_procfile(): - "Setup Procfile for bench start" from bench.config.procfile import setup_procfile - setup_procfile('.') + setup_procfile(".") -@click.command('socketio') +@click.command("socketio", help="Setup node dependencies for socketio server") def setup_socketio(): - "Setup node deps for socketio server" from bench.utils import setup_socketio setup_socketio() -@click.command('requirements', help="Update Python and Node packages") -@click.option('--node', help="Update only Node packages", default=False, is_flag=True) -@click.option('--python', help="Update only Python packages", default=False, is_flag=True) + +@click.command("requirements", help="Setup Python and Node dependencies") +@click.option("--node", help="Update only Node packages", default=False, is_flag=True) +@click.option("--python", help="Update only Python packages", default=False, is_flag=True) def setup_requirements(node=False, python=False): - "Setup python and node requirements" - if not node: - setup_python_requirements() + from bench.utils import update_requirements as setup_python_packages + setup_python_packages() + if not python: - setup_node_requirements() - -def setup_python_requirements(): - from bench.utils import update_requirements - update_requirements() - -def setup_node_requirements(): - from bench.utils import update_node_packages - update_node_packages() + from bench.utils import update_node_packages as setup_node_packages + setup_node_packages() -@click.command('manager') -@click.option('--yes', help='Yes to regeneration of nginx config file', default=False, is_flag=True) -@click.option('--port', help='Port on which you want to run bench manager', default=23624) -@click.option('--domain', help='Domain on which you want to run bench manager') +@click.command("manager", help="Setup bench-manager.local site with the bench_manager app installed on it") +@click.option("--yes", help="Yes to regeneration of nginx config file", default=False, is_flag=True) +@click.option("--port", help="Port on which you want to run bench manager", default=23624) +@click.option("--domain", help="Domain on which you want to run bench manager") def setup_manager(yes=False, port=23624, domain=None): - "Setup bench-manager.local site with the bench_manager app installed on it" from six.moves import input create_new_site = True - if 'bench-manager.local' in os.listdir('sites'): - ans = input('Site already exists. Overwrite existing site? [Y/n]: ').lower() - while ans not in ('y', 'n', ''): - ans = input( - 'Please enter "y" or "n". Site already exists. Overwrite existing site? [Y/n]: ').lower() - if ans == 'n': + if "bench-manager.local" in os.listdir("sites"): + ans = input("Site already exists. Overwrite existing site? [Y/n]: ").lower() + while ans not in ("y", "n", ""): + ans = input("Please enter 'y' or 'n'. Site already exists. Overwrite existing site? [Y/n]: ").lower() + if ans == "n": create_new_site = False if create_new_site: exec_cmd("bench new-site --force bench-manager.local") - if 'bench_manager' in os.listdir('apps'): - print('App already exists. Skipping app download.') + if "bench_manager" in os.listdir("apps"): + print("App already exists. Skipping app download.") else: exec_cmd("bench get-app bench_manager") exec_cmd("bench --site bench-manager.local install-app bench_manager") from bench.config.common_site_config import get_config - bench_path = '.' + bench_path = "." conf = get_config(bench_path) - if conf.get('restart_supervisor_on_update') or conf.get('restart_systemd_on_update'): + if conf.get("restart_supervisor_on_update") or conf.get("restart_systemd_on_update"): # implicates a production setup or so I presume if not domain: print("Please specify the site name on which you want to host bench-manager using the 'domain' flag") @@ -216,18 +206,17 @@ def setup_manager(yes=False, port=23624, domain=None): make_bench_manager_nginx_conf(bench_path, yes=yes, port=port, domain=domain) -@click.command('config') +@click.command("config", help="Generate or over-write sites/common_site_config.json") def setup_config(): - "overwrite or make config.json" from bench.config.common_site_config import make_config - make_config('.') + make_config(".") -@click.command('add-domain') -@click.argument('domain') -@click.option('--site', prompt=True) -@click.option('--ssl-certificate', help="Absolute path to SSL Certificate") -@click.option('--ssl-certificate-key', help="Absolute path to SSL Certificate Key") +@click.command("add-domain", help="Add a custom domain to a particular site") +@click.argument("domain") +@click.option("--site", prompt=True) +@click.option("--ssl-certificate", help="Absolute path to SSL Certificate") +@click.option("--ssl-certificate-key", help="Absolute path to SSL Certificate Key") def add_domain(domain, site=None, ssl_certificate=None, ssl_certificate_key=None): """Add custom domain to site""" from bench.config.site_config import add_domain @@ -236,24 +225,25 @@ def add_domain(domain, site=None, ssl_certificate=None, ssl_certificate_key=None print("Please specify site") sys.exit(1) - add_domain(site, domain, ssl_certificate, ssl_certificate_key, bench_path='.') + add_domain(site, domain, ssl_certificate, ssl_certificate_key, bench_path=".") -@click.command('remove-domain') -@click.argument('domain') -@click.option('--site', prompt=True) + +@click.command("remove-domain", help="Remove custom domain from a site") +@click.argument("domain") +@click.option("--site", prompt=True) def remove_domain(domain, site=None): - """Remove custom domain from a site""" from bench.config.site_config import remove_domain if not site: print("Please specify site") sys.exit(1) - remove_domain(site, domain, bench_path='.') + remove_domain(site, domain, bench_path=".") -@click.command('sync-domains') -@click.option('--domain', multiple=True) -@click.option('--site', prompt=True) + +@click.command("sync-domains", help="Check if there is a change in domains. If yes, updates the domains list.") +@click.option("--domain", multiple=True) +@click.option("--site", prompt=True) def sync_domains(domain=None, site=None): from bench.config.site_config import sync_domains @@ -262,53 +252,55 @@ def sync_domains(domain=None, site=None): sys.exit(1) try: - domains = list(map(str,domain)) + domains = list(map(str, domain)) except Exception: print("Domains should be a json list of strings or dictionaries") sys.exit(1) - changed = sync_domains(site, domains, bench_path='.') + changed = sync_domains(site, domains, bench_path=".") # if changed, success, else failure sys.exit(0 if changed else 1) -@click.command('role') -@click.argument('role') -@click.option('--admin_emails', default='') -@click.option('--mysql_root_password') -@click.option('--container', is_flag=True, default=False) + +@click.command("role", help="Install dependencies via ansible roles") +@click.argument("role") +@click.option("--admin_emails", default="") +@click.option("--mysql_root_password") +@click.option("--container", is_flag=True, default=False) def setup_roles(role, **kwargs): - "Install dependancies via roles" from bench.utils import run_playbook extra_vars = {"production": True} extra_vars.update(kwargs) if role: - run_playbook('site.yml', extra_vars=extra_vars, tag=role) + run_playbook("site.yml", extra_vars=extra_vars, tag=role) else: - run_playbook('site.yml', extra_vars=extra_vars) + run_playbook("site.yml", extra_vars=extra_vars) -@click.command('fail2ban') -@click.option('--maxretry', default=6, help="Number of matches (i.e. value of the counter) which triggers ban action on the IP. Default is 6 seconds" ) -@click.option('--bantime', default=600, help="The counter is set to zero if no match is found within 'findtime' seconds. Default is 600 seconds") -@click.option('--findtime', default=600, help='Duration (in seconds) for IP to be banned for. Negative number for "permanent" ban. Default is 600 seconds') + +@click.command("fail2ban", help="Setup fail2ban, an intrusion prevention software framework that protects computer servers from brute-force attacks") +@click.option("--maxretry", default=6, help="Number of matches (i.e. value of the counter) which triggers ban action on the IP. Default is 6 seconds" ) +@click.option("--bantime", default=600, help="The counter is set to zero if no match is found within 'findtime' seconds. Default is 600 seconds") +@click.option("--findtime", default=600, help="Duration (in seconds) for IP to be banned for. Negative number for 'permanent' ban. Default is 600 seconds") def setup_nginx_proxy_jail(**kwargs): from bench.utils import run_playbook - run_playbook('roles/fail2ban/tasks/configure_nginx_jail.yml', extra_vars=kwargs) + run_playbook("roles/fail2ban/tasks/configure_nginx_jail.yml", extra_vars=kwargs) -@click.command('systemd') -@click.option('--user') -@click.option('--yes', help='Yes to regeneration of systemd config files', is_flag=True, default=False) -@click.option('--stop', help='Stop bench services', is_flag=True, default=False) -@click.option('--create-symlinks', help='Create Symlinks', is_flag=True, default=False) -@click.option('--delete-symlinks', help='Delete Symlinks', is_flag=True, default=False) + +@click.command("systemd", help="Generate configuration for systemd") +@click.option("--user", help="Optional user argument") +@click.option("--yes", help="Yes to regeneration of systemd config files", is_flag=True, default=False) +@click.option("--stop", help="Stop bench services", is_flag=True, default=False) +@click.option("--create-symlinks", help="Create Symlinks", is_flag=True, default=False) +@click.option("--delete-symlinks", help="Delete Symlinks", is_flag=True, default=False) def setup_systemd(user=None, yes=False, stop=False, create_symlinks=False, delete_symlinks=False): - "generate configs for systemd with an optional user argument" from bench.config.systemd import generate_systemd_config generate_systemd_config(bench_path=".", user=user, yes=yes, stop=stop, create_symlinks=create_symlinks, delete_symlinks=delete_symlinks) + setup.add_command(setup_sudoers) setup.add_command(setup_nginx) setup.add_command(reload_nginx) From 1f488a8bfd391f677230c91e1fdfc1ad7034e88d Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 17 Feb 2020 17:22:30 +0530 Subject: [PATCH 05/44] docs: bench.command.make added help for make commands --- bench/commands/make.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/bench/commands/make.py b/bench/commands/make.py index b9b01b3e..f6ba1377 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -1,6 +1,8 @@ +# imports - third party imports import click -@click.command() + +@click.command('init', help='Initialize a new bench instance in the specified path') @click.argument('path') @click.option('--python', type = str, default = 'python3', help = 'Path to Python Executable.') @click.option('--ignore-exist', is_flag = True, default = False, help = "Ignore if Bench instance exists.") @@ -16,9 +18,6 @@ import click @click.option('--skip-assets',is_flag=True, default=False, help="Do not build assets") @click.option('--verbose',is_flag=True, help="Verbose output during install") def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, no_auto_update, clone_from, verbose, skip_redis_config_generation, clone_without_update, ignore_exist=False, skip_assets=False, python='python3'): - ''' - Create a New Bench Instance. - ''' from bench.utils import init, log try: @@ -52,42 +51,38 @@ def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, n shutil.rmtree(path) -@click.command('get-app') +@click.command('get-app', help='Clone an app from the internet or filesystem and set it up in your bench') @click.argument('name', nargs=-1) # Dummy argument for backward compatibility @click.argument('git-url') @click.option('--branch', default=None, help="branch to checkout") @click.option('--skip-assets', is_flag=True, default=False, help="Do not build assets") def get_app(git_url, branch, name=None, skip_assets=False): - "clone an app from the internet and set it up in your bench" from bench.app import get_app get_app(git_url, branch=branch, skip_assets=skip_assets) -@click.command('new-app') +@click.command('new-app', help='Create a new Frappe application under apps folder') @click.argument('app-name') def new_app(app_name): - "start a new app" from bench.app import new_app new_app(app_name) -@click.command('remove-app') +@click.command('remove-app', help='Completely remove app from bench and re-build assets if not installed on any site') @click.argument('app-name') def remove_app(app_name): - "completely remove app from bench" from bench.app import remove_app remove_app(app_name) -@click.command('exclude-app') +@click.command('exclude-app', help='Exclude app from updating') @click.argument('app_name') def exclude_app_for_update(app_name): - "Exclude app from updating" from bench.app import add_to_excluded_apps_txt add_to_excluded_apps_txt(app_name) -@click.command('include-app') +@click.command('include-app', help='Include app for updating') @click.argument('app_name') def include_app_for_update(app_name): "Include app from updating" From 214a2093ebff10f8277bc421e2ebdc438f7c2aee Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 18 Feb 2020 10:54:32 +0530 Subject: [PATCH 06/44] fix: added aliases for click commands to convert _ seperated commands to - seperated ones in config group eg: `bench config auto-update on` functions same as `bench config auto_update on` --- bench/commands/config.py | 65 +++++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/bench/commands/config.py b/bench/commands/config.py index a7a0f3ec..f8821b70 100644 --- a/bench/commands/config.py +++ b/bench/commands/config.py @@ -1,80 +1,87 @@ -import click, json -from bench.config.common_site_config import update_config +# imports - standard imports +import ast +import json -## Config -## Not DRY -@click.group() +# imports - module imports +from bench.config.common_site_config import update_config, get_config, put_config + +# imports - third party imports +import click + + + +class AliasedGroup(click.Group): + def get_command(self, ctx, cmd_name): + try: + cmd_name = ALIASES[cmd_name].name + except KeyError: + pass + return super(AliasedGroup, self).get_command(ctx, cmd_name) + + +@click.group(cls=AliasedGroup, help='Change bench configuration') def config(): - "change bench configuration" pass -@click.command('auto_update') + +@click.command('auto_update', help='Enable/Disable auto update for bench') @click.argument('state', type=click.Choice(['on', 'off'])) def config_auto_update(state): - "Enable/Disable auto update for bench" state = True if state == 'on' else False update_config({'auto_update': state}) -@click.command('restart_supervisor_on_update') +@click.command('restart_supervisor_on_update', help='Enable/Disable auto restart of supervisor processes') @click.argument('state', type=click.Choice(['on', 'off'])) def config_restart_supervisor_on_update(state): - "Enable/Disable auto restart of supervisor processes" state = True if state == 'on' else False update_config({'restart_supervisor_on_update': state}) -@click.command('restart_systemd_on_update') + +@click.command('restart_systemd_on_update', help='Enable/Disable auto restart of systemd units') @click.argument('state', type=click.Choice(['on', 'off'])) def config_restart_systemd_on_update(state): - "Enable/Disable auto restart of systemd units" state = True if state == 'on' else False update_config({'restart_systemd_on_update': state}) -@click.command('update_bench_on_update') + +@click.command('update_bench_on_update', help='Enable/Disable bench updates on running bench update') @click.argument('state', type=click.Choice(['on', 'off'])) def config_update_bench_on_update(state): - "Enable/Disable bench updates on running bench update" state = True if state == 'on' else False update_config({'update_bench_on_update': state}) -@click.command('dns_multitenant') +@click.command('dns_multitenant', help='Enable/Disable bench multitenancy on running bench update') @click.argument('state', type=click.Choice(['on', 'off'])) def config_dns_multitenant(state): - "Enable/Disable bench updates on running bench update" state = True if state == 'on' else False update_config({'dns_multitenant': state}) -@click.command('serve_default_site') +@click.command('serve_default_site', help='Configure nginx to serve the default site on port 80') @click.argument('state', type=click.Choice(['on', 'off'])) def config_serve_default_site(state): - "Configure nginx to serve the default site on port 80" state = True if state == 'on' else False update_config({'serve_default_site': state}) -@click.command('rebase_on_pull') +@click.command('rebase_on_pull', help='Rebase repositories on pulling') @click.argument('state', type=click.Choice(['on', 'off'])) def config_rebase_on_pull(state): - "Rebase repositories on pulling" state = True if state == 'on' else False update_config({'rebase_on_pull': state}) -@click.command('http_timeout') +@click.command('http_timeout', help='Set HTTP timeout') @click.argument('seconds', type=int) def config_http_timeout(seconds): - "set http timeout" update_config({'http_timeout': seconds}) -@click.command('set-common-config') +@click.command('set-common-config', help='Set value in common config') @click.option('configs', '-c', '--config', multiple=True, type=(str, str)) def set_common_config(configs): - import ast - from bench.config.common_site_config import update_config - common_site_config = {} for key, value in configs: if value in ("False", "True"): @@ -103,10 +110,9 @@ def set_common_config(configs): update_config(common_site_config, bench_path='.') -@click.command('remove-common-config') +@click.command('remove-common-config', help='Remove specific keys from current bench\'s common config') @click.argument('keys', nargs=-1) def remove_common_config(keys): - from bench.config.common_site_config import get_config, put_config common_site_config = get_config('.') for key in keys: if key in common_site_config: @@ -124,3 +130,6 @@ config.add_command(config_serve_default_site) config.add_command(config_http_timeout) config.add_command(set_common_config) config.add_command(remove_common_config) + +# aliases for _ seperated commands to - ones +ALIASES = {k.replace('_', '-'):v for k, v in config.commands.items() if '_' in k} From e4355001e73d543fe380394f02092600fefaf7f9 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 18 Feb 2020 10:56:37 +0530 Subject: [PATCH 07/44] fix: remove unnecessary - commands aliases --- bench/commands/config.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/bench/commands/config.py b/bench/commands/config.py index f8821b70..0ae01daa 100644 --- a/bench/commands/config.py +++ b/bench/commands/config.py @@ -9,17 +9,7 @@ from bench.config.common_site_config import update_config, get_config, put_confi import click - -class AliasedGroup(click.Group): - def get_command(self, ctx, cmd_name): - try: - cmd_name = ALIASES[cmd_name].name - except KeyError: - pass - return super(AliasedGroup, self).get_command(ctx, cmd_name) - - -@click.group(cls=AliasedGroup, help='Change bench configuration') +@click.group(help='Change bench configuration') def config(): pass @@ -130,6 +120,3 @@ config.add_command(config_serve_default_site) config.add_command(config_http_timeout) config.add_command(set_common_config) config.add_command(remove_common_config) - -# aliases for _ seperated commands to - ones -ALIASES = {k.replace('_', '-'):v for k, v in config.commands.items() if '_' in k} From 53e2272a66e8f0ac54ddd0eeb08b6080ac244aa5 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 18 Feb 2020 11:50:53 +0530 Subject: [PATCH 08/44] refactor: bench.commands.config * sorted imports * made it a little DRY * code formatting * added help --- bench/commands/config.py | 45 +++++++++++----------------------------- 1 file changed, 12 insertions(+), 33 deletions(-) diff --git a/bench/commands/config.py b/bench/commands/config.py index 0ae01daa..7dbaaed9 100644 --- a/bench/commands/config.py +++ b/bench/commands/config.py @@ -17,50 +17,43 @@ def config(): @click.command('auto_update', help='Enable/Disable auto update for bench') @click.argument('state', type=click.Choice(['on', 'off'])) def config_auto_update(state): - state = True if state == 'on' else False - update_config({'auto_update': state}) + update_config({'auto_update': state == 'on'}) @click.command('restart_supervisor_on_update', help='Enable/Disable auto restart of supervisor processes') @click.argument('state', type=click.Choice(['on', 'off'])) def config_restart_supervisor_on_update(state): - state = True if state == 'on' else False - update_config({'restart_supervisor_on_update': state}) + update_config({'restart_supervisor_on_update': state == 'on'}) @click.command('restart_systemd_on_update', help='Enable/Disable auto restart of systemd units') @click.argument('state', type=click.Choice(['on', 'off'])) def config_restart_systemd_on_update(state): - state = True if state == 'on' else False - update_config({'restart_systemd_on_update': state}) + update_config({'restart_systemd_on_update': state == 'on'}) @click.command('update_bench_on_update', help='Enable/Disable bench updates on running bench update') @click.argument('state', type=click.Choice(['on', 'off'])) def config_update_bench_on_update(state): - state = True if state == 'on' else False - update_config({'update_bench_on_update': state}) + update_config({'update_bench_on_update': state == 'on'}) @click.command('dns_multitenant', help='Enable/Disable bench multitenancy on running bench update') @click.argument('state', type=click.Choice(['on', 'off'])) def config_dns_multitenant(state): - state = True if state == 'on' else False - update_config({'dns_multitenant': state}) + update_config({'dns_multitenant': state == 'on'}) @click.command('serve_default_site', help='Configure nginx to serve the default site on port 80') @click.argument('state', type=click.Choice(['on', 'off'])) def config_serve_default_site(state): - state = True if state == 'on' else False - update_config({'serve_default_site': state}) + update_config({'serve_default_site': state == 'on'}) @click.command('rebase_on_pull', help='Rebase repositories on pulling') @click.argument('state', type=click.Choice(['on', 'off'])) def config_rebase_on_pull(state): - state = True if state == 'on' else False - update_config({'rebase_on_pull': state}) + update_config({'rebase_on_pull': state == 'on'}) @click.command('http_timeout', help='Set HTTP timeout') @@ -74,26 +67,12 @@ def config_http_timeout(seconds): def set_common_config(configs): common_site_config = {} for key, value in configs: - if value in ("False", "True"): + if value in ('true', 'false'): + value = value.title() + try: value = ast.literal_eval(value) - - elif "." in value: - try: - value = float(value) - except ValueError: - pass - - elif "{" in value or "[" in value: - try: - value = json.loads(value) - except ValueError: - pass - - else: - try: - value = int(value) - except ValueError: - pass + except ValueError: + pass common_site_config[key] = value From 962168659703b83c4b2f4fc66c592d430db825df Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 18 Feb 2020 11:56:46 +0530 Subject: [PATCH 09/44] refactor: consistent help action in click, formatting and sorted imports --- bench/commands/git.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/bench/commands/git.py b/bench/commands/git.py index ffaa4d5b..cc9812ab 100644 --- a/bench/commands/git.py +++ b/bench/commands/git.py @@ -1,28 +1,31 @@ -import click -import os, subprocess, re +# imports - standard imports +import os +import re +import subprocess +# imports - module imports from bench.app import get_repo_dir, get_apps, get_remote from bench.utils import set_git_remote_url +# imports - third party imports +import click -@click.command('remote-set-url') + +@click.command('remote-set-url', help="Set app remote url") @click.argument('git-url') def remote_set_url(git_url): - "Set app remote url" set_git_remote_url(git_url) -@click.command('remote-reset-url') +@click.command('remote-reset-url', help="Reset app remote url to frappe official") @click.argument('app') def remote_reset_url(app): - "Reset app remote url to frappe official" git_url = "https://github.com/frappe/{}.git".format(app) set_git_remote_url(git_url) -@click.command('remote-urls') +@click.command('remote-urls', help="Show apps remote url") def remote_urls(): - "Show apps remote url" for app in get_apps(): repo_dir = get_repo_dir(app) From 063e4a85be0aa61db4c0b1d72373a241df1b0e8e Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 18 Feb 2020 12:24:18 +0530 Subject: [PATCH 10/44] refactor: bench.command.install * sorted imports * consistent formatting and use of docstrings * help added for click commands --- bench/commands/install.py | 52 ++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/bench/commands/install.py b/bench/commands/install.py index e1fa75f1..acca340a 100644 --- a/bench/commands/install.py +++ b/bench/commands/install.py @@ -1,21 +1,34 @@ -import os, sys, json, click +# imports - standard imports +import os +import sys +import json + +# imports - module imports from bench.utils import run_playbook, setup_sudoers, is_root -extra_vars = {"production": True} +# imports - third party imports +import click -@click.group() + +extra_vars = { + "production": True +} + + +@click.group(help="Install system dependencies for setting up Frappe environment") def install(): - "Install system dependancies" pass + @click.command('prerequisites') def install_prerequisites(): run_playbook('site.yml', tag='common, redis') -@click.command('mariadb') -@click.option('--mysql_root_password') + +@click.command('mariadb', help="Install and setup MariaDB of specified version and root password") +@click.option('--mysql_root_password', '--mysql-root-password', default="") @click.option('--version', default="10.3") -def install_maridb(mysql_root_password='', version=''): +def install_maridb(mysql_root_password, version): if mysql_root_password: extra_vars.update({ "mysql_root_password": mysql_root_password, @@ -27,41 +40,49 @@ def install_maridb(mysql_root_password='', version=''): run_playbook('site.yml', extra_vars=extra_vars, tag='mariadb') -@click.command('wkhtmltopdf') + +@click.command('wkhtmltopdf', help="Installs wkhtmltopdf v0.12.3 for linux") def install_wkhtmltopdf(): run_playbook('site.yml', extra_vars=extra_vars, tag='wkhtmltopdf') -@click.command('nodejs') + +@click.command('nodejs', help="Installs Node.js v8") def install_nodejs(): run_playbook('site.yml', extra_vars=extra_vars, tag='nodejs') -@click.command('psutil') + +@click.command('psutil', help="Installs psutil via pip") def install_psutil(): run_playbook('site.yml', extra_vars=extra_vars, tag='psutil') -@click.command('supervisor') + +@click.command('supervisor', help="Installs supervisor. If user is specified, sudoers is setup for that user") @click.option('--user') def install_supervisor(user=None): run_playbook('site.yml', extra_vars=extra_vars, tag='supervisor') if user: setup_sudoers(user) -@click.command('nginx') + +@click.command('nginx', help="Installs NGINX. If user is specified, sudoers is setup for that user") @click.option('--user') def install_nginx(user=None): run_playbook('site.yml', extra_vars=extra_vars, tag='nginx') if user: setup_sudoers(user) -@click.command('virtualbox') + +@click.command('virtualbox', help="Installs supervisor") def install_virtualbox(): run_playbook('vm_build.yml', tag='virtualbox') -@click.command('packer') + +@click.command('packer', help="Installs Oracle virtualbox and packer 1.2.1") def install_packer(): run_playbook('vm_build.yml', tag='packer') -@click.command('fail2ban') + +@click.command("fail2ban", help="Install fail2ban, an intrusion prevention software framework that protects computer servers from brute-force attacks") @click.option('--maxretry', default=6, help="Number of matches (i.e. value of the counter) which triggers ban action on the IP.") @click.option('--bantime', default=600, help="The counter is set to zero if no match is found within 'findtime' seconds.") @click.option('--findtime', default=600, help='Duration (in seconds) for IP to be banned for. Negative number for "permanent" ban.') @@ -69,6 +90,7 @@ def install_failtoban(**kwargs): extra_vars.update(kwargs) run_playbook('site.yml', extra_vars=extra_vars, tag='fail2ban') + install.add_command(install_prerequisites) install.add_command(install_maridb) install.add_command(install_wkhtmltopdf) From f1ea6fce3ffeae0a311beecad1d514d003875548 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 18 Feb 2020 17:07:26 +0530 Subject: [PATCH 11/44] chore: drop auto update bench via cronjob --- bench/commands/config.py | 7 ------- bench/commands/make.py | 4 +--- bench/commands/setup.py | 7 ------- 3 files changed, 1 insertion(+), 17 deletions(-) diff --git a/bench/commands/config.py b/bench/commands/config.py index 7dbaaed9..bf912ed1 100644 --- a/bench/commands/config.py +++ b/bench/commands/config.py @@ -14,12 +14,6 @@ def config(): pass -@click.command('auto_update', help='Enable/Disable auto update for bench') -@click.argument('state', type=click.Choice(['on', 'off'])) -def config_auto_update(state): - update_config({'auto_update': state == 'on'}) - - @click.command('restart_supervisor_on_update', help='Enable/Disable auto restart of supervisor processes') @click.argument('state', type=click.Choice(['on', 'off'])) def config_restart_supervisor_on_update(state): @@ -90,7 +84,6 @@ def remove_common_config(keys): put_config(common_site_config) -config.add_command(config_auto_update) config.add_command(config_update_bench_on_update) config.add_command(config_restart_supervisor_on_update) config.add_command(config_restart_systemd_on_update) diff --git a/bench/commands/make.py b/bench/commands/make.py index f6ba1377..5f6e236f 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -13,11 +13,10 @@ import click @click.option('--clone-without-update', is_flag=True, help="copy repos from path without update") @click.option('--no-procfile', is_flag=True, help="Pull changes in all the apps in bench") @click.option('--no-backups',is_flag=True, help="Run migrations for all sites in the bench") -@click.option('--no-auto-update',is_flag=True, help="Build JS and CSS artifacts for the bench") @click.option('--skip-redis-config-generation', is_flag=True, help="Skip redis config generation if already specifying the common-site-config file") @click.option('--skip-assets',is_flag=True, default=False, help="Do not build assets") @click.option('--verbose',is_flag=True, help="Verbose output during install") -def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, no_auto_update, clone_from, verbose, skip_redis_config_generation, clone_without_update, ignore_exist=False, skip_assets=False, python='python3'): +def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, clone_from, verbose, skip_redis_config_generation, clone_without_update, ignore_exist=False, skip_assets=False, python='python3'): from bench.utils import init, log try: @@ -26,7 +25,6 @@ def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, n apps_path=apps_path, no_procfile=no_procfile, no_backups=no_backups, - no_auto_update=no_auto_update, frappe_path=frappe_path, frappe_branch=frappe_branch, verbose=verbose, diff --git a/bench/commands/setup.py b/bench/commands/setup.py index 440e1f5e..c83633b5 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -75,12 +75,6 @@ def setup_production(user, yes=False): setup_production(user=user, yes=yes) -@click.command("auto-update", help="Add cronjob for bench auto update") -def setup_auto_update(): - from bench.utils import setup_auto_update - setup_auto_update() - - @click.command("backups", help="Add cronjob for bench backups") def setup_backups(): from bench.utils import setup_backups @@ -309,7 +303,6 @@ setup.add_command(setup_redis) setup.add_command(setup_letsencrypt) setup.add_command(setup_wildcard_ssl) setup.add_command(setup_production) -setup.add_command(setup_auto_update) setup.add_command(setup_backups) setup.add_command(setup_env) setup.add_command(setup_procfile) From 2713a2e9cd7c254e6856e73dcc0389f6f8760fc0 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 18 Feb 2020 17:09:04 +0530 Subject: [PATCH 12/44] fix: move update to utils - update.py * chore: drop auto_update options * fix: reloading bench modules post requirements setup (seemed unecessary and randomly crashed `bench start` process) * docs: added help for commands * chore: deprecated switch-to-master function - utils.py * sorted imports for readability * formatting and consistency * dropped unused setup_auto_update * dropped broken pre_upgrade function Signed-off-by: Gavin D'souza --- bench/commands/update.py | 157 ++++++++------------------------------- bench/utils.py | 149 ++++++++++++++++++++++++++++++------- 2 files changed, 150 insertions(+), 156 deletions(-) diff --git a/bench/commands/update.py b/bench/commands/update.py index 82f857ba..e2a1b310 100755 --- a/bench/commands/update.py +++ b/bench/commands/update.py @@ -1,124 +1,33 @@ -import click -import sys +# imports - standard imports import os -from bench.config.common_site_config import get_config, update_config -from bench.app import pull_all_apps, is_version_upgrade, validate_branch -from bench.utils import (update_bench, validate_upgrade, pre_upgrade, post_upgrade, before_update, - update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets, - restart_supervisor_processes, restart_systemd_processes, is_bench_directory) -from bench import patches +import sys + +# imports - third party imports +import click from six.moves import reload_module +# imports - module imports +from bench.app import pull_all_apps +from bench.utils import post_upgrade, patch_sites, build_assets -@click.command('update') -@click.option('--pull', is_flag=True, help="Pull changes in all the apps in bench") + +@click.command('update', help="Updates bench tool and if executed in a bench directory, without any flags will backup, pull, setup requirements, build, run patches and restart bench. Using specific flags will only do certain tasks instead of all") +@click.option('--pull', is_flag=True, help="Pull updates for all the apps in bench") @click.option('--patch', is_flag=True, help="Run migrations for all sites in the bench") -@click.option('--build', is_flag=True, help="Build JS and CSS artifacts for the bench") -@click.option('--bench', is_flag=True, help="Update bench") -@click.option('--requirements', is_flag=True, help="Update requirements") -@click.option('--restart-supervisor', is_flag=True, help="restart supervisor processes after update") -@click.option('--restart-systemd', is_flag=True, help="restart systemd units after update") -@click.option('--auto', is_flag=True) -@click.option('--no-backup', is_flag=True) -@click.option('--force', is_flag=True) +@click.option('--build', is_flag=True, help="Build JS and CSS assets for the bench") +@click.option('--bench', is_flag=True, help="Update bench CLI tool") +@click.option('--requirements', is_flag=True, help="Update requirements. If run alone, equivalent to `bench setup requirements`") +@click.option('--restart-supervisor', is_flag=True, help="Restart supervisor processes after update") +@click.option('--restart-systemd', is_flag=True, help="Restart systemd units after update") +@click.option('--no-backup', is_flag=True, help="If this flag is set, sites won't be backed up prior to updates. Note: This is not recommended in production.") +@click.option('--force', is_flag=True, help="Forces major version upgrades") @click.option('--reset', is_flag=True, help="Hard resets git branch's to their new states overriding any changes and overriding rebase on pull") -def update(pull=False, patch=False, build=False, bench=False, auto=False, restart_supervisor=False, restart_systemd=False, requirements=False, no_backup=False, force=False, reset=False): - "Update bench" +def update(pull, patch, build, bench, requirements, restart_supervisor, restart_systemd, no_backup, force, reset): + from bench.utils import update + update(pull=pull, patch=patch, build=build, bench=bench, requirements=requirements, restart_supervisor=restart_supervisor, restart_systemd=restart_systemd, backup= not no_backup, force=force, reset=reset) - if not is_bench_directory(): - """Update only bench if bench update called from outside a bench""" - update_bench(bench_repo=True, requirements=True) - sys.exit() - if not (pull or patch or build or bench or requirements): - pull, patch, build, bench, requirements = True, True, True, True, True - - if auto: - sys.exit(1) - - patches.run(bench_path='.') - conf = get_config(".") - - if bench and conf.get('update_bench_on_update'): - update_bench(bench_repo=True, requirements=False) - restart_update({ - 'pull': pull, - 'patch': patch, - 'build': build, - 'requirements': requirements, - 'no-backup': no_backup, - 'restart-supervisor': restart_supervisor, - 'reset': reset - }) - - if conf.get('release_bench'): - print('Release bench, cannot update') - sys.exit(1) - - validate_branch() - - version_upgrade = is_version_upgrade() - if version_upgrade[0]: - print() - print() - print("This update will cause a major version change in Frappe/ERPNext from {0} to {1}.".format(*version_upgrade[1:])) - print("This would take significant time to migrate and might break custom apps.") - click.confirm('Do you want to continue?', abort=True) - - _update(pull, patch, build, bench, auto, restart_supervisor, restart_systemd, requirements, no_backup, force=force, reset=reset) - -def _update(pull=False, patch=False, build=False, update_bench=False, auto=False, restart_supervisor=False, - restart_systemd=False, requirements=False, no_backup=False, bench_path='.', force=False, reset=False): - conf = get_config(bench_path=bench_path) - version_upgrade = is_version_upgrade(bench_path=bench_path) - - if version_upgrade[0] or (not version_upgrade[0] and force): - validate_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) - - before_update(bench_path=bench_path, requirements=requirements) - - conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 }) - update_config(conf, bench_path=bench_path) - - if not no_backup: - print('Backing up sites...') - backup_all_sites(bench_path=bench_path) - - if pull: - pull_all_apps(bench_path=bench_path, reset=reset) - - if requirements: - update_requirements(bench_path=bench_path) - update_node_packages(bench_path=bench_path) - - if version_upgrade[0] or (not version_upgrade[0] and force): - pre_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) - import bench.utils, bench.app - print('Reloading bench...') - reload_module(bench.utils) - reload_module(bench.app) - - if patch: - print('Patching sites...') - patch_sites(bench_path=bench_path) - if build: - build_assets(bench_path=bench_path) - if version_upgrade[0] or (not version_upgrade[0] and force): - post_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) - if restart_supervisor or conf.get('restart_supervisor_on_update'): - restart_supervisor_processes(bench_path=bench_path) - if restart_systemd or conf.get('restart_systemd_on_update'): - restart_systemd_processes(bench_path=bench_path) - - conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 }) - update_config(conf, bench_path=bench_path) - - print("_"*80) - print("Bench: Deployment tool for Frappe and ERPNext (https://erpnext.org).") - print("Open source depends on your contributions, so please contribute bug reports, patches, fixes or cash and be a part of the community") - print() - -@click.command('retry-upgrade') +@click.command('retry-upgrade', help="Retry a failed upgrade") @click.option('--version', default=5) def retry_upgrade(version): pull_all_apps() @@ -126,35 +35,27 @@ def retry_upgrade(version): build_assets() post_upgrade(version-1, version) -def restart_update(kwargs): - args = ['--'+k for k, v in list(kwargs.items()) if v] - os.execv(sys.argv[0], sys.argv[:2] + args) -@click.command('switch-to-branch') +@click.command('switch-to-branch', help="Switch all apps to specified branch, or specify apps separated by space") @click.argument('branch') @click.argument('apps', nargs=-1) @click.option('--upgrade',is_flag=True) def switch_to_branch(branch, apps, upgrade=False): - "Switch all apps to specified branch, or specify apps separated by space" from bench.app import switch_to_branch switch_to_branch(branch=branch, apps=list(apps), upgrade=upgrade) print('Switched to ' + branch) print('Please run `bench update --patch` to be safe from any differences in database schema') -@click.command('switch-to-master') + +@click.command('switch-to-master', help="[DEPRECATED]: Switch frappe and erpnext to master branch") def switch_to_master(): - "Switch frappe and erpnext to master branch" - from bench.app import switch_to_master - switch_to_master(apps=['frappe', 'erpnext']) - print() - print('Switched to master') - print('Please run `bench update --patch` to be safe from any differences in database schema') + from bench.utils import log + log("`switch-to-master` has been deprecated as master branches were renamed to version-11") + @click.command('switch-to-develop') def switch_to_develop(upgrade=False): "Switch frappe and erpnext to develop branch" from bench.app import switch_to_develop switch_to_develop(apps=['frappe', 'erpnext']) - print() - print('Switched to develop') - print('Please run `bench update --patch` to be safe from any differences in database schema') + print('Switched to develop\nPlease run `bench update --patch` to be safe from any differences in database schema') diff --git a/bench/utils.py b/bench/utils.py index 10a4f801..c00d9488 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -1,14 +1,34 @@ -import errno, glob, grp, itertools, json, logging, multiprocessing, os, platform, pwd, re, select, shutil, site, subprocess, sys +# imports - standard imports +import errno +import glob +import grp +import itertools +import json +import logging +import multiprocessing +import os +import platform +import pwd +import re +import select +import shutil +import site +import subprocess +import sys +import time from datetime import datetime from distutils.spawn import find_executable +# imports - third party imports +import click import requests import semantic_version from six import iteritems +from six.moves import reload_module from six.moves.urllib.parse import urlparse +# imports - module imports import bench -from bench import env class PatchError(Exception): @@ -70,9 +90,11 @@ def get_frappe(bench_path='.'): def get_env_cmd(cmd, bench_path='.'): return os.path.abspath(os.path.join(bench_path, 'env', 'bin', cmd)) -def init(path, apps_path=None, no_procfile=False, no_backups=False, no_auto_update=False, - frappe_path=None, frappe_branch=None, wheel_cache_dir=None, verbose=False, clone_from=None, - skip_redis_config_generation=False, clone_without_update=False, ignore_exist = False, skip_assets=False, python='python3'): +def init(path, apps_path=None, no_procfile=False, no_backups=False, + frappe_path=None, frappe_branch=None, verbose=False, clone_from=None, + skip_redis_config_generation=False, clone_without_update=False, ignore_exist=False, skip_assets=False, + python='python3'): + """Initialize a new bench directory""" from bench.app import get_app, install_apps_from_path from bench.config import redis from bench.config.common_site_config import make_config @@ -127,10 +149,99 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, no_auto_upda setup_procfile(path) if not no_backups: setup_backups(bench_path=path) - if not no_auto_update: - setup_auto_update(bench_path=path) + copy_patches_txt(path) +def restart_update(kwargs): + args = ['--'+k for k, v in list(kwargs.items()) if v] + os.execv(sys.argv[0], sys.argv[:2] + args) + +def update(pull=False, patch=False, build=False, bench=False, restart_supervisor=False, + restart_systemd=False, requirements=False, backup=True, force=False, reset=False): + """command: bench update""" + + if not is_bench_directory(): + """Update only bench CLI if bench update called from outside a bench""" + update_bench(bench_repo=True, requirements=True) + sys.exit(0) + + from bench import patches + from bench.app import is_version_upgrade, pull_all_apps, validate_branch + from bench.config.common_site_config import get_config, update_config + + bench_path = os.path.abspath(".") + patches.run(bench_path=bench_path) + conf = get_config(bench_path) + + if conf.get('release_bench'): + print('Release bench detected, cannot update!') + sys.exit(1) + + if not (pull or patch or build or bench or requirements): + pull, patch, build, bench, requirements = True, True, True, True, True + + if bench and conf.get('update_bench_on_update'): + update_bench(bench_repo=True, requirements=False) + restart_update({ + 'pull': pull, + 'patch': patch, + 'build': build, + 'requirements': requirements, + 'backup': backup, + 'restart-supervisor': restart_supervisor, + 'reset': reset + }) + + validate_branch() + version_upgrade = is_version_upgrade() + + if version_upgrade[0]: + if force: + print("Force flag has been used for a major version change in Frappe and it's apps. \nThis will take significant time to migrate and might break custom apps.") + else: + print("This update will cause a major version change in Frappe/ERPNext from {0} to {1}. \nThis would take significant time to migrate and might break custom apps.".format(*version_upgrade[1:])) + click.confirm('Do you want to continue?', abort=True) + + if version_upgrade[0] or (not version_upgrade[0] and force): + validate_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) + + before_update(bench_path=bench_path, requirements=requirements) + + conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 }) + update_config(conf, bench_path=bench_path) + + if backup: + print('Backing up sites...') + backup_all_sites(bench_path=bench_path) + + if pull: + pull_all_apps(bench_path=bench_path, reset=reset) + + if requirements: + update_requirements(bench_path=bench_path) + update_node_packages(bench_path=bench_path) + + if patch: + print('Patching sites...') + patch_sites(bench_path=bench_path) + + if build: + build_assets(bench_path=bench_path) + + if version_upgrade[0] or (not version_upgrade[0] and force): + post_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) + + if restart_supervisor or conf.get('restart_supervisor_on_update'): + restart_supervisor_processes(bench_path=bench_path) + + if restart_systemd or conf.get('restart_systemd_on_update'): + restart_systemd_processes(bench_path=bench_path) + + conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 }) + update_config(conf, bench_path=bench_path) + + print("_" * 80 + "\nBench: Deployment tool for Frappe and Frappe Applications (https://frappe.io/bench).\nOpen source depends on your contributions, so please contribute bug reports, patches, fixes or cash and be a part of the community") + def copy_patches_txt(bench_path): shutil.copy(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'patches', 'patches.txt'), os.path.join(bench_path, 'patches.txt')) @@ -246,12 +357,6 @@ def get_sites(bench_path='.'): def get_bench_dir(bench_path='.'): return os.path.abspath(bench_path) -def setup_auto_update(bench_path='.'): - logger.info('setting up auto update') - add_to_crontab('0 10 * * * cd {bench_dir} && {bench} update --auto >> {logfile} 2>&1'.format(bench_dir=get_bench_dir(bench_path=bench_path), - bench=os.path.join(get_bench_dir(bench_path=bench_path), 'env', 'bin', 'bench'), - logfile=os.path.join(get_bench_dir(bench_path=bench_path), 'logs', 'auto_update_log.log'))) - def setup_backups(bench_path='.'): logger.info('setting up backups') bench_dir = get_bench_dir(bench_path=bench_path) @@ -302,6 +407,8 @@ def update_bench(bench_repo=True, requirements=True): logger.info("Bench Updated!") def setup_sudoers(user): + from bench import env + if not os.path.exists('/etc/sudoers.d'): os.makedirs('/etc/sudoers.d') @@ -690,20 +797,6 @@ def validate_upgrade(from_ver, to_ver, bench_path='.'): if not find_executable('npm') and not (find_executable('node') or find_executable('nodejs')): raise Exception("Please install nodejs and npm") -def pre_upgrade(from_ver, to_ver, bench_path='.'): - pip = os.path.join(bench_path, 'env', 'bin', 'pip') - - if from_ver <= 4 and to_ver >= 5: - from .migrate_to_v5 import remove_shopping_cart - apps = ('frappe', 'erpnext') - remove_shopping_cart(bench_path=bench_path) - - for app in apps: - cwd = os.path.abspath(os.path.join(bench_path, 'apps', app)) - if os.path.exists(cwd): - exec_cmd("git clean -dxf", cwd=cwd) - exec_cmd("{pip} install --upgrade -e {app}".format(pip=pip, app=cwd)) - def post_upgrade(from_ver, to_ver, bench_path='.'): from .config.common_site_config import get_config from .config import redis @@ -1042,4 +1135,4 @@ def migrate_env(python, backup=False): log.debug('Migration Successful to {}'.format(python)) except: log.debug('Migration Error') - raise \ No newline at end of file + raise From 9d00bc743e39acf9dd09ca43ddb12e2b6de3abfb Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 18 Feb 2020 18:04:20 +0530 Subject: [PATCH 13/44] style: help moved in bench.commands.utils docs --- bench/commands/utils.py | 38 +++++++++++++------------------------- 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/bench/commands/utils.py b/bench/commands/utils.py index f2bac0af..3309040d 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -2,22 +2,20 @@ import click import sys, os, copy -@click.command('start') +@click.command('start', help="Start Frappe development processes") @click.option('--no-dev', is_flag=True, default=False) @click.option('--concurrency', '-c', type=str) @click.option('--procfile', '-p', type=str) def start(no_dev, concurrency, procfile): - "Start Frappe development processes" from bench.utils import start start(no_dev=no_dev, concurrency=concurrency, procfile=procfile) -@click.command('restart') +@click.command('restart', help="Restart supervisor processes or systemd units") @click.option('--web', is_flag=True, default=False) @click.option('--supervisor', is_flag=True, default=False) @click.option('--systemd', is_flag=True, default=False) def restart(web, supervisor, systemd): - "Restart supervisor processes or systemd units" from bench.utils import restart_supervisor_processes, restart_systemd_processes from bench.config.common_site_config import get_config if get_config('.').get('restart_supervisor_on_update') or supervisor: @@ -25,20 +23,18 @@ def restart(web, supervisor, systemd): if get_config('.').get('restart_systemd_on_update') or systemd: restart_systemd_processes(bench_path='.', web_workers=web) -@click.command('set-nginx-port') +@click.command('set-nginx-port', help="Set nginx port for site") @click.argument('site') @click.argument('port', type=int) def set_nginx_port(site, port): - "Set nginx port for site" from bench.config.site_config import set_nginx_port set_nginx_port(site, port) -@click.command('set-ssl-certificate') +@click.command('set-ssl-certificate', help="Set ssl certificate path for site") @click.argument('site') @click.argument('ssl-certificate-path') def set_ssl_certificate(site, ssl_certificate_path): - "Set ssl certificate path for site" from bench.config.site_config import set_ssl_certificate set_ssl_certificate(site, ssl_certificate_path) @@ -134,25 +130,23 @@ def shell(bench_path='.'): os.execve(env['SHELL'], [env['SHELL']], env) -@click.command('backup') +@click.command('backup', help="Backup single site") @click.argument('site') def backup_site(site): - "backup site" from bench.utils import get_sites, backup_site if site not in get_sites(bench_path='.'): - print('site not found') + print('Site `{0}` not found'.format(site)) sys.exit(1) backup_site(site, bench_path='.') -@click.command('backup-all-sites') +@click.command('backup-all-sites', help="Backup all sites in current bench") def backup_all_sites(): - "backup all sites" from bench.utils import backup_all_sites backup_all_sites(bench_path='.') -@click.command('release') +@click.command('release', help="Release a Frappe app (internal to the Frappe team)") @click.argument('app') @click.argument('bump-type', type=click.Choice(['major', 'minor', 'patch', 'stable', 'prerelease'])) @click.option('--from-branch', default='develop') @@ -162,48 +156,42 @@ def backup_all_sites(): @click.option('--repo-name') @click.option('--dont-frontport', is_flag=True, default=False, help='Front port fixes to new branches, example merging hotfix(v10) into staging-fixes(v11)') def release(app, bump_type, from_branch, to_branch, owner, repo_name, remote, dont_frontport): - "Release app (internal to the Frappe team)" from bench.release import release frontport = not dont_frontport release(bench_path='.', app=app, bump_type=bump_type, from_branch=from_branch, to_branch=to_branch, remote=remote, owner=owner, repo_name=repo_name, frontport=frontport) -@click.command('prepare-beta-release') +@click.command('prepare-beta-release', help="Prepare major beta release from develop branch") @click.argument('app') @click.option('--owner', default='frappe') def prepare_beta_release(app, owner): - """Prepare major beta release from develop branch""" from bench.prepare_beta_release import prepare_beta_release prepare_beta_release(bench_path='.', app=app, owner=owner) -@click.command('disable-production') +@click.command('disable-production', help="Disables production environment for the bench.") def disable_production(): - """Disables production environment for the bench.""" from bench.config.production_setup import disable_production disable_production(bench_path='.') -@click.command('src') +@click.command('src', help="Prints bench source folder path, which can be used as: cd `bench src`") def bench_src(): - """Prints bench source folder path, which can be used as: cd `bench src` """ import bench print(os.path.dirname(bench.__path__[0])) -@click.command('find') +@click.command('find', help="Finds benches recursively from location") @click.argument('location', default='') def find_benches(location): - """Finds benches recursively from location""" from bench.utils import find_benches find_benches(directory=location) -@click.command('migrate-env') +@click.command('migrate-env', help="Migrate Virtual Environment to desired Python Version") @click.argument('python', type=str) @click.option('--no-backup', 'backup', is_flag=True, default=True) def migrate_env(python, backup=True): - """Migrate Virtual Environment to desired Python Version""" from bench.utils import migrate_env migrate_env(python=python, backup=backup) From 043f044853717e57a8ffe3449350731684c72a20 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 18 Feb 2020 18:47:38 +0530 Subject: [PATCH 14/44] style: bench.commands.utils help added --- bench/commands/utils.py | 53 ++++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/bench/commands/utils.py b/bench/commands/utils.py index 3309040d..8a78fea0 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -1,5 +1,10 @@ +# imports - standard imports +import copy +import os +import sys + +# imports - third party imports import click -import sys, os, copy @click.command('start', help="Start Frappe development processes") @@ -23,7 +28,8 @@ def restart(web, supervisor, systemd): if get_config('.').get('restart_systemd_on_update') or systemd: restart_systemd_processes(bench_path='.', web_workers=web) -@click.command('set-nginx-port', help="Set nginx port for site") + +@click.command('set-nginx-port', help="Set NGINX port for site") @click.argument('site') @click.argument('port', type=int) def set_nginx_port(site, port): @@ -31,7 +37,7 @@ def set_nginx_port(site, port): set_nginx_port(site, port) -@click.command('set-ssl-certificate', help="Set ssl certificate path for site") +@click.command('set-ssl-certificate', help="Set SSL certificate path for site") @click.argument('site') @click.argument('ssl-certificate-path') def set_ssl_certificate(site, ssl_certificate_path): @@ -39,79 +45,74 @@ def set_ssl_certificate(site, ssl_certificate_path): set_ssl_certificate(site, ssl_certificate_path) -@click.command('set-ssl-key') +@click.command('set-ssl-key', help="Set SSL certificate private key path for site") @click.argument('site') @click.argument('ssl-certificate-key-path') def set_ssl_certificate_key(site, ssl_certificate_key_path): - "Set ssl certificate private key path for site" from bench.config.site_config import set_ssl_certificate_key set_ssl_certificate_key(site, ssl_certificate_key_path) -@click.command('set-url-root') +@click.command('set-url-root', help="Set URL root for site") @click.argument('site') @click.argument('url-root') def set_url_root(site, url_root): - "Set url root for site" from bench.config.site_config import set_url_root set_url_root(site, url_root) -@click.command('set-mariadb-host') +@click.command('set-mariadb-host', help="Set MariaDB host for bench") @click.argument('host') def set_mariadb_host(host): - "Set MariaDB host for bench" from bench.utils import set_mariadb_host set_mariadb_host(host) -@click.command('set-redis-cache-host') + +@click.command('set-redis-cache-host', help="Set Redis cache host for bench") @click.argument('host') def set_redis_cache_host(host): """ - Set Redis cache host for bench - Eg: bench set-redis-cache-host localhost:6379/1 + Usage: bench set-redis-cache-host localhost:6379/1 """ from bench.utils import set_redis_cache_host set_redis_cache_host(host) -@click.command('set-redis-queue-host') + +@click.command('set-redis-queue-host', help="Set Redis queue host for bench") @click.argument('host') def set_redis_queue_host(host): """ - Set Redis queue host for bench - Eg: bench set-redis-queue-host localhost:6379/2 + Usage: bench set-redis-queue-host localhost:6379/2 """ from bench.utils import set_redis_queue_host set_redis_queue_host(host) -@click.command('set-redis-socketio-host') + +@click.command('set-redis-socketio-host', help="Set Redis socketio host for bench") @click.argument('host') def set_redis_socketio_host(host): """ - Set Redis socketio host for bench - Eg: bench set-redis-socketio-host localhost:6379/3 + Usage: bench set-redis-socketio-host localhost:6379/3 """ from bench.utils import set_redis_socketio_host set_redis_socketio_host(host) -@click.command('set-default-site') +@click.command('set-default-site', help="Set default site for bench") @click.argument('site') def set_default_site(site): - "Set default site for bench" from bench.utils import set_default_site set_default_site(site) -@click.command('download-translations') +@click.command('download-translations', help="Download latest translations") def download_translations(): - "Download latest translations" from bench.utils import download_translations_p download_translations_p() -@click.command('renew-lets-encrypt') + +@click.command('renew-lets-encrypt', help="Renew Let's Encrypt certificate") def renew_lets_encrypt(): - "Renew Let's Encrypt certificate" from bench.config.lets_encrypt import renew_certs renew_certs() @@ -129,7 +130,6 @@ def shell(bench_path='.'): os.chdir('sites') os.execve(env['SHELL'], [env['SHELL']], env) - @click.command('backup', help="Backup single site") @click.argument('site') def backup_site(site): @@ -158,8 +158,7 @@ def backup_all_sites(): def release(app, bump_type, from_branch, to_branch, owner, repo_name, remote, dont_frontport): from bench.release import release frontport = not dont_frontport - release(bench_path='.', app=app, bump_type=bump_type, from_branch=from_branch, to_branch=to_branch, - remote=remote, owner=owner, repo_name=repo_name, frontport=frontport) + release(bench_path='.', app=app, bump_type=bump_type, from_branch=from_branch, to_branch=to_branch, remote=remote, owner=owner, repo_name=repo_name, frontport=frontport) @click.command('prepare-beta-release', help="Prepare major beta release from develop branch") From 90d23139474d0ee8dc9b80d9b1b96192fc912218 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 12 Feb 2020 14:55:30 +0530 Subject: [PATCH 15/44] chore: drop `bench shell` command --- bench/commands/__init__.py | 3 +-- bench/commands/utils.py | 13 ------------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/bench/commands/__init__.py b/bench/commands/__init__.py index 7e514012..090013c2 100755 --- a/bench/commands/__init__.py +++ b/bench/commands/__init__.py @@ -40,7 +40,7 @@ bench_command.add_command(switch_to_develop) from bench.commands.utils import (start, restart, set_nginx_port, set_ssl_certificate, set_ssl_certificate_key, set_url_root, - set_mariadb_host, set_default_site, download_translations, shell, backup_site, backup_all_sites, release, renew_lets_encrypt, + set_mariadb_host, set_default_site, download_translations, backup_site, backup_all_sites, release, renew_lets_encrypt, disable_production, bench_src, prepare_beta_release, set_redis_cache_host, set_redis_queue_host, set_redis_socketio_host, find_benches, migrate_env) bench_command.add_command(start) bench_command.add_command(restart) @@ -54,7 +54,6 @@ bench_command.add_command(set_redis_queue_host) bench_command.add_command(set_redis_socketio_host) bench_command.add_command(set_default_site) bench_command.add_command(download_translations) -bench_command.add_command(shell) bench_command.add_command(backup_site) bench_command.add_command(backup_all_sites) bench_command.add_command(release) diff --git a/bench/commands/utils.py b/bench/commands/utils.py index 8a78fea0..38dc28bf 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -116,19 +116,6 @@ def renew_lets_encrypt(): from bench.config.lets_encrypt import renew_certs renew_certs() -@click.command() -def shell(bench_path='.'): - if not os.environ.get('SHELL'): - print("Cannot get shell") - sys.exit(1) - if not os.path.exists('sites'): - print("sites dir doesn't exist") - sys.exit(1) - env = copy.copy(os.environ) - env['PS1'] = '(' + os.path.basename(os.path.dirname(os.path.abspath(__file__))) + ')' + env.get('PS1', '') - env['PATH'] = os.path.dirname(os.path.abspath(os.path.join('env','bin')) + ':' + env['PATH']) - os.chdir('sites') - os.execve(env['SHELL'], [env['SHELL']], env) @click.command('backup', help="Backup single site") @click.argument('site') From b4f4e717197b08705f1f4b40c876172cbae3f471 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 13 Feb 2020 04:15:30 +0530 Subject: [PATCH 16/44] chore: move 'find_parent_bench' to utils style: consistent spaces and formatting, code cleanup --- bench/cli.py | 23 ++----------- bench/prepare_staging.py | 6 ++-- bench/release.py | 2 -- bench/utils.py | 74 +++++++++++++++++++++++++++++++++++++--- 4 files changed, 75 insertions(+), 30 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index 733a0dd6..243e2640 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -1,6 +1,6 @@ import click import os, sys, logging, json, pwd, subprocess -from bench.utils import is_root, PatchError, drop_privileges, get_env_cmd, get_cmd_output, get_frappe, log, is_bench_directory +from bench.utils import is_root, PatchError, drop_privileges, get_env_cmd, get_cmd_output, get_frappe, log, is_bench_directory, find_parent_bench from bench.app import get_apps from bench.config.common_site_config import get_config from bench.commands import bench_command @@ -29,7 +29,6 @@ def cli(): elif len(sys.argv) > 1 and sys.argv[1]=="--help": print(click.Context(bench_command).get_help()) - print() print(get_frappe_help()) return @@ -99,7 +98,6 @@ def get_frappe_commands(bench_path='.'): return [] try: output = get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-commands".format(python=python), cwd=sites_path) - # output = output.decode('utf-8') return json.loads(output) except subprocess.CalledProcessError as e: if hasattr(e, "stderr"): @@ -109,27 +107,12 @@ def get_frappe_commands(bench_path='.'): def get_frappe_help(bench_path='.'): python = get_env_cmd('python', bench_path=bench_path) sites_path = os.path.join(bench_path, 'sites') - if not os.path.exists(sites_path): - return [] try: out = get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-help".format(python=python), cwd=sites_path) - return "Framework commands:\n" + out.split('Commands:')[1] - except subprocess.CalledProcessError: + return "\n\nFramework commands:\n" + out.split('Commands:')[1] + except: return "" -def find_parent_bench(path): - """Checks if parent directories are benches""" - if is_bench_directory(directory=path): - return path - - home_path = os.path.expanduser("~") - root_path = os.path.abspath(os.sep) - - if path not in {home_path, root_path}: - # NOTE: the os.path.split assumes that given path is absolute - parent_dir = os.path.split(path)[0] - return find_parent_bench(parent_dir) - def change_working_directory(): """Allows bench commands to be run from anywhere inside a bench directory""" cur_dir = os.path.abspath(".") diff --git a/bench/prepare_staging.py b/bench/prepare_staging.py index baa2a7c3..9d004684 100755 --- a/bench/prepare_staging.py +++ b/bench/prepare_staging.py @@ -19,9 +19,7 @@ def prepare_staging(bench_path, app, remote='upstream'): print('No commits to release') return - print() print(message) - print() click.confirm('Do you want to continue?', abort=True) @@ -52,13 +50,13 @@ def create_staging(repo_path, from_branch='develop'): g.merge(from_branch, '--no-ff') except git.exc.GitCommandError as e: handle_merge_error(e, source=from_branch, target='staging') - + g.checkout(from_branch) try: g.merge('staging') except git.exc.GitCommandError as e: handle_merge_error(e, source='staging', target=from_branch) - + def push_commits(repo_path, remote='upstream'): print('pushing staging branch of', repo_path) diff --git a/bench/release.py b/bench/release.py index ebf60f6a..f026b379 100755 --- a/bench/release.py +++ b/bench/release.py @@ -83,9 +83,7 @@ def bump(bench_path, app, bump_type, from_branch, to_branch, remote, owner, repo print('No commits to release') return - print() print(message) - print() click.confirm('Do you want to continue?', abort=True) diff --git a/bench/utils.py b/bench/utils.py index c00d9488..35c1a319 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -80,6 +80,7 @@ def safe_decode(string, encoding = 'utf-8'): pass return string + def get_frappe(bench_path='.'): frappe = get_env_cmd('frappe', bench_path=bench_path) if not os.path.exists(frappe): @@ -87,9 +88,11 @@ def get_frappe(bench_path='.'): print('bench get-app https://github.com/frappe/frappe.git') return frappe + def get_env_cmd(cmd, bench_path='.'): return os.path.abspath(os.path.join(bench_path, 'env', 'bin', cmd)) + def init(path, apps_path=None, no_procfile=False, no_backups=False, frappe_path=None, frappe_branch=None, verbose=False, clone_from=None, skip_redis_config_generation=False, clone_without_update=False, ignore_exist=False, skip_assets=False, @@ -152,10 +155,12 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, copy_patches_txt(path) + def restart_update(kwargs): args = ['--'+k for k, v in list(kwargs.items()) if v] os.execv(sys.argv[0], sys.argv[:2] + args) + def update(pull=False, patch=False, build=False, bench=False, restart_supervisor=False, restart_systemd=False, requirements=False, backup=True, force=False, reset=False): """command: bench update""" @@ -242,10 +247,12 @@ def update(pull=False, patch=False, build=False, bench=False, restart_supervisor print("_" * 80 + "\nBench: Deployment tool for Frappe and Frappe Applications (https://frappe.io/bench).\nOpen source depends on your contributions, so please contribute bug reports, patches, fixes or cash and be a part of the community") + def copy_patches_txt(bench_path): shutil.copy(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'patches', 'patches.txt'), os.path.join(bench_path, 'patches.txt')) + def clone_apps_from(bench_path, clone_from, update_app=True): from .app import install_app print('Copying apps from {0}...'.format(clone_from)) @@ -282,6 +289,7 @@ def clone_apps_from(bench_path, clone_from, update_app=True): for app in apps: setup_app(app) + def exec_cmd(cmd, cwd='.'): from .cli import from_command_line @@ -304,6 +312,7 @@ def exec_cmd(cmd, cwd='.'): if return_code > 0: raise CommandFailedError(cmd) + def which(executable, raise_err = False): from distutils.spawn import find_executable exec_ = find_executable(executable) @@ -315,6 +324,7 @@ def which(executable, raise_err = False): return exec_ + def setup_env(bench_path='.', python = 'python3'): python = which(python, raise_err = True) pip = os.path.join('env', 'bin', 'pip') @@ -323,10 +333,12 @@ def setup_env(bench_path='.', python = 'python3'): exec_cmd('{} -q install -U pip wheel six'.format(pip), cwd=bench_path) exec_cmd('{} -q install -e git+https://github.com/frappe/python-pdfkit.git#egg=pdfkit'.format(pip), cwd=bench_path) + def setup_socketio(bench_path='.'): exec_cmd("npm install socket.io redis express superagent cookie babel-core less chokidar \ babel-cli babel-preset-es2015 babel-preset-es2016 babel-preset-es2017 babel-preset-babili", cwd=bench_path) + def patch_sites(bench_path='.'): bench.set_frappe_version(bench_path=bench_path) @@ -338,6 +350,7 @@ def patch_sites(bench_path='.'): except subprocess.CalledProcessError: raise PatchError + def build_assets(bench_path='.', app=None): bench.set_frappe_version(bench_path=bench_path) @@ -349,14 +362,17 @@ def build_assets(bench_path='.', app=None): command += ' --app {}'.format(app) exec_cmd(command, cwd=bench_path) + def get_sites(bench_path='.'): sites_path = os.path.join(bench_path, 'sites') sites = (site for site in os.listdir(sites_path) if os.path.exists(os.path.join(sites_path, site, 'site_config.json'))) return sites + def get_bench_dir(bench_path='.'): return os.path.abspath(bench_path) + def setup_backups(bench_path='.'): logger.info('setting up backups') bench_dir = get_bench_dir(bench_path=bench_path) @@ -370,6 +386,7 @@ def setup_backups(bench_path='.'): add_to_crontab('0 */6 * * * {backup_command} >> {logfile} 2>&1'.format(backup_command=backup_command, logfile=os.path.join(get_bench_dir(bench_path=bench_path), 'logs', 'backup.log'))) + def add_to_crontab(line): current_crontab = read_crontab() line = str.encode(line) @@ -382,12 +399,14 @@ def add_to_crontab(line): s.stdin.write(line + b'\n') s.stdin.close() + def read_crontab(): s = subprocess.Popen(["crontab", "-l"], stdin=subprocess.PIPE, stdout=subprocess.PIPE) out = s.stdout.read() s.stdout.close() return out + def update_bench(bench_repo=True, requirements=True): logger.info("Updating bench") @@ -406,6 +425,7 @@ def update_bench(bench_repo=True, requirements=True): logger.info("Bench Updated!") + def setup_sudoers(user): from bench import env @@ -440,6 +460,7 @@ def setup_sudoers(user): os.chmod(sudoers_file, 0o440) + def setup_logging(bench_path='.'): if os.path.exists(os.path.join(bench_path, 'logs')): logger = logging.getLogger('bench') @@ -450,6 +471,7 @@ def setup_logging(bench_path='.'): logger.addHandler(hdlr) logger.setLevel(logging.DEBUG) + def get_program(programs): program = None for p in programs: @@ -458,9 +480,11 @@ def get_program(programs): break return program + def get_process_manager(): return get_program(['foreman', 'forego', 'honcho']) + def start(no_dev=False, concurrency=None, procfile=None): program = get_process_manager() if not program: @@ -478,6 +502,7 @@ def start(no_dev=False, concurrency=None, procfile=None): os.execv(program, command) + def check_cmd(cmd, cwd='.'): try: subprocess.check_call(cmd, cwd=cwd, shell=True) @@ -485,6 +510,7 @@ def check_cmd(cmd, cwd='.'): except subprocess.CalledProcessError: return False + def get_git_version(): '''returns git version from `git --version` extracts version number from string `get version 1.9.1` etc''' @@ -494,6 +520,7 @@ def get_git_version(): version = '.'.join(version.split('.')[0:2]) return float(version) + def check_git_for_shallow_clone(): from .config.common_site_config import get_config config = get_config('.') @@ -509,6 +536,7 @@ def check_git_for_shallow_clone(): if git_version > 1.9: return True + def get_cmd_output(cmd, cwd='.'): try: output = subprocess.check_output(cmd, cwd=cwd, shell=True, stderr=subprocess.PIPE).strip() @@ -519,6 +547,7 @@ def get_cmd_output(cmd, cwd='.'): print(e.output) raise + def safe_encode(what, encoding = 'utf-8'): try: what = what.encode(encoding) @@ -527,6 +556,7 @@ def safe_encode(what, encoding = 'utf-8'): return what + def restart_supervisor_processes(bench_path='.', web_workers=False): from .config.common_site_config import get_config conf = get_config(bench_path=bench_path) @@ -556,26 +586,31 @@ def restart_supervisor_processes(bench_path='.', web_workers=False): exec_cmd('sudo supervisorctl restart {group}'.format(group=group), cwd=bench_path) + def restart_systemd_processes(bench_path='.', web_workers=False): from .config.common_site_config import get_config bench_name = get_bench_name(bench_path) exec_cmd('sudo systemctl stop -- $(systemctl show -p Requires {bench_name}.target | cut -d= -f2)'.format(bench_name=bench_name)) exec_cmd('sudo systemctl start -- $(systemctl show -p Requires {bench_name}.target | cut -d= -f2)'.format(bench_name=bench_name)) + def set_default_site(site, bench_path='.'): if site not in get_sites(bench_path=bench_path): raise Exception("Site not in bench") exec_cmd("{frappe} --use {site}".format(frappe=get_frappe(bench_path=bench_path), site=site), cwd=os.path.join(bench_path, 'sites')) + def update_bench_requirements(): bench_req_file = os.path.join(os.path.dirname(bench.__path__[0]), 'requirements.txt') install_requirements(bench_req_file, user=True) + def update_env_pip(bench_path): env_pip = os.path.join(bench_path, 'env', 'bin', 'pip') exec_cmd("{pip} install -q -U pip".format(pip=env_pip)) + def update_requirements(bench_path='.'): from bench.app import get_apps, install_app print('Updating Python libraries...') @@ -589,13 +624,13 @@ def update_requirements(bench_path='.'): for app in get_apps(): install_app(app, bench_path=bench_path) + def update_node_packages(bench_path='.'): print('Updating node packages...') from bench.app import get_develop_version from distutils.version import LooseVersion v = LooseVersion(get_develop_version('frappe', bench_path = bench_path)) - # After rollup was merged, frappe_version = 10.1 # if develop_verion is 11 and up, only then install yarn if v < LooseVersion('11.x.x-develop'): @@ -603,6 +638,7 @@ def update_node_packages(bench_path='.'): else: update_yarn_packages(bench_path) + def update_yarn_packages(bench_path='.'): apps_dir = os.path.join(bench_path, 'apps') @@ -663,6 +699,7 @@ def install_requirements(req_file, user=False): exec_cmd("{python} -m pip install {user_flag} -q -U -r {req_file}".format(python=python, user_flag=user_flag, req_file=req_file)) + def backup_site(site, bench_path='.'): bench.set_frappe_version(bench_path=bench_path) @@ -672,30 +709,38 @@ def backup_site(site, bench_path='.'): else: run_frappe_cmd('--site', site, 'backup', bench_path=bench_path) + def backup_all_sites(bench_path='.'): for site in get_sites(bench_path=bench_path): backup_site(site, bench_path=bench_path) + def is_root(): if os.getuid() == 0: return True return False + def set_mariadb_host(host, bench_path='.'): update_common_site_config({'db_host': host}, bench_path=bench_path) + def set_redis_cache_host(host, bench_path='.'): update_common_site_config({'redis_cache': "redis://{}".format(host)}, bench_path=bench_path) + def set_redis_queue_host(host, bench_path='.'): update_common_site_config({'redis_queue': "redis://{}".format(host)}, bench_path=bench_path) + def set_redis_socketio_host(host, bench_path='.'): update_common_site_config({'redis_socketio': "redis://{}".format(host)}, bench_path=bench_path) + def update_common_site_config(ddict, bench_path='.'): update_json_file(os.path.join(bench_path, 'sites', 'common_site_config.json'), ddict) + def update_json_file(filename, ddict): if os.path.exists(filename): with open(filename, 'r') as f: @@ -708,6 +753,7 @@ def update_json_file(filename, ddict): with open(filename, 'w') as f: json.dump(content, f, indent=1, sort_keys=True) + def drop_privileges(uid_name='nobody', gid_name='nogroup'): # from http://stackoverflow.com/a/2699996 if os.getuid() != 0: @@ -728,6 +774,7 @@ def drop_privileges(uid_name='nobody', gid_name='nogroup'): # Ensure a very conservative umask os.umask(0o22) + def fix_prod_setup_perms(bench_path='.', frappe_user=None): from .config.common_site_config import get_config @@ -745,6 +792,7 @@ def fix_prod_setup_perms(bench_path='.', frappe_user=None): gid = grp.getgrnam(frappe_user).gr_gid os.chown(path, uid, gid) + def fix_file_perms(): for dir_path, dirs, files in os.walk('.'): for _dir in dirs: @@ -757,10 +805,12 @@ def fix_file_perms(): if not _file.startswith('activate'): os.chmod(os.path.join(bin_dir, _file), 0o755) + def get_current_frappe_version(bench_path='.'): from .app import get_current_frappe_version as fv return fv(bench_path=bench_path) + def run_frappe_cmd(*args, **kwargs): from .cli import from_command_line @@ -784,7 +834,7 @@ def run_frappe_cmd(*args, **kwargs): if return_code > 0: sys.exit(return_code) - #raise CommandFailedError(args) + def get_frappe_cmd_output(*args, **kwargs): bench_path = kwargs.get('bench_path', '.') @@ -792,19 +842,20 @@ def get_frappe_cmd_output(*args, **kwargs): sites_dir = os.path.join(bench_path, 'sites') return subprocess.check_output((f, '-m', 'frappe.utils.bench_helper', 'frappe') + args, cwd=sites_dir) + def validate_upgrade(from_ver, to_ver, bench_path='.'): if to_ver >= 6: if not find_executable('npm') and not (find_executable('node') or find_executable('nodejs')): raise Exception("Please install nodejs and npm") + def post_upgrade(from_ver, to_ver, bench_path='.'): from .config.common_site_config import get_config from .config import redis from .config.supervisor import generate_supervisor_config from .config.nginx import make_nginx_conf conf = get_config(bench_path=bench_path) - print("-"*80) - print("Your bench was upgraded to version {0}".format(to_ver)) + print("-" * 80 + "Your bench was upgraded to version {0}".format(to_ver)) if conf.get('restart_supervisor_on_update'): redis.generate_config(bench_path=bench_path) @@ -1080,6 +1131,7 @@ def in_virtual_env(): return False + def migrate_env(python, backup=False): from bench.config.common_site_config import get_config from bench.app import get_apps @@ -1136,3 +1188,17 @@ def migrate_env(python, backup=False): except: log.debug('Migration Error') raise + + +def find_parent_bench(path): + """Checks if parent directories are benches""" + if is_bench_directory(directory=path): + return path + + home_path = os.path.expanduser("~") + root_path = os.path.abspath(os.sep) + + if path not in {home_path, root_path}: + # NOTE: the os.path.split assumes that given path is absolute + parent_dir = os.path.split(path)[0] + return find_parent_bench(parent_dir) From f9521efc50827aa6e72624e55b28164a15791583 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 15 Feb 2020 11:05:30 +0530 Subject: [PATCH 17/44] chore: removed unused and optimized imports --- bench/commands/config.py | 1 - bench/commands/git.py | 1 - bench/commands/install.py | 7 +------ bench/commands/setup.py | 15 ++++++++------- bench/utils.py | 4 ---- 5 files changed, 9 insertions(+), 19 deletions(-) diff --git a/bench/commands/config.py b/bench/commands/config.py index bf912ed1..1dc33a39 100644 --- a/bench/commands/config.py +++ b/bench/commands/config.py @@ -1,6 +1,5 @@ # imports - standard imports import ast -import json # imports - module imports from bench.config.common_site_config import update_config, get_config, put_config diff --git a/bench/commands/git.py b/bench/commands/git.py index cc9812ab..8b52661b 100644 --- a/bench/commands/git.py +++ b/bench/commands/git.py @@ -1,6 +1,5 @@ # imports - standard imports import os -import re import subprocess # imports - module imports diff --git a/bench/commands/install.py b/bench/commands/install.py index acca340a..b760719c 100644 --- a/bench/commands/install.py +++ b/bench/commands/install.py @@ -1,10 +1,5 @@ -# imports - standard imports -import os -import sys -import json - # imports - module imports -from bench.utils import run_playbook, setup_sudoers, is_root +from bench.utils import run_playbook, setup_sudoers # imports - third party imports import click diff --git a/bench/commands/setup.py b/bench/commands/setup.py index c83633b5..b28e704c 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -61,11 +61,10 @@ def setup_fonts(): @click.option("--yes", help="Yes to regeneration config", is_flag=True, default=False) def setup_production(user, yes=False): from bench.config.production_setup import setup_production - from bench.utils import run_playbook # Install prereqs for production from distutils.spawn import find_executable if not find_executable("ansible"): - exec_cmd("sudo {0} install ansible".format("pip3" if PY3 else "pip2")) + exec_cmd("sudo -H {0} -m pip install ansible".format(sys.executable)) if not find_executable("fail2ban-client"): exec_cmd("bench setup role fail2ban") if not find_executable("nginx"): @@ -164,13 +163,19 @@ def setup_requirements(node=False, python=False): @click.option("--domain", help="Domain on which you want to run bench manager") def setup_manager(yes=False, port=23624, domain=None): from six.moves import input + from bench.utils import get_sites + from bench.config.common_site_config import get_config + from bench.config.nginx import make_bench_manager_nginx_conf + create_new_site = True + if "bench-manager.local" in os.listdir("sites"): ans = input("Site already exists. Overwrite existing site? [Y/n]: ").lower() while ans not in ("y", "n", ""): ans = input("Please enter 'y' or 'n'. Site already exists. Overwrite existing site? [Y/n]: ").lower() if ans == "n": create_new_site = False + if create_new_site: exec_cmd("bench new-site --force bench-manager.local") @@ -181,22 +186,18 @@ def setup_manager(yes=False, port=23624, domain=None): exec_cmd("bench --site bench-manager.local install-app bench_manager") - from bench.config.common_site_config import get_config bench_path = "." conf = get_config(bench_path) + if conf.get("restart_supervisor_on_update") or conf.get("restart_systemd_on_update"): # implicates a production setup or so I presume if not domain: print("Please specify the site name on which you want to host bench-manager using the 'domain' flag") sys.exit(1) - from bench.utils import get_sites, get_bench_name - bench_name = get_bench_name(bench_path) - if domain not in get_sites(bench_path): raise Exception("No such site") - from bench.config.nginx import make_bench_manager_nginx_conf make_bench_manager_nginx_conf(bench_path, yes=yes, port=port, domain=domain) diff --git a/bench/utils.py b/bench/utils.py index 35c1a319..72b69054 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -15,16 +15,13 @@ import shutil import site import subprocess import sys -import time from datetime import datetime from distutils.spawn import find_executable # imports - third party imports import click import requests -import semantic_version from six import iteritems -from six.moves import reload_module from six.moves.urllib.parse import urlparse # imports - module imports @@ -314,7 +311,6 @@ def exec_cmd(cmd, cwd='.'): def which(executable, raise_err = False): - from distutils.spawn import find_executable exec_ = find_executable(executable) if not exec_ and raise_err: From 8fd534747a3e2893f4147cfb2b99b60a8a634dfe Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 19 Feb 2020 11:39:07 +0530 Subject: [PATCH 18/44] docs: added missing help for install prerequisites --- bench/commands/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/commands/install.py b/bench/commands/install.py index b760719c..e1fc93bb 100644 --- a/bench/commands/install.py +++ b/bench/commands/install.py @@ -15,7 +15,7 @@ def install(): pass -@click.command('prerequisites') +@click.command('prerequisites', help="Installs pre-requisite libraries, essential tools like b2zip, htop, screen, vim, x11-fonts, python libs, cups and Redis") def install_prerequisites(): run_playbook('site.yml', tag='common, redis') From 9107581a562118168ec6820d705e560be367c39b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 19 Feb 2020 15:33:10 +0530 Subject: [PATCH 19/44] docs: added bench usage docs --- docs/bench_usage.md | 249 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 249 insertions(+) create mode 100644 docs/bench_usage.md diff --git a/docs/bench_usage.md b/docs/bench_usage.md new file mode 100644 index 00000000..1cf71568 --- /dev/null +++ b/docs/bench_usage.md @@ -0,0 +1,249 @@ +# bench CLI Usage + +This may not be known to a lot of people but half the bench commands we're used to, exist in the Frappe Framework and not in bench directly. Those commands generally are the `--site` commands. This page is concerned only with the commands in the bench project. Any framework commands won't be a part of this consolidation. + + +## How are Frappe Framework commands available via bench? + +bench utilizes `frappe.utils.bench_manager` to get the framework's as well as those of any custom commands written in application installed in the Frappe environment. Currently, with *version 12* there are commands related to the scheduler, sites, translations and other utils in Frappe inherited by bench. + + +## Can I add CLI commands in my custom app and call them via bench? + +Along with the framework commands, Frappe's `bench_manager` module also searches for any commands in your custom applications. Thereby, bench communicates with the respective bench's Frappe which in turn checks for available commands in all of the applications. + +To make your custom command available to bench, just create a `commands` module under your parent module and write the command with a click wrapper and a variable commands which contains a list of click functions, which are your own commands. The directory strcuture may be visualized as: + +``` +frappe-bench +|──apps + |── frappe + ├── custom_app + │   ├── README.md + │   ├── custom_app + │   │   ├── commands <------ commands module + │   ├── license.txt + │   ├── requirements.txt + │   └── setup.py +``` + +The commands module maybe a single file such as `commands.py` or a directory with an `__init__.py` file. For a custom application of name 'flags', example may be given as + +```python +# file_path: frappe-bench/apps/flags/flags/commands.py +import click + +@click.command('set-flags') +@click.argument('state', type=click.Choice(['on', 'off'])) +def set_flags(state): + from flags.utils import set_flags + set_flags(state=state) + +commands = [ + set_flags +] +``` + +and with context of the current bench, this command maybe executed simply as + +```zsh +➜ bench set-flags +Flags are set to state: 'on' +``` + +# bench CLI Commands + +Under Click's structure, `bench` is the main command group, under which there are three main groups of commands in bench currently, namely + + - **install**: The install command group deals with commands used to install system dependencies for setting up Frappe environment + + - **setup**: This command group for consists of commands used to maipulate the requirements and environments required by your Frappe environment + + - **config**: The config command group deals with making changes in the current bench (not the CLI tool) configuration + + +## Using the bench command line + +```zsh +➜ bench +Usage: bench [OPTIONS] COMMAND [ARGS]... + + Bench manager for Frappe + +Options: + --version + --help Show this message and exit. + +Commands: + backup Backup single site + backup-all-sites Backup all sites in current bench + config Change bench configuration + disable-production Disables production environment for the bench. + download-translations Download latest translations + exclude-app Exclude app from updating + find Finds benches recursively from location + get-app Clone an app from the internet or filesystem and... +``` + +Similarly, all available flags and options can be checked for commands individually by executing them with the `--help` flag. The `init` command for instance: + +```zsh +➜ bench init --help +Usage: bench init [OPTIONS] PATH + + Initialize a new bench instance in the specified path + +Options: + --python TEXT Path to Python Executable. + --ignore-exist Ignore if Bench instance exists. + --apps_path TEXT path to json files with apps to install + after init +``` + + + +## bench and sudo + +Some bench commands may require sudo, such as some `setup` commands and everything else under the `install` commands group. For these commands, you may not be asked for your root password if sudoers setup has been done. The security implications, well we'll talk about those soon. + + + +## General Commands + +These commands belong directly to the bench group so they can be invoked directly prefixing each with `bench` in your shell. Therefore, the usage for these commands is as + +```zsh + bench COMMAND [ARGS]... +``` + +### The usual commands + + - **init**: Initialize a new bench instance in the specified path + - **restart**: Restart supervisor processes or systemd units. Used in production setup + - **update**: Updates bench tool and if executed in a bench directory, without any flags will backup, pull, setup requirements, build, run patches and restart bench. Using specific flags will only do certain tasks instead of all + - **migrate-env**: Migrate Virtual Environment to desired Python Version + - **retry-upgrade**: Retry a failed upgrade + - **disable-production**: Disables production environment for the bench. + - **renew-lets-encrypt**: Renew Let's Encrypt certificate + - **backup**: Backup single site + - **backup-all-sites**: Backup all sites in current bench + + - **get-app**: Clone an app from the internet or filesystem and set it up in your bench + - **remove-app**: Completely remove app from bench and re-build assets if not installed on any site + - **exclude-app**: Exclude app from updating + - **include-app**: Include app for updating + - **remote-set-url**: Set app remote url + - **remote-reset-url**: Reset app remote url to frappe official + - **remote-urls**: Show apps remote url + - **switch-to-branch**: Switch all apps to specified branch, or specify apps separated by space + - **switch-to-develop**: Switch frappe and erpnext to develop branch + + +### A little advanced + + - **set-nginx-port**: Set NGINX port for site + - **set-ssl-certificate**: Set SSL certificate path for site + - **set-ssl-key**: Set SSL certificate private key path for site + - **set-url-root**: Set URL root for site + - **set-mariadb-host**: Set MariaDB host for bench + - **set-redis-cache-host**: Set Redis cache host for bench + - **set-redis-queue-host**: Set Redis queue host for bench + - **set-redis-socketio-host**: Set Redis socketio host for bench + - **set-default-site**: Set default site for bench + - **download-translations**: Download latest translations + + +### Developer's commands + + - **start**: Start Frappe development processes + - **src**: Prints bench source folder path, which can be used as: cd `bench src` + - **find**: Finds benches recursively from location + - **pip**: For pip help use `bench pip help [COMMAND]` or `bench pip [COMMAND] -h` + - **new-app**: Create a new Frappe application under apps folder + + +### Release bench + - **release**: Release a Frappe application + - **prepare-beta-release**: Prepare major beta release from develop branch + + + +## Setup commands + +The setup commands used for setting up the Frappe environment in context of the current bench need to be executed using `bench setup` as the prefix. So, the general usage of these commands is as + +```zsh + bench setup COMMAND [ARGS]... +``` + + - **sudoers**: Add commands to sudoers list for execution without password + + - **env**: Setup virtualenv for bench + - **redis**: Generates configuration for Redis + - **fonts**: Add Frappe fonts to system + - **config**: Generate or over-write sites/common_site_config.json + - **backups**: Add cronjob for bench backups + - **socketio**: Setup node dependencies for socketio server + - **requirements**: Setup Python and Node dependencies + + - **manager**: Setup bench-manager.local site with the bench_manager app installed on it + + - **procfile**: Generate Procfile for bench start + + - **production**: Setup Frappe production environment for specific user + - **nginx**: Generate configuration files for NGINX + - **fail2ban**: Setup fail2ban, an intrusion prevention software framework that protects computer servers from brute-force attacks + - **systemd**: Generate configuration for systemd + - **firewall**: Setup firewall for system + - **ssh-port**: Set SSH Port for system + - **reload-nginx**: Checks NGINX config file and reloads service + - **supervisor**: Generate configuration for supervisor + - **lets-encrypt**: Setup lets-encrypt SSL for site + - **wildcard-ssl**: Setup wildcard SSL certificate for multi-tenant bench + + - **add-domain**: Add a custom domain to a particular site + - **remove-domain**: Remove custom domain from a site + - **sync-domains**: Check if there is a change in domains. If yes, updates the domains list. + + - **role**: Install dependencies via ansible roles + + + +## Config commands + +The config group commands are used for manipulating configurations in the current bench context. The usage for these commands is as + +```zsh + bench config COMMAND [ARGS]... +``` + + - **set-common-config**: Set value in common config + - **remove-common-config**: Remove specific keys from current bench's common config + + - **update_bench_on_update**: Enable/Disable bench updates on running bench update + - **restart_supervisor_on_update**: Enable/Disable auto restart of supervisor processes + - **restart_systemd_on_update**: Enable/Disable auto restart of systemd units + - **dns_multitenant**: Enable/Disable bench multitenancy on running bench update + - **serve_default_site**: Configure nginx to serve the default site on port 80 + - **http_timeout**: Set HTTP timeout + + + +## Install commands + +The install group commands are used for manipulating system level dependencies. The usage for these commands is as + +```zsh + bench install COMMAND [ARGS]... +``` + + - **prerequisites**: Installs pre-requisite libraries, essential tools like b2zip, htop, screen, vim, x11-fonts, python libs, cups and Redis + - **nodejs**: Installs Node.js v8 + - **nginx**: Installs NGINX. If user is specified, sudoers is setup for that user + - **packer**: Installs Oracle virtualbox and packer 1.2.1 + - **psutil**: Installs psutil via pip + - **mariadb**: Install and setup MariaDB of specified version and root password + - **wkhtmltopdf**: Installs wkhtmltopdf v0.12.3 for linux + - **supervisor**: Installs supervisor. If user is specified, sudoers is setup for that user + - **fail2ban**: Install fail2ban, an intrusion prevention software framework that protects computer servers from brute-force attacks + - **virtualbox**: Installs supervisor From d47e1ac330d27ba363e5594b029f0ef5ce368792 Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Thu, 20 Feb 2020 19:46:40 +0530 Subject: [PATCH 20/44] chore: fix easy_install import print from __future__ for python2 to use print's file param Signed-off-by: Chinmay D. Pai --- playbooks/install.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/playbooks/install.py b/playbooks/install.py index 0da571b5..c40aa579 100644 --- a/playbooks/install.py +++ b/playbooks/install.py @@ -1,5 +1,16 @@ #!/usr/bin/env python3 -import os, sys, subprocess, getpass, json, multiprocessing, shutil, platform, warnings, datetime +from __future__ import print_function +import os +import sys +import subprocess +import getpass +import json +import multiprocessing +import shutil +import platform +import warnings +import datetime + tmp_bench_repo = os.path.join('/', 'tmp', '.bench') tmp_log_folder = os.path.join('/', 'tmp', 'logs') From 0b3feb1f7b93462267f7f593a19799b5aa9bb0e3 Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Thu, 20 Feb 2020 19:55:50 +0530 Subject: [PATCH 21/44] chore: add check for CI Signed-off-by: Chinmay D. Pai --- playbooks/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/install.py b/playbooks/install.py index c40aa579..174004df 100644 --- a/playbooks/install.py +++ b/playbooks/install.py @@ -405,7 +405,7 @@ def parse_commandline_args(): return args if __name__ == '__main__': - if sys.version[0] == '2': + if sys.version[0] == '2' and not os.environ.get('CI'): if not raw_input("It is recommended to run this script with Python 3\nDo you still wish to continue? [Y/n]: ").lower() == "y": sys.exit() From 986015c13cf5b3ba4612280a232d5de4e890da0b Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Thu, 20 Feb 2020 20:02:35 +0530 Subject: [PATCH 22/44] chore: substitute shutil.which with find_executable on py2 Signed-off-by: Chinmay D. Pai --- playbooks/install.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/playbooks/install.py b/playbooks/install.py index 174004df..614a2087 100644 --- a/playbooks/install.py +++ b/playbooks/install.py @@ -405,9 +405,12 @@ def parse_commandline_args(): return args if __name__ == '__main__': - if sys.version[0] == '2' and not os.environ.get('CI'): - if not raw_input("It is recommended to run this script with Python 3\nDo you still wish to continue? [Y/n]: ").lower() == "y": - sys.exit() + if sys.version[0] == '2': + from disutils.spawn import find_executable + shutil.which = find_executable + if not os.environ.get('CI'): + if not raw_input("It is recommended to run this script with Python 3\nDo you still wish to continue? [Y/n]: ").lower() == "y": + sys.exit() if not is_sudo_user(): log("Please run this script as a non-root user with sudo privileges", level=3) From 0a9effdfb40e57bbba2020977884f3a76d0efeb4 Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Thu, 20 Feb 2020 20:12:28 +0530 Subject: [PATCH 23/44] chore: setup distutils inside travis Signed-off-by: Chinmay D. Pai --- .travis.yml | 2 +- playbooks/install.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9df3b6d4..a44f466b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ python: - "2.7" install: - - sudo pip install urllib3 pyOpenSSL ndg-httpsclient pyasn1 + - sudo pip install urllib3 pyOpenSSL ndg-httpsclient pyasn1 distutils - sudo apt-get purge -y mysql-common mysql-server mysql-client - sudo apt-get install --only-upgrade -y git - sudo apt-get install hhvm && rm -rf /home/travis/.kiex/ diff --git a/playbooks/install.py b/playbooks/install.py index 614a2087..bc9e4170 100644 --- a/playbooks/install.py +++ b/playbooks/install.py @@ -406,7 +406,12 @@ def parse_commandline_args(): if __name__ == '__main__': if sys.version[0] == '2': - from disutils.spawn import find_executable + try: + from disutils.spawn import find_executable + except ImportError: + print("Please install distutils or use Python3 to run the script") + print("$ pip install distutils") + sys.exit(1) shutil.which = find_executable if not os.environ.get('CI'): if not raw_input("It is recommended to run this script with Python 3\nDo you still wish to continue? [Y/n]: ").lower() == "y": From bf47661f3ed82e381c914eedbff80137818ca91e Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Thu, 20 Feb 2020 20:18:57 +0530 Subject: [PATCH 24/44] chore: setup distutils using apt instead of pip Signed-off-by: Chinmay D. Pai --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a44f466b..441c0f5a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,10 +5,10 @@ python: - "2.7" install: - - sudo pip install urllib3 pyOpenSSL ndg-httpsclient pyasn1 distutils + - sudo pip install urllib3 pyOpenSSL ndg-httpsclient pyasn1 - sudo apt-get purge -y mysql-common mysql-server mysql-client - sudo apt-get install --only-upgrade -y git - - sudo apt-get install hhvm && rm -rf /home/travis/.kiex/ + - sudo apt-get install hhvm python-distutils && rm -rf /home/travis/.kiex/ - mkdir -p ~/.bench - mkdir -p /tmp/.bench - cp -r $TRAVIS_BUILD_DIR/* ~/.bench From 5af3716560138b99717bb965ab41f8aea9df29fd Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 24 Feb 2020 12:51:53 +0530 Subject: [PATCH 25/44] docs: updated and re-link docs --- README.md | 15 +++++-- docs/bench_custom_cmd.md | 48 +++++++++++++++++++++ docs/bench_usage.md | 92 ++++++++++------------------------------ 3 files changed, 81 insertions(+), 74 deletions(-) create mode 100644 docs/bench_custom_cmd.md diff --git a/README.md b/README.md index 394448de..8eb96fed 100755 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ bench is a command-line utility that helps you to install apps, manage multiple - [bench CLI](#bench-cli) - [Usage](#usage) - [Installation](#installation) + - [Custom Bench commands](#custom-bench-commands) - [Easy Install Script](#easy-install-script) - [Release Bench](#release-bench) - [Guides](#guides) @@ -49,7 +50,7 @@ Bench is a command line tool that helps you install, setup, manage multiple site * Install apps on a particular site bench --site [site-name] install-app [app-name] - + * Start bench (only for development) bench start @@ -60,7 +61,7 @@ Bench is a command line tool that helps you install, setup, manage multiple site _Note:_ Apart from `bench init`, all other bench commands have to be run having the respective bench directory as the working directory. _(`bench update` may also be run, but it's behaviour is covered in depth in the docs)_ -For more in depth information on commands and usage follow [here](https://github.com/frappe/bench/blob/master/docs/commands_and_usage.md). +For more in depth information on commands and usage follow [here](https://github.com/frappe/bench/blob/master/docs/commands_and_usage.md). As for a consolidated list of bench commands, go through [this page](https://github.com/frappe/bench/blob/master/docs/bench_usage.md). --- @@ -68,8 +69,8 @@ For more in depth information on commands and usage follow [here](https://github To do this install, you must have basic information on how Linux works and should be able to use the command-line. bench will also create nginx and supervisor config files, setup backups and much more. If you are using on a VPS make sure it has >= 1Gb of RAM or has swap setup properly. - git clone https://github.com/frappe/bench ~/.bench - pip3 install --user -e ~/.bench + git clone https://github.com/frappe/bench ~/.bench + pip3 install --user -e ~/.bench As bench is a python application, its installation really depends on `python` + `pip` + `git`. The Frappe Framework, however has various other system dependencies like `nodejs`, `yarn`, `redis` and a database system like `mariadb` or `postgres`. Go through the [installation requirements](https://github.com/frappe/bench/blob/master/docs/installation.md) for an updated list. @@ -77,6 +78,12 @@ If you have questions, please ask them on the [forum](https://discuss.erpnext.co --- +## Custom Bench Commands + +Want to utilize a bench command you've added in your custom Frappe application? [This](https://github.com/frappe/bench/blob/master/docs/bench_custom_cmd.md) guide might be of some help. + +--- + # Easy Install Script - This is an opinionated setup so it is best to setup on a blank server. diff --git a/docs/bench_custom_cmd.md b/docs/bench_custom_cmd.md new file mode 100644 index 00000000..693739ba --- /dev/null +++ b/docs/bench_custom_cmd.md @@ -0,0 +1,48 @@ +## How are Frappe Framework commands available via bench? + +bench utilizes `frappe.utils.bench_manager` to get the framework's as well as those of any custom commands written in application installed in the Frappe environment. Currently, with *version 12* there are commands related to the scheduler, sites, translations and other utils in Frappe inherited by bench. + + +## Can I add CLI commands in my custom app and call them via bench? + +Along with the framework commands, Frappe's `bench_manager` module also searches for any commands in your custom applications. Thereby, bench communicates with the respective bench's Frappe which in turn checks for available commands in all of the applications. + +To make your custom command available to bench, just create a `commands` module under your parent module and write the command with a click wrapper and a variable commands which contains a list of click functions, which are your own commands. The directory strcuture may be visualized as: + +``` +frappe-bench +|──apps + |── frappe + ├── custom_app + │   ├── README.md + │   ├── custom_app + │   │   ├── commands <------ commands module + │   ├── license.txt + │   ├── requirements.txt + │   └── setup.py +``` + +The commands module maybe a single file such as `commands.py` or a directory with an `__init__.py` file. For a custom application of name 'flags', example may be given as + +```python +# file_path: frappe-bench/apps/flags/flags/commands.py +import click + +@click.command('set-flags') +@click.argument('state', type=click.Choice(['on', 'off'])) +def set_flags(state): + from flags.utils import set_flags + set_flags(state=state) + +commands = [ + set_flags +] +``` + +and with context of the current bench, this command maybe executed simply as + +```zsh +➜ bench set-flags +Flags are set to state: 'on' +``` + diff --git a/docs/bench_usage.md b/docs/bench_usage.md index 1cf71568..cf9a0364 100644 --- a/docs/bench_usage.md +++ b/docs/bench_usage.md @@ -3,54 +3,6 @@ This may not be known to a lot of people but half the bench commands we're used to, exist in the Frappe Framework and not in bench directly. Those commands generally are the `--site` commands. This page is concerned only with the commands in the bench project. Any framework commands won't be a part of this consolidation. -## How are Frappe Framework commands available via bench? - -bench utilizes `frappe.utils.bench_manager` to get the framework's as well as those of any custom commands written in application installed in the Frappe environment. Currently, with *version 12* there are commands related to the scheduler, sites, translations and other utils in Frappe inherited by bench. - - -## Can I add CLI commands in my custom app and call them via bench? - -Along with the framework commands, Frappe's `bench_manager` module also searches for any commands in your custom applications. Thereby, bench communicates with the respective bench's Frappe which in turn checks for available commands in all of the applications. - -To make your custom command available to bench, just create a `commands` module under your parent module and write the command with a click wrapper and a variable commands which contains a list of click functions, which are your own commands. The directory strcuture may be visualized as: - -``` -frappe-bench -|──apps - |── frappe - ├── custom_app - │   ├── README.md - │   ├── custom_app - │   │   ├── commands <------ commands module - │   ├── license.txt - │   ├── requirements.txt - │   └── setup.py -``` - -The commands module maybe a single file such as `commands.py` or a directory with an `__init__.py` file. For a custom application of name 'flags', example may be given as - -```python -# file_path: frappe-bench/apps/flags/flags/commands.py -import click - -@click.command('set-flags') -@click.argument('state', type=click.Choice(['on', 'off'])) -def set_flags(state): - from flags.utils import set_flags - set_flags(state=state) - -commands = [ - set_flags -] -``` - -and with context of the current bench, this command maybe executed simply as - -```zsh -➜ bench set-flags -Flags are set to state: 'on' -``` - # bench CLI Commands Under Click's structure, `bench` is the main command group, under which there are three main groups of commands in bench currently, namely @@ -118,25 +70,25 @@ These commands belong directly to the bench group so they can be invoked directl ### The usual commands - - **init**: Initialize a new bench instance in the specified path - - **restart**: Restart supervisor processes or systemd units. Used in production setup - - **update**: Updates bench tool and if executed in a bench directory, without any flags will backup, pull, setup requirements, build, run patches and restart bench. Using specific flags will only do certain tasks instead of all - - **migrate-env**: Migrate Virtual Environment to desired Python Version + - **init**: Initialize a new bench instance in the specified path. This sets up a complete bench folder with an `apps` folder which contains all the Frappe apps available in the current bench, `sites` folder that stores all site data seperated by individual site folders, `config` folder that contains your redis, NGINX and supervisor configuration files. The `env` folder consists of all python dependencies the current bench and installed Frappe applications have. + - **restart**: Restart web, supervisor, systemd processes units. Used in production setup. + - **update**: Updates bench tool and if executed in a bench directory, without any flags will backup, pull, setup requirements, build, run patches and restart bench. Using specific flags will only do certain tasks instead of all. + - **migrate-env**: Migrate Virtual Environment to desired Python version. This regenerates the `env` folder with the specified Python version. - **retry-upgrade**: Retry a failed upgrade - **disable-production**: Disables production environment for the bench. - - **renew-lets-encrypt**: Renew Let's Encrypt certificate - - **backup**: Backup single site - - **backup-all-sites**: Backup all sites in current bench + - **renew-lets-encrypt**: Renew Let's Encrypt certificate for site SSL. + - **backup**: Backup single site data. Can be used to backup files as well. + - **backup-all-sites**: Backup all sites in current bench. - - **get-app**: Clone an app from the internet or filesystem and set it up in your bench - - **remove-app**: Completely remove app from bench and re-build assets if not installed on any site - - **exclude-app**: Exclude app from updating - - **include-app**: Include app for updating + - **get-app**: Download an app from the internet or filesystem and set it up in your bench. This clones the git repo of the Frappe project and installs it in the bench environment. + - **remove-app**: Completely remove app from bench and re-build assets if not installed on any site. + - **exclude-app**: Exclude app from updating during a `bench update` + - **include-app**: Include app for updating. All Frappe applications are included by default when installed. - **remote-set-url**: Set app remote url - **remote-reset-url**: Reset app remote url to frappe official - **remote-urls**: Show apps remote url - **switch-to-branch**: Switch all apps to specified branch, or specify apps separated by space - - **switch-to-develop**: Switch frappe and erpnext to develop branch + - **switch-to-develop**: Switch Frappe and ERPNext to develop branch ### A little advanced @@ -155,15 +107,15 @@ These commands belong directly to the bench group so they can be invoked directl ### Developer's commands - - **start**: Start Frappe development processes - - **src**: Prints bench source folder path, which can be used as: cd `bench src` - - **find**: Finds benches recursively from location - - **pip**: For pip help use `bench pip help [COMMAND]` or `bench pip [COMMAND] -h` - - **new-app**: Create a new Frappe application under apps folder + - **start**: Start Frappe development processes. Uses the Procfile to start the Frappe development environment. + - **src**: Prints bench source folder path, which can be used to cd into the bench installation repository by `cd $(bench src)`. + - **find**: Finds benches recursively from location or specified path. + - **pip**: Use the current bench's pip to manage Python packages. For help about pip usage: `bench pip help [COMMAND]` or `bench pip [COMMAND] -h`. + - **new-app**: Create a new Frappe application under apps folder. ### Release bench - - **release**: Release a Frappe application + - **release**: Create a release of a Frappe application - **prepare-beta-release**: Prepare major beta release from develop branch @@ -176,9 +128,9 @@ The setup commands used for setting up the Frappe environment in context of the bench setup COMMAND [ARGS]... ``` - - **sudoers**: Add commands to sudoers list for execution without password + - **sudoers**: Add commands to sudoers list for allowing bench commands execution without root password - - **env**: Setup virtualenv for bench + - **env**: Setup virtualenv for bench. This sets up a `env` folder under the root of the bench directory. - **redis**: Generates configuration for Redis - **fonts**: Add Frappe fonts to system - **config**: Generate or over-write sites/common_site_config.json @@ -186,11 +138,11 @@ The setup commands used for setting up the Frappe environment in context of the - **socketio**: Setup node dependencies for socketio server - **requirements**: Setup Python and Node dependencies - - **manager**: Setup bench-manager.local site with the bench_manager app installed on it + - **manager**: Setup `bench-manager.local` site with the [Bench Manager](https://github.com/frappe/bench_manager) app, a GUI for bench installed on it. - **procfile**: Generate Procfile for bench start - - **production**: Setup Frappe production environment for specific user + - **production**: Setup Frappe production environment for specific user. This installs ansible, NGINX, supervisor, fail2ban and generates the respective configuration files. - **nginx**: Generate configuration files for NGINX - **fail2ban**: Setup fail2ban, an intrusion prevention software framework that protects computer servers from brute-force attacks - **systemd**: Generate configuration for systemd From 0c0ed767f73738e19692e4ea7667f0be5b7c05b7 Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Tue, 25 Feb 2020 11:45:03 +0530 Subject: [PATCH 26/44] chore: add overwrite flag, repo_name format, and exit gracefully Signed-off-by: Chinmay D. Pai --- bench/app.py | 4 ++-- bench/commands/make.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bench/app.py b/bench/app.py index b590fbf3..841c3800 100755 --- a/bench/app.py +++ b/bench/app.py @@ -123,13 +123,13 @@ def get_app(git_url, branch=None, bench_path='.', build_asset_files=True, verbos # application directory already exists # prompt user to overwrite it if overwrite or click.confirm('''A directory for the application "{0}" already exists. -Do you want to continue and overwrite it?'''): +Do you want to continue and overwrite it?'''.format(repo_name)): shutil.rmtree(os.path.join(bench_path, 'apps', repo_name)) elif click.confirm('''Do you want to reinstall the existing application?''', abort=True): app_name = get_app_name(bench_path, repo_name) print("Reinstalling {0}".format(app_name)) install_app(app=app_name, bench_path=bench_path, verbose=verbose, build_asset_files=build_asset_files) - sys.exit(1) + sys.exit() logger.info('Getting app {0}'.format(repo_name)) exec_cmd("git clone {git_url} {branch} {shallow_clone} --origin upstream".format( diff --git a/bench/commands/make.py b/bench/commands/make.py index c2d27431..cd7738cd 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -36,7 +36,7 @@ def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, @click.argument('git-url') @click.option('--branch', default=None, help="branch to checkout") @click.option('--overwrite', is_flag=True) -def get_app(git_url, branch, name=None): +def get_app(git_url, branch, overwrite, name=None): "clone an app from the internet and set it up in your bench" from bench.app import get_app get_app(git_url, branch=branch, overwrite=overwrite) From c660593a9ad0a99148a376913c5bec8bd77b675a Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 25 Feb 2020 11:57:28 +0530 Subject: [PATCH 27/44] fix: pin dependencies of requirements --- requirements.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/requirements.txt b/requirements.txt index a9b81fae..d677afd9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,11 @@ semantic_version==2.8.2 setuptools==40.8.0 six==1.12.0 virtualenv==16.6.0 +gitdb2==2.0.6 +MarkupSafe==1.1.1 +python-dateutil==2.8.1 +idna==2.8 +certifi==2019.9.11 +urllib3==1.25.7 +chardet==3.0.4 +smmap2==2.0.5 From e5b84be5660eba46f0c655821b6dc6a4c1c9e8ac Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Tue, 25 Feb 2020 12:11:24 +0530 Subject: [PATCH 28/44] chore: pin gitdb2 only for python < 3.4 pip allows pinning requirement using environment markers[0] [0]: https://www.python.org/dev/peps/pep-0508/#environment-markers Signed-off-by: Chinmay D. Pai --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d677afd9..1922bb98 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ semantic_version==2.8.2 setuptools==40.8.0 six==1.12.0 virtualenv==16.6.0 -gitdb2==2.0.6 +gitdb2==2.0.6;python_version<'3.4' MarkupSafe==1.1.1 python-dateutil==2.8.1 idna==2.8 From 3bd83da49b32fdc4d91dc3e5ce759a857b3f4a98 Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Wed, 26 Feb 2020 20:00:52 +0530 Subject: [PATCH 29/44] chore: remove redundant print messages Signed-off-by: Chinmay D. Pai --- bench/app.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bench/app.py b/bench/app.py index 500e8e7b..f9e5b7b5 100755 --- a/bench/app.py +++ b/bench/app.py @@ -127,7 +127,6 @@ Do you want to continue and overwrite it?'''.format(repo_name)): shutil.rmtree(os.path.join(bench_path, 'apps', repo_name)) elif click.confirm('''Do you want to reinstall the existing application?''', abort=True): app_name = get_app_name(bench_path, repo_name) - print("Reinstalling {0}".format(app_name)) install_app(app=app_name, bench_path=bench_path, verbose=verbose, skip_assets=skip_assets) sys.exit() @@ -139,7 +138,6 @@ Do you want to continue and overwrite it?'''.format(repo_name)): cwd=os.path.join(bench_path, 'apps')) app_name = get_app_name(bench_path, repo_name) - print("Installing {0}".format(app_name)) install_app(app=app_name, bench_path=bench_path, verbose=verbose, skip_assets=skip_assets) From 495a038c458834c395519ad5f1a13b684fa16d11 Mon Sep 17 00:00:00 2001 From: gavin Date: Thu, 27 Feb 2020 21:19:20 +0530 Subject: [PATCH 30/44] fix(easy-install): compatibility for easy install on PY2 (#937) * fix: compatibility for easy install on PY2 * test: remove python-distutils from travis * chore: handle specific CalledProcessError for setuptools Co-authored-by: Chinmay Pai --- .travis.yml | 2 +- playbooks/install.py | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 441c0f5a..9df3b6d4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ install: - sudo pip install urllib3 pyOpenSSL ndg-httpsclient pyasn1 - sudo apt-get purge -y mysql-common mysql-server mysql-client - sudo apt-get install --only-upgrade -y git - - sudo apt-get install hhvm python-distutils && rm -rf /home/travis/.kiex/ + - sudo apt-get install hhvm && rm -rf /home/travis/.kiex/ - mkdir -p ~/.bench - mkdir -p /tmp/.bench - cp -r $TRAVIS_BUILD_DIR/* ~/.bench diff --git a/playbooks/install.py b/playbooks/install.py index bc9e4170..9867d185 100644 --- a/playbooks/install.py +++ b/playbooks/install.py @@ -406,17 +406,21 @@ def parse_commandline_args(): if __name__ == '__main__': if sys.version[0] == '2': - try: - from disutils.spawn import find_executable - except ImportError: - print("Please install distutils or use Python3 to run the script") - print("$ pip install distutils") - sys.exit(1) - shutil.which = find_executable if not os.environ.get('CI'): if not raw_input("It is recommended to run this script with Python 3\nDo you still wish to continue? [Y/n]: ").lower() == "y": sys.exit() + try: + from distutils.spawn import find_executable + except ImportError: + try: + subprocess.check_call('pip install --upgrade setuptools') + except subprocess.CalledProcessError: + print("Install distutils or use Python3 to run the script") + sys.exit(1) + + shutil.which = find_executable + if not is_sudo_user(): log("Please run this script as a non-root user with sudo privileges", level=3) sys.exit() From ddcd2c11a185e28ea653e3f7a1c03d015a63b652 Mon Sep 17 00:00:00 2001 From: gavin Date: Fri, 28 Feb 2020 14:42:14 +0530 Subject: [PATCH 31/44] chore: update GitPython (#938) * Revert "chore: pin gitdb2 only for python < 3.4" * chore: update GitPython and remove second level dependencies --- requirements.txt | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1922bb98..e00b90ea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,18 +1,10 @@ Click==7.0 -GitPython==2.1.11 +GitPython==2.1.15 honcho==1.0.1 Jinja2==2.10.3 python-crontab==2.4.0 requests==2.22.0 -semantic_version==2.8.2 +semantic-version==2.8.2 setuptools==40.8.0 six==1.12.0 virtualenv==16.6.0 -gitdb2==2.0.6;python_version<'3.4' -MarkupSafe==1.1.1 -python-dateutil==2.8.1 -idna==2.8 -certifi==2019.9.11 -urllib3==1.25.7 -chardet==3.0.4 -smmap2==2.0.5 From ec1343c0fae59388a64a22e6289005cb3e7801b2 Mon Sep 17 00:00:00 2001 From: gavin Date: Fri, 28 Feb 2020 15:33:00 +0530 Subject: [PATCH 32/44] fix: dont abort script if not overwriting scripts (#919) * fix: dont abort script if not overwriting scripts why: currently, running of `bench ` or `bench setup production` creates nginx conf files and asks for confirmation before overwriting them. in case user doesnt want to overwrite files, `setup production` and `lets-encrypt` is exitted abrubtly and nginx isnt started again after * fix: use click.confirm instead of six.moves.input * fix: use click.confirm instead of six.moves.input * chore: remove unused imports --- bench/config/nginx.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/bench/config/nginx.py b/bench/config/nginx.py index 7e584c3a..9f10c6f7 100644 --- a/bench/config/nginx.py +++ b/bench/config/nginx.py @@ -1,8 +1,24 @@ -import os, json, click, random, string, hashlib -from bench.utils import get_sites, get_bench_name, exec_cmd +# imports - standard imports +import hashlib +import os +import random +import string + +# imports - third party imports +import click from six import string_types +# imports - module imports +from bench.utils import get_bench_name, get_sites + + def make_nginx_conf(bench_path, yes=False): + conf_path = os.path.join(bench_path, "config", "nginx.conf") + + if not yes and os.path.exists(conf_path): + if not click.confirm('nginx.conf already exists and this will overwrite it. Do you want to continue?'): + return + from bench import env from bench.config.common_site_config import get_config @@ -37,10 +53,6 @@ def make_nginx_conf(bench_path, yes=False): nginx_conf = template.render(**template_vars) - conf_path = os.path.join(bench_path, "config", "nginx.conf") - if not yes and os.path.exists(conf_path): - click.confirm('nginx.conf already exists and this will overwrite it. Do you want to continue?', - abort=True) with open(conf_path, "w") as f: f.write(nginx_conf) From 9b5e1c9fdc9365a6c74e7721446ac93d8558bb4f Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 2 Mar 2020 19:24:41 +0530 Subject: [PATCH 33/44] style: removed unused imports --- bench/cli.py | 2 +- bench/commands/setup.py | 1 - bench/commands/update.py | 1 - bench/commands/utils.py | 1 - 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index 243e2640..ad1c90fa 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -1,6 +1,6 @@ import click import os, sys, logging, json, pwd, subprocess -from bench.utils import is_root, PatchError, drop_privileges, get_env_cmd, get_cmd_output, get_frappe, log, is_bench_directory, find_parent_bench +from bench.utils import is_root, PatchError, drop_privileges, get_env_cmd, get_cmd_output, get_frappe, log, find_parent_bench from bench.app import get_apps from bench.config.common_site_config import get_config from bench.commands import bench_command diff --git a/bench/commands/setup.py b/bench/commands/setup.py index b28e704c..352a781b 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -1,7 +1,6 @@ # imports - standard imports import os import sys -import json # imports - module imports from bench.utils import exec_cmd diff --git a/bench/commands/update.py b/bench/commands/update.py index e2a1b310..6dac7fa5 100755 --- a/bench/commands/update.py +++ b/bench/commands/update.py @@ -1,6 +1,5 @@ # imports - standard imports import os -import sys # imports - third party imports import click diff --git a/bench/commands/utils.py b/bench/commands/utils.py index 38dc28bf..822086a7 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -1,5 +1,4 @@ # imports - standard imports -import copy import os import sys From b0649870eecaf7e00e4f4e122c2111016f1b8d59 Mon Sep 17 00:00:00 2001 From: gavin Date: Mon, 2 Mar 2020 19:32:25 +0530 Subject: [PATCH 34/44] fix: random failings from subprocesses and command formatting (#935) * fix: random failings from subprocesses async executions of muliple processes leads to various errors. eg: get-app operation runs a "git clone" at the same time "pip install" is run. This causes failures as the folder either doesnt exist or isnt complete. Proposed solution to this is to execute blocking subprocesseses * style: command logging format via exec_cmd --- bench/utils.py | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/bench/utils.py b/bench/utils.py index 10a25988..c37b0806 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -28,6 +28,7 @@ class color: green = '\033[92m' yellow = '\033[93m' red = '\033[91m' + silver = '\033[90m' def is_bench_directory(directory=os.path.curdir): @@ -172,26 +173,11 @@ def clone_apps_from(bench_path, clone_from, update_app=True): setup_app(app) def exec_cmd(cmd, cwd='.'): - from .cli import from_command_line - - is_async = False if from_command_line else True - if is_async: - stderr = stdout = subprocess.PIPE - else: - stderr = stdout = None - + import shlex logger.info(cmd) - - p = subprocess.Popen(cmd, cwd=cwd, shell=True, stdout=stdout, stderr=stderr, - universal_newlines=True) - - if is_async: - return_code = print_output(p) - else: - return_code = p.wait() - - if return_code > 0: - raise CommandFailedError(cmd) + cmd = shlex.split(cmd) + print("{0}$ {1}{2}".format(color.silver, cmd, color.nc)) + subprocess.call(cmd, cwd=cwd, universal_newlines=True) def which(executable, raise_err = False): from distutils.spawn import find_executable From 0088d1565df73f2868825fc00410f485e190b99a Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 2 Mar 2020 23:00:15 +0530 Subject: [PATCH 35/44] style: fixed cmd log formatting via exec_cmd --- bench/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bench/utils.py b/bench/utils.py index c37b0806..aae06314 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -174,9 +174,8 @@ def clone_apps_from(bench_path, clone_from, update_app=True): def exec_cmd(cmd, cwd='.'): import shlex - logger.info(cmd) - cmd = shlex.split(cmd) print("{0}$ {1}{2}".format(color.silver, cmd, color.nc)) + cmd = shlex.split(cmd) subprocess.call(cmd, cwd=cwd, universal_newlines=True) def which(executable, raise_err = False): From 76943e070c1ef53e7448024ad5b78073d3d80b58 Mon Sep 17 00:00:00 2001 From: Chris Ian Fiel Date: Wed, 4 Mar 2020 11:14:45 +0800 Subject: [PATCH 36/44] fix: "Error: no such option: --backup" (#946) * fix issue: Error: no such option: --backup https://discuss.erpnext.com/t/bench-update-fails-due-to-no-bench-backup-option/58697 --- bench/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/utils.py b/bench/utils.py index eccda651..d986c034 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -190,7 +190,7 @@ def update(pull=False, patch=False, build=False, bench=False, restart_supervisor 'patch': patch, 'build': build, 'requirements': requirements, - 'backup': backup, + 'no-backup': backup, 'restart-supervisor': restart_supervisor, 'reset': reset }) From 016b7da9ffcfeb342d9ca28ed2aae51329ee87c4 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 4 Mar 2020 13:12:39 +0530 Subject: [PATCH 37/44] fix: remove deprecated pre_upgrade usage --- bench/app.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bench/app.py b/bench/app.py index f9e5b7b5..72431249 100755 --- a/bench/app.py +++ b/bench/app.py @@ -358,7 +358,7 @@ def get_repo_dir(app, bench_path='.'): return os.path.join(bench_path, 'apps', app) def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrade=True): - from .utils import update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets, pre_upgrade, post_upgrade + from .utils import update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets, post_upgrade from . import utils apps_dir = os.path.join(bench_path, 'apps') version_upgrade = (False,) @@ -399,7 +399,6 @@ def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrad if version_upgrade[0] and upgrade: update_requirements() update_node_packages() - pre_upgrade(version_upgrade[1], version_upgrade[2]) reload_module(utils) backup_all_sites() patch_sites() From d0ebf7f4c4b17d96f8ee798cb83c7a6122db37ef Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 4 Mar 2020 14:04:54 +0530 Subject: [PATCH 38/44] fix: setup requirements --python skips assets --- bench/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bench/utils.py b/bench/utils.py index eccda651..ab6ed165 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -596,14 +596,14 @@ def update_requirements(bench_path='.'): from bench.app import get_apps, install_app print('Updating Python libraries...') - # update env pip - update_env_pip(bench_path) - # Update bench requirements (at user level) update_bench_requirements() + # update env pip + update_env_pip(bench_path) + for app in get_apps(): - install_app(app, bench_path=bench_path) + install_app(app, bench_path=bench_path, skip_assets=True) def update_node_packages(bench_path='.'): From 4c173ded26a2e36651706439f58401ffafea6ac7 Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Thu, 5 Mar 2020 18:26:48 +0530 Subject: [PATCH 39/44] feat(procfile): dont add redis to procfile if skipping config gen Signed-off-by: Chinmay D. Pai --- bench/config/procfile.py | 5 +++-- bench/config/templates/Procfile | 2 ++ bench/utils.py | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/bench/config/procfile.py b/bench/config/procfile.py index 3dd7cece..9dbe55b3 100755 --- a/bench/config/procfile.py +++ b/bench/config/procfile.py @@ -3,7 +3,7 @@ from bench.utils import find_executable from bench.app import use_rq from bench.config.common_site_config import get_config -def setup_procfile(bench_path, yes=False): +def setup_procfile(bench_path, yes=False, setup_redis=True): config = get_config(bench_path=bench_path) procfile_path = os.path.join(bench_path, 'Procfile') if not yes and os.path.exists(procfile_path): @@ -14,7 +14,8 @@ def setup_procfile(bench_path, yes=False): node=find_executable("node") or find_executable("nodejs"), use_rq=use_rq(bench_path), webserver_port=config.get('webserver_port'), - CI=os.environ.get('CI')) + CI=os.environ.get('CI'), + setup_redis=setup_redis) with open(procfile_path, 'w') as f: f.write(procfile) diff --git a/bench/config/templates/Procfile b/bench/config/templates/Procfile index b9c81118..f1083dcc 100644 --- a/bench/config/templates/Procfile +++ b/bench/config/templates/Procfile @@ -1,6 +1,8 @@ +{% if setup_redis %} redis_cache: redis-server config/redis_cache.conf redis_socketio: redis-server config/redis_socketio.conf redis_queue: redis-server config/redis_queue.conf +{% endif %} web: bench serve {% if webserver_port -%} --port {{ webserver_port }} {%- endif %} socketio: {{ node }} apps/frappe/socketio.js diff --git a/bench/utils.py b/bench/utils.py index 4b4b518a..4f6c4dff 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -147,7 +147,7 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, redis.generate_config(path) if not no_procfile: - setup_procfile(path) + setup_procfile(path, setup_redis=skip_redis_config_generation) if not no_backups: setup_backups(bench_path=path) From fceb2725c4f085788f684a8a1229b08a90c9db27 Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Thu, 5 Mar 2020 22:59:59 +0530 Subject: [PATCH 40/44] chore: rename setup_redis to skip_redis Signed-off-by: Chinmay D. Pai --- bench/config/procfile.py | 4 ++-- bench/config/templates/Procfile | 2 +- bench/utils.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bench/config/procfile.py b/bench/config/procfile.py index 9dbe55b3..a6982c83 100755 --- a/bench/config/procfile.py +++ b/bench/config/procfile.py @@ -3,7 +3,7 @@ from bench.utils import find_executable from bench.app import use_rq from bench.config.common_site_config import get_config -def setup_procfile(bench_path, yes=False, setup_redis=True): +def setup_procfile(bench_path, yes=False, skip_redis=False): config = get_config(bench_path=bench_path) procfile_path = os.path.join(bench_path, 'Procfile') if not yes and os.path.exists(procfile_path): @@ -15,7 +15,7 @@ def setup_procfile(bench_path, yes=False, setup_redis=True): use_rq=use_rq(bench_path), webserver_port=config.get('webserver_port'), CI=os.environ.get('CI'), - setup_redis=setup_redis) + skip_redis=skip_redis) with open(procfile_path, 'w') as f: f.write(procfile) diff --git a/bench/config/templates/Procfile b/bench/config/templates/Procfile index f1083dcc..e557f561 100644 --- a/bench/config/templates/Procfile +++ b/bench/config/templates/Procfile @@ -1,4 +1,4 @@ -{% if setup_redis %} +{% if not skip_redis %} redis_cache: redis-server config/redis_cache.conf redis_socketio: redis-server config/redis_socketio.conf redis_queue: redis-server config/redis_queue.conf diff --git a/bench/utils.py b/bench/utils.py index 4f6c4dff..c455e03e 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -147,7 +147,7 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, redis.generate_config(path) if not no_procfile: - setup_procfile(path, setup_redis=skip_redis_config_generation) + setup_procfile(path, skip_redis=skip_redis_config_generation) if not no_backups: setup_backups(bench_path=path) From dc307b6880d61956754d665aae37b6f9c3fab4b8 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 9 Mar 2020 13:53:13 +0530 Subject: [PATCH 41/44] fix: switch branches "better" --- bench/app.py | 87 +++++++++++++++++++++++----------------- bench/commands/update.py | 3 -- 2 files changed, 50 insertions(+), 40 deletions(-) diff --git a/bench/app.py b/bench/app.py index 72431249..9933ae14 100755 --- a/bench/app.py +++ b/bench/app.py @@ -1,25 +1,32 @@ +# imports - compatibility imports from __future__ import print_function -import os -from .utils import (exec_cmd, get_frappe, check_git_for_shallow_clone, build_assets, - restart_supervisor_processes, get_cmd_output, run_frappe_cmd, CommandFailedError, - restart_systemd_processes) -from .config.common_site_config import get_config -from six.moves import reload_module +# imports - standard imports +import json import logging +import os +import re +import shutil +import subprocess +import sys + +# imports - third party imports +import click +import git import requests import semantic_version -import json -import re -import subprocess -import bench -import sys -import shutil -import click +from six.moves import reload_module -logging.basicConfig(level="DEBUG") +# imports - module imports +import bench +from bench.config.common_site_config import get_config +from bench.utils import CommandFailedError, build_assets, check_git_for_shallow_clone, exec_cmd, get_cmd_output, get_frappe, restart_supervisor_processes, restart_systemd_processes, run_frappe_cmd + + +logging.basicConfig(level="INFO") logger = logging.getLogger(__name__) + class InvalidBranchException(Exception): pass class InvalidRemoteException(Exception): pass @@ -358,8 +365,7 @@ def get_repo_dir(app, bench_path='.'): return os.path.join(bench_path, 'apps', app) def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrade=True): - from .utils import update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets, post_upgrade - from . import utils + from bench.utils import update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets, post_upgrade apps_dir = os.path.join(bench_path, 'apps') version_upgrade = (False,) switched_apps = [] @@ -372,29 +378,35 @@ def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrad for app in apps: app_dir = os.path.join(apps_dir, app) - if os.path.exists(app_dir): - try: - if check_upgrade: - version_upgrade = is_version_upgrade(app=app, bench_path=bench_path, branch=branch) - if version_upgrade[0] and not upgrade: - raise MajorVersionUpgradeException("Switching to {0} will cause upgrade from {1} to {2}. Pass --upgrade to confirm".format(branch, version_upgrade[1], version_upgrade[2]), version_upgrade[1], version_upgrade[2]) - print("Switching for "+app) - unshallow = "--unshallow" if os.path.exists(os.path.join(app_dir, ".git", "shallow")) else "" - exec_cmd("git config --unset-all remote.upstream.fetch", cwd=app_dir) - exec_cmd("git config --add remote.upstream.fetch '+refs/heads/*:refs/remotes/upstream/*'", cwd=app_dir) - exec_cmd("git fetch upstream {unshallow}".format(unshallow=unshallow), cwd=app_dir) - exec_cmd("git checkout {branch}".format(branch=branch), cwd=app_dir) - exec_cmd("git merge upstream/{branch}".format(branch=branch), cwd=app_dir) - switched_apps.append(app) - except CommandFailedError: - print("Error switching to branch {0} for {1}".format(branch, app)) - except InvalidRemoteException: - print("Remote does not exist for app {0}".format(app)) - except InvalidBranchException: - print("Branch {0} does not exist in Upstream for {1}".format(branch, app)) + + if not os.path.exists(app_dir): + bench.utils.log("{} does not exist!".format(app), level=2) + continue + + repo = git.Repo(app_dir) + unshallow_flag = os.path.exists(os.path.join(app_dir, ".git", "shallow")) + bench.utils.log("Fetching upstream {0}for {1}".format("unshallow " if unshallow_flag else "", app)) + + bench.utils.exec_cmd("git remote set-branches upstream '*'", cwd=app_dir) + bench.utils.exec_cmd("git fetch --all{0}".format(" --unshallow" if unshallow_flag else ""), cwd=app_dir) + + if check_upgrade: + version_upgrade = is_version_upgrade(app=app, bench_path=bench_path, branch=branch) + if version_upgrade[0] and not upgrade: + bench.utils.log("Switching to {0} will cause upgrade from {1} to {2}. Pass --upgrade to confirm".format(branch, version_upgrade[1], version_upgrade[2]), level=2) + sys.exit(1) + + print("Switching for "+app) + bench.utils.exec_cmd("git checkout {0}".format(branch), cwd=app_dir) + + if str(repo.active_branch) == branch: + switched_apps.append(app) + else: + bench.utils.log("Switching branches failed for: {}".format(app), level=2) if switched_apps: - print("Successfully switched branches for:\n" + "\n".join(switched_apps)) + bench.utils.log("Successfully switched branches for: " + ", ".join(switched_apps), level=1) + print('Please run `bench update --patch` to be safe from any differences in database schema') if version_upgrade[0] and upgrade: update_requirements() @@ -405,6 +417,7 @@ def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrad build_assets() post_upgrade(version_upgrade[1], version_upgrade[2]) + def switch_to_branch(branch=None, apps=None, bench_path='.', upgrade=False): switch_branch(branch, apps=apps, bench_path=bench_path, upgrade=upgrade) diff --git a/bench/commands/update.py b/bench/commands/update.py index 6dac7fa5..8461da2c 100755 --- a/bench/commands/update.py +++ b/bench/commands/update.py @@ -42,8 +42,6 @@ def retry_upgrade(version): def switch_to_branch(branch, apps, upgrade=False): from bench.app import switch_to_branch switch_to_branch(branch=branch, apps=list(apps), upgrade=upgrade) - print('Switched to ' + branch) - print('Please run `bench update --patch` to be safe from any differences in database schema') @click.command('switch-to-master', help="[DEPRECATED]: Switch frappe and erpnext to master branch") @@ -57,4 +55,3 @@ def switch_to_develop(upgrade=False): "Switch frappe and erpnext to develop branch" from bench.app import switch_to_develop switch_to_develop(apps=['frappe', 'erpnext']) - print('Switched to develop\nPlease run `bench update --patch` to be safe from any differences in database schema') From 1f5c7ec20163968e7edb3faa69d2da7215da6ed6 Mon Sep 17 00:00:00 2001 From: gavin Date: Mon, 9 Mar 2020 17:47:43 +0530 Subject: [PATCH 42/44] chore: add travis, deepsource (#900) * chore: added deepsource config * chore: updated travis config * chore: updated travis config * chore: updated travis tests for 2.7, 3.6, 3.7, 3.8 * chore: quiet installs and sudo easy install * fix(tests): add mariadb versioning * fix(travis): print compat for pyhton 3 * fix: drop deprecated function usage * chore(tests): update test_init * tests: update branch to use version-12 for testing and use git module * chore: quieten git command outputs on branch switch * fix: execute 'setup production' via cli * style: sort imports * chore: update mariadb variables in .travis.yml * chore: seperate jobs for easy install and production setup * chore: use exec_cmd to log command output * chore: pin tests to ubuntu trusty * chore: use playbooks to install mariadb * chore: mariadb fixes in travis * chore: pin dist to ubuntu 18.04 * chore: setup envs according by type of test * chore: ignore auto backup, procfile, assets build * chore: change app frammeclient to frappe_theme * test: try travis_wait * tests: update and restructure tests * tests: restructure flow of setUp, tearDown * fix: python class inheritence fix * chore: use local frappe repo instead of remote pull * tests: skip assets during get-app * fix(tests): remove reinstalling app after get_app called * fix(tests): updated test_install_app * fix: broken remove_app tests: broken tests * tests: no backup while dropping new_sites * tests: added certain tests and py obj fixes * tests: seperate basic and production setup tests * tests: update travis, remove basic setup * chore: move from function calls to exec_cmd * chore: tests fixes * chore: removed sudo from basic setup runs * chore: use "sudo" for setting up sudoers * fix: allow get_app from local folder * fix: use gitpython to switch branch instead of exec_cmd * chore: use test to check for file existing * chore: restructure bench tests * fix: fetch app repo before checking for version upgrade during switch_branch * fix: gitpython error in usage * fix: boolean return value of file_exists * fix: dont decode file contents * fix: bench names and close open files * chore: update bench config multiple benches * chore: check where production fails * chore: mention python version for init * chore: remove node production test in CI * fix: compatibility and permissions fixes * chore: setup sudoers after prod setup * fix: set remote upstream after local clone * fix: disable production via cli * chore(tests): fix upstream link * chore: split tests and remove unused imports --- .deepsource.toml | 16 ++ .travis.yml | 110 +++++++++--- bench/app.py | 46 +++-- bench/tests/test_base.py | 96 +++++++++++ bench/tests/test_init.py | 243 +++++++-------------------- bench/tests/test_setup_production.py | 92 +++++----- bench/utils.py | 2 +- 7 files changed, 325 insertions(+), 280 deletions(-) create mode 100644 .deepsource.toml create mode 100644 bench/tests/test_base.py diff --git a/.deepsource.toml b/.deepsource.toml new file mode 100644 index 00000000..90b9b9bb --- /dev/null +++ b/.deepsource.toml @@ -0,0 +1,16 @@ +version = 1 + +exclude_patterns = [ + ".*" +] + +test_patterns = [ + "bench/tests/**" +] + +[[analyzers]] +name = "python" +enabled = true +dependency_file_paths = [ + "requirements.txt" +] \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 9df3b6d4..550aa53e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,26 +1,96 @@ language: python -dist: xenial +dist: bionic +sudo: true -python: - - "2.7" +git: + depth: 1 + +cache: + - pip + - npm + - yarn + +addons: + mariadb: '10.3' + +matrix: + include: + - name: "Python 2.7 Basic Setup" + python: 2.7 + env: TEST=bench + script: python -m unittest -v bench.tests.test_init + + - name: "Python 3.6 Basic Setup" + python: 3.6 + env: TEST=bench + script: python -m unittest -v bench.tests.test_init + + - name: "Python 3.7 Basic Setup" + python: 3.7 + env: TEST=bench + script: python -m unittest -v bench.tests.test_init + + - name: "Python 3.8 Production Setup" + python: 3.8 + env: TEST=bench + script: python -m unittest -v bench.tests.test_setup_production + + - name: "Python 2.7 Production Setup" + python: 2.7 + env: TEST=bench + script: python -m unittest -v bench.tests.test_setup_production + + - name: "Python 3.6 Production Setup" + python: 3.6 + env: TEST=bench + script: python -m unittest -v bench.tests.test_setup_production + + - name: "Python 3.7 Production Setup" + python: 3.7 + env: TEST=bench + script: python -m unittest -v bench.tests.test_setup_production + + - name: "Python 3.8 Production Setup" + python: 3.8 + env: TEST=bench + script: python -m unittest -v bench.tests.test_setup_production + + - name: "Python 3.6 Easy Install" + python: 3.6 + env: TEST=easy_install + script: sudo python $TRAVIS_BUILD_DIR/playbooks/install.py --user travis --run-travis --production --verbose + + - name: "Python 3.7 Easy Install" + python: 3.7 + env: TEST=easy_install + script: sudo python $TRAVIS_BUILD_DIR/playbooks/install.py --user travis --run-travis --production --verbose + + - name: "Python 3.8 Easy Install" + python: 3.8 + env: TEST=easy_install + script: sudo python $TRAVIS_BUILD_DIR/playbooks/install.py --user travis --run-travis --production --verbose install: - - sudo pip install urllib3 pyOpenSSL ndg-httpsclient pyasn1 - - sudo apt-get purge -y mysql-common mysql-server mysql-client - - sudo apt-get install --only-upgrade -y git - - sudo apt-get install hhvm && rm -rf /home/travis/.kiex/ - - mkdir -p ~/.bench - - mkdir -p /tmp/.bench - - cp -r $TRAVIS_BUILD_DIR/* ~/.bench - - cp -r $TRAVIS_BUILD_DIR/* /tmp/.bench + - pip install urllib3 pyOpenSSL ndg-httpsclient pyasn1 - - sudo python $TRAVIS_BUILD_DIR/playbooks/install.py --user travis --run-travis --production --verbose - # - sudo bash $TRAVIS_BUILD_DIR/install_scripts/setup_frappe.sh --skip-install-bench --mysql-root-password travis - # - cd ~ && sudo python bench-repo/installer/install.py --only-dependencies + - if [ $TEST == "bench" ];then + wget -q -O /tmp/wkhtmltox.tar.xz https://github.com/frappe/wkhtmltopdf/raw/master/wkhtmltox-0.12.3_linux-generic-amd64.tar.xz; + tar -xf /tmp/wkhtmltox.tar.xz -C /tmp; + sudo mv /tmp/wkhtmltox/bin/wkhtmltopdf /usr/local/bin/wkhtmltopdf; + sudo chmod o+x /usr/local/bin/wkhtmltopdf; -script: - - cd ~ - - sudo pip install --upgrade pip - - sudo pip install -e ~/.bench - # - sudo python -m unittest bench.tests.test_setup_production.TestSetupProduction.test_setup_production_v6 - - sudo python -m unittest -v bench.tests.test_setup_production + mkdir -p ~/.bench; + cp -r $TRAVIS_BUILD_DIR/* ~/.bench; + pip install -q -U -e ~/.bench; + sudo pip install -q -U -e ~/.bench; + + mysql -u root -e "SET GLOBAL character_set_server = 'utf8mb4'"; + mysql -u root -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'"; + mysql -u root -e "UPDATE mysql.user SET Password=PASSWORD('travis') WHERE User='root'"; + mysql -u root -e "FLUSH PRIVILEGES"; + fi + + - if [ $TEST == "easy_install" ];then + mkdir -p /tmp/.bench; + cp -r $TRAVIS_BUILD_DIR/* /tmp/.bench; + fi diff --git a/bench/app.py b/bench/app.py index 9933ae14..211388db 100755 --- a/bench/app.py +++ b/bench/app.py @@ -101,30 +101,28 @@ def remove_from_excluded_apps_txt(app, bench_path='.'): apps.remove(app) return write_excluded_apps_txt(apps, bench_path=bench_path) -def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=False, - postprocess=True, overwrite=False): - # from bench.utils import check_url - try: - from urlparse import urljoin - except ImportError: - from urllib.parse import urljoin +def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=False, postprocess=True, overwrite=False): + if not os.path.exists(git_url): + if not check_url(git_url, raise_err=False): + orgs = ['frappe', 'erpnext'] + for org in orgs: + url = 'https://api.github.com/repos/{org}/{app}'.format(org=org, app=git_url) + res = requests.get(url) + if res.ok: + data = res.json() + if 'name' in data: + if git_url == data['name']: + git_url = 'https://github.com/{org}/{app}'.format(org=org, app=git_url) + break - if not check_url(git_url, raise_err=False): - orgs = ['frappe', 'erpnext'] - for org in orgs: - url = 'https://api.github.com/repos/{org}/{app}'.format(org=org, app=git_url) - res = requests.get(url) - if res.ok: - data = res.json() - if 'name' in data: - if git_url == data['name']: - git_url = 'https://github.com/{org}/{app}'.format(org=org, app=git_url) - break - - # Gets repo name from URL - repo_name = git_url.rsplit('/', 1)[1].rsplit('.', 1)[0] - shallow_clone = '--depth 1' if check_git_for_shallow_clone() else '' - branch = '--branch {branch}'.format(branch=branch) if branch else '' + # Gets repo name from URL + repo_name = git_url.rsplit('/', 1)[1].rsplit('.', 1)[0] + shallow_clone = '--depth 1' if check_git_for_shallow_clone() else '' + branch = '--branch {branch}'.format(branch=branch) if branch else '' + else: + repo_name = git_url.split(os.sep)[-1] + shallow_clone = '' + branch = '--branch {branch}'.format(branch=branch) if branch else '' if os.path.isdir(os.path.join(bench_path, 'apps', repo_name)): # application directory already exists @@ -214,7 +212,7 @@ def remove_app(app, bench_path='.'): print("Cannot remove, app is installed on site: {0}".format(site)) sys.exit(1) - exec_cmd(["{0} uninstall -y {1}".format(pip, app)]) + exec_cmd("{0} uninstall -y {1}".format(pip, app), cwd=bench_path) remove_from_appstxt(app, bench_path) shutil.rmtree(app_path) run_frappe_cmd("build", bench_path=bench_path) diff --git a/bench/tests/test_base.py b/bench/tests/test_base.py new file mode 100644 index 00000000..b5022398 --- /dev/null +++ b/bench/tests/test_base.py @@ -0,0 +1,96 @@ +# imports - standard imports +import json +import os +import shutil +import subprocess +import sys +import unittest +import getpass + +# imports - module imports +import bench +import bench.utils + + +class TestBenchBase(unittest.TestCase): + def setUp(self): + self.benches_path = "." + self.benches = [] + + def tearDown(self): + for bench_name in self.benches: + bench_path = os.path.join(self.benches_path, bench_name) + mariadb_password = "travis" if os.environ.get("CI") else getpass.getpass(prompt="Enter MariaDB root Password: ") + if os.path.exists(bench_path): + sites = bench.utils.get_sites(bench_path=bench_path) + for site in sites: + subprocess.call(["bench", "drop-site", site, "--force", "--no-backup", "--root-password", mariadb_password], cwd=bench_path) + shutil.rmtree(bench_path, ignore_errors=True) + + def assert_folders(self, bench_name): + for folder in bench.utils.folders_in_bench: + self.assert_exists(bench_name, folder) + self.assert_exists(bench_name, "apps", "frappe") + + def assert_virtual_env(self, bench_name): + bench_path = os.path.abspath(bench_name) + python_path = os.path.abspath(os.path.join(bench_path, "env", "bin", "python")) + self.assertTrue(python_path.startswith(bench_path)) + for subdir in ("bin", "include", "lib", "share"): + self.assert_exists(bench_name, "env", subdir) + + def assert_config(self, bench_name): + for config, search_key in ( + ("redis_queue.conf", "redis_queue.rdb"), + ("redis_socketio.conf", "redis_socketio.rdb"), + ("redis_cache.conf", "redis_cache.rdb")): + + self.assert_exists(bench_name, "config", config) + + with open(os.path.join(bench_name, "config", config), "r") as f: + self.assertTrue(search_key in f.read()) + + def assert_common_site_config(self, bench_name, expected_config): + common_site_config_path = os.path.join(self.benches_path, bench_name, 'sites', 'common_site_config.json') + self.assertTrue(os.path.exists(common_site_config_path)) + + with open(common_site_config_path, "r") as f: + config = json.load(f) + + for key, value in list(expected_config.items()): + self.assertEqual(config.get(key), value) + + def assert_exists(self, *args): + self.assertTrue(os.path.exists(os.path.join(*args))) + + def new_site(self, site_name, bench_name): + new_site_cmd = ["bench", "new-site", site_name, "--admin-password", "admin"] + + if os.environ.get('CI'): + new_site_cmd.extend(["--mariadb-root-password", "travis"]) + + subprocess.call(new_site_cmd, cwd=os.path.join(self.benches_path, bench_name)) + + def init_bench(self, bench_name, **kwargs): + self.benches.append(bench_name) + frappe_tmp_path = "/tmp/frappe" + + if not os.path.exists(frappe_tmp_path): + bench.utils.exec_cmd("git clone https://github.com/frappe/frappe --depth 1 --origin upstream {location}".format(location=frappe_tmp_path)) + + kwargs.update(dict( + python=sys.executable, + no_procfile=True, + no_backups=True, + skip_assets=True, + frappe_path=frappe_tmp_path + )) + + if not os.path.exists(os.path.join(self.benches_path, bench_name)): + bench.utils.init(bench_name, **kwargs) + bench.utils.exec_cmd("git remote set-url upstream https://github.com/frappe/frappe", cwd=os.path.join(self.benches_path, bench_name, "apps", "frappe")) + + def file_exists(self, path): + if os.environ.get("CI"): + return not subprocess.call(["sudo", "test", "-f", path]) + return os.path.isfile(path) diff --git a/bench/tests/test_init.py b/bench/tests/test_init.py index 170ccf17..195f90dc 100755 --- a/bench/tests/test_init.py +++ b/bench/tests/test_init.py @@ -1,26 +1,20 @@ - +# imports - standard imports +import json +import os +import subprocess import unittest -import json, os, shutil, subprocess + +# imports - third paty imports +import git + +# imports - module imports import bench import bench.utils -import bench.app -import bench.config.common_site_config -import bench.cli from bench.release import get_bumped_version +from bench.tests.test_base import TestBenchBase -bench.cli.from_command_line = True - -class TestBenchInit(unittest.TestCase): - def setUp(self): - self.benches_path = "." - self.benches = [] - - def tearDown(self): - for bench_name in self.benches: - bench_path = os.path.join(self.benches_path, bench_name) - if os.path.exists(bench_path): - shutil.rmtree(bench_path, ignore_errors=True) +class TestBenchInit(TestBenchBase): def test_semantic_version(self): self.assertEqual( get_bumped_version('11.0.4', 'major'), '12.0.0' ) self.assertEqual( get_bumped_version('11.0.4', 'minor'), '11.1.0' ) @@ -35,20 +29,14 @@ class TestBenchInit(unittest.TestCase): def test_init(self, bench_name="test-bench", **kwargs): self.init_bench(bench_name, **kwargs) - self.assert_folders(bench_name) - self.assert_virtual_env(bench_name) - - self.assert_common_site_config(bench_name, bench.config.common_site_config.default_config) - self.assert_config(bench_name) - self.assert_socketio(bench_name) def test_multiple_benches(self): - # 1st bench - self.test_init("test-bench-1") + for bench_name in ("test-bench-1", "test-bench-2"): + self.init_bench(bench_name) self.assert_common_site_config("test-bench-1", { "webserver_port": 8000, @@ -59,9 +47,6 @@ class TestBenchInit(unittest.TestCase): "redis_cache": "redis://localhost:13000" }) - # 2nd bench - self.test_init("test-bench-2") - self.assert_common_site_config("test-bench-2", { "webserver_port": 8001, "socketio_port": 9001, @@ -71,196 +56,92 @@ class TestBenchInit(unittest.TestCase): "redis_cache": "redis://localhost:13001" }) + + def test_new_site(self): - self.init_bench('test-bench') - self.new_site("test-site-1.dev") + bench_name = "test-bench" + site_name = "test-site.local" + bench_path = os.path.join(self.benches_path, bench_name) + site_path = os.path.join(bench_path, "sites", site_name) + site_config_path = os.path.join(site_path, "site_config.json") - def new_site(self, site_name): - new_site_cmd = ["bench", "new-site", site_name, "--admin-password", "admin"] - - # set in CI - if os.environ.get('CI'): - new_site_cmd.extend(["--mariadb-root-password", "travis"]) - - subprocess.check_output(new_site_cmd, cwd=os.path.join(self.benches_path, "test-bench")) - - site_path = os.path.join(self.benches_path, "test-bench", "sites", site_name) + self.init_bench(bench_name) + bench.utils.exec_cmd("bench setup requirements --node", cwd=bench_path) + self.new_site(site_name, bench_name) self.assertTrue(os.path.exists(site_path)) self.assertTrue(os.path.exists(os.path.join(site_path, "private", "backups"))) self.assertTrue(os.path.exists(os.path.join(site_path, "private", "files"))) self.assertTrue(os.path.exists(os.path.join(site_path, "public", "files"))) - - site_config_path = os.path.join(site_path, "site_config.json") self.assertTrue(os.path.exists(site_config_path)) + with open(site_config_path, "r") as f: site_config = json.loads(f.read()) - for key in ("db_name", "db_password"): - self.assertTrue(key in site_config) - self.assertTrue(site_config[key]) + for key in ("db_name", "db_password"): + self.assertTrue(key in site_config) + self.assertTrue(site_config[key]) def test_get_app(self): - site_name = "test-site-2.dev" - self.init_bench('test-bench') - - self.new_site(site_name) + self.init_bench("test-bench") bench_path = os.path.join(self.benches_path, "test-bench") + bench.utils.exec_cmd("bench get-app frappe_theme --skip-assets", cwd=bench_path) + self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "frappe_theme"))) + app_installed_in_env = "frappe_theme" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8') + self.assertTrue(app_installed_in_env) - bench.app.get_app("https://github.com/frappe/frappe-client", bench_path=bench_path) - self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "frappeclient"))) def test_install_app(self): - site_name = "test-site-3.dev" - self.init_bench('test-bench') - - self.new_site(site_name) + bench_name = "test-bench" + site_name = "install-app.test" bench_path = os.path.join(self.benches_path, "test-bench") - # get app - bench.app.get_app("https://github.com/frappe/erpnext", "develop", bench_path=bench_path) + self.init_bench(bench_name) + bench.utils.exec_cmd("bench setup requirements --node", cwd=bench_path) + bench.utils.exec_cmd("bench build", cwd=bench_path) + bench.utils.exec_cmd("bench get-app erpnext", cwd=bench_path) self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "erpnext"))) - # install app - bench.app.install_app("erpnext", bench_path=bench_path) + # check if app is installed + app_installed_in_env = "erpnext" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8') + self.assertTrue(app_installed_in_env) - # install it to site - subprocess.check_output(["bench", "--site", site_name, "install-app", "erpnext"], cwd=bench_path) + # create and install app on site + self.new_site(site_name, bench_name) + bench.utils.exec_cmd("bench --site {0} install-app erpnext".format(site_name), cwd=bench_path) - out = subprocess.check_output(["bench", "--site", site_name, "list-apps"], cwd=bench_path) - self.assertTrue("erpnext" in out) + app_installed_on_site = subprocess.check_output(["bench", "--site", site_name, "list-apps"], cwd=bench_path).decode('utf8') + self.assertTrue("erpnext" in app_installed_on_site) def test_remove_app(self): - self.init_bench('test-bench') - + self.init_bench("test-bench") bench_path = os.path.join(self.benches_path, "test-bench") - # get app - bench.app.get_app("https://github.com/frappe/erpnext", "develop", bench_path=bench_path) - - self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "erpnext"))) - - # remove it - bench.app.remove_app("erpnext", bench_path=bench_path) + bench.utils.exec_cmd("bench setup requirements --node", cwd=bench_path) + bench.utils.exec_cmd("bench get-app erpnext --branch version-12 --skip-assets --overwrite", cwd=bench_path) + bench.utils.exec_cmd("bench remove-app erpnext", cwd=bench_path) + with open(os.path.join(bench_path, "sites", "apps.txt")) as f: + self.assertFalse("erpnext" in f.read()) + self.assertFalse("erpnext" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8')) self.assertFalse(os.path.exists(os.path.join(bench_path, "apps", "erpnext"))) def test_switch_to_branch(self): - self.init_bench('test-bench') - + self.init_bench("test-bench") bench_path = os.path.join(self.benches_path, "test-bench") app_path = os.path.join(bench_path, "apps", "frappe") - bench.app.switch_branch(branch="master", apps=["frappe"], bench_path=bench_path, check_upgrade=False) - out = subprocess.check_output(['git', 'status'], cwd=app_path) - self.assertTrue("master" in out) + bench.utils.exec_cmd("bench switch-to-branch version-12 frappe", cwd=bench_path) + app_branch_after_switch = str(git.Repo(path=app_path).active_branch) + self.assertEqual("version-12", app_branch_after_switch) - # bring it back to develop! - bench.app.switch_branch(branch="develop", apps=["frappe"], bench_path=bench_path, check_upgrade=False) - out = subprocess.check_output(['git', 'status'], cwd=app_path) - self.assertTrue("develop" in out) + bench.utils.exec_cmd("bench switch-to-branch develop frappe", cwd=bench_path) + app_branch_after_second_switch = str(git.Repo(path=app_path).active_branch) + self.assertEqual("develop", app_branch_after_second_switch) - def init_bench(self, bench_name, **kwargs): - self.benches.append(bench_name) - bench.utils.init(bench_name, **kwargs) - def test_drop_site(self): - self.init_bench('test-bench') - # Check without archive_path given to drop-site command - self.drop_site("test-drop-without-archive-path") - - # Check with archive_path given to drop-site command - home = os.path.abspath(os.path.expanduser('~')) - archived_sites_path = os.path.join(home, 'archived_sites') - - self.drop_site("test-drop-with-archive-path", archived_sites_path=archived_sites_path) - - def drop_site(self, site_name, archived_sites_path=None): - self.new_site(site_name) - - drop_site_cmd = ['bench', 'drop-site', site_name] - - if archived_sites_path: - drop_site_cmd.extend(['--archived-sites-path', archived_sites_path]) - - if os.environ.get('CI'): - drop_site_cmd.extend(['--root-password', 'travis']) - - bench_path = os.path.join(self.benches_path, 'test-bench') - try: - subprocess.check_output(drop_site_cmd, cwd=bench_path) - except subprocess.CalledProcessError as err: - print(err.output) - - if not archived_sites_path: - archived_sites_path = os.path.join(bench_path, 'archived_sites') - self.assertTrue(os.path.exists(archived_sites_path)) - self.assertTrue(os.path.exists(os.path.join(archived_sites_path, site_name))) - - else: - self.assertTrue(os.path.exists(archived_sites_path)) - self.assertTrue(os.path.exists(os.path.join(archived_sites_path, site_name))) - - def assert_folders(self, bench_name): - for folder in bench.utils.folders_in_bench: - self.assert_exists(bench_name, folder) - - self.assert_exists(bench_name, "sites", "assets") - self.assert_exists(bench_name, "apps", "frappe") - self.assert_exists(bench_name, "apps", "frappe", "setup.py") - - def assert_virtual_env(self, bench_name): - bench_path = os.path.abspath(bench_name) - python = os.path.join(bench_path, "env", "bin", "python") - python_path = bench.utils.get_cmd_output('{python} -c "import os; print os.path.dirname(os.__file__)"'.format(python=python)) - - # part of bench's virtualenv - self.assertTrue(python_path.startswith(bench_path)) - self.assert_exists(python_path) - self.assert_exists(python_path, "site-packages") - self.assert_exists(python_path, "site-packages", "IPython") - self.assert_exists(python_path, "site-packages", "pip") - - site_packages = os.listdir(os.path.join(python_path, "site-packages")) - # removing test case temporarily - # as develop and master branch havin differnt version of mysqlclient - #self.assertTrue(any(package.startswith("mysqlclient-1.3.12") for package in site_packages)) - - def assert_config(self, bench_name): - for config, search_key in ( - ("redis_queue.conf", "redis_queue.rdb"), - ("redis_socketio.conf", "redis_socketio.rdb"), - ("redis_cache.conf", "redis_cache.rdb")): - - self.assert_exists(bench_name, "config", config) - - with open(os.path.join(bench_name, "config", config), "r") as f: - f = f.read().decode("utf-8") - self.assertTrue(search_key in f) - - def assert_socketio(self, bench_name): - try: # for v10 and under - self.assert_exists(bench_name, "node_modules") - self.assert_exists(bench_name, "node_modules", "socket.io") - except: # for v11 and above - self.assert_exists(bench_name, "apps", "frappe", "node_modules") - self.assert_exists(bench_name, "apps", "frappe", "node_modules", "socket.io") - - def assert_common_site_config(self, bench_name, expected_config): - common_site_config_path = os.path.join(bench_name, 'sites', 'common_site_config.json') - self.assertTrue(os.path.exists(common_site_config_path)) - - config = self.load_json(common_site_config_path) - - for key, value in list(expected_config.items()): - self.assertEqual(config.get(key), value) - - def assert_exists(self, *args): - self.assertTrue(os.path.exists(os.path.join(*args))) - - def load_json(self, path): - with open(path, "r") as f: - return json.loads(f.read().decode("utf-8")) +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/bench/tests/test_setup_production.py b/bench/tests/test_setup_production.py index e22a5858..ca5b4abc 100644 --- a/bench/tests/test_setup_production.py +++ b/bench/tests/test_setup_production.py @@ -1,69 +1,51 @@ - -from bench.tests import test_init -from bench.config.production_setup import setup_production, get_supervisor_confdir, disable_production -import bench.utils -import os +# imports - standard imports import getpass +import os import re -import unittest +import subprocess import time +import unittest -class TestSetupProduction(test_init.TestBenchInit): - # setUp, tearDown and other tests are defiend in TestBenchInit +# imports - module imports +import bench.utils +from bench.config.production_setup import get_supervisor_confdir +from bench.tests.test_base import TestBenchBase + +class TestSetupProduction(TestBenchBase): def test_setup_production(self): - self.test_multiple_benches() - user = getpass.getuser() for bench_name in ("test-bench-1", "test-bench-2"): bench_path = os.path.join(os.path.abspath(self.benches_path), bench_name) - setup_production(user, bench_path) + self.init_bench(bench_name) + bench.utils.exec_cmd("sudo bench setup production {0}".format(user), cwd=bench_path) self.assert_nginx_config(bench_name) self.assert_supervisor_config(bench_name) - - # test after start of both benches - for bench_name in ("test-bench-1", "test-bench-2"): self.assert_supervisor_process(bench_name) self.assert_nginx_process() - - # sudoers - bench.utils.setup_sudoers(user) + bench.utils.exec_cmd("sudo bench setup sudoers {0}".format(user)) self.assert_sudoers(user) - for bench_name in ("test-bench-1", "test-bench-2"): + for bench_name in self.benches: bench_path = os.path.join(os.path.abspath(self.benches_path), bench_name) - disable_production(bench_path) + bench.utils.exec_cmd("sudo bench disable-production", cwd=bench_path) - def test_disable_production(self): - bench_name = 'test-disable-prod' - self.test_init(bench_name, frappe_branch='master') - - user = getpass.getuser() - - bench_path = os.path.join(os.path.abspath(self.benches_path), bench_name) - setup_production(user, bench_path) - - disable_production(bench_path) - - self.assert_nginx_link(bench_name) - self.assert_supervisor_link(bench_name) - self.assert_supervisor_process(bench_name=bench_name, disable_production=True) def assert_nginx_config(self, bench_name): conf_src = os.path.join(os.path.abspath(self.benches_path), bench_name, 'config', 'nginx.conf') conf_dest = "/etc/nginx/conf.d/{bench_name}.conf".format(bench_name=bench_name) - self.assertTrue(os.path.exists(conf_src)) - self.assertTrue(os.path.exists(conf_dest)) + self.assertTrue(self.file_exists(conf_src)) + self.assertTrue(self.file_exists(conf_dest)) # symlink matches self.assertEqual(os.path.realpath(conf_dest), conf_src) # file content with open(conf_src, "r") as f: - f = f.read().decode("utf-8") + f = f.read() for key in ( "upstream {bench_name}-frappe", @@ -71,48 +53,56 @@ class TestSetupProduction(test_init.TestBenchInit): ): self.assertTrue(key.format(bench_name=bench_name) in f) + def assert_nginx_process(self): out = bench.utils.get_cmd_output("sudo nginx -t 2>&1") self.assertTrue("nginx: configuration file /etc/nginx/nginx.conf test is successful" in out) + def assert_sudoers(self, user): sudoers_file = '/etc/sudoers.d/frappe' - self.assertTrue(os.path.exists(sudoers_file)) + self.assertTrue(self.file_exists(sudoers_file)) - with open(sudoers_file, 'r') as f: - sudoers = f.read().decode('utf-8') + if os.environ.get("CI"): + sudoers = subprocess.check_output(["sudo", "cat", sudoers_file]).decode("utf-8") + else: + with open(sudoers_file, 'r') as f: + sudoers = f.read() self.assertTrue('{user} ALL = (root) NOPASSWD: /usr/sbin/service nginx *'.format(user=user) in sudoers) self.assertTrue('{user} ALL = (root) NOPASSWD: /usr/bin/supervisorctl'.format(user=user) in sudoers) self.assertTrue('{user} ALL = (root) NOPASSWD: /usr/sbin/nginx'.format(user=user) in sudoers) + def assert_supervisor_config(self, bench_name, use_rq=True): conf_src = os.path.join(os.path.abspath(self.benches_path), bench_name, 'config', 'supervisor.conf') supervisor_conf_dir = get_supervisor_confdir() conf_dest = "{supervisor_conf_dir}/{bench_name}.conf".format(supervisor_conf_dir=supervisor_conf_dir, bench_name=bench_name) - self.assertTrue(os.path.exists(conf_src)) - self.assertTrue(os.path.exists(conf_dest)) + self.assertTrue(self.file_exists(conf_src)) + self.assertTrue(self.file_exists(conf_dest)) # symlink matches self.assertEqual(os.path.realpath(conf_dest), conf_src) # file content with open(conf_src, "r") as f: - f = f.read().decode("utf-8") + f = f.read() tests = [ "program:{bench_name}-frappe-web", "program:{bench_name}-redis-cache", "program:{bench_name}-redis-queue", "program:{bench_name}-redis-socketio", - "program:{bench_name}-node-socketio", "group:{bench_name}-web", "group:{bench_name}-workers", "group:{bench_name}-redis" ] + if not os.environ.get("CI"): + tests.append("program:{bench_name}-node-socketio") + if use_rq: tests.extend([ "program:{bench_name}-frappe-schedule", @@ -130,8 +120,11 @@ class TestSetupProduction(test_init.TestBenchInit): ]) for key in tests: + if key.format(bench_name=bench_name) not in f: + print(key.format(bench_name=bench_name)) self.assertTrue(key.format(bench_name=bench_name) in f) + def assert_supervisor_process(self, bench_name, use_rq=True, disable_production=False): out = bench.utils.get_cmd_output("sudo supervisorctl status") @@ -172,15 +165,6 @@ class TestSetupProduction(test_init.TestBenchInit): else: self.assertTrue(re.search(key.format(bench_name=bench_name), out)) - def assert_nginx_link(self, bench_name): - nginx_conf_name = '{bench_name}.conf'.format(bench_name=bench_name) - nginx_conf_path = os.path.join('/etc/nginx/conf.d', nginx_conf_name) - self.assertFalse(os.path.islink(nginx_conf_path)) - - def assert_supervisor_link(self, bench_name): - supervisor_conf_dir = get_supervisor_confdir() - supervisor_conf_name = '{bench_name}.conf'.format(bench_name=bench_name) - supervisor_conf_path = os.path.join(supervisor_conf_dir, supervisor_conf_name) - - self.assertFalse(os.path.islink(supervisor_conf_path)) +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/bench/utils.py b/bench/utils.py index 4b4b518a..bd42556f 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -373,7 +373,7 @@ def add_to_crontab(line): line = str.encode(line) if not line in current_crontab: cmd = ["crontab"] - if platform.system() == 'FreeBSD' or platform.linux_distribution()[0]=="arch": + if platform.system() == 'FreeBSD': cmd = ["crontab", "-"] s = subprocess.Popen(cmd, stdin=subprocess.PIPE) s.stdin.write(current_crontab) From 06d4303a9f24e54b157b6da240165337040cd7bb Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 11 Mar 2020 12:30:37 +0530 Subject: [PATCH 43/44] chore: print error message in case of failures --- bench/commands/make.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bench/commands/make.py b/bench/commands/make.py index 8963dde9..7fc799e6 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -38,10 +38,11 @@ def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, c log('Bench {} initialized'.format(path), level=1) except SystemExit: pass - except: + except Exception as e: import os, shutil, time, six # add a sleep here so that the traceback of other processes doesnt overlap with the prompts time.sleep(1) + print(e) log("There was a problem while creating {}".format(path), level=2) if six.moves.input("Do you want to rollback these changes? [Y/n]: ").lower() == "y": print('Rolling back Bench "{}"'.format(path)) From 72756236a5745a15acb72ff06b55272d6c327e90 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 13 Mar 2020 16:34:01 +0530 Subject: [PATCH 44/44] chore: update README for docker (#954) --- README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 8eb96fed..294520d8 100755 --- a/README.md +++ b/README.md @@ -117,12 +117,11 @@ Releases can be made for [Frappe](https://github.com/frappe/frappe) apps using b --- -# Docker Install +# Docker -1. For developer setup, you can also use the official [Frappe Docker](https://github.com/frappe/frappe_docker/). -2. The app, mariadb and redis run on individual containers. -3. This setup supports multi-tenancy and exposes the frappe-bench volume as a external storage. -4. For more details, [ead the instructions on the [Frappe Docker README](https://github.com/frappe/frappe_docker/) +- For official images and resources [Frappe Docker](https://github.com/frappe/frappe_docker) +- Production Installation [README](https://github.com/frappe/frappe_docker/blob/develop/README.md) +- Developer Setup [README](https://github.com/frappe/frappe_docker/blob/develop/development/README.md) ---