diff --git a/bench/app.py b/bench/app.py index b6e83444..7ebaf8ff 100755 --- a/bench/app.py +++ b/bench/app.py @@ -443,9 +443,6 @@ def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrad def switch_to_branch(branch=None, apps=None, bench_path='.', upgrade=False): switch_branch(branch, apps=apps, bench_path=bench_path, upgrade=upgrade) -def switch_to_master(apps=None, bench_path='.', upgrade=True): - switch_branch('master', apps=apps, bench_path=bench_path, upgrade=upgrade) - def switch_to_develop(apps=None, bench_path='.', upgrade=True): switch_branch('develop', apps=apps, bench_path=bench_path, upgrade=upgrade) diff --git a/bench/commands/__init__.py b/bench/commands/__init__.py index 2b073b65..54347f37 100755 --- a/bench/commands/__init__.py +++ b/bench/commands/__init__.py @@ -27,11 +27,10 @@ bench_command.add_command(include_app_for_update) bench_command.add_command(pip) -from bench.commands.update import update, retry_upgrade, switch_to_branch, switch_to_master, switch_to_develop +from bench.commands.update import update, retry_upgrade, switch_to_branch, switch_to_develop bench_command.add_command(update) bench_command.add_command(retry_upgrade) bench_command.add_command(switch_to_branch) -bench_command.add_command(switch_to_master) bench_command.add_command(switch_to_develop) diff --git a/bench/commands/update.py b/bench/commands/update.py index e1113b40..16ffefbf 100755 --- a/bench/commands/update.py +++ b/bench/commands/update.py @@ -41,12 +41,6 @@ def switch_to_branch(branch, apps, upgrade=False): switch_to_branch(branch=branch, apps=list(apps), upgrade=upgrade) -@click.command('switch-to-master', help="[DEPRECATED]: Switch frappe and erpnext to master branch") -def switch_to_master(): - from bench.utils import log - log("`switch-to-master` has been deprecated as master branches were renamed to version-11") - - @click.command('switch-to-develop') def switch_to_develop(upgrade=False): "Switch frappe and erpnext to develop branch" diff --git a/bench/config/procfile.py b/bench/config/procfile.py index e57e579d..50a45560 100755 --- a/bench/config/procfile.py +++ b/bench/config/procfile.py @@ -8,7 +8,7 @@ import click import bench from bench.app import use_rq from bench.config.common_site_config import get_config -from bench.utils import find_executable +from bench.utils import which def setup_procfile(bench_path, yes=False, skip_redis=False): @@ -19,7 +19,7 @@ def setup_procfile(bench_path, yes=False, skip_redis=False): abort=True) procfile = bench.config.env().get_template('Procfile').render( - node=find_executable("node") or find_executable("nodejs"), + node=which("node") or which("nodejs"), use_rq=use_rq(bench_path), webserver_port=config.get('webserver_port'), CI=os.environ.get('CI'), diff --git a/bench/config/production_setup.py b/bench/config/production_setup.py index 6809b733..13640e6f 100755 --- a/bench/config/production_setup.py +++ b/bench/config/production_setup.py @@ -9,7 +9,7 @@ from bench.config.common_site_config import get_config from bench.config.nginx import make_nginx_conf from bench.config.supervisor import generate_supervisor_config, update_supervisord_config from bench.config.systemd import generate_systemd_config -from bench.utils import CommandFailedError, exec_cmd, find_executable, fix_prod_setup_perms, get_bench_name, get_cmd_output, log +from bench.utils import CommandFailedError, exec_cmd, which, fix_prod_setup_perms, get_bench_name, get_cmd_output, log logger = logging.getLogger(bench.PROJECT_NAME) @@ -17,13 +17,13 @@ logger = logging.getLogger(bench.PROJECT_NAME) def setup_production_prerequisites(): """Installs ansible, fail2banc, NGINX and supervisor""" - if not find_executable("ansible"): + if not which("ansible"): exec_cmd("sudo {0} -m pip install ansible".format(sys.executable)) - if not find_executable("fail2ban-client"): + if not which("fail2ban-client"): exec_cmd("bench setup role fail2ban") - if not find_executable("nginx"): + if not which("nginx"): exec_cmd("bench setup role nginx") - if not find_executable("supervisord"): + if not which("supervisord"): exec_cmd("bench setup role supervisor") @@ -95,11 +95,11 @@ def disable_production(bench_path='.'): def service(service_name, service_option): - if os.path.basename(find_executable('systemctl') or '') == 'systemctl' and is_running_systemd(): + if os.path.basename(which('systemctl') or '') == 'systemctl' and is_running_systemd(): systemctl_cmd = "sudo {service_manager} {service_option} {service_name}" exec_cmd(systemctl_cmd.format(service_manager='systemctl', service_option=service_option, service_name=service_name)) - elif os.path.basename(find_executable('service') or '') == 'service': + elif os.path.basename(which('service') or '') == 'service': service_cmd = "sudo {service_manager} {service_name} {service_option}" exec_cmd(service_cmd.format(service_manager='service', service_name=service_name, service_option=service_option)) @@ -145,7 +145,7 @@ def is_running_systemd(): def reload_supervisor(): - supervisorctl = find_executable('supervisorctl') + supervisorctl = which('supervisorctl') try: # first try reread/update @@ -178,7 +178,7 @@ def reload_supervisor(): def reload_nginx(): try: - exec_cmd('sudo {0} -t'.format(find_executable('nginx'))) + exec_cmd('sudo {0} -t'.format(which('nginx'))) except: raise diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 76ffd877..0cea009e 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -6,7 +6,7 @@ import os # imports - module imports import bench from bench.app import use_rq -from bench.utils import get_bench_name, find_executable +from bench.utils import get_bench_name, which from bench.config.common_site_config import get_config, update_config, get_gunicorn_workers # imports - third party imports @@ -31,8 +31,8 @@ def generate_supervisor_config(bench_path, user=None, yes=False, skip_redis=Fals "user": user, "use_rq": use_rq(bench_path), "http_timeout": config.get("http_timeout", 120), - "redis_server": find_executable('redis-server'), - "node": find_executable('node') or find_executable('nodejs'), + "redis_server": which('redis-server'), + "node": which('node') or which('nodejs'), "redis_cache_config": os.path.join(bench_dir, 'config', 'redis_cache.conf'), "redis_socketio_config": os.path.join(bench_dir, 'config', 'redis_socketio.conf'), "redis_queue_config": os.path.join(bench_dir, 'config', 'redis_queue.conf'), @@ -40,7 +40,7 @@ def generate_supervisor_config(bench_path, user=None, yes=False, skip_redis=Fals "gunicorn_workers": config.get('gunicorn_workers', get_gunicorn_workers()["gunicorn_workers"]), "bench_name": get_bench_name(bench_path), "background_workers": config.get('background_workers') or 1, - "bench_cmd": find_executable('bench'), + "bench_cmd": which('bench'), "skip_redis": skip_redis, }) diff --git a/bench/config/systemd.py b/bench/config/systemd.py index c0a0053c..1501d455 100644 --- a/bench/config/systemd.py +++ b/bench/config/systemd.py @@ -9,7 +9,7 @@ import click import bench from bench.app import use_rq from bench.config.common_site_config import get_config, get_gunicorn_workers, update_config -from bench.utils import exec_cmd, find_executable, get_bench_name +from bench.utils import exec_cmd, which, get_bench_name def generate_systemd_config(bench_path, user=None, yes=False, @@ -53,8 +53,8 @@ def generate_systemd_config(bench_path, user=None, yes=False, "user": user, "use_rq": use_rq(bench_path), "http_timeout": config.get("http_timeout", 120), - "redis_server": find_executable('redis-server'), - "node": find_executable('node') or find_executable('nodejs'), + "redis_server": which('redis-server'), + "node": which('node') or which('nodejs'), "redis_cache_config": os.path.join(bench_dir, 'config', 'redis_cache.conf'), "redis_socketio_config": os.path.join(bench_dir, 'config', 'redis_socketio.conf'), "redis_queue_config": os.path.join(bench_dir, 'config', 'redis_queue.conf'), @@ -62,7 +62,7 @@ def generate_systemd_config(bench_path, user=None, yes=False, "gunicorn_workers": config.get('gunicorn_workers', get_gunicorn_workers()["gunicorn_workers"]), "bench_name": get_bench_name(bench_path), "worker_target_wants": " ".join(background_workers), - "bench_cmd": find_executable('bench') + "bench_cmd": which('bench') } if not yes: diff --git a/bench/tests/test_base.py b/bench/tests/test_base.py index 73cc90f5..ec38c813 100644 --- a/bench/tests/test_base.py +++ b/bench/tests/test_base.py @@ -90,7 +90,6 @@ class TestBenchBase(unittest.TestCase): python=sys.executable, no_procfile=True, no_backups=True, - skip_assets=True, frappe_path=frappe_tmp_path )) diff --git a/bench/tests/test_init.py b/bench/tests/test_init.py index 2f409899..8ac7f548 100755 --- a/bench/tests/test_init.py +++ b/bench/tests/test_init.py @@ -97,7 +97,7 @@ class TestBenchInit(TestBenchBase): def test_get_app(self): self.init_bench("test-bench") bench_path = os.path.join(self.benches_path, "test-bench") - bench.utils.exec_cmd("bench get-app frappe_theme --skip-assets", cwd=bench_path) + bench.utils.exec_cmd("bench get-app frappe_theme", cwd=bench_path) self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "frappe_theme"))) app_installed_in_env = "frappe_theme" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8') self.assertTrue(app_installed_in_env) @@ -111,22 +111,22 @@ class TestBenchInit(TestBenchBase): self.init_bench(bench_name) bench.utils.exec_cmd("bench setup requirements --node", cwd=bench_path) bench.utils.exec_cmd("bench build", cwd=bench_path) - bench.utils.exec_cmd("bench get-app erpnext --branch {0}".format(FRAPPE_BRANCH), cwd=bench_path) + bench.utils.exec_cmd("bench get-app frappe_theme --branch master", cwd=bench_path) - self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "erpnext"))) + self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "frappe_theme"))) # check if app is installed - app_installed_in_env = "erpnext" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8') + app_installed_in_env = "frappe_theme" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8') self.assertTrue(app_installed_in_env) # create and install app on site self.new_site(site_name, bench_name) - installed_erpnext = not bench.utils.exec_cmd("bench --site {0} install-app erpnext".format(site_name), cwd=bench_path) + installed_app = not bench.utils.exec_cmd("bench --site {0} install-app frappe_theme".format(site_name), cwd=bench_path) app_installed_on_site = subprocess.check_output(["bench", "--site", site_name, "list-apps"], cwd=bench_path).decode('utf8') - if installed_erpnext: - self.assertTrue("erpnext" in app_installed_on_site) + if installed_app: + self.assertTrue("frappe_theme" in app_installed_on_site) def test_remove_app(self): @@ -134,13 +134,13 @@ class TestBenchInit(TestBenchBase): bench_path = os.path.join(self.benches_path, "test-bench") bench.utils.exec_cmd("bench setup requirements --node", cwd=bench_path) - bench.utils.exec_cmd("bench get-app erpnext --branch version-12 --skip-assets --overwrite", cwd=bench_path) - bench.utils.exec_cmd("bench remove-app erpnext", cwd=bench_path) + bench.utils.exec_cmd("bench get-app frappe_theme --branch master --overwrite", cwd=bench_path) + bench.utils.exec_cmd("bench remove-app frappe_theme", cwd=bench_path) with open(os.path.join(bench_path, "sites", "apps.txt")) as f: - self.assertFalse("erpnext" in f.read()) - self.assertFalse("erpnext" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8')) - self.assertFalse(os.path.exists(os.path.join(bench_path, "apps", "erpnext"))) + self.assertFalse("frappe_theme" in f.read()) + self.assertFalse("frappe_theme" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8')) + self.assertFalse(os.path.exists(os.path.join(bench_path, "apps", "frappe_theme"))) def test_switch_to_branch(self): diff --git a/bench/utils.py b/bench/utils.py index 417eb12f..ede9b169 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -2,26 +2,17 @@ # -*- coding: utf-8 -*- # imports - standard imports -import compileall -import errno -import glob import grp import itertools import json import logging import os import pwd -import re -import select -import site import subprocess import sys -from datetime import datetime -from distutils.spawn import find_executable # imports - third party imports import click -from six import iteritems # imports - module imports import bench @@ -143,6 +134,8 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, try: os.makedirs(os.path.join(path, dirname)) except OSError as e: + import errno + if e.errno == errno.EEXIST: pass @@ -184,6 +177,7 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, def update(pull=False, apps=None, patch=False, build=False, requirements=False, backup=True, compile=True, force=False, reset=False, restart_supervisor=False, restart_systemd=False): """command: bench update""" + import re from bench import patches from bench.app import is_version_upgrade, pull_apps, validate_branch from bench.config.common_site_config import get_config, update_config @@ -244,9 +238,11 @@ def update(pull=False, apps=None, patch=False, build=False, requirements=False, post_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) if pull and compile: + from compileall import compile_dir + print('Compiling Python files...') apps_dir = os.path.join(bench_path, 'apps') - compileall.compile_dir(apps_dir, quiet=1, rx=re.compile('.*node_modules.*')) + compile_dir(apps_dir, quiet=1, rx=re.compile('.*node_modules.*')) if restart_supervisor or conf.get('restart_supervisor_on_update'): restart_supervisor_processes(bench_path=bench_path) @@ -268,7 +264,7 @@ def copy_patches_txt(bench_path): def clone_apps_from(bench_path, clone_from, update_app=True): - from .app import install_app + from bench.app import install_app print('Copying apps from {0}...'.format(clone_from)) subprocess.check_output(['cp', '-R', os.path.join(clone_from, 'apps'), bench_path]) @@ -316,7 +312,9 @@ def exec_cmd(cmd, cwd='.'): logger.warning("{0} executed with exit code {1}".format(cmd_log, return_code)) -def which(executable, raise_err = False): +def which(executable, raise_err=False): + from distutils.spawn import find_executable + exec_ = find_executable(executable) if not exec_ and raise_err: @@ -356,10 +354,11 @@ def setup_socketio(bench_path='.'): def patch_sites(bench_path='.'): - try: - run_frappe_cmd('--site', 'all', 'migrate', bench_path=bench_path) - except subprocess.CalledProcessError: - raise PatchError + for site in get_sites(bench_path=bench_path): + try: + migrate_site(site, bench_path=bench_path) + except subprocess.CalledProcessError: + raise PatchError def build_assets(bench_path='.', app=None): @@ -410,9 +409,9 @@ def setup_sudoers(user): template = bench.config.env().get_template('frappe_sudoers') frappe_sudoers = template.render(**{ 'user': user, - 'service': find_executable('service'), - 'systemctl': find_executable('systemctl'), - 'nginx': find_executable('nginx'), + 'service': which('service'), + 'systemctl': which('systemctl'), + 'nginx': which('nginx'), }) frappe_sudoers = safe_decode(frappe_sudoers) @@ -448,7 +447,7 @@ def setup_logging(bench_path='.'): def get_process_manager(): for proc_man in ['honcho', 'foreman', 'forego']: - proc_man_path = find_executable(proc_man) + proc_man_path = which(proc_man) if proc_man_path: return proc_man_path @@ -485,7 +484,7 @@ def get_git_version(): def check_git_for_shallow_clone(): - from .config.common_site_config import get_config + from bench.config.common_site_config import get_config config = get_config('.') if config: @@ -513,7 +512,7 @@ def get_cmd_output(cmd, cwd='.', _raise=True): def restart_supervisor_processes(bench_path='.', web_workers=False): - from .config.common_site_config import get_config + from bench.config.common_site_config import get_config conf = get_config(bench_path=bench_path) bench_name = get_bench_name(bench_path) @@ -599,7 +598,7 @@ def update_node_packages(bench_path='.'): def update_yarn_packages(bench_path='.'): apps_dir = os.path.join(bench_path, 'apps') - if not find_executable('yarn'): + if not which('yarn'): print("Please install yarn using below command and try again.") print("`npm install -g yarn`") return @@ -620,6 +619,8 @@ def update_npm_packages(bench_path='.'): if os.path.exists(package_json_path): with open(package_json_path, "r") as f: + from six import iteritems + app_package_json = json.loads(f.read()) # package.json is usually a dict in a dict for key, value in iteritems(app_package_json): @@ -643,6 +644,10 @@ def update_npm_packages(bench_path='.'): exec_cmd('npm install', cwd=bench_path) +def migrate_site(site, bench_path='.'): + run_frappe_cmd('--site', site, 'migrate', bench_path=bench_path) + + def backup_site(site, bench_path='.'): run_frappe_cmd('--site', site, 'backup', bench_path=bench_path) @@ -653,9 +658,7 @@ def backup_all_sites(bench_path='.'): def is_root(): - if os.getuid() == 0: - return True - return False + return os.getuid() == 0 def set_mariadb_host(host, bench_path='.'): @@ -713,7 +716,8 @@ def drop_privileges(uid_name='nobody', gid_name='nogroup'): def fix_prod_setup_perms(bench_path='.', frappe_user=None): - from .config.common_site_config import get_config + from glob import glob + from bench.config.common_site_config import get_config if not frappe_user: frappe_user = get_config(bench_path).get('frappe_user') @@ -724,14 +728,14 @@ def fix_prod_setup_perms(bench_path='.', frappe_user=None): globs = ["logs/*", "config/*"] for glob_name in globs: - for path in glob.glob(glob_name): + for path in glob(glob_name): uid = pwd.getpwnam(frappe_user).pw_uid gid = grp.getgrnam(frappe_user).gr_gid os.chown(path, uid, gid) def run_frappe_cmd(*args, **kwargs): - from .cli import from_command_line + from bench.cli import from_command_line bench_path = kwargs.get('bench_path', '.') f = get_env_cmd('python', bench_path=bench_path) @@ -757,15 +761,15 @@ def run_frappe_cmd(*args, **kwargs): def validate_upgrade(from_ver, to_ver, bench_path='.'): if to_ver >= 6: - if not find_executable('npm') and not (find_executable('node') or find_executable('nodejs')): + if not which('npm') and not (which('node') or which('nodejs')): raise Exception("Please install nodejs and npm") def post_upgrade(from_ver, to_ver, bench_path='.'): - from .config.common_site_config import get_config - from .config import redis - from .config.supervisor import generate_supervisor_config - from .config.nginx import make_nginx_conf + from bench.config.common_site_config import get_config + from bench.config import redis + from bench.config.supervisor import generate_supervisor_config + from bench.config.nginx import make_nginx_conf conf = get_config(bench_path=bench_path) print("-" * 80 + "Your bench was upgraded to version {0}".format(to_ver)) @@ -843,8 +847,10 @@ def update_translations(app, lang): def print_output(p): + from select import select + while p.poll() is None: - readx = select.select([p.stdout.fileno(), p.stderr.fileno()], [], [])[0] + readx = select([p.stdout.fileno(), p.stderr.fileno()], [], [])[0] send_buffer = [] for fd in readx: if fd == p.stdout.fileno(): @@ -905,7 +911,7 @@ def set_git_remote_url(git_url, bench_path='.'): def run_playbook(playbook_name, extra_vars=None, tag=None): - if not find_executable('ansible'): + if not which('ansible'): print("Ansible is needed to run this command, please install it using 'pip install ansible'") sys.exit(1) args = ['ansible-playbook', '-c', 'local', playbook_name, '-vvvv'] @@ -977,6 +983,8 @@ def migrate_env(python, backup=False): # Backup venv: restore using `virtualenv --relocatable` if needed if backup: + from datetime import datetime + parch = os.path.join(path, 'archived_envs') if not os.path.exists(parch): os.mkdir(parch)