Merge pull request #1157 from gavindsouza/b-update-breaking-patch

fix: Misc fixes

bench/app.py

@@ -443,9 +443,6 @@ def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrad
 def switch_to_branch(branch=None, apps=None, bench_path='.', upgrade=False):
 	switch_branch(branch, apps=apps, bench_path=bench_path, upgrade=upgrade)
 
-def switch_to_master(apps=None, bench_path='.', upgrade=True):
-	switch_branch('master', apps=apps, bench_path=bench_path, upgrade=upgrade)
-
 def switch_to_develop(apps=None, bench_path='.', upgrade=True):
 	switch_branch('develop', apps=apps, bench_path=bench_path, upgrade=upgrade)
 

bench/commands/__init__.py

@@ -27,11 +27,10 @@ bench_command.add_command(include_app_for_update)
 bench_command.add_command(pip)
 
 
-from bench.commands.update import update, retry_upgrade, switch_to_branch, switch_to_master, switch_to_develop
+from bench.commands.update import update, retry_upgrade, switch_to_branch, switch_to_develop
 bench_command.add_command(update)
 bench_command.add_command(retry_upgrade)
 bench_command.add_command(switch_to_branch)
-bench_command.add_command(switch_to_master)
 bench_command.add_command(switch_to_develop)
 
 

bench/commands/update.py

@@ -41,12 +41,6 @@ def switch_to_branch(branch, apps, upgrade=False):
 	switch_to_branch(branch=branch, apps=list(apps), upgrade=upgrade)
 
 
-@click.command('switch-to-master', help="[DEPRECATED]: Switch frappe and erpnext to master branch")
-def switch_to_master():
-	from bench.utils import log
-	log("`switch-to-master` has been deprecated as master branches were renamed to version-11")
-
-
 @click.command('switch-to-develop')
 def switch_to_develop(upgrade=False):
 	"Switch frappe and erpnext to develop branch"

bench/config/procfile.py

@@ -8,7 +8,7 @@ import click
 import bench
 from bench.app import use_rq
 from bench.config.common_site_config import get_config
-from bench.utils import find_executable
+from bench.utils import which
 
 
 def setup_procfile(bench_path, yes=False, skip_redis=False):
@@ -19,7 +19,7 @@ def setup_procfile(bench_path, yes=False, skip_redis=False):
 			abort=True)
 
 	procfile = bench.config.env().get_template('Procfile').render(
-		node=find_executable("node") or find_executable("nodejs"),
+		node=which("node") or which("nodejs"),
 		use_rq=use_rq(bench_path),
 		webserver_port=config.get('webserver_port'),
 		CI=os.environ.get('CI'),

bench/config/production_setup.py

@@ -9,7 +9,7 @@ from bench.config.common_site_config import get_config
 from bench.config.nginx import make_nginx_conf
 from bench.config.supervisor import generate_supervisor_config, update_supervisord_config
 from bench.config.systemd import generate_systemd_config
-from bench.utils import CommandFailedError, exec_cmd, find_executable, fix_prod_setup_perms, get_bench_name, get_cmd_output, log
+from bench.utils import CommandFailedError, exec_cmd, which, fix_prod_setup_perms, get_bench_name, get_cmd_output, log
 
 
 logger = logging.getLogger(bench.PROJECT_NAME)
@@ -17,13 +17,13 @@ logger = logging.getLogger(bench.PROJECT_NAME)
 
 def setup_production_prerequisites():
 	"""Installs ansible, fail2banc, NGINX and supervisor"""
-	if not find_executable("ansible"):
+	if not which("ansible"):
 		exec_cmd("sudo {0} -m pip install ansible".format(sys.executable))
-	if not find_executable("fail2ban-client"):
+	if not which("fail2ban-client"):
 		exec_cmd("bench setup role fail2ban")
-	if not find_executable("nginx"):
+	if not which("nginx"):
 		exec_cmd("bench setup role nginx")
-	if not find_executable("supervisord"):
+	if not which("supervisord"):
 		exec_cmd("bench setup role supervisor")
 
 
@@ -95,11 +95,11 @@ def disable_production(bench_path='.'):
 
 
 def service(service_name, service_option):
-	if os.path.basename(find_executable('systemctl') or '') == 'systemctl' and is_running_systemd():
+	if os.path.basename(which('systemctl') or '') == 'systemctl' and is_running_systemd():
 		systemctl_cmd = "sudo {service_manager} {service_option} {service_name}"
 		exec_cmd(systemctl_cmd.format(service_manager='systemctl', service_option=service_option, service_name=service_name))
 
-	elif os.path.basename(find_executable('service') or '') == 'service':
+	elif os.path.basename(which('service') or '') == 'service':
 		service_cmd = "sudo {service_manager} {service_name} {service_option}"
 		exec_cmd(service_cmd.format(service_manager='service', service_name=service_name, service_option=service_option))
 
@@ -145,7 +145,7 @@ def is_running_systemd():
 
 
 def reload_supervisor():
-	supervisorctl = find_executable('supervisorctl')
+	supervisorctl = which('supervisorctl')
 
 	try:
 		# first try reread/update
@@ -178,7 +178,7 @@ def reload_supervisor():
 
 def reload_nginx():
 	try:
-		exec_cmd('sudo {0} -t'.format(find_executable('nginx')))
+		exec_cmd('sudo {0} -t'.format(which('nginx')))
 	except:
 		raise
 

bench/config/supervisor.py

@@ -6,7 +6,7 @@ import os
 # imports - module imports
 import bench
 from bench.app import use_rq
-from bench.utils import get_bench_name, find_executable
+from bench.utils import get_bench_name, which
 from bench.config.common_site_config import get_config, update_config, get_gunicorn_workers
 
 # imports - third party imports
@@ -31,8 +31,8 @@ def generate_supervisor_config(bench_path, user=None, yes=False, skip_redis=Fals
 		"user": user,
 		"use_rq": use_rq(bench_path),
 		"http_timeout": config.get("http_timeout", 120),
-		"redis_server": find_executable('redis-server'),
+		"redis_server": which('redis-server'),
-		"node": find_executable('node') or find_executable('nodejs'),
+		"node": which('node') or which('nodejs'),
 		"redis_cache_config": os.path.join(bench_dir, 'config', 'redis_cache.conf'),
 		"redis_socketio_config": os.path.join(bench_dir, 'config', 'redis_socketio.conf'),
 		"redis_queue_config": os.path.join(bench_dir, 'config', 'redis_queue.conf'),
@@ -40,7 +40,7 @@ def generate_supervisor_config(bench_path, user=None, yes=False, skip_redis=Fals
 		"gunicorn_workers": config.get('gunicorn_workers', get_gunicorn_workers()["gunicorn_workers"]),
 		"bench_name": get_bench_name(bench_path),
 		"background_workers": config.get('background_workers') or 1,
-		"bench_cmd": find_executable('bench'),
+		"bench_cmd": which('bench'),
 		"skip_redis": skip_redis,
 	})
 

bench/config/systemd.py

@@ -9,7 +9,7 @@ import click
 import bench
 from bench.app import use_rq
 from bench.config.common_site_config import get_config, get_gunicorn_workers, update_config
-from bench.utils import exec_cmd, find_executable, get_bench_name
+from bench.utils import exec_cmd, which, get_bench_name
 
 
 def generate_systemd_config(bench_path, user=None, yes=False,
@@ -53,8 +53,8 @@ def generate_systemd_config(bench_path, user=None, yes=False,
 		"user": user,
 		"use_rq": use_rq(bench_path),
 		"http_timeout": config.get("http_timeout", 120),
-		"redis_server": find_executable('redis-server'),
+		"redis_server": which('redis-server'),
-		"node": find_executable('node') or find_executable('nodejs'),
+		"node": which('node') or which('nodejs'),
 		"redis_cache_config": os.path.join(bench_dir, 'config', 'redis_cache.conf'),
 		"redis_socketio_config": os.path.join(bench_dir, 'config', 'redis_socketio.conf'),
 		"redis_queue_config": os.path.join(bench_dir, 'config', 'redis_queue.conf'),
@@ -62,7 +62,7 @@ def generate_systemd_config(bench_path, user=None, yes=False,
 		"gunicorn_workers": config.get('gunicorn_workers', get_gunicorn_workers()["gunicorn_workers"]),
 		"bench_name": get_bench_name(bench_path),
 		"worker_target_wants": " ".join(background_workers),
-		"bench_cmd": find_executable('bench')
+		"bench_cmd": which('bench')
 	}
 
 	if not yes:

bench/tests/test_base.py

@@ -90,7 +90,6 @@ class TestBenchBase(unittest.TestCase):
 			python=sys.executable,
 			no_procfile=True,
 			no_backups=True,
-			skip_assets=True,
 			frappe_path=frappe_tmp_path
 		))
 

bench/tests/test_init.py

@@ -97,7 +97,7 @@ class TestBenchInit(TestBenchBase):
 	def test_get_app(self):
 		self.init_bench("test-bench")
 		bench_path = os.path.join(self.benches_path, "test-bench")
-		bench.utils.exec_cmd("bench get-app frappe_theme --skip-assets", cwd=bench_path)
+		bench.utils.exec_cmd("bench get-app frappe_theme", cwd=bench_path)
 		self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "frappe_theme")))
 		app_installed_in_env = "frappe_theme" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8')
 		self.assertTrue(app_installed_in_env)
@@ -111,22 +111,22 @@ class TestBenchInit(TestBenchBase):
 		self.init_bench(bench_name)
 		bench.utils.exec_cmd("bench setup requirements --node", cwd=bench_path)
 		bench.utils.exec_cmd("bench build", cwd=bench_path)
-		bench.utils.exec_cmd("bench get-app erpnext --branch {0}".format(FRAPPE_BRANCH), cwd=bench_path)
+		bench.utils.exec_cmd("bench get-app frappe_theme --branch master", cwd=bench_path)
 
-		self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "erpnext")))
+		self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "frappe_theme")))
 
 		# check if app is installed
-		app_installed_in_env = "erpnext" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8')
+		app_installed_in_env = "frappe_theme" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8')
 		self.assertTrue(app_installed_in_env)
 
 		# create and install app on site
 		self.new_site(site_name, bench_name)
-		installed_erpnext = not bench.utils.exec_cmd("bench --site {0} install-app erpnext".format(site_name), cwd=bench_path)
+		installed_app = not bench.utils.exec_cmd("bench --site {0} install-app frappe_theme".format(site_name), cwd=bench_path)
 
 		app_installed_on_site = subprocess.check_output(["bench", "--site", site_name, "list-apps"], cwd=bench_path).decode('utf8')
 
-		if installed_erpnext:
+		if installed_app:
-			self.assertTrue("erpnext" in app_installed_on_site)
+			self.assertTrue("frappe_theme" in app_installed_on_site)
 
 
 	def test_remove_app(self):
@@ -134,13 +134,13 @@ class TestBenchInit(TestBenchBase):
 		bench_path = os.path.join(self.benches_path, "test-bench")
 
 		bench.utils.exec_cmd("bench setup requirements --node", cwd=bench_path)
-		bench.utils.exec_cmd("bench get-app erpnext --branch version-12 --skip-assets --overwrite", cwd=bench_path)
+		bench.utils.exec_cmd("bench get-app frappe_theme --branch master --overwrite", cwd=bench_path)
-		bench.utils.exec_cmd("bench remove-app erpnext", cwd=bench_path)
+		bench.utils.exec_cmd("bench remove-app frappe_theme", cwd=bench_path)
 
 		with open(os.path.join(bench_path, "sites", "apps.txt")) as f:
-			self.assertFalse("erpnext" in f.read())
+			self.assertFalse("frappe_theme" in f.read())
-		self.assertFalse("erpnext" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8'))
+		self.assertFalse("frappe_theme" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8'))
-		self.assertFalse(os.path.exists(os.path.join(bench_path, "apps", "erpnext")))
+		self.assertFalse(os.path.exists(os.path.join(bench_path, "apps", "frappe_theme")))
 
 
 	def test_switch_to_branch(self):

bench/utils.py

@@ -2,26 +2,17 @@
 # -*- coding: utf-8 -*-
 
 # imports - standard imports
-import compileall
-import errno
-import glob
 import grp
 import itertools
 import json
 import logging
 import os
 import pwd
-import re
-import select
-import site
 import subprocess
 import sys
-from datetime import datetime
-from distutils.spawn import find_executable
 
 # imports - third party imports
 import click
-from six import iteritems
 
 # imports - module imports
 import bench
@@ -143,6 +134,8 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False,
 		try:
 			os.makedirs(os.path.join(path, dirname))
 		except OSError as e:
+			import errno
+
 			if e.errno == errno.EEXIST:
 				pass
 
@@ -184,6 +177,7 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False,
 def update(pull=False, apps=None, patch=False, build=False, requirements=False, backup=True, compile=True,
 	force=False, reset=False, restart_supervisor=False, restart_systemd=False):
 	"""command: bench update"""
+	import re
 	from bench import patches
 	from bench.app import is_version_upgrade, pull_apps, validate_branch
 	from bench.config.common_site_config import get_config, update_config
@@ -244,9 +238,11 @@ def update(pull=False, apps=None, patch=False, build=False, requirements=False,
 		post_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path)
 
 	if pull and compile:
+		from compileall import compile_dir
+
 		print('Compiling Python files...')
 		apps_dir = os.path.join(bench_path, 'apps')
-		compileall.compile_dir(apps_dir, quiet=1, rx=re.compile('.*node_modules.*'))
+		compile_dir(apps_dir, quiet=1, rx=re.compile('.*node_modules.*'))
 
 	if restart_supervisor or conf.get('restart_supervisor_on_update'):
 		restart_supervisor_processes(bench_path=bench_path)
@@ -268,7 +264,7 @@ def copy_patches_txt(bench_path):
 
 
 def clone_apps_from(bench_path, clone_from, update_app=True):
-	from .app import install_app
+	from bench.app import install_app
 	print('Copying apps from {0}...'.format(clone_from))
 	subprocess.check_output(['cp', '-R', os.path.join(clone_from, 'apps'), bench_path])
 
@@ -316,7 +312,9 @@ def exec_cmd(cmd, cwd='.'):
 		logger.warning("{0} executed with exit code {1}".format(cmd_log, return_code))
 
 
-def which(executable, raise_err = False):
+def which(executable, raise_err=False):
+	from distutils.spawn import find_executable
+
 	exec_ = find_executable(executable)
 
 	if not exec_ and raise_err:
@@ -356,10 +354,11 @@ def setup_socketio(bench_path='.'):
 
 
 def patch_sites(bench_path='.'):
-	try:
+	for site in get_sites(bench_path=bench_path):
-		run_frappe_cmd('--site', 'all', 'migrate', bench_path=bench_path)
+		try:
-	except subprocess.CalledProcessError:
+			migrate_site(site, bench_path=bench_path)
-		raise PatchError
+		except subprocess.CalledProcessError:
+			raise PatchError
 
 
 def build_assets(bench_path='.', app=None):
@@ -410,9 +409,9 @@ def setup_sudoers(user):
 	template = bench.config.env().get_template('frappe_sudoers')
 	frappe_sudoers = template.render(**{
 		'user': user,
-		'service': find_executable('service'),
+		'service': which('service'),
-		'systemctl': find_executable('systemctl'),
+		'systemctl': which('systemctl'),
-		'nginx': find_executable('nginx'),
+		'nginx': which('nginx'),
 	})
 	frappe_sudoers = safe_decode(frappe_sudoers)
 
@@ -448,7 +447,7 @@ def setup_logging(bench_path='.'):
 
 def get_process_manager():
 	for proc_man in ['honcho', 'foreman', 'forego']:
-		proc_man_path = find_executable(proc_man)
+		proc_man_path = which(proc_man)
 		if proc_man_path:
 			return proc_man_path
 
@@ -485,7 +484,7 @@ def get_git_version():
 
 
 def check_git_for_shallow_clone():
-	from .config.common_site_config import get_config
+	from bench.config.common_site_config import get_config
 	config = get_config('.')
 
 	if config:
@@ -513,7 +512,7 @@ def get_cmd_output(cmd, cwd='.', _raise=True):
 
 
 def restart_supervisor_processes(bench_path='.', web_workers=False):
-	from .config.common_site_config import get_config
+	from bench.config.common_site_config import get_config
 	conf = get_config(bench_path=bench_path)
 	bench_name = get_bench_name(bench_path)
 
@@ -599,7 +598,7 @@ def update_node_packages(bench_path='.'):
 def update_yarn_packages(bench_path='.'):
 	apps_dir = os.path.join(bench_path, 'apps')
 
-	if not find_executable('yarn'):
+	if not which('yarn'):
 		print("Please install yarn using below command and try again.")
 		print("`npm install -g yarn`")
 		return
@@ -620,6 +619,8 @@ def update_npm_packages(bench_path='.'):
 
 		if os.path.exists(package_json_path):
 			with open(package_json_path, "r") as f:
+				from six import iteritems
+
 				app_package_json = json.loads(f.read())
 				# package.json is usually a dict in a dict
 				for key, value in iteritems(app_package_json):
@@ -643,6 +644,10 @@ def update_npm_packages(bench_path='.'):
 	exec_cmd('npm install', cwd=bench_path)
 
 
+def migrate_site(site, bench_path='.'):
+	run_frappe_cmd('--site', site, 'migrate', bench_path=bench_path)
+
+
 def backup_site(site, bench_path='.'):
 	run_frappe_cmd('--site', site, 'backup', bench_path=bench_path)
 
@@ -653,9 +658,7 @@ def backup_all_sites(bench_path='.'):
 
 
 def is_root():
-	if os.getuid() == 0:
+	return os.getuid() == 0
-		return True
-	return False
 
 
 def set_mariadb_host(host, bench_path='.'):
@@ -713,7 +716,8 @@ def drop_privileges(uid_name='nobody', gid_name='nogroup'):
 
 
 def fix_prod_setup_perms(bench_path='.', frappe_user=None):
-	from .config.common_site_config import get_config
+	from glob import glob
+	from bench.config.common_site_config import get_config
 
 	if not frappe_user:
 		frappe_user = get_config(bench_path).get('frappe_user')
@@ -724,14 +728,14 @@ def fix_prod_setup_perms(bench_path='.', frappe_user=None):
 
 	globs = ["logs/*", "config/*"]
 	for glob_name in globs:
-		for path in glob.glob(glob_name):
+		for path in glob(glob_name):
 			uid = pwd.getpwnam(frappe_user).pw_uid
 			gid = grp.getgrnam(frappe_user).gr_gid
 			os.chown(path, uid, gid)
 
 
 def run_frappe_cmd(*args, **kwargs):
-	from .cli import from_command_line
+	from bench.cli import from_command_line
 
 	bench_path = kwargs.get('bench_path', '.')
 	f = get_env_cmd('python', bench_path=bench_path)
@@ -757,15 +761,15 @@ def run_frappe_cmd(*args, **kwargs):
 
 def validate_upgrade(from_ver, to_ver, bench_path='.'):
 	if to_ver >= 6:
-		if not find_executable('npm') and not (find_executable('node') or find_executable('nodejs')):
+		if not which('npm') and not (which('node') or which('nodejs')):
 			raise Exception("Please install nodejs and npm")
 
 
 def post_upgrade(from_ver, to_ver, bench_path='.'):
-	from .config.common_site_config import get_config
+	from bench.config.common_site_config import get_config
-	from .config import redis
+	from bench.config import redis
-	from .config.supervisor import generate_supervisor_config
+	from bench.config.supervisor import generate_supervisor_config
-	from .config.nginx import make_nginx_conf
+	from bench.config.nginx import make_nginx_conf
 	conf = get_config(bench_path=bench_path)
 	print("-" * 80 + "Your bench was upgraded to version {0}".format(to_ver))
 
@@ -843,8 +847,10 @@ def update_translations(app, lang):
 
 
 def print_output(p):
+	from select import select
+
 	while p.poll() is None:
-		readx = select.select([p.stdout.fileno(), p.stderr.fileno()], [], [])[0]
+		readx = select([p.stdout.fileno(), p.stderr.fileno()], [], [])[0]
 		send_buffer = []
 		for fd in readx:
 			if fd == p.stdout.fileno():
@@ -905,7 +911,7 @@ def set_git_remote_url(git_url, bench_path='.'):
 
 
 def run_playbook(playbook_name, extra_vars=None, tag=None):
-	if not find_executable('ansible'):
+	if not which('ansible'):
 		print("Ansible is needed to run this command, please install it using 'pip install ansible'")
 		sys.exit(1)
 	args = ['ansible-playbook', '-c', 'local', playbook_name, '-vvvv']
@@ -977,6 +983,8 @@ def migrate_env(python, backup=False):
 
 	# Backup venv: restore using `virtualenv --relocatable` if needed
 	if backup:
+		from datetime import datetime
+
 		parch = os.path.join(path, 'archived_envs')
 		if not os.path.exists(parch):
 			os.mkdir(parch)