Merge branch master into wkhtmltopdf-playbooks-update-v0.12.5

.deepsource.toml

@@ -0,0 +1,16 @@
+version = 1
+
+exclude_patterns = [
+  ".*"
+]
+
+test_patterns = [
+  "bench/tests/**"
+]
+
+[[analyzers]]
+name = "python"
+enabled = true
+dependency_file_paths = [
+  "requirements.txt"
+]

.travis.yml

@@ -1,26 +1,96 @@
 language: python
-dist: xenial
+dist: bionic
+sudo: true
 
-python:
-  - "2.7"
+git:
+  depth: 1
+
+cache:
+  - pip
+  - npm
+  - yarn
+
+addons:
+  mariadb: '10.3'
+
+matrix:
+  include:
+  - name: "Python 2.7 Basic Setup"
+    python: 2.7
+    env: TEST=bench
+    script: python -m unittest -v bench.tests.test_init
+
+  - name: "Python 3.6 Basic Setup"
+    python: 3.6
+    env: TEST=bench
+    script: python -m unittest -v bench.tests.test_init
+
+  - name: "Python 3.7 Basic Setup"
+    python: 3.7
+    env: TEST=bench
+    script: python -m unittest -v bench.tests.test_init
+
+  - name: "Python 3.8 Production Setup"
+    python: 3.8
+    env: TEST=bench
+    script: python -m unittest -v bench.tests.test_setup_production
+
+  - name: "Python 2.7 Production Setup"
+    python: 2.7
+    env: TEST=bench
+    script: python -m unittest -v bench.tests.test_setup_production
+
+  - name: "Python 3.6 Production Setup"
+    python: 3.6
+    env: TEST=bench
+    script: python -m unittest -v bench.tests.test_setup_production
+
+  - name: "Python 3.7 Production Setup"
+    python: 3.7
+    env: TEST=bench
+    script: python -m unittest -v bench.tests.test_setup_production
+
+  - name: "Python 3.8 Production Setup"
+    python: 3.8
+    env: TEST=bench
+    script: python -m unittest -v bench.tests.test_setup_production
+
+  - name: "Python 3.6 Easy Install"
+    python: 3.6
+    env: TEST=easy_install
+    script: sudo python $TRAVIS_BUILD_DIR/playbooks/install.py --user travis --run-travis --production --verbose
+
+  - name: "Python 3.7 Easy Install"
+    python: 3.7
+    env: TEST=easy_install
+    script: sudo python $TRAVIS_BUILD_DIR/playbooks/install.py --user travis --run-travis --production --verbose
+
+  - name: "Python 3.8 Easy Install"
+    python: 3.8
+    env: TEST=easy_install
+    script: sudo python $TRAVIS_BUILD_DIR/playbooks/install.py --user travis --run-travis --production --verbose
 
 install:
-  - sudo pip install urllib3 pyOpenSSL ndg-httpsclient pyasn1
-  - sudo apt-get purge -y mysql-common mysql-server mysql-client
-  - sudo apt-get install --only-upgrade -y git
-  - sudo apt-get install hhvm && rm -rf /home/travis/.kiex/
-  - mkdir -p ~/.bench
-  - mkdir -p /tmp/.bench
-  - cp -r $TRAVIS_BUILD_DIR/* ~/.bench
-  - cp -r $TRAVIS_BUILD_DIR/* /tmp/.bench
+  - pip install urllib3 pyOpenSSL ndg-httpsclient pyasn1
 
-  - sudo python $TRAVIS_BUILD_DIR/playbooks/install.py --user travis --run-travis --production --verbose
-  # - sudo bash $TRAVIS_BUILD_DIR/install_scripts/setup_frappe.sh --skip-install-bench --mysql-root-password travis
-  # - cd ~ && sudo python bench-repo/installer/install.py --only-dependencies
+  - if [ $TEST == "bench" ];then
+       wget -q -O /tmp/wkhtmltox.tar.xz https://github.com/frappe/wkhtmltopdf/raw/master/wkhtmltox-0.12.3_linux-generic-amd64.tar.xz;
+       tar -xf /tmp/wkhtmltox.tar.xz -C /tmp;
+       sudo mv /tmp/wkhtmltox/bin/wkhtmltopdf /usr/local/bin/wkhtmltopdf;
+       sudo chmod o+x /usr/local/bin/wkhtmltopdf;
 
-script:
-  - cd ~
-  - sudo pip install --upgrade pip
-  - sudo pip install -e ~/.bench
-  # - sudo python -m unittest bench.tests.test_setup_production.TestSetupProduction.test_setup_production_v6
-  - sudo python -m unittest -v bench.tests.test_setup_production
+      mkdir -p ~/.bench;
+      cp -r $TRAVIS_BUILD_DIR/* ~/.bench;
+      pip install -q -U -e ~/.bench;
+      sudo pip install -q -U -e ~/.bench;
+
+      mysql -u root -e "SET GLOBAL character_set_server = 'utf8mb4'";
+      mysql -u root -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'";
+      mysql -u root -e "UPDATE mysql.user SET Password=PASSWORD('travis') WHERE User='root'";
+      mysql -u root -e "FLUSH PRIVILEGES";
+    fi
+
+  - if [ $TEST == "easy_install" ];then
+      mkdir -p /tmp/.bench;
+      cp -r $TRAVIS_BUILD_DIR/* /tmp/.bench;
+    fi

README.md

@@ -117,12 +117,11 @@ Releases can be made for [Frappe](https://github.com/frappe/frappe) apps using b
 
 ---
 
-# Docker Install
+# Docker
 
-1. For developer setup, you can also use the official [Frappe Docker](https://github.com/frappe/frappe_docker/).
-2. The app, mariadb and redis run on individual containers.
-3. This setup supports multi-tenancy and exposes the frappe-bench volume as a external storage.
-4. For more details, [ead the instructions on the [Frappe Docker README](https://github.com/frappe/frappe_docker/)
+- For official images and resources [Frappe Docker](https://github.com/frappe/frappe_docker)
+- Production Installation [README](https://github.com/frappe/frappe_docker/blob/develop/README.md)
+- Developer Setup [README](https://github.com/frappe/frappe_docker/blob/develop/development/README.md)
 
 ---
 

bench/app.py

@@ -1,23 +1,32 @@
+# imports - compatibility imports
 from __future__ import print_function
-import os
-from .utils import (exec_cmd, get_frappe, check_git_for_shallow_clone, build_assets,
-	restart_supervisor_processes, get_cmd_output, run_frappe_cmd, CommandFailedError,
-	restart_systemd_processes)
-from .config.common_site_config import get_config
 
+# imports - standard imports
+import json
 import logging
+import os
+import re
+import shutil
+import subprocess
+import sys
+
+# imports - third party imports
+import click
+import git
 import requests
 import semantic_version
-import json
-import re
-import subprocess
-import bench
-import sys
-import shutil
+from six.moves import reload_module
 
-logging.basicConfig(level="DEBUG")
+# imports - module imports
+import bench
+from bench.config.common_site_config import get_config
+from bench.utils import CommandFailedError, build_assets, check_git_for_shallow_clone, exec_cmd, get_cmd_output, get_frappe, restart_supervisor_processes, restart_systemd_processes, run_frappe_cmd
+
+
+logging.basicConfig(level="INFO")
 logger = logging.getLogger(__name__)
 
+
 class InvalidBranchException(Exception): pass
 class InvalidRemoteException(Exception): pass
 
@@ -50,7 +59,7 @@ def write_appstxt(apps, bench_path='.'):
 	with open(os.path.join(bench_path, 'sites', 'apps.txt'), 'w') as f:
 		return f.write('\n'.join(apps))
 
-def check_url(url, raise_err = True):
+def check_url(url, raise_err=True):
 	try:
 		from urlparse import urlparse
 	except ImportError:
@@ -59,7 +68,7 @@ def check_url(url, raise_err = True):
 	parsed = urlparse(url)
 	if not parsed.scheme:
 		if raise_err:
-			raise TypeError('{url} Not a valid URL'.format(url = url))
+			raise TypeError('{url} Not a valid URL'.format(url=url))
 		else:
 			return False
 
@@ -92,59 +101,61 @@ def remove_from_excluded_apps_txt(app, bench_path='.'):
 		apps.remove(app)
 		return write_excluded_apps_txt(apps, bench_path=bench_path)
 
-def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=False,
-	postprocess = True):
-	# from bench.utils import check_url
-	try:
-		from urlparse import urljoin
-	except ImportError:
-		from urllib.parse import urljoin
+def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=False, postprocess=True, overwrite=False):
+	if not os.path.exists(git_url):
+		if not check_url(git_url, raise_err=False):
+			orgs = ['frappe', 'erpnext']
+			for org in orgs:
+				url = 'https://api.github.com/repos/{org}/{app}'.format(org=org, app=git_url)
+				res = requests.get(url)
+				if res.ok:
+					data = res.json()
+					if 'name' in data:
+						if git_url == data['name']:
+							git_url = 'https://github.com/{org}/{app}'.format(org=org, app=git_url)
+							break
 
-	if not check_url(git_url, raise_err = False):
-		orgs = ['frappe', 'erpnext']
-		for org in orgs:
-			url = 'https://api.github.com/repos/{org}/{app}'.format(org = org, app = git_url)
-			res = requests.get(url)
-			if res.ok:
-				data    = res.json()
-				if 'name' in data:
-					if git_url == data['name']:
-						git_url = 'https://github.com/{org}/{app}'.format(org = org, app = git_url)
-						break
+		# Gets repo name from URL
+		repo_name = git_url.rsplit('/', 1)[1].rsplit('.', 1)[0]
+		shallow_clone = '--depth 1' if check_git_for_shallow_clone() else ''
+		branch = '--branch {branch}'.format(branch=branch) if branch else ''
+	else:
+		repo_name = git_url.split(os.sep)[-1]
+		shallow_clone = ''
+		branch = '--branch {branch}'.format(branch=branch) if branch else ''
 
-	#Gets repo name from URL
-	repo_name = git_url.rsplit('/', 1)[1].rsplit('.', 1)[0]
-	logger.info('getting app {}'.format(repo_name))
-	shallow_clone = '--depth 1' if check_git_for_shallow_clone() else ''
-	branch = '--branch {branch}'.format(branch=branch) if branch else ''
+	if os.path.isdir(os.path.join(bench_path, 'apps', repo_name)):
+		# application directory already exists
+		# prompt user to overwrite it
+		if overwrite or click.confirm('''A directory for the application "{0}" already exists.
+Do you want to continue and overwrite it?'''.format(repo_name)):
+			shutil.rmtree(os.path.join(bench_path, 'apps', repo_name))
+		elif click.confirm('''Do you want to reinstall the existing application?''', abort=True):
+			app_name = get_app_name(bench_path, repo_name)
+			install_app(app=app_name, bench_path=bench_path, verbose=verbose, skip_assets=skip_assets)
+			sys.exit()
 
-	exec_cmd("git clone -q {git_url} {branch} {shallow_clone} --origin upstream".format(
-				git_url=git_url,
-				shallow_clone=shallow_clone,
-				branch=branch),
-			cwd=os.path.join(bench_path, 'apps'))
+	logger.info('Getting app {0}'.format(repo_name))
+	exec_cmd("git clone {git_url} {branch} {shallow_clone} --origin upstream".format(
+		git_url=git_url,
+		shallow_clone=shallow_clone,
+		branch=branch),
+		cwd=os.path.join(bench_path, 'apps'))
 
-	#Retrieves app name from setup.py
+	app_name = get_app_name(bench_path, repo_name)
+	install_app(app=app_name, bench_path=bench_path, verbose=verbose, skip_assets=skip_assets)
+
+
+def get_app_name(bench_path, repo_name):
+	# retrieves app name from setup.py
 	app_path = os.path.join(bench_path, 'apps', repo_name, 'setup.py')
 	with open(app_path, 'rb') as f:
 		app_name = re.search(r'name\s*=\s*[\'"](.*)[\'"]', f.read().decode('utf-8')).group(1)
 		if repo_name != app_name:
 			apps_path = os.path.join(os.path.abspath(bench_path), 'apps')
 			os.rename(os.path.join(apps_path, repo_name), os.path.join(apps_path, app_name))
+		return app_name
 
-	print('installing', app_name)
-	install_app(app=app_name, bench_path=bench_path, verbose=verbose)
-
-	if postprocess:
-
-		if not skip_assets:
-			build_assets(bench_path=bench_path, app=app_name)
-		conf = get_config(bench_path=bench_path)
-
-		if conf.get('restart_supervisor_on_update'):
-			restart_supervisor_processes(bench_path=bench_path)
-		if conf.get('restart_systemd_on_update'):
-			restart_systemd_processes(bench_path=bench_path)
 
 def new_app(app, bench_path='.'):
 	# For backwards compatibility
@@ -160,7 +171,8 @@ def new_app(app, bench_path='.'):
 		run_frappe_cmd('make-app', apps, app, bench_path=bench_path)
 	install_app(app, bench_path=bench_path)
 
-def install_app(app, bench_path=".", verbose=False, no_cache=False):
+
+def install_app(app, bench_path=".", verbose=False, no_cache=False, postprocess=True, skip_assets=False):
 	logger.info("installing {}".format(app))
 
 	pip_path = os.path.join(bench_path, "env", "bin", "pip")
@@ -168,11 +180,23 @@ def install_app(app, bench_path=".", verbose=False, no_cache=False):
 	app_path = os.path.join(bench_path, "apps", app)
 	cache_flag = "--no-cache-dir" if no_cache else ""
 
-	exec_cmd("{pip} install {quiet} -U -e {app} {no_cache}".format(pip=pip_path, quiet=quiet_flag, app=app_path, no_cache=cache_flag))
+	exec_cmd("{pip} install {quiet} -U -e {app} {no_cache}".format(pip=pip_path,
+									quiet=quiet_flag, app=app_path, no_cache=cache_flag))
 	add_to_appstxt(app, bench_path=bench_path)
 
+	if postprocess:
+		if not skip_assets:
+			build_assets(bench_path=bench_path, app=app)
+		conf = get_config(bench_path=bench_path)
+
+		if conf.get('restart_supervisor_on_update'):
+			restart_supervisor_processes(bench_path=bench_path)
+		if conf.get('restart_systemd_on_update'):
+			restart_systemd_processes(bench_path=bench_path)
+
+
 def remove_app(app, bench_path='.'):
-	if not app in get_apps(bench_path):
+	if app not in get_apps(bench_path):
 		print("No app named {0}".format(app))
 		sys.exit(1)
 
@@ -188,7 +212,7 @@ def remove_app(app, bench_path='.'):
 				print("Cannot remove, app is installed on site: {0}".format(site))
 				sys.exit(1)
 
-	exec_cmd(["{0} uninstall -y {1}".format(pip, app)])
+	exec_cmd("{0} uninstall -y {1}".format(pip, app), cwd=bench_path)
 	remove_from_appstxt(app, bench_path)
 	shutil.rmtree(app_path)
 	run_frappe_cmd("build", bench_path=bench_path)
@@ -281,8 +305,7 @@ def get_current_branch(app, bench_path='.'):
 
 def get_remote(app, bench_path='.'):
 	repo_dir = get_repo_dir(app, bench_path=bench_path)
-	contents = subprocess.check_output(['git', 'remote', '-v'], cwd=repo_dir,
-									   stderr=subprocess.STDOUT)
+	contents = subprocess.check_output(['git', 'remote', '-v'], cwd=repo_dir, stderr=subprocess.STDOUT)
 	contents = contents.decode('utf-8')
 	if re.findall('upstream[\s]+', contents):
 		return 'upstream'
@@ -340,8 +363,7 @@ def get_repo_dir(app, bench_path='.'):
 	return os.path.join(bench_path, 'apps', app)
 
 def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrade=True):
-	from .utils import update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets, pre_upgrade, post_upgrade
-	from . import utils
+	from bench.utils import update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets, post_upgrade
 	apps_dir = os.path.join(bench_path, 'apps')
 	version_upgrade = (False,)
 	switched_apps = []
@@ -354,44 +376,46 @@ def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrad
 
 	for app in apps:
 		app_dir = os.path.join(apps_dir, app)
-		if os.path.exists(app_dir):
-			try:
-				if check_upgrade:
-					version_upgrade = is_version_upgrade(app=app, bench_path=bench_path, branch=branch)
-					if version_upgrade[0] and not upgrade:
-						raise MajorVersionUpgradeException("Switching to {0} will cause upgrade from {1} to {2}. Pass --upgrade to confirm".format(branch, version_upgrade[1], version_upgrade[2]), version_upgrade[1], version_upgrade[2])
-				print("Switching for "+app)
-				unshallow = "--unshallow" if os.path.exists(os.path.join(app_dir, ".git", "shallow")) else ""
-				exec_cmd("git config --unset-all remote.upstream.fetch", cwd=app_dir)
-				exec_cmd("git config --add remote.upstream.fetch '+refs/heads/*:refs/remotes/upstream/*'", cwd=app_dir)
-				exec_cmd("git fetch upstream {unshallow}".format(unshallow=unshallow), cwd=app_dir)
-				exec_cmd("git checkout {branch}".format(branch=branch), cwd=app_dir)
-				exec_cmd("git merge upstream/{branch}".format(branch=branch), cwd=app_dir)
-				switched_apps.append(app)
-			except CommandFailedError:
-				print("Error switching to branch {0} for {1}".format(branch, app))
-			except InvalidRemoteException:
-				print("Remote does not exist for app "+app)
-			except InvalidBranchException:
-				print("Branch {0} does not exist in Upstream for {1}".format(branch, app))
+
+		if not os.path.exists(app_dir):
+			bench.utils.log("{} does not exist!".format(app), level=2)
+			continue
+
+		repo = git.Repo(app_dir)
+		unshallow_flag = os.path.exists(os.path.join(app_dir, ".git", "shallow"))
+		bench.utils.log("Fetching upstream {0}for {1}".format("unshallow " if unshallow_flag else "", app))
+
+		bench.utils.exec_cmd("git remote set-branches upstream  '*'", cwd=app_dir)
+		bench.utils.exec_cmd("git fetch --all{0}".format(" --unshallow" if unshallow_flag else ""), cwd=app_dir)
+
+		if check_upgrade:
+			version_upgrade = is_version_upgrade(app=app, bench_path=bench_path, branch=branch)
+			if version_upgrade[0] and not upgrade:
+				bench.utils.log("Switching to {0} will cause upgrade from {1} to {2}. Pass --upgrade to confirm".format(branch, version_upgrade[1], version_upgrade[2]), level=2)
+				sys.exit(1)
+
+		print("Switching for "+app)
+		bench.utils.exec_cmd("git checkout {0}".format(branch), cwd=app_dir)
+
+		if str(repo.active_branch) == branch:
+			switched_apps.append(app)
+		else:
+			bench.utils.log("Switching branches failed for: {}".format(app), level=2)
 
 	if switched_apps:
-		print("Successfully switched branches for:\n" + "\n".join(switched_apps))
+		bench.utils.log("Successfully switched branches for: " + ", ".join(switched_apps), level=1)
+		print('Please run `bench update --patch` to be safe from any differences in database schema')
 
 	if version_upgrade[0] and upgrade:
 		update_requirements()
 		update_node_packages()
-		pre_upgrade(version_upgrade[1], version_upgrade[2])
-		if sys.version_info >= (3, 4):
-			import importlib
-			importlib.reload(utils)
-		else:
-			reload(utils)
+		reload_module(utils)
 		backup_all_sites()
 		patch_sites()
 		build_assets()
 		post_upgrade(version_upgrade[1], version_upgrade[2])
 
+
 def switch_to_branch(branch=None, apps=None, bench_path='.', upgrade=False):
 	switch_branch(branch, apps=apps, bench_path=bench_path, upgrade=upgrade)
 
@@ -402,8 +426,7 @@ def switch_to_develop(apps=None, bench_path='.', upgrade=True):
 	switch_branch('develop', apps=apps, bench_path=bench_path, upgrade=upgrade)
 
 def get_version_from_string(contents, field='__version__'):
-	match = re.search(r"^(\s*%s\s*=\s*['\\\"])(.+?)(['\"])(?sm)" % field,
-			contents)
+	match = re.search(r"^(\s*%s\s*=\s*['\\\"])(.+?)(['\"])(?sm)" % field, contents)
 	return match.group(2)
 
 def get_major_version(version):

bench/cli.py

@@ -1,6 +1,6 @@
 import click
 import os, sys, logging, json, pwd, subprocess
-from bench.utils import is_root, PatchError, drop_privileges, get_env_cmd, get_cmd_output, get_frappe, log, is_bench_directory
+from bench.utils import is_root, PatchError, drop_privileges, get_env_cmd, get_cmd_output, get_frappe, log, find_parent_bench
 from bench.app import get_apps
 from bench.config.common_site_config import get_config
 from bench.commands import bench_command
@@ -29,7 +29,6 @@ def cli():
 
 	elif len(sys.argv) > 1 and sys.argv[1]=="--help":
 		print(click.Context(bench_command).get_help())
-		print()
 		print(get_frappe_help())
 		return
 
@@ -99,7 +98,6 @@ def get_frappe_commands(bench_path='.'):
 		return []
 	try:
 		output = get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-commands".format(python=python), cwd=sites_path)
-		# output = output.decode('utf-8')
 		return json.loads(output)
 	except subprocess.CalledProcessError as e:
 		if hasattr(e, "stderr"):
@@ -109,27 +107,12 @@ def get_frappe_commands(bench_path='.'):
 def get_frappe_help(bench_path='.'):
 	python = get_env_cmd('python', bench_path=bench_path)
 	sites_path = os.path.join(bench_path, 'sites')
-	if not os.path.exists(sites_path):
-		return []
 	try:
 		out = get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-help".format(python=python), cwd=sites_path)
-		return "Framework commands:\n" + out.split('Commands:')[1]
-	except subprocess.CalledProcessError:
+		return "\n\nFramework commands:\n" + out.split('Commands:')[1]
+	except:
 		return ""
 
-def find_parent_bench(path):
-	"""Checks if parent directories are benches"""
-	if is_bench_directory(directory=path):
-		return path
-
-	home_path = os.path.expanduser("~")
-	root_path = os.path.abspath(os.sep)
-
-	if path not in {home_path, root_path}:
-		# NOTE: the os.path.split assumes that given path is absolute
-		parent_dir = os.path.split(path)[0]
-		return find_parent_bench(parent_dir)
-
 def change_working_directory():
 	"""Allows bench commands to be run from anywhere inside a bench directory"""
 	cur_dir = os.path.abspath(".")

bench/commands/__init__.py

@@ -40,7 +40,7 @@ bench_command.add_command(switch_to_develop)
 
 
 from bench.commands.utils import (start, restart, set_nginx_port, set_ssl_certificate, set_ssl_certificate_key, set_url_root,
-	set_mariadb_host, set_default_site, download_translations, shell, backup_site, backup_all_sites, release, renew_lets_encrypt,
+	set_mariadb_host, set_default_site, download_translations, backup_site, backup_all_sites, release, renew_lets_encrypt,
 	disable_production, bench_src, prepare_beta_release, set_redis_cache_host, set_redis_queue_host, set_redis_socketio_host, find_benches, migrate_env)
 bench_command.add_command(start)
 bench_command.add_command(restart)
@@ -54,7 +54,6 @@ bench_command.add_command(set_redis_queue_host)
 bench_command.add_command(set_redis_socketio_host)
 bench_command.add_command(set_default_site)
 bench_command.add_command(download_translations)
-bench_command.add_command(shell)
 bench_command.add_command(backup_site)
 bench_command.add_command(backup_all_sites)
 bench_command.add_command(release)

bench/commands/config.py

@@ -1,112 +1,80 @@
-import click, json
-from bench.config.common_site_config import update_config
+# imports - standard imports
+import ast
 
-## Config
-## Not DRY
-@click.group()
+# imports - module imports
+from bench.config.common_site_config import update_config, get_config, put_config
+
+# imports - third party imports
+import click
+
+
+@click.group(help='Change bench configuration')
 def config():
-	"change bench configuration"
 	pass
 
-@click.command('auto_update')
-@click.argument('state', type=click.Choice(['on', 'off']))
-def config_auto_update(state):
-	"Enable/Disable auto update for bench"
-	state = True if state == 'on' else False
-	update_config({'auto_update': state})
 
-
-@click.command('restart_supervisor_on_update')
+@click.command('restart_supervisor_on_update', help='Enable/Disable auto restart of supervisor processes')
 @click.argument('state', type=click.Choice(['on', 'off']))
 def config_restart_supervisor_on_update(state):
-	"Enable/Disable auto restart of supervisor processes"
-	state = True if state == 'on' else False
-	update_config({'restart_supervisor_on_update': state})
+	update_config({'restart_supervisor_on_update': state == 'on'})
 
-@click.command('restart_systemd_on_update')
+
+@click.command('restart_systemd_on_update', help='Enable/Disable auto restart of systemd units')
 @click.argument('state', type=click.Choice(['on', 'off']))
 def config_restart_systemd_on_update(state):
-	"Enable/Disable auto restart of systemd units"
-	state = True if state == 'on' else False
-	update_config({'restart_systemd_on_update': state})
+	update_config({'restart_systemd_on_update': state == 'on'})
 
-@click.command('update_bench_on_update')
+
+@click.command('update_bench_on_update', help='Enable/Disable bench updates on running bench update')
 @click.argument('state', type=click.Choice(['on', 'off']))
 def config_update_bench_on_update(state):
-	"Enable/Disable bench updates on running bench update"
-	state = True if state == 'on' else False
-	update_config({'update_bench_on_update': state})
+	update_config({'update_bench_on_update': state == 'on'})
 
 
-@click.command('dns_multitenant')
+@click.command('dns_multitenant', help='Enable/Disable bench multitenancy on running bench update')
 @click.argument('state', type=click.Choice(['on', 'off']))
 def config_dns_multitenant(state):
-	"Enable/Disable bench updates on running bench update"
-	state = True if state == 'on' else False
-	update_config({'dns_multitenant': state})
+	update_config({'dns_multitenant': state == 'on'})
 
 
-@click.command('serve_default_site')
+@click.command('serve_default_site', help='Configure nginx to serve the default site on port 80')
 @click.argument('state', type=click.Choice(['on', 'off']))
 def config_serve_default_site(state):
-	"Configure nginx to serve the default site on port 80"
-	state = True if state == 'on' else False
-	update_config({'serve_default_site': state})
+	update_config({'serve_default_site': state == 'on'})
 
 
-@click.command('rebase_on_pull')
+@click.command('rebase_on_pull', help='Rebase repositories on pulling')
 @click.argument('state', type=click.Choice(['on', 'off']))
 def config_rebase_on_pull(state):
-	"Rebase repositories on pulling"
-	state = True if state == 'on' else False
-	update_config({'rebase_on_pull': state})
+	update_config({'rebase_on_pull': state == 'on'})
 
 
-@click.command('http_timeout')
+@click.command('http_timeout', help='Set HTTP timeout')
 @click.argument('seconds', type=int)
 def config_http_timeout(seconds):
-	"set http timeout"
 	update_config({'http_timeout': seconds})
 
 
-@click.command('set-common-config')
+@click.command('set-common-config', help='Set value in common config')
 @click.option('configs', '-c', '--config', multiple=True, type=(str, str))
 def set_common_config(configs):
-	import ast
-	from bench.config.common_site_config import update_config
-
 	common_site_config = {}
 	for key, value in configs:
-		if value in ("False", "True"):
+		if value in ('true', 'false'):
+			value = value.title()
+		try:
 			value = ast.literal_eval(value)
-
-		elif "." in value:
-			try:
-				value = float(value)
-			except ValueError:
-				pass
-
-		elif "{" in value or "[" in value:
-			try:
-				value = json.loads(value)
-			except ValueError:
-				pass
-
-		else:
-			try:
-				value = int(value)
-			except ValueError:
-				pass
+		except ValueError:
+			pass
 
 		common_site_config[key] = value
 
 	update_config(common_site_config, bench_path='.')
 
 
-@click.command('remove-common-config')
+@click.command('remove-common-config', help='Remove specific keys from current bench\'s common config')
 @click.argument('keys', nargs=-1)
 def remove_common_config(keys):
-	from bench.config.common_site_config import get_config, put_config
 	common_site_config = get_config('.')
 	for key in keys:
 		if key in common_site_config:
@@ -115,7 +83,6 @@ def remove_common_config(keys):
 	put_config(common_site_config)
 
 
-config.add_command(config_auto_update)
 config.add_command(config_update_bench_on_update)
 config.add_command(config_restart_supervisor_on_update)
 config.add_command(config_restart_systemd_on_update)

bench/commands/git.py

@@ -1,28 +1,30 @@
-import click
-import os, subprocess, re
+# imports - standard imports
+import os
+import subprocess
 
+# imports - module imports
 from bench.app import get_repo_dir, get_apps, get_remote
 from bench.utils import set_git_remote_url
 
+# imports - third party imports
+import click
 
-@click.command('remote-set-url')
+
+@click.command('remote-set-url', help="Set app remote url")
 @click.argument('git-url')
 def remote_set_url(git_url):
-	"Set app remote url"
 	set_git_remote_url(git_url)
 
 
-@click.command('remote-reset-url')
+@click.command('remote-reset-url', help="Reset app remote url to frappe official")
 @click.argument('app')
 def remote_reset_url(app):
-	"Reset app remote url to frappe official"
 	git_url = "https://github.com/frappe/{}.git".format(app)
 	set_git_remote_url(git_url)
 
 
-@click.command('remote-urls')
+@click.command('remote-urls', help="Show apps remote url")
 def remote_urls():
-	"Show apps remote url"
 	for app in get_apps():
 		repo_dir = get_repo_dir(app)
 

bench/commands/install.py

@@ -1,21 +1,29 @@
-import os, sys, json, click
-from bench.utils import run_playbook, setup_sudoers, is_root
+# imports - module imports
+from bench.utils import run_playbook, setup_sudoers
 
-extra_vars = {"production": True}
+# imports - third party imports
+import click
 
-@click.group()
+
+extra_vars = {
+	"production": True
+}
+
+
+@click.group(help="Install system dependencies for setting up Frappe environment")
 def install():
-	"Install system dependancies"
 	pass
 
-@click.command('prerequisites')
+
+@click.command('prerequisites', help="Installs pre-requisite libraries, essential tools like b2zip, htop, screen, vim, x11-fonts, python libs, cups and Redis")
 def install_prerequisites():
 	run_playbook('site.yml', tag='common, redis')
 
-@click.command('mariadb')
-@click.option('--mysql_root_password')
+
+@click.command('mariadb', help="Install and setup MariaDB of specified version and root password")
+@click.option('--mysql_root_password', '--mysql-root-password', default="")
 @click.option('--version', default="10.3")
-def install_maridb(mysql_root_password='', version=''):
+def install_maridb(mysql_root_password, version):
 	if mysql_root_password:
 		extra_vars.update({
 			"mysql_root_password": mysql_root_password,
@@ -27,41 +35,49 @@ def install_maridb(mysql_root_password='', version=''):
 
 	run_playbook('site.yml', extra_vars=extra_vars, tag='mariadb')
 
-@click.command('wkhtmltopdf')
+
+@click.command('wkhtmltopdf', help="Installs wkhtmltopdf v0.12.3 for linux")
 def install_wkhtmltopdf():
 	run_playbook('site.yml', extra_vars=extra_vars, tag='wkhtmltopdf')
 
-@click.command('nodejs')
+
+@click.command('nodejs', help="Installs Node.js v8")
 def install_nodejs():
 	run_playbook('site.yml', extra_vars=extra_vars, tag='nodejs')
 
-@click.command('psutil')
+
+@click.command('psutil', help="Installs psutil via pip")
 def install_psutil():
 	run_playbook('site.yml', extra_vars=extra_vars, tag='psutil')
 
-@click.command('supervisor')
+
+@click.command('supervisor', help="Installs supervisor. If user is specified, sudoers is setup for that user")
 @click.option('--user')
 def install_supervisor(user=None):
 	run_playbook('site.yml', extra_vars=extra_vars, tag='supervisor')
 	if user:
 		setup_sudoers(user)
 
-@click.command('nginx')
+
+@click.command('nginx', help="Installs NGINX. If user is specified, sudoers is setup for that user")
 @click.option('--user')
 def install_nginx(user=None):
 	run_playbook('site.yml', extra_vars=extra_vars, tag='nginx')
 	if user:
 		setup_sudoers(user)
 
-@click.command('virtualbox')
+
+@click.command('virtualbox', help="Installs supervisor")
 def install_virtualbox():
 	run_playbook('vm_build.yml', tag='virtualbox')
 
-@click.command('packer')
+
+@click.command('packer', help="Installs Oracle virtualbox and packer 1.2.1")
 def install_packer():
 	run_playbook('vm_build.yml', tag='packer')
 
-@click.command('fail2ban')
+
+@click.command("fail2ban", help="Install fail2ban, an intrusion prevention software framework that protects computer servers from brute-force attacks")
 @click.option('--maxretry', default=6, help="Number of matches (i.e. value of the counter) which triggers ban action on the IP.")
 @click.option('--bantime', default=600, help="The counter is set to zero if no match is found within 'findtime' seconds.")
 @click.option('--findtime', default=600, help='Duration (in seconds) for IP to be banned for. Negative number for "permanent" ban.')
@@ -69,6 +85,7 @@ def install_failtoban(**kwargs):
 	extra_vars.update(kwargs)
 	run_playbook('site.yml', extra_vars=extra_vars, tag='fail2ban')
 
+
 install.add_command(install_prerequisites)
 install.add_command(install_maridb)
 install.add_command(install_wkhtmltopdf)

bench/commands/make.py

@@ -1,6 +1,8 @@
+# imports - third party imports
 import click
 
-@click.command()
+
+@click.command('init', help='Initialize a new bench instance in the specified path')
 @click.argument('path')
 @click.option('--python', type = str, default = 'python3', help = 'Path to Python Executable.')
 @click.option('--ignore-exist', is_flag = True, default = False, help = "Ignore if Bench instance exists.")
@@ -11,14 +13,10 @@ import click
 @click.option('--clone-without-update', is_flag=True, help="copy repos from path without update")
 @click.option('--no-procfile', is_flag=True, help="Pull changes in all the apps in bench")
 @click.option('--no-backups',is_flag=True, help="Run migrations for all sites in the bench")
-@click.option('--no-auto-update',is_flag=True, help="Build JS and CSS artifacts for the bench")
 @click.option('--skip-redis-config-generation', is_flag=True, help="Skip redis config generation if already specifying the common-site-config file")
 @click.option('--skip-assets',is_flag=True, default=False, help="Do not build assets")
 @click.option('--verbose',is_flag=True, help="Verbose output during install")
-def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, no_auto_update, clone_from, verbose, skip_redis_config_generation, clone_without_update, ignore_exist=False, skip_assets=False, python='python3'):
-	'''
-	Create a New Bench Instance.
-	'''
+def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, clone_from, verbose, skip_redis_config_generation, clone_without_update, ignore_exist=False, skip_assets=False, python='python3'):
 	from bench.utils import init, log
 
 	try:
@@ -27,7 +25,6 @@ def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, n
 			apps_path=apps_path,
 			no_procfile=no_procfile,
 			no_backups=no_backups,
-			no_auto_update=no_auto_update,
 			frappe_path=frappe_path,
 			frappe_branch=frappe_branch,
 			verbose=verbose,
@@ -41,10 +38,11 @@ def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, n
 		log('Bench {} initialized'.format(path), level=1)
 	except SystemExit:
 		pass
-	except:
+	except Exception as e:
 		import os, shutil, time, six
 		# add a sleep here so that the traceback of other processes doesnt overlap with the prompts
 		time.sleep(1)
+		print(e)
 		log("There was a problem while creating {}".format(path), level=2)
 		if six.moves.input("Do you want to rollback these changes? [Y/n]: ").lower() == "y":
 			print('Rolling back Bench "{}"'.format(path))
@@ -52,42 +50,40 @@ def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, n
 				shutil.rmtree(path)
 
 
-@click.command('get-app')
+@click.command('get-app', help='Clone an app from the internet or filesystem and set it up in your bench')
 @click.argument('name', nargs=-1) # Dummy argument for backward compatibility
 @click.argument('git-url')
 @click.option('--branch', default=None, help="branch to checkout")
+@click.option('--overwrite', is_flag=True, default=False)
 @click.option('--skip-assets', is_flag=True, default=False, help="Do not build assets")
-def get_app(git_url, branch, name=None, skip_assets=False):
+def get_app(git_url, branch, name=None, overwrite=False, skip_assets=False):
 	"clone an app from the internet and set it up in your bench"
 	from bench.app import get_app
-	get_app(git_url, branch=branch, skip_assets=skip_assets)
+	get_app(git_url, branch=branch, skip_assets=skip_assets, overwrite=overwrite)
 
 
-@click.command('new-app')
+@click.command('new-app', help='Create a new Frappe application under apps folder')
 @click.argument('app-name')
 def new_app(app_name):
-	"start a new app"
 	from bench.app import new_app
 	new_app(app_name)
 
 
-@click.command('remove-app')
+@click.command('remove-app', help='Completely remove app from bench and re-build assets if not installed on any site')
 @click.argument('app-name')
 def remove_app(app_name):
-	"completely remove app from bench"
 	from bench.app import remove_app
 	remove_app(app_name)
 
 
-@click.command('exclude-app')
+@click.command('exclude-app', help='Exclude app from updating')
 @click.argument('app_name')
 def exclude_app_for_update(app_name):
-	"Exclude app from updating"
 	from bench.app import add_to_excluded_apps_txt
 	add_to_excluded_apps_txt(app_name)
 
 
-@click.command('include-app')
+@click.command('include-app', help='Include app for updating')
 @click.argument('app_name')
 def include_app_for_update(app_name):
 	"Include app from updating"

bench/commands/setup.py

@@ -1,233 +1,216 @@
-from bench.utils import exec_cmd
-from six import PY3
-import click, sys, json
+# imports - standard imports
 import os
+import sys
 
-@click.group()
+# imports - module imports
+from bench.utils import exec_cmd
+
+# imports - third party imports
+from six import PY3
+import click
+
+
+@click.group(help="Setup command group for enabling setting up a Frappe environment")
 def setup():
-	"Setup bench"
 	pass
 
-@click.command('sudoers')
-@click.argument('user')
+
+@click.command("sudoers", help="Add commands to sudoers list for execution without password")
+@click.argument("user")
 def setup_sudoers(user):
-	"Add commands to sudoers list for execution without password"
 	from bench.utils import setup_sudoers
 	setup_sudoers(user)
 
-@click.command('nginx')
-@click.option('--yes', help='Yes to regeneration of nginx config file', default=False, is_flag=True)
+
+@click.command("nginx", help="Generate configuration files for NGINX")
+@click.option("--yes", help="Yes to regeneration of nginx config file", default=False, is_flag=True)
 def setup_nginx(yes=False):
-	"generate config for nginx"
 	from bench.config.nginx import make_nginx_conf
 	make_nginx_conf(bench_path=".", yes=yes)
 
-@click.command('reload-nginx')
+
+@click.command("reload-nginx", help="Checks NGINX config file and reloads service")
 def reload_nginx():
 	from bench.config.production_setup import reload_nginx
 	reload_nginx()
 
-@click.command('supervisor')
-@click.option('--user')
-@click.option('--yes', help='Yes to regeneration of supervisor config', is_flag=True, default=False)
+
+@click.command("supervisor", help="Generate configuration for supervisor")
+@click.option("--user", help="optional user argument")
+@click.option("--yes", help="Yes to regeneration of supervisor config", is_flag=True, default=False)
 def setup_supervisor(user=None, yes=False):
-	"generate config for supervisor with an optional user argument"
 	from bench.config.supervisor import generate_supervisor_config
 	generate_supervisor_config(bench_path=".", user=user, yes=yes)
 
-@click.command('redis')
+
+@click.command("redis", help="Generates configuration for Redis")
 def setup_redis():
-	"generate config for redis cache"
 	from bench.config.redis import generate_config
-	generate_config('.')
+	generate_config(".")
 
 
-@click.command('fonts')
+@click.command("fonts", help="Add Frappe fonts to system")
 def setup_fonts():
-	"Add frappe fonts to system"
 	from bench.utils import setup_fonts
 	setup_fonts()
 
-@click.command('production')
-@click.argument('user')
-@click.option('--yes', help='Yes to regeneration config', is_flag=True, default=False)
+
+@click.command("production", help="Setup Frappe production environment for specific user")
+@click.argument("user")
+@click.option("--yes", help="Yes to regeneration config", is_flag=True, default=False)
 def setup_production(user, yes=False):
-	"setup bench for production"
 	from bench.config.production_setup import setup_production
-	from bench.utils import run_playbook
 	# Install prereqs for production
 	from distutils.spawn import find_executable
-	if not find_executable('ansible'):
-		exec_cmd("sudo {0} install ansible".format("pip3" if PY3 else "pip2"))
-	if not find_executable('fail2ban-client'):
+	if not find_executable("ansible"):
+		exec_cmd("sudo -H {0} -m pip install ansible".format(sys.executable))
+	if not find_executable("fail2ban-client"):
 		exec_cmd("bench setup role fail2ban")
-	if not find_executable('nginx'):
+	if not find_executable("nginx"):
 		exec_cmd("bench setup role nginx")
-	if not find_executable('supervisord'):
+	if not find_executable("supervisord"):
 		exec_cmd("bench setup role supervisor")
 	setup_production(user=user, yes=yes)
 
 
-@click.command('auto-update')
-def setup_auto_update():
-	"Add cronjob for bench auto update"
-	from bench.utils import setup_auto_update
-	setup_auto_update()
-
-
-@click.command('backups')
+@click.command("backups", help="Add cronjob for bench backups")
 def setup_backups():
-	"Add cronjob for bench backups"
 	from bench.utils import setup_backups
 	setup_backups()
 
-@click.command('env')
-@click.option('--python', type = str, default = 'python3', help = 'Path to Python Executable.')
-def setup_env(python='python3'):
-	"Setup virtualenv for bench"
+
+@click.command("env", help="Setup virtualenv for bench")
+@click.option("--python", type = str, default = "python3", help = "Path to Python Executable.")
+def setup_env(python="python3"):
 	from bench.utils import setup_env
 	setup_env(python=python)
 
-@click.command('firewall')
-@click.option('--ssh_port')
-@click.option('--force')
+
+@click.command("firewall", help="Setup firewall for system")
+@click.option("--ssh_port")
+@click.option("--force")
 def setup_firewall(ssh_port=None, force=False):
-	"Setup firewall"
 	from bench.utils import run_playbook
 
 	if not force:
-		click.confirm('Setting up the firewall will block all ports except 80, 443 and 22\n'
-			'Do you want to continue?',
-			abort=True)
+		click.confirm("Setting up the firewall will block all ports except 80, 443 and {0}\nDo you want to continue?".format(ssh_port), abort=True)
 
 	if not ssh_port:
 		ssh_port = 22
 
-	run_playbook('roles/bench/tasks/setup_firewall.yml', {"ssh_port": ssh_port})
+	run_playbook("roles/bench/tasks/setup_firewall.yml", {"ssh_port": ssh_port})
 
-@click.command('ssh-port')
-@click.argument('port')
-@click.option('--force')
+
+@click.command("ssh-port", help="Set SSH Port for system")
+@click.argument("port")
+@click.option("--force")
 def set_ssh_port(port, force=False):
-	"Set SSH Port"
 	from bench.utils import run_playbook
 
 	if not force:
-		click.confirm('This will change your SSH Port to {}\n'
-			'Do you want to continue?'.format(port),
-			abort=True)
+		click.confirm("This will change your SSH Port to {}\nDo you want to continue?".format(port), abort=True)
 
-	run_playbook('roles/bench/tasks/change_ssh_port.yml', {"ssh_port": port})
+	run_playbook("roles/bench/tasks/change_ssh_port.yml", {"ssh_port": port})
 
-@click.command('lets-encrypt')
-@click.argument('site')
-@click.option('--custom-domain')
+
+@click.command("lets-encrypt", help="Setup lets-encrypt SSL for site")
+@click.argument("site")
+@click.option("--custom-domain")
 @click.option('-n', '--non-interactive', default=False, is_flag=True, help="Run command non-interactively. This flag restarts nginx and runs certbot non interactively. Shouldn't be used on 1'st attempt")
 def setup_letsencrypt(site, custom_domain, non_interactive):
-	"Setup lets-encrypt for site"
 	from bench.config.lets_encrypt import setup_letsencrypt
-	setup_letsencrypt(site, custom_domain, bench_path='.', interactive=not non_interactive)
+	setup_letsencrypt(site, custom_domain, bench_path=".", interactive=not non_interactive)
 
 
-@click.command('wildcard-ssl')
-@click.argument('domain')
-@click.option('--email')
-@click.option('--exclude-base-domain', default=False, is_flag=True, help="SSL Certificate not applicable for base domain")
+@click.command("wildcard-ssl", help="Setup wildcard SSL certificate for multi-tenant bench")
+@click.argument("domain")
+@click.option("--email")
+@click.option("--exclude-base-domain", default=False, is_flag=True, help="SSL Certificate not applicable for base domain")
 def setup_wildcard_ssl(domain, email, exclude_base_domain):
-	''' Setup wildcard ssl certificate '''
 	from bench.config.lets_encrypt import setup_wildcard_ssl
-	setup_wildcard_ssl(domain, email, bench_path='.', exclude_base_domain=exclude_base_domain)
+	setup_wildcard_ssl(domain, email, bench_path=".", exclude_base_domain=exclude_base_domain)
 
 
-@click.command('procfile')
+@click.command("procfile", help="Generate Procfile for bench start")
 def setup_procfile():
-	"Setup Procfile for bench start"
 	from bench.config.procfile import setup_procfile
-	setup_procfile('.')
+	setup_procfile(".")
 
 
-@click.command('socketio')
+@click.command("socketio", help="Setup node dependencies for socketio server")
 def setup_socketio():
-	"Setup node deps for socketio server"
 	from bench.utils import setup_socketio
 	setup_socketio()
 
-@click.command('requirements', help="Update Python and Node packages")
-@click.option('--node', help="Update only Node packages", default=False, is_flag=True)
-@click.option('--python', help="Update only Python packages", default=False, is_flag=True)
+
+@click.command("requirements", help="Setup Python and Node dependencies")
+@click.option("--node", help="Update only Node packages", default=False, is_flag=True)
+@click.option("--python", help="Update only Python packages", default=False, is_flag=True)
 def setup_requirements(node=False, python=False):
-	"Setup python and node requirements"
-
 	if not node:
-		setup_python_requirements()
+		from bench.utils import update_requirements as setup_python_packages
+		setup_python_packages()
+
 	if not python:
-		setup_node_requirements()
-
-def setup_python_requirements():
-	from bench.utils import update_requirements
-	update_requirements()
-
-def setup_node_requirements():
-	from bench.utils import update_node_packages
-	update_node_packages()
+		from bench.utils import update_node_packages as setup_node_packages
+		setup_node_packages()
 
 
-@click.command('manager')
-@click.option('--yes', help='Yes to regeneration of nginx config file', default=False, is_flag=True)
-@click.option('--port', help='Port on which you want to run bench manager', default=23624)
-@click.option('--domain', help='Domain on which you want to run bench manager')
+@click.command("manager", help="Setup bench-manager.local site with the bench_manager app installed on it")
+@click.option("--yes", help="Yes to regeneration of nginx config file", default=False, is_flag=True)
+@click.option("--port", help="Port on which you want to run bench manager", default=23624)
+@click.option("--domain", help="Domain on which you want to run bench manager")
 def setup_manager(yes=False, port=23624, domain=None):
-	"Setup bench-manager.local site with the bench_manager app installed on it"
 	from six.moves import input
+	from bench.utils import get_sites
+	from bench.config.common_site_config import get_config
+	from bench.config.nginx import make_bench_manager_nginx_conf
+
 	create_new_site = True
-	if 'bench-manager.local' in os.listdir('sites'):
-		ans = input('Site already exists. Overwrite existing site? [Y/n]: ').lower()
-		while ans not in ('y', 'n', ''):
-			ans = input(
-				'Please enter "y" or "n". Site already exists. Overwrite existing site? [Y/n]: ').lower()
-		if ans == 'n':
+
+	if "bench-manager.local" in os.listdir("sites"):
+		ans = input("Site already exists. Overwrite existing site? [Y/n]: ").lower()
+		while ans not in ("y", "n", ""):
+			ans = input("Please enter 'y' or 'n'. Site already exists. Overwrite existing site? [Y/n]: ").lower()
+		if ans == "n":
 			create_new_site = False
+
 	if create_new_site:
 		exec_cmd("bench new-site --force bench-manager.local")
 
-	if 'bench_manager' in os.listdir('apps'):
-		print('App already exists. Skipping app download.')
+	if "bench_manager" in os.listdir("apps"):
+		print("App already exists. Skipping app download.")
 	else:
 		exec_cmd("bench get-app bench_manager")
 
 	exec_cmd("bench --site bench-manager.local install-app bench_manager")
 
-	from bench.config.common_site_config import get_config
-	bench_path = '.'
+	bench_path = "."
 	conf = get_config(bench_path)
-	if conf.get('restart_supervisor_on_update') or conf.get('restart_systemd_on_update'):
+
+	if conf.get("restart_supervisor_on_update") or conf.get("restart_systemd_on_update"):
 		# implicates a production setup or so I presume
 		if not domain:
 			print("Please specify the site name on which you want to host bench-manager using the 'domain' flag")
 			sys.exit(1)
 
-		from bench.utils import get_sites, get_bench_name
-		bench_name = get_bench_name(bench_path)
-
 		if domain not in get_sites(bench_path):
 			raise Exception("No such site")
 
-		from bench.config.nginx import make_bench_manager_nginx_conf
 		make_bench_manager_nginx_conf(bench_path, yes=yes, port=port, domain=domain)
 
 
-@click.command('config')
+@click.command("config", help="Generate or over-write sites/common_site_config.json")
 def setup_config():
-	"overwrite or make config.json"
 	from bench.config.common_site_config import make_config
-	make_config('.')
+	make_config(".")
 
 
-@click.command('add-domain')
-@click.argument('domain')
-@click.option('--site', prompt=True)
-@click.option('--ssl-certificate', help="Absolute path to SSL Certificate")
-@click.option('--ssl-certificate-key', help="Absolute path to SSL Certificate Key")
+@click.command("add-domain", help="Add a custom domain to a particular site")
+@click.argument("domain")
+@click.option("--site", prompt=True)
+@click.option("--ssl-certificate", help="Absolute path to SSL Certificate")
+@click.option("--ssl-certificate-key", help="Absolute path to SSL Certificate Key")
 def add_domain(domain, site=None, ssl_certificate=None, ssl_certificate_key=None):
 	"""Add custom domain to site"""
 	from bench.config.site_config import add_domain
@@ -236,24 +219,25 @@ def add_domain(domain, site=None, ssl_certificate=None, ssl_certificate_key=None
 		print("Please specify site")
 		sys.exit(1)
 
-	add_domain(site, domain, ssl_certificate, ssl_certificate_key, bench_path='.')
+	add_domain(site, domain, ssl_certificate, ssl_certificate_key, bench_path=".")
 
-@click.command('remove-domain')
-@click.argument('domain')
-@click.option('--site', prompt=True)
+
+@click.command("remove-domain", help="Remove custom domain from a site")
+@click.argument("domain")
+@click.option("--site", prompt=True)
 def remove_domain(domain, site=None):
-	"""Remove custom domain from a site"""
 	from bench.config.site_config import remove_domain
 
 	if not site:
 		print("Please specify site")
 		sys.exit(1)
 
-	remove_domain(site, domain, bench_path='.')
+	remove_domain(site, domain, bench_path=".")
 
-@click.command('sync-domains')
-@click.option('--domain', multiple=True)
-@click.option('--site', prompt=True)
+
+@click.command("sync-domains", help="Check if there is a change in domains. If yes, updates the domains list.")
+@click.option("--domain", multiple=True)
+@click.option("--site", prompt=True)
 def sync_domains(domain=None, site=None):
 	from bench.config.site_config import sync_domains
 
@@ -262,53 +246,55 @@ def sync_domains(domain=None, site=None):
 		sys.exit(1)
 
 	try:
-		domains = list(map(str,domain))
+		domains = list(map(str, domain))
 	except Exception:
 		print("Domains should be a json list of strings or dictionaries")
 		sys.exit(1)
 
-	changed = sync_domains(site, domains, bench_path='.')
+	changed = sync_domains(site, domains, bench_path=".")
 
 	# if changed, success, else failure
 	sys.exit(0 if changed else 1)
 
-@click.command('role')
-@click.argument('role')
-@click.option('--admin_emails', default='')
-@click.option('--mysql_root_password')
-@click.option('--container', is_flag=True, default=False)
+
+@click.command("role", help="Install dependencies via ansible roles")
+@click.argument("role")
+@click.option("--admin_emails", default="")
+@click.option("--mysql_root_password")
+@click.option("--container", is_flag=True, default=False)
 def setup_roles(role, **kwargs):
-	"Install dependancies via roles"
 	from bench.utils import run_playbook
 
 	extra_vars = {"production": True}
 	extra_vars.update(kwargs)
 
 	if role:
-		run_playbook('site.yml', extra_vars=extra_vars, tag=role)
+		run_playbook("site.yml", extra_vars=extra_vars, tag=role)
 	else:
-		run_playbook('site.yml', extra_vars=extra_vars)
+		run_playbook("site.yml", extra_vars=extra_vars)
 
-@click.command('fail2ban')
-@click.option('--maxretry', default=6, help="Number of matches (i.e. value of the counter) which triggers ban action on the IP. Default is 6 seconds" )
-@click.option('--bantime', default=600, help="The counter is set to zero if no match is found within 'findtime' seconds. Default is 600 seconds")
-@click.option('--findtime', default=600, help='Duration (in seconds) for IP to be banned for. Negative number for "permanent" ban. Default is 600 seconds')
+
+@click.command("fail2ban", help="Setup fail2ban, an intrusion prevention software framework that protects computer servers from brute-force attacks")
+@click.option("--maxretry", default=6, help="Number of matches (i.e. value of the counter) which triggers ban action on the IP. Default is 6 seconds" )
+@click.option("--bantime", default=600, help="The counter is set to zero if no match is found within 'findtime' seconds. Default is 600 seconds")
+@click.option("--findtime", default=600, help="Duration (in seconds) for IP to be banned for. Negative number for 'permanent' ban. Default is 600 seconds")
 def setup_nginx_proxy_jail(**kwargs):
 	from bench.utils import run_playbook
-	run_playbook('roles/fail2ban/tasks/configure_nginx_jail.yml', extra_vars=kwargs)
+	run_playbook("roles/fail2ban/tasks/configure_nginx_jail.yml", extra_vars=kwargs)
 
-@click.command('systemd')
-@click.option('--user')
-@click.option('--yes', help='Yes to regeneration of systemd config files', is_flag=True, default=False)
-@click.option('--stop', help='Stop bench services', is_flag=True, default=False)
-@click.option('--create-symlinks', help='Create Symlinks', is_flag=True, default=False)
-@click.option('--delete-symlinks', help='Delete Symlinks', is_flag=True, default=False)
+
+@click.command("systemd", help="Generate configuration for systemd")
+@click.option("--user", help="Optional user argument")
+@click.option("--yes", help="Yes to regeneration of systemd config files", is_flag=True, default=False)
+@click.option("--stop", help="Stop bench services", is_flag=True, default=False)
+@click.option("--create-symlinks", help="Create Symlinks", is_flag=True, default=False)
+@click.option("--delete-symlinks", help="Delete Symlinks", is_flag=True, default=False)
 def setup_systemd(user=None, yes=False, stop=False, create_symlinks=False, delete_symlinks=False):
-	"generate configs for systemd with an optional user argument"
 	from bench.config.systemd import generate_systemd_config
 	generate_systemd_config(bench_path=".", user=user, yes=yes,
 		stop=stop, create_symlinks=create_symlinks, delete_symlinks=delete_symlinks)
 
+
 setup.add_command(setup_sudoers)
 setup.add_command(setup_nginx)
 setup.add_command(reload_nginx)
@@ -317,7 +303,6 @@ setup.add_command(setup_redis)
 setup.add_command(setup_letsencrypt)
 setup.add_command(setup_wildcard_ssl)
 setup.add_command(setup_production)
-setup.add_command(setup_auto_update)
 setup.add_command(setup_backups)
 setup.add_command(setup_env)
 setup.add_command(setup_procfile)

bench/commands/update.py

@@ -1,124 +1,32 @@
-import click
-import sys
+# imports - standard imports
 import os
-from bench.config.common_site_config import get_config, update_config
-from bench.app import pull_all_apps, is_version_upgrade, validate_branch
-from bench.utils import (update_bench, validate_upgrade, pre_upgrade, post_upgrade, before_update,
-	update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets,
-	restart_supervisor_processes, restart_systemd_processes, is_bench_directory)
-from bench import patches
+
+# imports - third party imports
+import click
 from six.moves import reload_module
 
+# imports - module imports
+from bench.app import pull_all_apps
+from bench.utils import post_upgrade, patch_sites, build_assets
 
-@click.command('update')
-@click.option('--pull', is_flag=True, help="Pull changes in all the apps in bench")
+
+@click.command('update', help="Updates bench tool and if executed in a bench directory, without any flags will backup, pull, setup requirements, build, run patches and restart bench. Using specific flags will only do certain tasks instead of all")
+@click.option('--pull', is_flag=True, help="Pull updates for all the apps in bench")
 @click.option('--patch', is_flag=True, help="Run migrations for all sites in the bench")
-@click.option('--build', is_flag=True, help="Build JS and CSS artifacts for the bench")
-@click.option('--bench', is_flag=True, help="Update bench")
-@click.option('--requirements', is_flag=True, help="Update requirements")
-@click.option('--restart-supervisor', is_flag=True, help="restart supervisor processes after update")
-@click.option('--restart-systemd', is_flag=True, help="restart systemd units after update")
-@click.option('--auto', is_flag=True)
-@click.option('--no-backup', is_flag=True)
-@click.option('--force', is_flag=True)
+@click.option('--build', is_flag=True, help="Build JS and CSS assets for the bench")
+@click.option('--bench', is_flag=True, help="Update bench CLI tool")
+@click.option('--requirements', is_flag=True, help="Update requirements. If run alone, equivalent to `bench setup requirements`")
+@click.option('--restart-supervisor', is_flag=True, help="Restart supervisor processes after update")
+@click.option('--restart-systemd', is_flag=True, help="Restart systemd units after update")
+@click.option('--no-backup', is_flag=True, help="If this flag is set, sites won't be backed up prior to updates. Note: This is not recommended in production.")
+@click.option('--force', is_flag=True, help="Forces major version upgrades")
 @click.option('--reset', is_flag=True, help="Hard resets git branch's to their new states overriding any changes and overriding rebase on pull")
-def update(pull=False, patch=False, build=False, bench=False, auto=False, restart_supervisor=False, restart_systemd=False, requirements=False, no_backup=False, force=False, reset=False):
-	"Update bench"
+def update(pull, patch, build, bench, requirements, restart_supervisor, restart_systemd, no_backup, force, reset):
+	from bench.utils import update
+	update(pull=pull, patch=patch, build=build, bench=bench, requirements=requirements, restart_supervisor=restart_supervisor, restart_systemd=restart_systemd, backup= not no_backup, force=force, reset=reset)
 
-	if not is_bench_directory():
-		"""Update only bench if bench update called from outside a bench"""
-		update_bench(bench_repo=True, requirements=True)
-		sys.exit()
 
-	if not (pull or patch or build or bench or requirements):
-		pull, patch, build, bench, requirements = True, True, True, True, True
-
-	if auto:
-		sys.exit(1)
-
-	patches.run(bench_path='.')
-	conf = get_config(".")
-
-	if bench and conf.get('update_bench_on_update'):
-		update_bench(bench_repo=True, requirements=False)
-		restart_update({
-				'pull': pull,
-				'patch': patch,
-				'build': build,
-				'requirements': requirements,
-				'no-backup': no_backup,
-				'restart-supervisor': restart_supervisor,
-				'reset': reset
-		})
-
-	if conf.get('release_bench'):
-		print('Release bench, cannot update')
-		sys.exit(1)
-
-	validate_branch()
-
-	version_upgrade = is_version_upgrade()
-	if version_upgrade[0]:
-		print()
-		print()
-		print("This update will cause a major version change in Frappe/ERPNext from {0} to {1}.".format(*version_upgrade[1:]))
-		print("This would take significant time to migrate and might break custom apps.")
-		click.confirm('Do you want to continue?', abort=True)
-
-	_update(pull, patch, build, bench, auto, restart_supervisor, restart_systemd, requirements, no_backup, force=force, reset=reset)
-
-def _update(pull=False, patch=False, build=False, update_bench=False, auto=False, restart_supervisor=False,
-		restart_systemd=False, requirements=False, no_backup=False, bench_path='.', force=False, reset=False):
-	conf = get_config(bench_path=bench_path)
-	version_upgrade = is_version_upgrade(bench_path=bench_path)
-
-	if version_upgrade[0] or (not version_upgrade[0] and force):
-		validate_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path)
-
-	before_update(bench_path=bench_path, requirements=requirements)
-
-	conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 })
-	update_config(conf, bench_path=bench_path)
-
-	if not no_backup:
-		print('Backing up sites...')
-		backup_all_sites(bench_path=bench_path)
-
-	if pull:
-		pull_all_apps(bench_path=bench_path, reset=reset)
-
-	if requirements:
-		update_requirements(bench_path=bench_path)
-		update_node_packages(bench_path=bench_path)
-
-	if version_upgrade[0] or (not version_upgrade[0] and force):
-		pre_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path)
-		import bench.utils, bench.app
-		print('Reloading bench...')
-		reload_module(bench.utils)
-		reload_module(bench.app)
-
-	if patch:
-		print('Patching sites...')
-		patch_sites(bench_path=bench_path)
-	if build:
-		build_assets(bench_path=bench_path)
-	if version_upgrade[0] or (not version_upgrade[0] and force):
-		post_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path)
-	if restart_supervisor or conf.get('restart_supervisor_on_update'):
-		restart_supervisor_processes(bench_path=bench_path)
-	if restart_systemd or conf.get('restart_systemd_on_update'):
-		restart_systemd_processes(bench_path=bench_path)
-
-	conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 })
-	update_config(conf, bench_path=bench_path)
-
-	print("_"*80)
-	print("Bench: Deployment tool for Frappe and ERPNext (https://erpnext.org).")
-	print("Open source depends on your contributions, so please contribute bug reports, patches, fixes or cash and be a part of the community")
-	print()
-
-@click.command('retry-upgrade')
+@click.command('retry-upgrade', help="Retry a failed upgrade")
 @click.option('--version', default=5)
 def retry_upgrade(version):
 	pull_all_apps()
@@ -126,35 +34,24 @@ def retry_upgrade(version):
 	build_assets()
 	post_upgrade(version-1, version)
 
-def restart_update(kwargs):
-	args = ['--'+k for k, v in list(kwargs.items()) if v]
-	os.execv(sys.argv[0], sys.argv[:2] + args)
 
-@click.command('switch-to-branch')
+@click.command('switch-to-branch', help="Switch all apps to specified branch, or specify apps separated by space")
 @click.argument('branch')
 @click.argument('apps', nargs=-1)
 @click.option('--upgrade',is_flag=True)
 def switch_to_branch(branch, apps, upgrade=False):
-	"Switch all apps to specified branch, or specify apps separated by space"
 	from bench.app import switch_to_branch
 	switch_to_branch(branch=branch, apps=list(apps), upgrade=upgrade)
-	print('Switched to ' + branch)
-	print('Please run `bench update --patch` to be safe from any differences in database schema')
 
-@click.command('switch-to-master')
+
+@click.command('switch-to-master', help="[DEPRECATED]: Switch frappe and erpnext to master branch")
 def switch_to_master():
-	"Switch frappe and erpnext to master branch"
-	from bench.app import switch_to_master
-	switch_to_master(apps=['frappe', 'erpnext'])
-	print()
-	print('Switched to master')
-	print('Please run `bench update --patch` to be safe from any differences in database schema')
+	from bench.utils import log
+	log("`switch-to-master` has been deprecated as master branches were renamed to version-11")
+
 
 @click.command('switch-to-develop')
 def switch_to_develop(upgrade=False):
 	"Switch frappe and erpnext to develop branch"
 	from bench.app import switch_to_develop
 	switch_to_develop(apps=['frappe', 'erpnext'])
-	print()
-	print('Switched to develop')
-	print('Please run `bench update --patch` to be safe from any differences in database schema')

bench/commands/utils.py

@@ -1,23 +1,25 @@
+# imports - standard imports
+import os
+import sys
+
+# imports - third party imports
 import click
-import sys, os, copy
 
 
-@click.command('start')
+@click.command('start', help="Start Frappe development processes")
 @click.option('--no-dev', is_flag=True, default=False)
 @click.option('--concurrency', '-c', type=str)
 @click.option('--procfile', '-p', type=str)
 def start(no_dev, concurrency, procfile):
-	"Start Frappe development processes"
 	from bench.utils import start
 	start(no_dev=no_dev, concurrency=concurrency, procfile=procfile)
 
 
-@click.command('restart')
+@click.command('restart', help="Restart supervisor processes or systemd units")
 @click.option('--web', is_flag=True, default=False)
 @click.option('--supervisor', is_flag=True, default=False)
 @click.option('--systemd', is_flag=True, default=False)
 def restart(web, supervisor, systemd):
-	"Restart supervisor processes or systemd units"
 	from bench.utils import restart_supervisor_processes, restart_systemd_processes
 	from bench.config.common_site_config import get_config
 	if get_config('.').get('restart_supervisor_on_update') or supervisor:
@@ -25,134 +27,112 @@ def restart(web, supervisor, systemd):
 	if get_config('.').get('restart_systemd_on_update') or systemd:
 		restart_systemd_processes(bench_path='.', web_workers=web)
 
-@click.command('set-nginx-port')
+
+@click.command('set-nginx-port', help="Set NGINX port for site")
 @click.argument('site')
 @click.argument('port', type=int)
 def set_nginx_port(site, port):
-	"Set nginx port for site"
 	from bench.config.site_config import set_nginx_port
 	set_nginx_port(site, port)
 
 
-@click.command('set-ssl-certificate')
+@click.command('set-ssl-certificate', help="Set SSL certificate path for site")
 @click.argument('site')
 @click.argument('ssl-certificate-path')
 def set_ssl_certificate(site, ssl_certificate_path):
-	"Set ssl certificate path for site"
 	from bench.config.site_config import set_ssl_certificate
 	set_ssl_certificate(site, ssl_certificate_path)
 
 
-@click.command('set-ssl-key')
+@click.command('set-ssl-key', help="Set SSL certificate private key path for site")
 @click.argument('site')
 @click.argument('ssl-certificate-key-path')
 def set_ssl_certificate_key(site, ssl_certificate_key_path):
-	"Set ssl certificate private key path for site"
 	from bench.config.site_config import set_ssl_certificate_key
 	set_ssl_certificate_key(site, ssl_certificate_key_path)
 
 
-@click.command('set-url-root')
+@click.command('set-url-root', help="Set URL root for site")
 @click.argument('site')
 @click.argument('url-root')
 def set_url_root(site, url_root):
-	"Set url root for site"
 	from bench.config.site_config import set_url_root
 	set_url_root(site, url_root)
 
 
-@click.command('set-mariadb-host')
+@click.command('set-mariadb-host', help="Set MariaDB host for bench")
 @click.argument('host')
 def set_mariadb_host(host):
-	"Set MariaDB host for bench"
 	from bench.utils import set_mariadb_host
 	set_mariadb_host(host)
 
-@click.command('set-redis-cache-host')
+
+@click.command('set-redis-cache-host', help="Set Redis cache host for bench")
 @click.argument('host')
 def set_redis_cache_host(host):
 	"""
-	Set Redis cache host for bench
-	Eg: bench set-redis-cache-host localhost:6379/1
+	Usage: bench set-redis-cache-host localhost:6379/1
 	"""
 	from bench.utils import set_redis_cache_host
 	set_redis_cache_host(host)
 
-@click.command('set-redis-queue-host')
+
+@click.command('set-redis-queue-host', help="Set Redis queue host for bench")
 @click.argument('host')
 def set_redis_queue_host(host):
 	"""
-	Set Redis queue host for bench
-	Eg: bench set-redis-queue-host localhost:6379/2
+	Usage: bench set-redis-queue-host localhost:6379/2
 	"""
 	from bench.utils import set_redis_queue_host
 	set_redis_queue_host(host)
 
-@click.command('set-redis-socketio-host')
+
+@click.command('set-redis-socketio-host', help="Set Redis socketio host for bench")
 @click.argument('host')
 def set_redis_socketio_host(host):
 	"""
-	Set Redis socketio host for bench
-	Eg: bench set-redis-socketio-host localhost:6379/3
+	Usage: bench set-redis-socketio-host localhost:6379/3
 	"""
 	from bench.utils import set_redis_socketio_host
 	set_redis_socketio_host(host)
 
 
-@click.command('set-default-site')
+@click.command('set-default-site', help="Set default site for bench")
 @click.argument('site')
 def set_default_site(site):
-	"Set default site for bench"
 	from bench.utils import set_default_site
 	set_default_site(site)
 
 
-@click.command('download-translations')
+@click.command('download-translations', help="Download latest translations")
 def download_translations():
-	"Download latest translations"
 	from bench.utils import download_translations_p
 	download_translations_p()
 
-@click.command('renew-lets-encrypt')
+
+@click.command('renew-lets-encrypt', help="Renew Let's Encrypt certificate")
 def renew_lets_encrypt():
-	"Renew Let's Encrypt certificate"
 	from bench.config.lets_encrypt import renew_certs
 	renew_certs()
 
-@click.command()
-def shell(bench_path='.'):
-	if not os.environ.get('SHELL'):
-		print("Cannot get shell")
-		sys.exit(1)
-	if not os.path.exists('sites'):
-		print("sites dir doesn't exist")
-		sys.exit(1)
-	env = copy.copy(os.environ)
-	env['PS1'] = '(' + os.path.basename(os.path.dirname(os.path.abspath(__file__))) + ')' + env.get('PS1', '')
-	env['PATH'] = os.path.dirname(os.path.abspath(os.path.join('env','bin')) + ':' + env['PATH'])
-	os.chdir('sites')
-	os.execve(env['SHELL'], [env['SHELL']], env)
 
-
-@click.command('backup')
+@click.command('backup', help="Backup single site")
 @click.argument('site')
 def backup_site(site):
-	"backup site"
 	from bench.utils import get_sites, backup_site
 	if site not in get_sites(bench_path='.'):
-		print('site not found')
+		print('Site `{0}` not found'.format(site))
 		sys.exit(1)
 	backup_site(site, bench_path='.')
 
 
-@click.command('backup-all-sites')
+@click.command('backup-all-sites', help="Backup all sites in current bench")
 def backup_all_sites():
-	"backup all sites"
 	from bench.utils import backup_all_sites
 	backup_all_sites(bench_path='.')
 
 
-@click.command('release')
+@click.command('release', help="Release a Frappe app (internal to the Frappe team)")
 @click.argument('app')
 @click.argument('bump-type', type=click.Choice(['major', 'minor', 'patch', 'stable', 'prerelease']))
 @click.option('--from-branch', default='develop')
@@ -162,48 +142,41 @@ def backup_all_sites():
 @click.option('--repo-name')
 @click.option('--dont-frontport', is_flag=True, default=False, help='Front port fixes to new branches, example merging hotfix(v10) into staging-fixes(v11)')
 def release(app, bump_type, from_branch, to_branch, owner, repo_name, remote, dont_frontport):
-	"Release app (internal to the Frappe team)"
 	from bench.release import release
 	frontport = not dont_frontport
-	release(bench_path='.', app=app, bump_type=bump_type, from_branch=from_branch, to_branch=to_branch,
-		remote=remote, owner=owner, repo_name=repo_name, frontport=frontport)
+	release(bench_path='.', app=app, bump_type=bump_type, from_branch=from_branch, to_branch=to_branch, remote=remote, owner=owner, repo_name=repo_name, frontport=frontport)
 
 
-@click.command('prepare-beta-release')
+@click.command('prepare-beta-release', help="Prepare major beta release from develop branch")
 @click.argument('app')
 @click.option('--owner', default='frappe')
 def prepare_beta_release(app, owner):
-	"""Prepare major beta release from develop branch"""
 	from bench.prepare_beta_release import prepare_beta_release
 	prepare_beta_release(bench_path='.', app=app, owner=owner)
 
 
-@click.command('disable-production')
+@click.command('disable-production', help="Disables production environment for the bench.")
 def disable_production():
-	"""Disables production environment for the bench."""
 	from bench.config.production_setup import disable_production
 	disable_production(bench_path='.')
 
 
-@click.command('src')
+@click.command('src', help="Prints bench source folder path, which can be used as: cd `bench src`")
 def bench_src():
-	"""Prints bench source folder path, which can be used as: cd `bench src` """
 	import bench
 	print(os.path.dirname(bench.__path__[0]))
 
 
-@click.command('find')
+@click.command('find', help="Finds benches recursively from location")
 @click.argument('location', default='')
 def find_benches(location):
-	"""Finds benches recursively from location"""
 	from bench.utils import find_benches
 	find_benches(directory=location)
 
 
-@click.command('migrate-env')
+@click.command('migrate-env', help="Migrate Virtual Environment to desired Python Version")
 @click.argument('python', type=str)
 @click.option('--no-backup', 'backup', is_flag=True, default=True)
 def migrate_env(python, backup=True):
-	"""Migrate Virtual Environment to desired Python Version"""
 	from bench.utils import migrate_env
 	migrate_env(python=python, backup=backup)

bench/config/nginx.py

@@ -1,8 +1,24 @@
-import os, json, click, random, string, hashlib
-from bench.utils import get_sites, get_bench_name, exec_cmd
+# imports - standard imports
+import hashlib
+import os
+import random
+import string
+
+# imports - third party imports
+import click
 from six import string_types
 
+# imports - module imports
+from bench.utils import get_bench_name, get_sites
+
+
 def make_nginx_conf(bench_path, yes=False):
+	conf_path = os.path.join(bench_path, "config", "nginx.conf")
+
+	if not yes and os.path.exists(conf_path):
+		if not click.confirm('nginx.conf already exists and this will overwrite it. Do you want to continue?'):
+			return
+
 	from bench import env
 	from bench.config.common_site_config import get_config
 
@@ -37,10 +53,6 @@ def make_nginx_conf(bench_path, yes=False):
 
 	nginx_conf = template.render(**template_vars)
 
-	conf_path = os.path.join(bench_path, "config", "nginx.conf")
-	if not yes and os.path.exists(conf_path):
-		click.confirm('nginx.conf already exists and this will overwrite it. Do you want to continue?',
-			abort=True)
 
 	with open(conf_path, "w") as f:
 		f.write(nginx_conf)

bench/config/procfile.py

@@ -3,7 +3,7 @@ from bench.utils import find_executable
 from bench.app import use_rq
 from bench.config.common_site_config import get_config
 
-def setup_procfile(bench_path, yes=False):
+def setup_procfile(bench_path, yes=False, skip_redis=False):
 	config = get_config(bench_path=bench_path)
 	procfile_path = os.path.join(bench_path, 'Procfile')
 	if not yes and os.path.exists(procfile_path):
@@ -14,7 +14,8 @@ def setup_procfile(bench_path, yes=False):
 		node=find_executable("node") or find_executable("nodejs"),
 		use_rq=use_rq(bench_path),
 		webserver_port=config.get('webserver_port'),
-		CI=os.environ.get('CI'))
+		CI=os.environ.get('CI'),
+		skip_redis=skip_redis)
 
 	with open(procfile_path, 'w') as f:
 		f.write(procfile)

bench/config/templates/Procfile

@@ -1,6 +1,8 @@
+{% if not skip_redis %}
 redis_cache: redis-server config/redis_cache.conf
 redis_socketio: redis-server config/redis_socketio.conf
 redis_queue: redis-server config/redis_queue.conf
+{% endif %}
 web: bench serve {% if webserver_port -%} --port {{ webserver_port }} {%- endif %}
 
 socketio: {{ node }} apps/frappe/socketio.js

bench/prepare_staging.py

@@ -19,9 +19,7 @@ def prepare_staging(bench_path, app, remote='upstream'):
 		print('No commits to release')
 		return
 
-	print()
 	print(message)
-	print()
 
 	click.confirm('Do you want to continue?', abort=True)
 
@@ -52,13 +50,13 @@ def create_staging(repo_path, from_branch='develop'):
 		g.merge(from_branch, '--no-ff')
 	except git.exc.GitCommandError as e:
 		handle_merge_error(e, source=from_branch, target='staging')
-	
+
 	g.checkout(from_branch)
 	try:
 		g.merge('staging')
 	except git.exc.GitCommandError as e:
 		handle_merge_error(e, source='staging', target=from_branch)
-	
+
 def push_commits(repo_path, remote='upstream'):
 	print('pushing staging branch of', repo_path)
 

bench/release.py

@@ -83,9 +83,7 @@ def bump(bench_path, app, bump_type, from_branch, to_branch, remote, owner, repo
 		print('No commits to release')
 		return
 
-	print()
 	print(message)
-	print()
 
 	click.confirm('Do you want to continue?', abort=True)
 

bench/tests/test_base.py

@@ -0,0 +1,96 @@
+# imports - standard imports
+import json
+import os
+import shutil
+import subprocess
+import sys
+import unittest
+import getpass
+
+# imports - module imports
+import bench
+import bench.utils
+
+
+class TestBenchBase(unittest.TestCase):
+	def setUp(self):
+		self.benches_path = "."
+		self.benches = []
+
+	def tearDown(self):
+		for bench_name in self.benches:
+			bench_path = os.path.join(self.benches_path, bench_name)
+			mariadb_password = "travis" if os.environ.get("CI") else getpass.getpass(prompt="Enter MariaDB root Password: ")
+			if os.path.exists(bench_path):
+				sites = bench.utils.get_sites(bench_path=bench_path)
+				for site in sites:
+					subprocess.call(["bench", "drop-site", site, "--force", "--no-backup", "--root-password", mariadb_password], cwd=bench_path)
+				shutil.rmtree(bench_path, ignore_errors=True)
+
+	def assert_folders(self, bench_name):
+		for folder in bench.utils.folders_in_bench:
+			self.assert_exists(bench_name, folder)
+		self.assert_exists(bench_name, "apps", "frappe")
+
+	def assert_virtual_env(self, bench_name):
+		bench_path = os.path.abspath(bench_name)
+		python_path = os.path.abspath(os.path.join(bench_path, "env", "bin", "python"))
+		self.assertTrue(python_path.startswith(bench_path))
+		for subdir in ("bin", "include", "lib", "share"):
+			self.assert_exists(bench_name, "env", subdir)
+
+	def assert_config(self, bench_name):
+		for config, search_key in (
+			("redis_queue.conf", "redis_queue.rdb"),
+			("redis_socketio.conf", "redis_socketio.rdb"),
+			("redis_cache.conf", "redis_cache.rdb")):
+
+			self.assert_exists(bench_name, "config", config)
+
+			with open(os.path.join(bench_name, "config", config), "r") as f:
+				self.assertTrue(search_key in f.read())
+
+	def assert_common_site_config(self, bench_name, expected_config):
+		common_site_config_path = os.path.join(self.benches_path, bench_name, 'sites', 'common_site_config.json')
+		self.assertTrue(os.path.exists(common_site_config_path))
+
+		with open(common_site_config_path, "r") as f:
+			config = json.load(f)
+
+		for key, value in list(expected_config.items()):
+			self.assertEqual(config.get(key), value)
+
+	def assert_exists(self, *args):
+		self.assertTrue(os.path.exists(os.path.join(*args)))
+
+	def new_site(self, site_name, bench_name):
+		new_site_cmd = ["bench", "new-site", site_name, "--admin-password", "admin"]
+
+		if os.environ.get('CI'):
+			new_site_cmd.extend(["--mariadb-root-password", "travis"])
+
+		subprocess.call(new_site_cmd, cwd=os.path.join(self.benches_path, bench_name))
+
+	def init_bench(self, bench_name, **kwargs):
+		self.benches.append(bench_name)
+		frappe_tmp_path = "/tmp/frappe"
+
+		if not os.path.exists(frappe_tmp_path):
+			bench.utils.exec_cmd("git clone https://github.com/frappe/frappe --depth 1 --origin upstream {location}".format(location=frappe_tmp_path))
+
+		kwargs.update(dict(
+			python=sys.executable,
+			no_procfile=True,
+			no_backups=True,
+			skip_assets=True,
+			frappe_path=frappe_tmp_path
+		))
+
+		if not os.path.exists(os.path.join(self.benches_path, bench_name)):
+			bench.utils.init(bench_name, **kwargs)
+			bench.utils.exec_cmd("git remote set-url upstream https://github.com/frappe/frappe", cwd=os.path.join(self.benches_path, bench_name, "apps", "frappe"))
+
+	def file_exists(self, path):
+		if os.environ.get("CI"):
+			return not subprocess.call(["sudo", "test", "-f", path])
+		return os.path.isfile(path)

bench/tests/test_init.py

@@ -1,26 +1,20 @@
-
+# imports - standard imports
+import json
+import os
+import subprocess
 import unittest
-import json, os, shutil, subprocess
+
+# imports - third paty imports
+import git
+
+# imports - module imports
 import bench
 import bench.utils
-import bench.app
-import bench.config.common_site_config
-import bench.cli
 from bench.release import get_bumped_version
+from bench.tests.test_base import TestBenchBase
 
-bench.cli.from_command_line = True
-
-class TestBenchInit(unittest.TestCase):
-	def setUp(self):
-		self.benches_path = "."
-		self.benches = []
-
-	def tearDown(self):
-		for bench_name in self.benches:
-			bench_path = os.path.join(self.benches_path, bench_name)
-			if os.path.exists(bench_path):
-				shutil.rmtree(bench_path, ignore_errors=True)
 
+class TestBenchInit(TestBenchBase):
 	def test_semantic_version(self):
 		self.assertEqual( get_bumped_version('11.0.4', 'major'), '12.0.0' )
 		self.assertEqual( get_bumped_version('11.0.4', 'minor'), '11.1.0' )
@@ -35,20 +29,14 @@ class TestBenchInit(unittest.TestCase):
 
 	def test_init(self, bench_name="test-bench", **kwargs):
 		self.init_bench(bench_name, **kwargs)
-
 		self.assert_folders(bench_name)
-
 		self.assert_virtual_env(bench_name)
-
-		self.assert_common_site_config(bench_name, bench.config.common_site_config.default_config)
-
 		self.assert_config(bench_name)
 
-		self.assert_socketio(bench_name)
 
 	def test_multiple_benches(self):
-		# 1st bench
-		self.test_init("test-bench-1")
+		for bench_name in ("test-bench-1", "test-bench-2"):
+			self.init_bench(bench_name)
 
 		self.assert_common_site_config("test-bench-1", {
 			"webserver_port": 8000,
@@ -59,9 +47,6 @@ class TestBenchInit(unittest.TestCase):
 			"redis_cache": "redis://localhost:13000"
 		})
 
-		# 2nd bench
-		self.test_init("test-bench-2")
-
 		self.assert_common_site_config("test-bench-2", {
 			"webserver_port": 8001,
 			"socketio_port": 9001,
@@ -71,196 +56,92 @@ class TestBenchInit(unittest.TestCase):
 			"redis_cache": "redis://localhost:13001"
 		})
 
+
+
 	def test_new_site(self):
-		self.init_bench('test-bench')
-		self.new_site("test-site-1.dev")
+		bench_name = "test-bench"
+		site_name = "test-site.local"
+		bench_path = os.path.join(self.benches_path, bench_name)
+		site_path = os.path.join(bench_path, "sites", site_name)
+		site_config_path = os.path.join(site_path, "site_config.json")
 
-	def new_site(self, site_name):
-		new_site_cmd = ["bench", "new-site", site_name, "--admin-password", "admin"]
-
-		# set in CI
-		if os.environ.get('CI'):
-			new_site_cmd.extend(["--mariadb-root-password", "travis"])
-
-		subprocess.check_output(new_site_cmd, cwd=os.path.join(self.benches_path, "test-bench"))
-
-		site_path = os.path.join(self.benches_path, "test-bench", "sites", site_name)
+		self.init_bench(bench_name)
+		bench.utils.exec_cmd("bench setup requirements --node", cwd=bench_path)
+		self.new_site(site_name, bench_name)
 
 		self.assertTrue(os.path.exists(site_path))
 		self.assertTrue(os.path.exists(os.path.join(site_path, "private", "backups")))
 		self.assertTrue(os.path.exists(os.path.join(site_path, "private", "files")))
 		self.assertTrue(os.path.exists(os.path.join(site_path, "public", "files")))
-
-		site_config_path = os.path.join(site_path, "site_config.json")
 		self.assertTrue(os.path.exists(site_config_path))
+
 		with open(site_config_path, "r") as f:
 			site_config = json.loads(f.read())
 
-		for key in ("db_name", "db_password"):
-			self.assertTrue(key in site_config)
-			self.assertTrue(site_config[key])
+			for key in ("db_name", "db_password"):
+				self.assertTrue(key in site_config)
+				self.assertTrue(site_config[key])
 
 	def test_get_app(self):
-		site_name = "test-site-2.dev"
-		self.init_bench('test-bench')
-
-		self.new_site(site_name)
+		self.init_bench("test-bench")
 		bench_path = os.path.join(self.benches_path, "test-bench")
+		bench.utils.exec_cmd("bench get-app frappe_theme --skip-assets", cwd=bench_path)
+		self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "frappe_theme")))
+		app_installed_in_env = "frappe_theme" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8')
+		self.assertTrue(app_installed_in_env)
 
-		bench.app.get_app("https://github.com/frappe/frappe-client", bench_path=bench_path)
-		self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "frappeclient")))
 
 	def test_install_app(self):
-		site_name = "test-site-3.dev"
-		self.init_bench('test-bench')
-
-		self.new_site(site_name)
+		bench_name = "test-bench"
+		site_name = "install-app.test"
 		bench_path = os.path.join(self.benches_path, "test-bench")
 
-		# get app
-		bench.app.get_app("https://github.com/frappe/erpnext", "develop", bench_path=bench_path)
+		self.init_bench(bench_name)
+		bench.utils.exec_cmd("bench setup requirements --node", cwd=bench_path)
+		bench.utils.exec_cmd("bench build", cwd=bench_path)
+		bench.utils.exec_cmd("bench get-app erpnext", cwd=bench_path)
 
 		self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "erpnext")))
 
-		# install app
-		bench.app.install_app("erpnext", bench_path=bench_path)
+		# check if app is installed
+		app_installed_in_env = "erpnext" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8')
+		self.assertTrue(app_installed_in_env)
 
-		# install it to site
-		subprocess.check_output(["bench", "--site", site_name, "install-app", "erpnext"], cwd=bench_path)
+		# create and install app on site
+		self.new_site(site_name, bench_name)
+		bench.utils.exec_cmd("bench --site {0} install-app erpnext".format(site_name), cwd=bench_path)
 
-		out = subprocess.check_output(["bench", "--site", site_name, "list-apps"], cwd=bench_path)
-		self.assertTrue("erpnext" in out)
+		app_installed_on_site = subprocess.check_output(["bench", "--site", site_name, "list-apps"], cwd=bench_path).decode('utf8')
+		self.assertTrue("erpnext" in app_installed_on_site)
 
 
 	def test_remove_app(self):
-		self.init_bench('test-bench')
-
+		self.init_bench("test-bench")
 		bench_path = os.path.join(self.benches_path, "test-bench")
 
-		# get app
-		bench.app.get_app("https://github.com/frappe/erpnext", "develop", bench_path=bench_path)
-
-		self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "erpnext")))
-
-		# remove it
-		bench.app.remove_app("erpnext", bench_path=bench_path)
+		bench.utils.exec_cmd("bench setup requirements --node", cwd=bench_path)
+		bench.utils.exec_cmd("bench get-app erpnext --branch version-12 --skip-assets --overwrite", cwd=bench_path)
+		bench.utils.exec_cmd("bench remove-app erpnext", cwd=bench_path)
 
+		with open(os.path.join(bench_path, "sites", "apps.txt")) as f:
+			self.assertFalse("erpnext" in f.read())
+		self.assertFalse("erpnext" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8'))
 		self.assertFalse(os.path.exists(os.path.join(bench_path, "apps", "erpnext")))
 
 
 	def test_switch_to_branch(self):
-		self.init_bench('test-bench')
-
+		self.init_bench("test-bench")
 		bench_path = os.path.join(self.benches_path, "test-bench")
 		app_path = os.path.join(bench_path, "apps", "frappe")
 
-		bench.app.switch_branch(branch="master", apps=["frappe"], bench_path=bench_path, check_upgrade=False)
-		out = subprocess.check_output(['git', 'status'], cwd=app_path)
-		self.assertTrue("master" in out)
+		bench.utils.exec_cmd("bench switch-to-branch version-12 frappe", cwd=bench_path)
+		app_branch_after_switch = str(git.Repo(path=app_path).active_branch)
+		self.assertEqual("version-12", app_branch_after_switch)
 
-		# bring it back to develop!
-		bench.app.switch_branch(branch="develop", apps=["frappe"], bench_path=bench_path, check_upgrade=False)
-		out = subprocess.check_output(['git', 'status'], cwd=app_path)
-		self.assertTrue("develop" in out)
+		bench.utils.exec_cmd("bench switch-to-branch develop frappe", cwd=bench_path)
+		app_branch_after_second_switch = str(git.Repo(path=app_path).active_branch)
+		self.assertEqual("develop", app_branch_after_second_switch)
 
-	def init_bench(self, bench_name, **kwargs):
-		self.benches.append(bench_name)
-		bench.utils.init(bench_name, **kwargs)
 
-	def test_drop_site(self):
-		self.init_bench('test-bench')
-		# Check without archive_path given to drop-site command
-		self.drop_site("test-drop-without-archive-path")
-
-		# Check with archive_path given to drop-site command
-		home = os.path.abspath(os.path.expanduser('~'))
-		archived_sites_path = os.path.join(home, 'archived_sites')
-
-		self.drop_site("test-drop-with-archive-path", archived_sites_path=archived_sites_path)
-
-	def drop_site(self, site_name, archived_sites_path=None):
-		self.new_site(site_name)
-
-		drop_site_cmd = ['bench', 'drop-site', site_name]
-
-		if archived_sites_path:
-			drop_site_cmd.extend(['--archived-sites-path', archived_sites_path])
-
-		if os.environ.get('CI'):
-			drop_site_cmd.extend(['--root-password', 'travis'])
-
-		bench_path = os.path.join(self.benches_path, 'test-bench')
-		try:
-			subprocess.check_output(drop_site_cmd, cwd=bench_path)
-		except subprocess.CalledProcessError as err:
-			print(err.output)
-
-		if not archived_sites_path:
-			archived_sites_path = os.path.join(bench_path, 'archived_sites')
-			self.assertTrue(os.path.exists(archived_sites_path))
-			self.assertTrue(os.path.exists(os.path.join(archived_sites_path, site_name)))
-
-		else:
-			self.assertTrue(os.path.exists(archived_sites_path))
-			self.assertTrue(os.path.exists(os.path.join(archived_sites_path, site_name)))
-
-	def assert_folders(self, bench_name):
-		for folder in bench.utils.folders_in_bench:
-			self.assert_exists(bench_name, folder)
-
-		self.assert_exists(bench_name, "sites", "assets")
-		self.assert_exists(bench_name, "apps", "frappe")
-		self.assert_exists(bench_name, "apps", "frappe", "setup.py")
-
-	def assert_virtual_env(self, bench_name):
-		bench_path = os.path.abspath(bench_name)
-		python = os.path.join(bench_path, "env", "bin", "python")
-		python_path = bench.utils.get_cmd_output('{python} -c "import os; print os.path.dirname(os.__file__)"'.format(python=python))
-
-		# part of bench's virtualenv
-		self.assertTrue(python_path.startswith(bench_path))
-		self.assert_exists(python_path)
-		self.assert_exists(python_path, "site-packages")
-		self.assert_exists(python_path, "site-packages", "IPython")
-		self.assert_exists(python_path, "site-packages", "pip")
-
-		site_packages = os.listdir(os.path.join(python_path, "site-packages"))
-		# removing test case temporarily
-		# as develop and master branch havin differnt version of mysqlclient
-		#self.assertTrue(any(package.startswith("mysqlclient-1.3.12") for package in site_packages))
-
-	def assert_config(self, bench_name):
-		for config, search_key in (
-			("redis_queue.conf", "redis_queue.rdb"),
-			("redis_socketio.conf", "redis_socketio.rdb"),
-			("redis_cache.conf", "redis_cache.rdb")):
-
-			self.assert_exists(bench_name, "config", config)
-
-			with open(os.path.join(bench_name, "config", config), "r") as f:
-				f = f.read().decode("utf-8")
-				self.assertTrue(search_key in f)
-
-	def assert_socketio(self, bench_name):
-		try: # for v10 and under
-			self.assert_exists(bench_name, "node_modules")
-			self.assert_exists(bench_name, "node_modules", "socket.io")
-		except: # for v11 and above
-			self.assert_exists(bench_name, "apps", "frappe", "node_modules")
-			self.assert_exists(bench_name, "apps", "frappe", "node_modules", "socket.io")
-
-	def assert_common_site_config(self, bench_name, expected_config):
-		common_site_config_path = os.path.join(bench_name, 'sites', 'common_site_config.json')
-		self.assertTrue(os.path.exists(common_site_config_path))
-
-		config = self.load_json(common_site_config_path)
-
-		for key, value in list(expected_config.items()):
-			self.assertEqual(config.get(key), value)
-
-	def assert_exists(self, *args):
-		self.assertTrue(os.path.exists(os.path.join(*args)))
-
-	def load_json(self, path):
-		with open(path, "r") as f:
-			return json.loads(f.read().decode("utf-8"))
+if __name__ == '__main__':
+	unittest.main()

bench/tests/test_setup_production.py

@@ -1,69 +1,51 @@
-
-from bench.tests import test_init
-from bench.config.production_setup import setup_production, get_supervisor_confdir, disable_production
-import bench.utils
-import os
+# imports - standard imports
 import getpass
+import os
 import re
-import unittest
+import subprocess
 import time
+import unittest
 
-class TestSetupProduction(test_init.TestBenchInit):
-	# setUp, tearDown and other tests are defiend in TestBenchInit
+# imports - module imports
+import bench.utils
+from bench.config.production_setup import get_supervisor_confdir
+from bench.tests.test_base import TestBenchBase
 
+
+class TestSetupProduction(TestBenchBase):
 	def test_setup_production(self):
-		self.test_multiple_benches()
-
 		user = getpass.getuser()
 
 		for bench_name in ("test-bench-1", "test-bench-2"):
 			bench_path = os.path.join(os.path.abspath(self.benches_path), bench_name)
-			setup_production(user, bench_path)
+			self.init_bench(bench_name)
+			bench.utils.exec_cmd("sudo bench setup production {0}".format(user), cwd=bench_path)
 			self.assert_nginx_config(bench_name)
 			self.assert_supervisor_config(bench_name)
-
-		# test after start of both benches
-		for bench_name in ("test-bench-1", "test-bench-2"):
 			self.assert_supervisor_process(bench_name)
 
 		self.assert_nginx_process()
-
-		# sudoers
-		bench.utils.setup_sudoers(user)
+		bench.utils.exec_cmd("sudo bench setup sudoers {0}".format(user))
 		self.assert_sudoers(user)
 
-		for bench_name in ("test-bench-1", "test-bench-2"):
+		for bench_name in self.benches:
 			bench_path = os.path.join(os.path.abspath(self.benches_path), bench_name)
-			disable_production(bench_path)
+			bench.utils.exec_cmd("sudo bench disable-production", cwd=bench_path)
 
-	def test_disable_production(self):
-		bench_name = 'test-disable-prod'
-		self.test_init(bench_name, frappe_branch='master')
-
-		user = getpass.getuser()
-
-		bench_path = os.path.join(os.path.abspath(self.benches_path), bench_name)
-		setup_production(user, bench_path)
-
-		disable_production(bench_path)
-
-		self.assert_nginx_link(bench_name)
-		self.assert_supervisor_link(bench_name)
-		self.assert_supervisor_process(bench_name=bench_name, disable_production=True)
 
 	def assert_nginx_config(self, bench_name):
 		conf_src = os.path.join(os.path.abspath(self.benches_path), bench_name, 'config', 'nginx.conf')
 		conf_dest = "/etc/nginx/conf.d/{bench_name}.conf".format(bench_name=bench_name)
 
-		self.assertTrue(os.path.exists(conf_src))
-		self.assertTrue(os.path.exists(conf_dest))
+		self.assertTrue(self.file_exists(conf_src))
+		self.assertTrue(self.file_exists(conf_dest))
 
 		# symlink matches
 		self.assertEqual(os.path.realpath(conf_dest), conf_src)
 
 		# file content
 		with open(conf_src, "r") as f:
-			f = f.read().decode("utf-8")
+			f = f.read()
 
 			for key in (
 					"upstream {bench_name}-frappe",
@@ -71,48 +53,56 @@ class TestSetupProduction(test_init.TestBenchInit):
 				):
 				self.assertTrue(key.format(bench_name=bench_name) in f)
 
+
 	def assert_nginx_process(self):
 		out = bench.utils.get_cmd_output("sudo nginx -t 2>&1")
 		self.assertTrue("nginx: configuration file /etc/nginx/nginx.conf test is successful" in out)
 
+
 	def assert_sudoers(self, user):
 		sudoers_file = '/etc/sudoers.d/frappe'
-		self.assertTrue(os.path.exists(sudoers_file))
+		self.assertTrue(self.file_exists(sudoers_file))
 
-		with open(sudoers_file, 'r') as f:
-			sudoers = f.read().decode('utf-8')
+		if os.environ.get("CI"):
+			sudoers = subprocess.check_output(["sudo", "cat", sudoers_file]).decode("utf-8")
+		else:
+			with open(sudoers_file, 'r') as f:
+				sudoers = f.read()
 
 		self.assertTrue('{user} ALL = (root) NOPASSWD: /usr/sbin/service nginx *'.format(user=user) in sudoers)
 		self.assertTrue('{user} ALL = (root) NOPASSWD: /usr/bin/supervisorctl'.format(user=user) in sudoers)
 		self.assertTrue('{user} ALL = (root) NOPASSWD: /usr/sbin/nginx'.format(user=user) in sudoers)
 
+
 	def assert_supervisor_config(self, bench_name, use_rq=True):
 		conf_src = os.path.join(os.path.abspath(self.benches_path), bench_name, 'config', 'supervisor.conf')
 
 		supervisor_conf_dir = get_supervisor_confdir()
 		conf_dest = "{supervisor_conf_dir}/{bench_name}.conf".format(supervisor_conf_dir=supervisor_conf_dir, bench_name=bench_name)
 
-		self.assertTrue(os.path.exists(conf_src))
-		self.assertTrue(os.path.exists(conf_dest))
+		self.assertTrue(self.file_exists(conf_src))
+		self.assertTrue(self.file_exists(conf_dest))
 
 		# symlink matches
 		self.assertEqual(os.path.realpath(conf_dest), conf_src)
 
 		# file content
 		with open(conf_src, "r") as f:
-			f = f.read().decode("utf-8")
+			f = f.read()
 
 			tests = [
 				"program:{bench_name}-frappe-web",
 				"program:{bench_name}-redis-cache",
 				"program:{bench_name}-redis-queue",
 				"program:{bench_name}-redis-socketio",
-				"program:{bench_name}-node-socketio",
 				"group:{bench_name}-web",
 				"group:{bench_name}-workers",
 				"group:{bench_name}-redis"
 			]
 
+			if not os.environ.get("CI"):
+				tests.append("program:{bench_name}-node-socketio")
+
 			if use_rq:
 				tests.extend([
 					"program:{bench_name}-frappe-schedule",
@@ -130,8 +120,11 @@ class TestSetupProduction(test_init.TestBenchInit):
 				])
 
 			for key in tests:
+				if key.format(bench_name=bench_name) not in f:
+					print(key.format(bench_name=bench_name))
 				self.assertTrue(key.format(bench_name=bench_name) in f)
 
+
 	def assert_supervisor_process(self, bench_name, use_rq=True, disable_production=False):
 		out = bench.utils.get_cmd_output("sudo supervisorctl status")
 
@@ -172,15 +165,6 @@ class TestSetupProduction(test_init.TestBenchInit):
 			else:
 				self.assertTrue(re.search(key.format(bench_name=bench_name), out))
 
-	def assert_nginx_link(self, bench_name):
-		nginx_conf_name = '{bench_name}.conf'.format(bench_name=bench_name)
-		nginx_conf_path = os.path.join('/etc/nginx/conf.d', nginx_conf_name)
 
-		self.assertFalse(os.path.islink(nginx_conf_path))
-
-	def assert_supervisor_link(self, bench_name):
-		supervisor_conf_dir = get_supervisor_confdir()
-		supervisor_conf_name = '{bench_name}.conf'.format(bench_name=bench_name)
-		supervisor_conf_path = os.path.join(supervisor_conf_dir, supervisor_conf_name)
-
-		self.assertFalse(os.path.islink(supervisor_conf_path))
+if __name__ == '__main__':
+	unittest.main()

bench/utils.py

@@ -1,14 +1,31 @@
-import errno, glob, grp, itertools, json, logging, multiprocessing, os, platform, pwd, re, select, shutil, site, subprocess, sys
+# imports - standard imports
+import errno
+import glob
+import grp
+import itertools
+import json
+import logging
+import multiprocessing
+import os
+import platform
+import pwd
+import re
+import select
+import shutil
+import site
+import subprocess
+import sys
 from datetime import datetime
 from distutils.spawn import find_executable
 
+# imports - third party imports
+import click
 import requests
-import semantic_version
 from six import iteritems
 from six.moves.urllib.parse import urlparse
 
+# imports - module imports
 import bench
-from bench import env
 
 
 class PatchError(Exception):
@@ -28,6 +45,7 @@ class color:
 	green = '\033[92m'
 	yellow = '\033[93m'
 	red = '\033[91m'
+	silver = '\033[90m'
 
 
 def is_bench_directory(directory=os.path.curdir):
@@ -60,6 +78,7 @@ def safe_decode(string, encoding = 'utf-8'):
 		pass
 	return string
 
+
 def get_frappe(bench_path='.'):
 	frappe = get_env_cmd('frappe', bench_path=bench_path)
 	if not os.path.exists(frappe):
@@ -67,12 +86,16 @@ def get_frappe(bench_path='.'):
 		print('bench get-app https://github.com/frappe/frappe.git')
 	return frappe
 
+
 def get_env_cmd(cmd, bench_path='.'):
 	return os.path.abspath(os.path.join(bench_path, 'env', 'bin', cmd))
 
-def init(path, apps_path=None, no_procfile=False, no_backups=False, no_auto_update=False,
-		frappe_path=None, frappe_branch=None, wheel_cache_dir=None, verbose=False, clone_from=None,
-		skip_redis_config_generation=False, clone_without_update=False, ignore_exist = False, skip_assets=False, python='python3'):
+
+def init(path, apps_path=None, no_procfile=False, no_backups=False,
+		frappe_path=None, frappe_branch=None, verbose=False, clone_from=None,
+		skip_redis_config_generation=False, clone_without_update=False, ignore_exist=False, skip_assets=False,
+		python='python3'):
+	"""Initialize a new bench directory"""
 	from bench.app import get_app, install_apps_from_path
 	from bench.config import redis
 	from bench.config.common_site_config import make_config
@@ -124,17 +147,110 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, no_auto_upda
 		redis.generate_config(path)
 
 	if not no_procfile:
-		setup_procfile(path)
+		setup_procfile(path, skip_redis=skip_redis_config_generation)
 	if not no_backups:
 		setup_backups(bench_path=path)
-	if not no_auto_update:
-		setup_auto_update(bench_path=path)
+
 	copy_patches_txt(path)
 
+
+def restart_update(kwargs):
+	args = ['--'+k for k, v in list(kwargs.items()) if v]
+	os.execv(sys.argv[0], sys.argv[:2] + args)
+
+
+def update(pull=False, patch=False, build=False, bench=False, restart_supervisor=False,
+	restart_systemd=False, requirements=False, backup=True, force=False, reset=False):
+	"""command: bench update"""
+
+	if not is_bench_directory():
+		"""Update only bench CLI if bench update called from outside a bench"""
+		update_bench(bench_repo=True, requirements=True)
+		sys.exit(0)
+
+	from bench import patches
+	from bench.app import is_version_upgrade, pull_all_apps, validate_branch
+	from bench.config.common_site_config import get_config, update_config
+
+	bench_path = os.path.abspath(".")
+	patches.run(bench_path=bench_path)
+	conf = get_config(bench_path)
+
+	if conf.get('release_bench'):
+		print('Release bench detected, cannot update!')
+		sys.exit(1)
+
+	if not (pull or patch or build or bench or requirements):
+		pull, patch, build, bench, requirements = True, True, True, True, True
+
+	if bench and conf.get('update_bench_on_update'):
+		update_bench(bench_repo=True, requirements=False)
+		restart_update({
+			'pull': pull,
+			'patch': patch,
+			'build': build,
+			'requirements': requirements,
+			'no-backup': backup,
+			'restart-supervisor': restart_supervisor,
+			'reset': reset
+		})
+
+	validate_branch()
+	version_upgrade = is_version_upgrade()
+
+	if version_upgrade[0]:
+		if force:
+			print("Force flag has been used for a major version change in Frappe and it's apps. \nThis will take significant time to migrate and might break custom apps.")
+		else:
+			print("This update will cause a major version change in Frappe/ERPNext from {0} to {1}. \nThis would take significant time to migrate and might break custom apps.".format(*version_upgrade[1:]))
+			click.confirm('Do you want to continue?', abort=True)
+
+	if version_upgrade[0] or (not version_upgrade[0] and force):
+		validate_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path)
+
+	before_update(bench_path=bench_path, requirements=requirements)
+
+	conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 })
+	update_config(conf, bench_path=bench_path)
+
+	if backup:
+		print('Backing up sites...')
+		backup_all_sites(bench_path=bench_path)
+
+	if pull:
+		pull_all_apps(bench_path=bench_path, reset=reset)
+
+	if requirements:
+		update_requirements(bench_path=bench_path)
+		update_node_packages(bench_path=bench_path)
+
+	if patch:
+		print('Patching sites...')
+		patch_sites(bench_path=bench_path)
+
+	if build:
+		build_assets(bench_path=bench_path)
+
+	if version_upgrade[0] or (not version_upgrade[0] and force):
+		post_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path)
+
+	if restart_supervisor or conf.get('restart_supervisor_on_update'):
+		restart_supervisor_processes(bench_path=bench_path)
+
+	if restart_systemd or conf.get('restart_systemd_on_update'):
+		restart_systemd_processes(bench_path=bench_path)
+
+	conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 })
+	update_config(conf, bench_path=bench_path)
+
+	print("_" * 80 + "\nBench: Deployment tool for Frappe and Frappe Applications (https://frappe.io/bench).\nOpen source depends on your contributions, so please contribute bug reports, patches, fixes or cash and be a part of the community")
+
+
 def copy_patches_txt(bench_path):
 	shutil.copy(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'patches', 'patches.txt'),
 		os.path.join(bench_path, 'patches.txt'))
 
+
 def clone_apps_from(bench_path, clone_from, update_app=True):
 	from .app import install_app
 	print('Copying apps from {0}...'.format(clone_from))
@@ -171,30 +287,15 @@ def clone_apps_from(bench_path, clone_from, update_app=True):
 	for app in apps:
 		setup_app(app)
 
+
 def exec_cmd(cmd, cwd='.'):
-	from .cli import from_command_line
+	import shlex
+	print("{0}$ {1}{2}".format(color.silver, cmd, color.nc))
+	cmd = shlex.split(cmd)
+	subprocess.call(cmd, cwd=cwd, universal_newlines=True)
 
-	is_async = False if from_command_line else True
-	if is_async:
-		stderr = stdout = subprocess.PIPE
-	else:
-		stderr = stdout = None
-
-	logger.info(cmd)
-
-	p = subprocess.Popen(cmd, cwd=cwd, shell=True, stdout=stdout, stderr=stderr,
-		universal_newlines=True)
-
-	if is_async:
-		return_code = print_output(p)
-	else:
-		return_code = p.wait()
-
-	if return_code > 0:
-		raise CommandFailedError(cmd)
 
 def which(executable, raise_err = False):
-	from distutils.spawn import find_executable
 	exec_ = find_executable(executable)
 
 	if not exec_ and raise_err:
@@ -204,6 +305,7 @@ def which(executable, raise_err = False):
 
 	return exec_
 
+
 def setup_env(bench_path='.', python = 'python3'):
 	python = which(python, raise_err = True)
 	pip    = os.path.join('env', 'bin', 'pip')
@@ -212,10 +314,12 @@ def setup_env(bench_path='.', python = 'python3'):
 	exec_cmd('{} -q install -U pip wheel six'.format(pip), cwd=bench_path)
 	exec_cmd('{} -q install -e git+https://github.com/frappe/python-pdfkit.git#egg=pdfkit'.format(pip), cwd=bench_path)
 
+
 def setup_socketio(bench_path='.'):
 	exec_cmd("npm install socket.io redis express superagent cookie babel-core less chokidar \
 		babel-cli babel-preset-es2015 babel-preset-es2016 babel-preset-es2017 babel-preset-babili", cwd=bench_path)
 
+
 def patch_sites(bench_path='.'):
 	bench.set_frappe_version(bench_path=bench_path)
 
@@ -227,6 +331,7 @@ def patch_sites(bench_path='.'):
 	except subprocess.CalledProcessError:
 		raise PatchError
 
+
 def build_assets(bench_path='.', app=None):
 	bench.set_frappe_version(bench_path=bench_path)
 
@@ -238,19 +343,16 @@ def build_assets(bench_path='.', app=None):
 			command += ' --app {}'.format(app)
 		exec_cmd(command, cwd=bench_path)
 
+
 def get_sites(bench_path='.'):
 	sites_path = os.path.join(bench_path, 'sites')
 	sites = (site for site in os.listdir(sites_path) if os.path.exists(os.path.join(sites_path, site, 'site_config.json')))
 	return sites
 
+
 def get_bench_dir(bench_path='.'):
 	return os.path.abspath(bench_path)
 
-def setup_auto_update(bench_path='.'):
-	logger.info('setting up auto update')
-	add_to_crontab('0 10 * * * cd {bench_dir} &&  {bench} update --auto >> {logfile} 2>&1'.format(bench_dir=get_bench_dir(bench_path=bench_path),
-		bench=os.path.join(get_bench_dir(bench_path=bench_path), 'env', 'bin', 'bench'),
-		logfile=os.path.join(get_bench_dir(bench_path=bench_path), 'logs', 'auto_update_log.log')))
 
 def setup_backups(bench_path='.'):
 	logger.info('setting up backups')
@@ -265,24 +367,27 @@ def setup_backups(bench_path='.'):
 	add_to_crontab('0 */6 * * *  {backup_command} >> {logfile} 2>&1'.format(backup_command=backup_command,
 		logfile=os.path.join(get_bench_dir(bench_path=bench_path), 'logs', 'backup.log')))
 
+
 def add_to_crontab(line):
 	current_crontab = read_crontab()
 	line = str.encode(line)
 	if not line in current_crontab:
 		cmd = ["crontab"]
-		if platform.system() == 'FreeBSD' or platform.linux_distribution()[0]=="arch":
+		if platform.system() == 'FreeBSD':
 			cmd = ["crontab", "-"]
 		s = subprocess.Popen(cmd, stdin=subprocess.PIPE)
 		s.stdin.write(current_crontab)
 		s.stdin.write(line + b'\n')
 		s.stdin.close()
 
+
 def read_crontab():
 	s = subprocess.Popen(["crontab", "-l"], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
 	out = s.stdout.read()
 	s.stdout.close()
 	return out
 
+
 def update_bench(bench_repo=True, requirements=True):
 	logger.info("Updating bench")
 
@@ -301,7 +406,10 @@ def update_bench(bench_repo=True, requirements=True):
 
 	logger.info("Bench Updated!")
 
+
 def setup_sudoers(user):
+	from bench import env
+
 	if not os.path.exists('/etc/sudoers.d'):
 		os.makedirs('/etc/sudoers.d')
 
@@ -333,6 +441,7 @@ def setup_sudoers(user):
 
 	os.chmod(sudoers_file, 0o440)
 
+
 def setup_logging(bench_path='.'):
 	if os.path.exists(os.path.join(bench_path, 'logs')):
 		logger = logging.getLogger('bench')
@@ -343,6 +452,7 @@ def setup_logging(bench_path='.'):
 		logger.addHandler(hdlr)
 		logger.setLevel(logging.DEBUG)
 
+
 def get_program(programs):
 	program = None
 	for p in programs:
@@ -351,9 +461,11 @@ def get_program(programs):
 			break
 	return program
 
+
 def get_process_manager():
 	return get_program(['foreman', 'forego', 'honcho'])
 
+
 def start(no_dev=False, concurrency=None, procfile=None):
 	program = get_process_manager()
 	if not program:
@@ -371,6 +483,7 @@ def start(no_dev=False, concurrency=None, procfile=None):
 
 	os.execv(program, command)
 
+
 def check_cmd(cmd, cwd='.'):
 	try:
 		subprocess.check_call(cmd, cwd=cwd, shell=True)
@@ -378,6 +491,7 @@ def check_cmd(cmd, cwd='.'):
 	except subprocess.CalledProcessError:
 		return False
 
+
 def get_git_version():
 	'''returns git version from `git --version`
 	extracts version number from string `get version 1.9.1` etc'''
@@ -387,6 +501,7 @@ def get_git_version():
 	version = '.'.join(version.split('.')[0:2])
 	return float(version)
 
+
 def check_git_for_shallow_clone():
 	from .config.common_site_config import get_config
 	config = get_config('.')
@@ -402,6 +517,7 @@ def check_git_for_shallow_clone():
 	if git_version > 1.9:
 		return True
 
+
 def get_cmd_output(cmd, cwd='.'):
 	try:
 		output = subprocess.check_output(cmd, cwd=cwd, shell=True, stderr=subprocess.PIPE).strip()
@@ -412,6 +528,7 @@ def get_cmd_output(cmd, cwd='.'):
 			print(e.output)
 		raise
 
+
 def safe_encode(what, encoding = 'utf-8'):
 	try:
 		what = what.encode(encoding)
@@ -420,6 +537,7 @@ def safe_encode(what, encoding = 'utf-8'):
 
 	return what
 
+
 def restart_supervisor_processes(bench_path='.', web_workers=False):
 	from .config.common_site_config import get_config
 	conf = get_config(bench_path=bench_path)
@@ -449,38 +567,44 @@ def restart_supervisor_processes(bench_path='.', web_workers=False):
 
 		exec_cmd('sudo supervisorctl restart {group}'.format(group=group), cwd=bench_path)
 
+
 def restart_systemd_processes(bench_path='.', web_workers=False):
 	from .config.common_site_config import get_config
 	bench_name = get_bench_name(bench_path)
 	exec_cmd('sudo systemctl stop -- $(systemctl show -p Requires {bench_name}.target | cut -d= -f2)'.format(bench_name=bench_name))
 	exec_cmd('sudo systemctl start -- $(systemctl show -p Requires {bench_name}.target | cut -d= -f2)'.format(bench_name=bench_name))
 
+
 def set_default_site(site, bench_path='.'):
 	if site not in get_sites(bench_path=bench_path):
 		raise Exception("Site not in bench")
 	exec_cmd("{frappe} --use {site}".format(frappe=get_frappe(bench_path=bench_path), site=site),
 			cwd=os.path.join(bench_path, 'sites'))
 
+
 def update_bench_requirements():
 	bench_req_file = os.path.join(os.path.dirname(bench.__path__[0]), 'requirements.txt')
 	install_requirements(bench_req_file, user=True)
 
+
 def update_env_pip(bench_path):
 	env_pip = os.path.join(bench_path, 'env', 'bin', 'pip')
 	exec_cmd("{pip} install -q -U pip".format(pip=env_pip))
 
+
 def update_requirements(bench_path='.'):
 	from bench.app import get_apps, install_app
 	print('Updating Python libraries...')
 
-	# update env pip
-	update_env_pip(bench_path)
-
 	# Update bench requirements (at user level)
 	update_bench_requirements()
 
+	# update env pip
+	update_env_pip(bench_path)
+
 	for app in get_apps():
-		install_app(app, bench_path=bench_path)
+		install_app(app, bench_path=bench_path, skip_assets=True)
+
 
 def update_node_packages(bench_path='.'):
 	print('Updating node packages...')
@@ -488,7 +612,6 @@ def update_node_packages(bench_path='.'):
 	from distutils.version import LooseVersion
 	v = LooseVersion(get_develop_version('frappe', bench_path = bench_path))
 
-
 	# After rollup was merged, frappe_version = 10.1
 	# if develop_verion is 11 and up, only then install yarn
 	if v < LooseVersion('11.x.x-develop'):
@@ -496,6 +619,7 @@ def update_node_packages(bench_path='.'):
 	else:
 		update_yarn_packages(bench_path)
 
+
 def update_yarn_packages(bench_path='.'):
 	apps_dir = os.path.join(bench_path, 'apps')
 
@@ -556,6 +680,7 @@ def install_requirements(req_file, user=False):
 
 		exec_cmd("{python} -m pip install {user_flag} -q -U -r {req_file}".format(python=python, user_flag=user_flag, req_file=req_file))
 
+
 def backup_site(site, bench_path='.'):
 	bench.set_frappe_version(bench_path=bench_path)
 
@@ -565,30 +690,38 @@ def backup_site(site, bench_path='.'):
 	else:
 		run_frappe_cmd('--site', site, 'backup', bench_path=bench_path)
 
+
 def backup_all_sites(bench_path='.'):
 	for site in get_sites(bench_path=bench_path):
 		backup_site(site, bench_path=bench_path)
 
+
 def is_root():
 	if os.getuid() == 0:
 		return True
 	return False
 
+
 def set_mariadb_host(host, bench_path='.'):
 	update_common_site_config({'db_host': host}, bench_path=bench_path)
 
+
 def set_redis_cache_host(host, bench_path='.'):
 	update_common_site_config({'redis_cache': "redis://{}".format(host)}, bench_path=bench_path)
 
+
 def set_redis_queue_host(host, bench_path='.'):
 	update_common_site_config({'redis_queue': "redis://{}".format(host)}, bench_path=bench_path)
 
+
 def set_redis_socketio_host(host, bench_path='.'):
 	update_common_site_config({'redis_socketio': "redis://{}".format(host)}, bench_path=bench_path)
 
+
 def update_common_site_config(ddict, bench_path='.'):
 	update_json_file(os.path.join(bench_path, 'sites', 'common_site_config.json'), ddict)
 
+
 def update_json_file(filename, ddict):
 	if os.path.exists(filename):
 		with open(filename, 'r') as f:
@@ -601,6 +734,7 @@ def update_json_file(filename, ddict):
 	with open(filename, 'w') as f:
 		json.dump(content, f, indent=1, sort_keys=True)
 
+
 def drop_privileges(uid_name='nobody', gid_name='nogroup'):
 	# from http://stackoverflow.com/a/2699996
 	if os.getuid() != 0:
@@ -621,6 +755,7 @@ def drop_privileges(uid_name='nobody', gid_name='nogroup'):
 	# Ensure a very conservative umask
 	os.umask(0o22)
 
+
 def fix_prod_setup_perms(bench_path='.', frappe_user=None):
 	from .config.common_site_config import get_config
 
@@ -638,6 +773,7 @@ def fix_prod_setup_perms(bench_path='.', frappe_user=None):
 			gid = grp.getgrnam(frappe_user).gr_gid
 			os.chown(path, uid, gid)
 
+
 def fix_file_perms():
 	for dir_path, dirs, files in os.walk('.'):
 		for _dir in dirs:
@@ -650,10 +786,12 @@ def fix_file_perms():
 			if not _file.startswith('activate'):
 				os.chmod(os.path.join(bin_dir, _file), 0o755)
 
+
 def get_current_frappe_version(bench_path='.'):
 	from .app import get_current_frappe_version as fv
 	return fv(bench_path=bench_path)
 
+
 def run_frappe_cmd(*args, **kwargs):
 	from .cli import from_command_line
 
@@ -677,7 +815,7 @@ def run_frappe_cmd(*args, **kwargs):
 
 	if return_code > 0:
 		sys.exit(return_code)
-		#raise CommandFailedError(args)
+
 
 def get_frappe_cmd_output(*args, **kwargs):
 	bench_path = kwargs.get('bench_path', '.')
@@ -685,24 +823,12 @@ def get_frappe_cmd_output(*args, **kwargs):
 	sites_dir = os.path.join(bench_path, 'sites')
 	return subprocess.check_output((f, '-m', 'frappe.utils.bench_helper', 'frappe') + args, cwd=sites_dir)
 
+
 def validate_upgrade(from_ver, to_ver, bench_path='.'):
 	if to_ver >= 6:
 		if not find_executable('npm') and not (find_executable('node') or find_executable('nodejs')):
 			raise Exception("Please install nodejs and npm")
 
-def pre_upgrade(from_ver, to_ver, bench_path='.'):
-	pip = os.path.join(bench_path, 'env', 'bin', 'pip')
-
-	if from_ver <= 4 and to_ver >= 5:
-		from .migrate_to_v5 import remove_shopping_cart
-		apps = ('frappe', 'erpnext')
-		remove_shopping_cart(bench_path=bench_path)
-
-		for app in apps:
-			cwd = os.path.abspath(os.path.join(bench_path, 'apps', app))
-			if os.path.exists(cwd):
-				exec_cmd("git clean -dxf", cwd=cwd)
-				exec_cmd("{pip} install --upgrade -e {app}".format(pip=pip, app=cwd))
 
 def post_upgrade(from_ver, to_ver, bench_path='.'):
 	from .config.common_site_config import get_config
@@ -710,8 +836,7 @@ def post_upgrade(from_ver, to_ver, bench_path='.'):
 	from .config.supervisor import generate_supervisor_config
 	from .config.nginx import make_nginx_conf
 	conf = get_config(bench_path=bench_path)
-	print("-"*80)
-	print("Your bench was upgraded to version {0}".format(to_ver))
+	print("-" * 80 + "Your bench was upgraded to version {0}".format(to_ver))
 
 	if conf.get('restart_supervisor_on_update'):
 		redis.generate_config(bench_path=bench_path)
@@ -987,6 +1112,7 @@ def in_virtual_env():
 
 	return False
 
+
 def migrate_env(python, backup=False):
 	from bench.config.common_site_config import get_config
 	from bench.app import get_apps
@@ -1042,4 +1168,18 @@ def migrate_env(python, backup=False):
 		log.debug('Migration Successful to {}'.format(python))
 	except:
 		log.debug('Migration Error')
-		raise
+		raise
+
+
+def find_parent_bench(path):
+	"""Checks if parent directories are benches"""
+	if is_bench_directory(directory=path):
+		return path
+
+	home_path = os.path.expanduser("~")
+	root_path = os.path.abspath(os.sep)
+
+	if path not in {home_path, root_path}:
+		# NOTE: the os.path.split assumes that given path is absolute
+		parent_dir = os.path.split(path)[0]
+		return find_parent_bench(parent_dir)

playbooks/install.py

@@ -1,5 +1,16 @@
 #!/usr/bin/env python3
-import os, sys, subprocess, getpass, json, multiprocessing, shutil, platform, warnings, datetime
+from __future__ import print_function
+import os
+import sys
+import subprocess
+import getpass
+import json
+import multiprocessing
+import shutil
+import platform
+import warnings
+import datetime
+
 
 tmp_bench_repo = os.path.join('/', 'tmp', '.bench')
 tmp_log_folder = os.path.join('/', 'tmp', 'logs')
@@ -395,8 +406,20 @@ def parse_commandline_args():
 
 if __name__ == '__main__':
 	if sys.version[0] == '2':
-		if not raw_input("It is recommended to run this script with Python 3\nDo you still wish to continue? [Y/n]: ").lower() == "y":
-			sys.exit()
+		if not os.environ.get('CI'):
+			if not raw_input("It is recommended to run this script with Python 3\nDo you still wish to continue? [Y/n]: ").lower() == "y":
+				sys.exit()
+
+		try:
+			from distutils.spawn import find_executable
+		except ImportError:
+			try:
+				subprocess.check_call('pip install --upgrade setuptools')
+			except subprocess.CalledProcessError:
+				print("Install distutils or use Python3 to run the script")
+				sys.exit(1)
+
+		shutil.which = find_executable
 
 	if not is_sudo_user():
 		log("Please run this script as a non-root user with sudo privileges", level=3)

requirements.txt

@@ -1,18 +1,10 @@
 Click==7.0
-GitPython==2.1.11
+GitPython==2.1.15
 honcho==1.0.1
 Jinja2==2.10.3
 python-crontab==2.4.0
 requests==2.22.0
-semantic_version==2.8.2
+semantic-version==2.8.2
 setuptools==40.8.0
 six==1.12.0
 virtualenv==16.6.0
-gitdb2==2.0.6;python_version<'3.4'
-MarkupSafe==1.1.1
-python-dateutil==2.8.1
-idna==2.8
-certifi==2019.9.11
-urllib3==1.25.7
-chardet==3.0.4
-smmap2==2.0.5