Merge branch 'develop' of github.com:frappe/bench into netchampfaris/minor-message

.circleci/config.yml

@@ -24,7 +24,7 @@ jobs:
           name: Setup Tests
           command: |
             cd ~
-            sudo pip install --upgrade pip==19.3.1
+            sudo pip install --upgrade pip
             sudo pip install -e ~/.bench
 
       - run:

.deepsource.toml

@@ -0,0 +1,16 @@
+version = 1
+
+exclude_patterns = [
+  ".*"
+]
+
+test_patterns = [
+  "bench/tests/**"
+]
+
+[[analyzers]]
+name = "python"
+enabled = true
+dependency_file_paths = [
+  "requirements.txt"
+]

.travis.yml

@@ -1,26 +1,96 @@
 language: python
-dist: xenial
+dist: bionic
+sudo: true
 
-python:
+git:
-  - "2.7"
+  depth: 1
+
+cache:
+  - pip
+  - npm
+  - yarn
+
+addons:
+  mariadb: '10.3'
+
+matrix:
+  include:
+  - name: "Python 2.7 Basic Setup"
+    python: 2.7
+    env: TEST=bench
+    script: python -m unittest -v bench.tests.test_init
+
+  - name: "Python 3.6 Basic Setup"
+    python: 3.6
+    env: TEST=bench
+    script: python -m unittest -v bench.tests.test_init
+
+  - name: "Python 3.7 Basic Setup"
+    python: 3.7
+    env: TEST=bench
+    script: python -m unittest -v bench.tests.test_init
+
+  - name: "Python 3.8 Production Setup"
+    python: 3.8
+    env: TEST=bench
+    script: python -m unittest -v bench.tests.test_setup_production
+
+  - name: "Python 2.7 Production Setup"
+    python: 2.7
+    env: TEST=bench
+    script: python -m unittest -v bench.tests.test_setup_production
+
+  - name: "Python 3.6 Production Setup"
+    python: 3.6
+    env: TEST=bench
+    script: python -m unittest -v bench.tests.test_setup_production
+
+  - name: "Python 3.7 Production Setup"
+    python: 3.7
+    env: TEST=bench
+    script: python -m unittest -v bench.tests.test_setup_production
+
+  - name: "Python 3.8 Production Setup"
+    python: 3.8
+    env: TEST=bench
+    script: python -m unittest -v bench.tests.test_setup_production
+
+  - name: "Python 3.6 Easy Install"
+    python: 3.6
+    env: TEST=easy_install
+    script: sudo python $TRAVIS_BUILD_DIR/playbooks/install.py --user travis --run-travis --production --verbose
+
+  - name: "Python 3.7 Easy Install"
+    python: 3.7
+    env: TEST=easy_install
+    script: sudo python $TRAVIS_BUILD_DIR/playbooks/install.py --user travis --run-travis --production --verbose
+
+  - name: "Python 3.8 Easy Install"
+    python: 3.8
+    env: TEST=easy_install
+    script: sudo python $TRAVIS_BUILD_DIR/playbooks/install.py --user travis --run-travis --production --verbose
 
 install:
-  - sudo pip install urllib3 pyOpenSSL ndg-httpsclient pyasn1
+  - pip install urllib3 pyOpenSSL ndg-httpsclient pyasn1
-  - sudo apt-get purge -y mysql-common mysql-server mysql-client
-  - sudo apt-get install --only-upgrade -y git
-  - sudo apt-get install hhvm && rm -rf /home/travis/.kiex/
-  - mkdir -p ~/.bench
-  - mkdir -p /tmp/.bench
-  - cp -r $TRAVIS_BUILD_DIR/* ~/.bench
-  - cp -r $TRAVIS_BUILD_DIR/* /tmp/.bench
 
-  - sudo python $TRAVIS_BUILD_DIR/playbooks/install.py --user travis --run-travis --production --verbose
+  - if [ $TEST == "bench" ];then
-  # - sudo bash $TRAVIS_BUILD_DIR/install_scripts/setup_frappe.sh --skip-install-bench --mysql-root-password travis
+       wget -q -O /tmp/wkhtmltox.tar.xz https://github.com/frappe/wkhtmltopdf/raw/master/wkhtmltox-0.12.3_linux-generic-amd64.tar.xz;
-  # - cd ~ && sudo python bench-repo/installer/install.py --only-dependencies
+       tar -xf /tmp/wkhtmltox.tar.xz -C /tmp;
+       sudo mv /tmp/wkhtmltox/bin/wkhtmltopdf /usr/local/bin/wkhtmltopdf;
+       sudo chmod o+x /usr/local/bin/wkhtmltopdf;
 
-script:
+      mkdir -p ~/.bench;
-  - cd ~
+      cp -r $TRAVIS_BUILD_DIR/* ~/.bench;
-  - sudo pip install --upgrade pip==19.3.1
+      pip install -q -U -e ~/.bench;
-  - sudo pip install -e ~/.bench
+      sudo pip install -q -U -e ~/.bench;
-  # - sudo python -m unittest bench.tests.test_setup_production.TestSetupProduction.test_setup_production_v6
+
-  - sudo python -m unittest -v bench.tests.test_setup_production
+      mysql -u root -e "SET GLOBAL character_set_server = 'utf8mb4'";
+      mysql -u root -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'";
+      mysql -u root -e "UPDATE mysql.user SET Password=PASSWORD('travis') WHERE User='root'";
+      mysql -u root -e "FLUSH PRIVILEGES";
+    fi
+
+  - if [ $TEST == "easy_install" ];then
+      mkdir -p /tmp/.bench;
+      cp -r $TRAVIS_BUILD_DIR/* /tmp/.bench;
+    fi

README.md

@@ -15,6 +15,7 @@ bench is a command-line utility that helps you to install apps, manage multiple
  - [bench CLI](#bench-cli)
 	- [Usage](#usage)
 	- [Installation](#installation)
+	- [Custom Bench commands](#custom-bench-commands)
  - [Easy Install Script](#easy-install-script)
  - [Release Bench](#release-bench)
  - [Guides](#guides)
@@ -60,7 +61,7 @@ Bench is a command line tool that helps you install, setup, manage multiple site
 
 _Note:_ Apart from `bench init`, all other bench commands have to be run having the respective bench directory as the working directory. _(`bench update` may also be run, but it's behaviour is covered in depth in the docs)_
 
-For more in depth information on commands and usage follow [here](https://github.com/frappe/bench/blob/master/docs/commands_and_usage.md).
+For more in depth information on commands and usage follow [here](https://github.com/frappe/bench/blob/master/docs/commands_and_usage.md). As for a consolidated list of bench commands, go through [this page](https://github.com/frappe/bench/blob/master/docs/bench_usage.md).
 
 ---
 
@@ -68,8 +69,8 @@ For more in depth information on commands and usage follow [here](https://github
 
 To do this install, you must have basic information on how Linux works and should be able to use the command-line. bench will also create nginx and supervisor config files, setup backups and much more. If you are using on a VPS make sure it has >= 1Gb of RAM or has swap setup properly.
 
-		git clone https://github.com/frappe/bench ~/.bench
+	git clone https://github.com/frappe/bench ~/.bench
-		pip3 install --user -e ~/.bench
+	pip3 install --user -e ~/.bench
 
 As bench is a python application, its installation really depends on `python` + `pip` + `git`. The Frappe Framework, however has various other system dependencies like `nodejs`, `yarn`, `redis` and a database system like `mariadb` or `postgres`. Go through the [installation requirements](https://github.com/frappe/bench/blob/master/docs/installation.md) for an updated list.
 
@@ -77,6 +78,12 @@ If you have questions, please ask them on the [forum](https://discuss.erpnext.co
 
 ---
 
+## Custom Bench Commands
+
+Want to utilize a bench command you've added in your custom Frappe application? [This](https://github.com/frappe/bench/blob/master/docs/bench_custom_cmd.md) guide might be of some help.
+
+---
+
 # Easy Install Script
 
 - This is an opinionated setup so it is best to setup on a blank server.
@@ -110,12 +117,11 @@ Releases can be made for [Frappe](https://github.com/frappe/frappe) apps using b
 
 ---
 
-# Docker Install
+# Docker
 
-1. For developer setup, you can also use the official [Frappe Docker](https://github.com/frappe/frappe_docker/).
+- For official images and resources [Frappe Docker](https://github.com/frappe/frappe_docker)
-2. The app, mariadb and redis run on individual containers.
+- Production Installation [README](https://github.com/frappe/frappe_docker/blob/develop/README.md)
-3. This setup supports multi-tenancy and exposes the frappe-bench volume as a external storage.
+- Developer Setup [README](https://github.com/frappe/frappe_docker/blob/develop/development/README.md)
-4. For more details, [ead the instructions on the [Frappe Docker README](https://github.com/frappe/frappe_docker/)
 
 ---
 

bench/app.py

@@ -1,23 +1,32 @@
+# imports - compatibility imports
 from __future__ import print_function
-import os
-from .utils import (exec_cmd, get_frappe, check_git_for_shallow_clone, build_assets,
-	restart_supervisor_processes, get_cmd_output, run_frappe_cmd, CommandFailedError,
-	restart_systemd_processes)
-from .config.common_site_config import get_config
 
+# imports - standard imports
+import json
 import logging
+import os
+import re
+import shutil
+import subprocess
+import sys
+
+# imports - third party imports
+import click
+import git
 import requests
 import semantic_version
-import json
+from six.moves import reload_module
-import re
-import subprocess
-import bench
-import sys
-import shutil
 
-logging.basicConfig(level="DEBUG")
+# imports - module imports
+import bench
+from bench.config.common_site_config import get_config
+from bench.utils import CommandFailedError, build_assets, check_git_for_shallow_clone, exec_cmd, get_cmd_output, get_frappe, restart_supervisor_processes, restart_systemd_processes, run_frappe_cmd
+
+
+logging.basicConfig(level="INFO")
 logger = logging.getLogger(__name__)
 
+
 class InvalidBranchException(Exception): pass
 class InvalidRemoteException(Exception): pass
 
@@ -50,16 +59,13 @@ def write_appstxt(apps, bench_path='.'):
 	with open(os.path.join(bench_path, 'sites', 'apps.txt'), 'w') as f:
 		return f.write('\n'.join(apps))
 
-def check_url(url, raise_err = True):
+def check_url(url, raise_err=True):
-	try:
+	from six.moves.urllib.parse import urlparse
-		from urlparse import urlparse
-	except ImportError:
-		from urllib.parse import urlparse
 
 	parsed = urlparse(url)
 	if not parsed.scheme:
 		if raise_err:
-			raise TypeError('{url} Not a valid URL'.format(url = url))
+			raise TypeError('{url} Not a valid URL'.format(url=url))
 		else:
 			return False
 
@@ -92,59 +98,61 @@ def remove_from_excluded_apps_txt(app, bench_path='.'):
 		apps.remove(app)
 		return write_excluded_apps_txt(apps, bench_path=bench_path)
 
-def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=False,
+def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=False, postprocess=True, overwrite=False):
-	postprocess = True):
+	if not os.path.exists(git_url):
-	# from bench.utils import check_url
+		if not check_url(git_url, raise_err=False):
-	try:
+			orgs = ['frappe', 'erpnext']
-		from urlparse import urljoin
+			for org in orgs:
-	except ImportError:
+				url = 'https://api.github.com/repos/{org}/{app}'.format(org=org, app=git_url)
-		from urllib.parse import urljoin
+				res = requests.get(url)
+				if res.ok:
+					data = res.json()
+					if 'name' in data:
+						if git_url == data['name']:
+							git_url = 'https://github.com/{org}/{app}'.format(org=org, app=git_url)
+							break
 
-	if not check_url(git_url, raise_err = False):
+		# Gets repo name from URL
-		orgs = ['frappe', 'erpnext']
+		repo_name = git_url.rsplit('/', 1)[1].rsplit('.', 1)[0]
-		for org in orgs:
+		shallow_clone = '--depth 1' if check_git_for_shallow_clone() else ''
-			url = 'https://api.github.com/repos/{org}/{app}'.format(org = org, app = git_url)
+		branch = '--branch {branch}'.format(branch=branch) if branch else ''
-			res = requests.get(url)
+	else:
-			if res.ok:
+		repo_name = git_url.split(os.sep)[-1]
-				data    = res.json()
+		shallow_clone = ''
-				if 'name' in data:
+		branch = '--branch {branch}'.format(branch=branch) if branch else ''
-					if git_url == data['name']:
-						git_url = 'https://github.com/{org}/{app}'.format(org = org, app = git_url)
-						break
 
-	#Gets repo name from URL
+	if os.path.isdir(os.path.join(bench_path, 'apps', repo_name)):
-	repo_name = git_url.rsplit('/', 1)[1].rsplit('.', 1)[0]
+		# application directory already exists
-	logger.info('getting app {}'.format(repo_name))
+		# prompt user to overwrite it
-	shallow_clone = '--depth 1' if check_git_for_shallow_clone() else ''
+		if overwrite or click.confirm('''A directory for the application "{0}" already exists.
-	branch = '--branch {branch}'.format(branch=branch) if branch else ''
+Do you want to continue and overwrite it?'''.format(repo_name)):
+			shutil.rmtree(os.path.join(bench_path, 'apps', repo_name))
+		elif click.confirm('''Do you want to reinstall the existing application?''', abort=True):
+			app_name = get_app_name(bench_path, repo_name)
+			install_app(app=app_name, bench_path=bench_path, verbose=verbose, skip_assets=skip_assets)
+			sys.exit()
 
-	exec_cmd("git clone -q {git_url} {branch} {shallow_clone} --origin upstream".format(
+	logger.info('Getting app {0}'.format(repo_name))
-				git_url=git_url,
+	exec_cmd("git clone {git_url} {branch} {shallow_clone} --origin upstream".format(
-				shallow_clone=shallow_clone,
+		git_url=git_url,
-				branch=branch),
+		shallow_clone=shallow_clone,
-			cwd=os.path.join(bench_path, 'apps'))
+		branch=branch),
+		cwd=os.path.join(bench_path, 'apps'))
 
-	#Retrieves app name from setup.py
+	app_name = get_app_name(bench_path, repo_name)
+	install_app(app=app_name, bench_path=bench_path, verbose=verbose, skip_assets=skip_assets)
+
+
+def get_app_name(bench_path, repo_name):
+	# retrieves app name from setup.py
 	app_path = os.path.join(bench_path, 'apps', repo_name, 'setup.py')
 	with open(app_path, 'rb') as f:
 		app_name = re.search(r'name\s*=\s*[\'"](.*)[\'"]', f.read().decode('utf-8')).group(1)
 		if repo_name != app_name:
 			apps_path = os.path.join(os.path.abspath(bench_path), 'apps')
 			os.rename(os.path.join(apps_path, repo_name), os.path.join(apps_path, app_name))
+		return app_name
 
-	print('installing', app_name)
-	install_app(app=app_name, bench_path=bench_path, verbose=verbose)
-
-	if postprocess:
-
-		if not skip_assets:
-			build_assets(bench_path=bench_path, app=app_name)
-		conf = get_config(bench_path=bench_path)
-
-		if conf.get('restart_supervisor_on_update'):
-			restart_supervisor_processes(bench_path=bench_path)
-		if conf.get('restart_systemd_on_update'):
-			restart_systemd_processes(bench_path=bench_path)
 
 def new_app(app, bench_path='.'):
 	# For backwards compatibility
@@ -160,7 +168,8 @@ def new_app(app, bench_path='.'):
 		run_frappe_cmd('make-app', apps, app, bench_path=bench_path)
 	install_app(app, bench_path=bench_path)
 
-def install_app(app, bench_path=".", verbose=False, no_cache=False):
+
+def install_app(app, bench_path=".", verbose=False, no_cache=False, postprocess=True, skip_assets=False):
 	logger.info("installing {}".format(app))
 
 	pip_path = os.path.join(bench_path, "env", "bin", "pip")
@@ -168,11 +177,23 @@ def install_app(app, bench_path=".", verbose=False, no_cache=False):
 	app_path = os.path.join(bench_path, "apps", app)
 	cache_flag = "--no-cache-dir" if no_cache else ""
 
-	exec_cmd("{pip} install {quiet} -U -e {app} {no_cache}".format(pip=pip_path, quiet=quiet_flag, app=app_path, no_cache=cache_flag))
+	exec_cmd("{pip} install {quiet} -U -e {app} {no_cache}".format(pip=pip_path,
+									quiet=quiet_flag, app=app_path, no_cache=cache_flag))
 	add_to_appstxt(app, bench_path=bench_path)
 
+	if postprocess:
+		if not skip_assets:
+			build_assets(bench_path=bench_path, app=app)
+		conf = get_config(bench_path=bench_path)
+
+		if conf.get('restart_supervisor_on_update'):
+			restart_supervisor_processes(bench_path=bench_path)
+		if conf.get('restart_systemd_on_update'):
+			restart_systemd_processes(bench_path=bench_path)
+
+
 def remove_app(app, bench_path='.'):
-	if not app in get_apps(bench_path):
+	if app not in get_apps(bench_path):
 		print("No app named {0}".format(app))
 		sys.exit(1)
 
@@ -188,7 +209,7 @@ def remove_app(app, bench_path='.'):
 				print("Cannot remove, app is installed on site: {0}".format(site))
 				sys.exit(1)
 
-	exec_cmd(["{0} uninstall -y {1}".format(pip, app)])
+	exec_cmd("{0} uninstall -y {1}".format(pip, app), cwd=bench_path)
 	remove_from_appstxt(app, bench_path)
 	shutil.rmtree(app_path)
 	run_frappe_cmd("build", bench_path=bench_path)
@@ -281,8 +302,7 @@ def get_current_branch(app, bench_path='.'):
 
 def get_remote(app, bench_path='.'):
 	repo_dir = get_repo_dir(app, bench_path=bench_path)
-	contents = subprocess.check_output(['git', 'remote', '-v'], cwd=repo_dir,
+	contents = subprocess.check_output(['git', 'remote', '-v'], cwd=repo_dir, stderr=subprocess.STDOUT)
-									   stderr=subprocess.STDOUT)
 	contents = contents.decode('utf-8')
 	if re.findall('upstream[\s]+', contents):
 		return 'upstream'
@@ -340,8 +360,7 @@ def get_repo_dir(app, bench_path='.'):
 	return os.path.join(bench_path, 'apps', app)
 
 def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrade=True):
-	from .utils import update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets, pre_upgrade, post_upgrade
+	from bench.utils import update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets, post_upgrade
-	from . import utils
 	apps_dir = os.path.join(bench_path, 'apps')
 	version_upgrade = (False,)
 	switched_apps = []
@@ -354,44 +373,46 @@ def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrad
 
 	for app in apps:
 		app_dir = os.path.join(apps_dir, app)
-		if os.path.exists(app_dir):
+
-			try:
+		if not os.path.exists(app_dir):
-				if check_upgrade:
+			bench.utils.log("{} does not exist!".format(app), level=2)
-					version_upgrade = is_version_upgrade(app=app, bench_path=bench_path, branch=branch)
+			continue
-					if version_upgrade[0] and not upgrade:
+
-						raise MajorVersionUpgradeException("Switching to {0} will cause upgrade from {1} to {2}. Pass --upgrade to confirm".format(branch, version_upgrade[1], version_upgrade[2]), version_upgrade[1], version_upgrade[2])
+		repo = git.Repo(app_dir)
-				print("Switching for "+app)
+		unshallow_flag = os.path.exists(os.path.join(app_dir, ".git", "shallow"))
-				unshallow = "--unshallow" if os.path.exists(os.path.join(app_dir, ".git", "shallow")) else ""
+		bench.utils.log("Fetching upstream {0}for {1}".format("unshallow " if unshallow_flag else "", app))
-				exec_cmd("git config --unset-all remote.upstream.fetch", cwd=app_dir)
+
-				exec_cmd("git config --add remote.upstream.fetch '+refs/heads/*:refs/remotes/upstream/*'", cwd=app_dir)
+		bench.utils.exec_cmd("git remote set-branches upstream  '*'", cwd=app_dir)
-				exec_cmd("git fetch upstream {unshallow}".format(unshallow=unshallow), cwd=app_dir)
+		bench.utils.exec_cmd("git fetch --all{0}".format(" --unshallow" if unshallow_flag else ""), cwd=app_dir)
-				exec_cmd("git checkout {branch}".format(branch=branch), cwd=app_dir)
+
-				exec_cmd("git merge upstream/{branch}".format(branch=branch), cwd=app_dir)
+		if check_upgrade:
-				switched_apps.append(app)
+			version_upgrade = is_version_upgrade(app=app, bench_path=bench_path, branch=branch)
-			except CommandFailedError:
+			if version_upgrade[0] and not upgrade:
-				print("Error switching to branch {0} for {1}".format(branch, app))
+				bench.utils.log("Switching to {0} will cause upgrade from {1} to {2}. Pass --upgrade to confirm".format(branch, version_upgrade[1], version_upgrade[2]), level=2)
-			except InvalidRemoteException:
+				sys.exit(1)
-				print("Remote does not exist for app "+app)
+
-			except InvalidBranchException:
+		print("Switching for "+app)
-				print("Branch {0} does not exist in Upstream for {1}".format(branch, app))
+		bench.utils.exec_cmd("git checkout {0}".format(branch), cwd=app_dir)
+
+		if str(repo.active_branch) == branch:
+			switched_apps.append(app)
+		else:
+			bench.utils.log("Switching branches failed for: {}".format(app), level=2)
 
 	if switched_apps:
-		print("Successfully switched branches for:\n" + "\n".join(switched_apps))
+		bench.utils.log("Successfully switched branches for: " + ", ".join(switched_apps), level=1)
+		print('Please run `bench update --patch` to be safe from any differences in database schema')
 
 	if version_upgrade[0] and upgrade:
 		update_requirements()
 		update_node_packages()
-		pre_upgrade(version_upgrade[1], version_upgrade[2])
+		reload_module(utils)
-		if sys.version_info >= (3, 4):
-			import importlib
-			importlib.reload(utils)
-		else:
-			reload(utils)
 		backup_all_sites()
 		patch_sites()
 		build_assets()
 		post_upgrade(version_upgrade[1], version_upgrade[2])
 
+
 def switch_to_branch(branch=None, apps=None, bench_path='.', upgrade=False):
 	switch_branch(branch, apps=apps, bench_path=bench_path, upgrade=upgrade)
 
@@ -402,8 +423,7 @@ def switch_to_develop(apps=None, bench_path='.', upgrade=True):
 	switch_branch('develop', apps=apps, bench_path=bench_path, upgrade=upgrade)
 
 def get_version_from_string(contents, field='__version__'):
-	match = re.search(r"^(\s*%s\s*=\s*['\\\"])(.+?)(['\"])(?sm)" % field,
+	match = re.search(r"^(\s*%s\s*=\s*['\\\"])(.+?)(['\"])(?sm)" % field, contents)
-			contents)
 	return match.group(2)
 
 def get_major_version(version):

bench/cli.py

@@ -1,6 +1,6 @@
 import click
 import os, sys, logging, json, pwd, subprocess
-from bench.utils import is_root, PatchError, drop_privileges, get_env_cmd, get_cmd_output, get_frappe, log, is_bench_directory
+from bench.utils import is_root, PatchError, drop_privileges, get_env_cmd, get_cmd_output, get_frappe, log, find_parent_bench
 from bench.app import get_apps
 from bench.config.common_site_config import get_config
 from bench.commands import bench_command
@@ -29,7 +29,6 @@ def cli():
 
 	elif len(sys.argv) > 1 and sys.argv[1]=="--help":
 		print(click.Context(bench_command).get_help())
-		print()
 		print(get_frappe_help())
 		return
 
@@ -99,7 +98,6 @@ def get_frappe_commands(bench_path='.'):
 		return []
 	try:
 		output = get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-commands".format(python=python), cwd=sites_path)
-		# output = output.decode('utf-8')
 		return json.loads(output)
 	except subprocess.CalledProcessError as e:
 		if hasattr(e, "stderr"):
@@ -109,27 +107,12 @@ def get_frappe_commands(bench_path='.'):
 def get_frappe_help(bench_path='.'):
 	python = get_env_cmd('python', bench_path=bench_path)
 	sites_path = os.path.join(bench_path, 'sites')
-	if not os.path.exists(sites_path):
-		return []
 	try:
 		out = get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-help".format(python=python), cwd=sites_path)
-		return "Framework commands:\n" + out.split('Commands:')[1]
+		return "\n\nFramework commands:\n" + out.split('Commands:')[1]
-	except subprocess.CalledProcessError:
+	except:
 		return ""
 
-def find_parent_bench(path):
-	"""Checks if parent directories are benches"""
-	if is_bench_directory(directory=path):
-		return path
-
-	home_path = os.path.expanduser("~")
-	root_path = os.path.abspath(os.sep)
-
-	if path not in {home_path, root_path}:
-		# NOTE: the os.path.split assumes that given path is absolute
-		parent_dir = os.path.split(path)[0]
-		return find_parent_bench(parent_dir)
-
 def change_working_directory():
 	"""Allows bench commands to be run from anywhere inside a bench directory"""
 	cur_dir = os.path.abspath(".")

bench/commands/__init__.py

@@ -1,15 +1,5 @@
 import click
 
-import os, shutil
-import os.path as osp
-import logging
-
-from datetime import datetime
-
-from bench.utils import which, exec_cmd
-
-log = logging.getLogger(__name__)
-log.setLevel(logging.DEBUG)
 
 def print_bench_version(ctx, param, value):
 	"""Prints current bench version"""
@@ -31,11 +21,14 @@ def bench_command(bench_path='.'):
 	setup_logging(bench_path=bench_path)
 
 
-from bench.commands.make import init, get_app, new_app, remove_app
+from bench.commands.make import init, get_app, new_app, remove_app, exclude_app_for_update, include_app_for_update, pip
 bench_command.add_command(init)
 bench_command.add_command(get_app)
 bench_command.add_command(new_app)
 bench_command.add_command(remove_app)
+bench_command.add_command(exclude_app_for_update)
+bench_command.add_command(include_app_for_update)
+bench_command.add_command(pip)
 
 
 from bench.commands.update import update, retry_upgrade, switch_to_branch, switch_to_master, switch_to_develop
@@ -45,9 +38,10 @@ bench_command.add_command(switch_to_branch)
 bench_command.add_command(switch_to_master)
 bench_command.add_command(switch_to_develop)
 
+
 from bench.commands.utils import (start, restart, set_nginx_port, set_ssl_certificate, set_ssl_certificate_key, set_url_root,
-	set_mariadb_host, set_default_site, download_translations, shell, backup_site, backup_all_sites, release, renew_lets_encrypt,
+	set_mariadb_host, set_default_site, download_translations, backup_site, backup_all_sites, release, renew_lets_encrypt,
-	disable_production, bench_src, prepare_beta_release, set_redis_cache_host, set_redis_queue_host, set_redis_socketio_host, find_benches)
+	disable_production, bench_src, prepare_beta_release, set_redis_cache_host, set_redis_queue_host, set_redis_socketio_host, find_benches, migrate_env)
 bench_command.add_command(start)
 bench_command.add_command(restart)
 bench_command.add_command(set_nginx_port)
@@ -60,7 +54,6 @@ bench_command.add_command(set_redis_queue_host)
 bench_command.add_command(set_redis_socketio_host)
 bench_command.add_command(set_default_site)
 bench_command.add_command(download_translations)
-bench_command.add_command(shell)
 bench_command.add_command(backup_site)
 bench_command.add_command(backup_all_sites)
 bench_command.add_command(release)
@@ -69,6 +62,8 @@ bench_command.add_command(disable_production)
 bench_command.add_command(bench_src)
 bench_command.add_command(prepare_beta_release)
 bench_command.add_command(find_benches)
+bench_command.add_command(migrate_env)
+
 
 from bench.commands.setup import setup
 bench_command.add_command(setup)
@@ -84,108 +79,3 @@ bench_command.add_command(remote_urls)
 
 from bench.commands.install import install
 bench_command.add_command(install)
-
-from bench.config.common_site_config import get_config
-try:
-	from urlparse 	  import urlparse
-except ImportError:
-	from urllib.parse import urlparse
-
-@click.command('migrate-env')
-@click.argument('python', type = str)
-@click.option('--no-backup', is_flag=True)
-def migrate_env(python, no_backup = False):
-	"""
-	Migrate Virtual Environment to desired Python Version.
-	"""
-	try:
-		# Clear Cache before Bench Dies.
-		config = get_config(bench_path = os.getcwd())
-		rredis = urlparse(config['redis_cache'])
-
-		redis  = '{redis} -p {port}'.format(
-			redis = which('redis-cli'),
-			port  = rredis.port
-		)
-
-		log.debug('Clearing Redis Cache...')
-		exec_cmd('{redis} FLUSHALL'.format(redis = redis))
-		log.debug('Clearing Redis DataBase...')
-		exec_cmd('{redis} FLUSHDB'.format(redis = redis))
-	except Exception:
-		log.warn('Please ensure Redis Connections are running or Daemonized.')
-
-	try:
-		# This is with the assumption that a bench is set-up within path.
-		path       = os.getcwd()
-
-		# I know, bad name for a flag. Thanks, Ameya! :| - <achilles@frappe.io>
-		if not no_backup:
-			# Back, the f*ck up.
-			parch = osp.join(path, 'archived_envs')
-			if not osp.exists(parch):
-				os.mkdir(parch)
-
-			# Simply moving. Thanks, Ameya.
-			# I'm keen to zip.
-			source = osp.join(path, 'env')
-			target = parch
-
-			log.debug('Backing up Virtual Environment')
-			stamp  = datetime.now().strftime('%Y%m%d_%H%M%S')
-			dest   = osp.join(path, str(stamp))
-
-			# WARNING: This is an archive, you might have to use virtualenv --relocate
-			# That's because virtualenv creates symlinks with shebangs pointing to executables.
-			# shebangs, shebangs - ricky martin.
-
-			# ...and shutil.copytree is a f*cking mess.
-			os.rename(source, dest)
-			shutil.move(dest, target)
-
-		log.debug('Setting up a New Virtual {python} Environment'.format(
-			python = python
-		))
-
-		# Path to Python Executable (Basically $PYTHONPTH)
-		python     = which(python)
-
-
-		virtualenv = which('virtualenv')
-
-		nvenv      = 'env'
-		pvenv      = osp.join(path, nvenv)
-
-		exec_cmd('{virtualenv} --python {python} {pvenv}'.format(
-			virtualenv = virtualenv,
-			python     = python,
-			pvenv      = pvenv
-		), cwd = path)
-
-		pip = osp.join(pvenv, 'bin', 'pip')
-		exec_cmd('{pip} install --upgrade pip'.format(pip=pip))
-		exec_cmd('{pip} install --upgrade setuptools'.format(pip=pip))
-		# TODO: Options
-
-		papps  = osp.join(path, 'apps')
-		apps   = ['frappe', 'erpnext'] + [app for app in os.listdir(papps) if app not in ['frappe', 'erpnext']]
-
-		for app in apps:
-			papp = osp.join(papps, app)
-			if osp.isdir(papp) and osp.exists(osp.join(papp, 'setup.py')):
-				exec_cmd('{pip} install -e {app}'.format(
-					pip = pip, app = papp
-				))
-
-		log.debug('Migration Successful to {python}'.format(
-			python = python
-		))
-	except:
-		log.debug('Migration Error')
-		raise
-
-bench_command.add_command(migrate_env)
-
-from bench.commands.make import exclude_app_for_update, include_app_for_update
-bench_command.add_command(exclude_app_for_update)
-bench_command.add_command(include_app_for_update)

bench/commands/config.py

@@ -1,112 +1,74 @@
-import click, json
+# imports - standard imports
-from bench.config.common_site_config import update_config
+import ast
 
-## Config
+# imports - module imports
-## Not DRY
+from bench.config.common_site_config import update_config, get_config, put_config
-@click.group()
+
+# imports - third party imports
+import click
+
+
+@click.group(help='Change bench configuration')
 def config():
-	"change bench configuration"
 	pass
 
-@click.command('auto_update')
-@click.argument('state', type=click.Choice(['on', 'off']))
-def config_auto_update(state):
-	"Enable/Disable auto update for bench"
-	state = True if state == 'on' else False
-	update_config({'auto_update': state})
 
-
+@click.command('restart_supervisor_on_update', help='Enable/Disable auto restart of supervisor processes')
-@click.command('restart_supervisor_on_update')
 @click.argument('state', type=click.Choice(['on', 'off']))
 def config_restart_supervisor_on_update(state):
-	"Enable/Disable auto restart of supervisor processes"
+	update_config({'restart_supervisor_on_update': state == 'on'})
-	state = True if state == 'on' else False
-	update_config({'restart_supervisor_on_update': state})
 
-@click.command('restart_systemd_on_update')
+
+@click.command('restart_systemd_on_update', help='Enable/Disable auto restart of systemd units')
 @click.argument('state', type=click.Choice(['on', 'off']))
 def config_restart_systemd_on_update(state):
-	"Enable/Disable auto restart of systemd units"
+	update_config({'restart_systemd_on_update': state == 'on'})
-	state = True if state == 'on' else False
-	update_config({'restart_systemd_on_update': state})
-
-@click.command('update_bench_on_update')
-@click.argument('state', type=click.Choice(['on', 'off']))
-def config_update_bench_on_update(state):
-	"Enable/Disable bench updates on running bench update"
-	state = True if state == 'on' else False
-	update_config({'update_bench_on_update': state})
 
 
-@click.command('dns_multitenant')
+@click.command('dns_multitenant', help='Enable/Disable bench multitenancy on running bench update')
 @click.argument('state', type=click.Choice(['on', 'off']))
 def config_dns_multitenant(state):
-	"Enable/Disable bench updates on running bench update"
+	update_config({'dns_multitenant': state == 'on'})
-	state = True if state == 'on' else False
-	update_config({'dns_multitenant': state})
 
 
-@click.command('serve_default_site')
+@click.command('serve_default_site', help='Configure nginx to serve the default site on port 80')
 @click.argument('state', type=click.Choice(['on', 'off']))
 def config_serve_default_site(state):
-	"Configure nginx to serve the default site on port 80"
+	update_config({'serve_default_site': state == 'on'})
-	state = True if state == 'on' else False
-	update_config({'serve_default_site': state})
 
 
-@click.command('rebase_on_pull')
+@click.command('rebase_on_pull', help='Rebase repositories on pulling')
 @click.argument('state', type=click.Choice(['on', 'off']))
 def config_rebase_on_pull(state):
-	"Rebase repositories on pulling"
+	update_config({'rebase_on_pull': state == 'on'})
-	state = True if state == 'on' else False
-	update_config({'rebase_on_pull': state})
 
 
-@click.command('http_timeout')
+@click.command('http_timeout', help='Set HTTP timeout')
 @click.argument('seconds', type=int)
 def config_http_timeout(seconds):
-	"set http timeout"
 	update_config({'http_timeout': seconds})
 
 
-@click.command('set-common-config')
+@click.command('set-common-config', help='Set value in common config')
 @click.option('configs', '-c', '--config', multiple=True, type=(str, str))
 def set_common_config(configs):
-	import ast
-	from bench.config.common_site_config import update_config
-
 	common_site_config = {}
 	for key, value in configs:
-		if value in ("False", "True"):
+		if value in ('true', 'false'):
+			value = value.title()
+		try:
 			value = ast.literal_eval(value)
-
+		except ValueError:
-		elif "." in value:
+			pass
-			try:
-				value = float(value)
-			except ValueError:
-				pass
-
-		elif "{" in value or "[" in value:
-			try:
-				value = json.loads(value)
-			except ValueError:
-				pass
-
-		else:
-			try:
-				value = int(value)
-			except ValueError:
-				pass
 
 		common_site_config[key] = value
 
 	update_config(common_site_config, bench_path='.')
 
 
-@click.command('remove-common-config')
+@click.command('remove-common-config', help='Remove specific keys from current bench\'s common config')
 @click.argument('keys', nargs=-1)
 def remove_common_config(keys):
-	from bench.config.common_site_config import get_config, put_config
 	common_site_config = get_config('.')
 	for key in keys:
 		if key in common_site_config:
@@ -115,8 +77,6 @@ def remove_common_config(keys):
 	put_config(common_site_config)
 
 
-config.add_command(config_auto_update)
-config.add_command(config_update_bench_on_update)
 config.add_command(config_restart_supervisor_on_update)
 config.add_command(config_restart_systemd_on_update)
 config.add_command(config_dns_multitenant)

bench/commands/git.py

@@ -1,28 +1,30 @@
-import click
+# imports - standard imports
-import os, subprocess, re
+import os
+import subprocess
 
+# imports - module imports
 from bench.app import get_repo_dir, get_apps, get_remote
 from bench.utils import set_git_remote_url
 
+# imports - third party imports
+import click
 
-@click.command('remote-set-url')
+
+@click.command('remote-set-url', help="Set app remote url")
 @click.argument('git-url')
 def remote_set_url(git_url):
-	"Set app remote url"
 	set_git_remote_url(git_url)
 
 
-@click.command('remote-reset-url')
+@click.command('remote-reset-url', help="Reset app remote url to frappe official")
 @click.argument('app')
 def remote_reset_url(app):
-	"Reset app remote url to frappe official"
 	git_url = "https://github.com/frappe/{}.git".format(app)
 	set_git_remote_url(git_url)
 
 
-@click.command('remote-urls')
+@click.command('remote-urls', help="Show apps remote url")
 def remote_urls():
-	"Show apps remote url"
 	for app in get_apps():
 		repo_dir = get_repo_dir(app)
 

bench/commands/install.py

@@ -1,21 +1,29 @@
-import os, sys, json, click
+# imports - module imports
-from bench.utils import run_playbook, setup_sudoers, is_root
+from bench.utils import run_playbook, setup_sudoers
 
-extra_vars = {"production": True}
+# imports - third party imports
+import click
 
-@click.group()
+
+extra_vars = {
+	"production": True
+}
+
+
+@click.group(help="Install system dependencies for setting up Frappe environment")
 def install():
-	"Install system dependancies"
 	pass
 
-@click.command('prerequisites')
+
+@click.command('prerequisites', help="Installs pre-requisite libraries, essential tools like b2zip, htop, screen, vim, x11-fonts, python libs, cups and Redis")
 def install_prerequisites():
 	run_playbook('site.yml', tag='common, redis')
 
-@click.command('mariadb')
+
-@click.option('--mysql_root_password')
+@click.command('mariadb', help="Install and setup MariaDB of specified version and root password")
+@click.option('--mysql_root_password', '--mysql-root-password', default="")
 @click.option('--version', default="10.3")
-def install_maridb(mysql_root_password='', version=''):
+def install_maridb(mysql_root_password, version):
 	if mysql_root_password:
 		extra_vars.update({
 			"mysql_root_password": mysql_root_password,
@@ -27,41 +35,49 @@ def install_maridb(mysql_root_password='', version=''):
 
 	run_playbook('site.yml', extra_vars=extra_vars, tag='mariadb')
 
-@click.command('wkhtmltopdf')
+
+@click.command('wkhtmltopdf', help="Installs wkhtmltopdf v0.12.3 for linux")
 def install_wkhtmltopdf():
 	run_playbook('site.yml', extra_vars=extra_vars, tag='wkhtmltopdf')
 
-@click.command('nodejs')
+
+@click.command('nodejs', help="Installs Node.js v8")
 def install_nodejs():
 	run_playbook('site.yml', extra_vars=extra_vars, tag='nodejs')
 
-@click.command('psutil')
+
+@click.command('psutil', help="Installs psutil via pip")
 def install_psutil():
 	run_playbook('site.yml', extra_vars=extra_vars, tag='psutil')
 
-@click.command('supervisor')
+
+@click.command('supervisor', help="Installs supervisor. If user is specified, sudoers is setup for that user")
 @click.option('--user')
 def install_supervisor(user=None):
 	run_playbook('site.yml', extra_vars=extra_vars, tag='supervisor')
 	if user:
 		setup_sudoers(user)
 
-@click.command('nginx')
+
+@click.command('nginx', help="Installs NGINX. If user is specified, sudoers is setup for that user")
 @click.option('--user')
 def install_nginx(user=None):
 	run_playbook('site.yml', extra_vars=extra_vars, tag='nginx')
 	if user:
 		setup_sudoers(user)
 
-@click.command('virtualbox')
+
+@click.command('virtualbox', help="Installs supervisor")
 def install_virtualbox():
 	run_playbook('vm_build.yml', tag='virtualbox')
 
-@click.command('packer')
+
+@click.command('packer', help="Installs Oracle virtualbox and packer 1.2.1")
 def install_packer():
 	run_playbook('vm_build.yml', tag='packer')
 
-@click.command('fail2ban')
+
+@click.command("fail2ban", help="Install fail2ban, an intrusion prevention software framework that protects computer servers from brute-force attacks")
 @click.option('--maxretry', default=6, help="Number of matches (i.e. value of the counter) which triggers ban action on the IP.")
 @click.option('--bantime', default=600, help="The counter is set to zero if no match is found within 'findtime' seconds.")
 @click.option('--findtime', default=600, help='Duration (in seconds) for IP to be banned for. Negative number for "permanent" ban.')
@@ -69,6 +85,7 @@ def install_failtoban(**kwargs):
 	extra_vars.update(kwargs)
 	run_playbook('site.yml', extra_vars=extra_vars, tag='fail2ban')
 
+
 install.add_command(install_prerequisites)
 install.add_command(install_maridb)
 install.add_command(install_wkhtmltopdf)

bench/commands/make.py

@@ -1,6 +1,8 @@
+# imports - third party imports
 import click
 
-@click.command()
+
+@click.command('init', help='Initialize a new bench instance in the specified path')
 @click.argument('path')
 @click.option('--python', type = str, default = 'python3', help = 'Path to Python Executable.')
 @click.option('--ignore-exist', is_flag = True, default = False, help = "Ignore if Bench instance exists.")
@@ -11,14 +13,10 @@ import click
 @click.option('--clone-without-update', is_flag=True, help="copy repos from path without update")
 @click.option('--no-procfile', is_flag=True, help="Pull changes in all the apps in bench")
 @click.option('--no-backups',is_flag=True, help="Run migrations for all sites in the bench")
-@click.option('--no-auto-update',is_flag=True, help="Build JS and CSS artifacts for the bench")
 @click.option('--skip-redis-config-generation', is_flag=True, help="Skip redis config generation if already specifying the common-site-config file")
 @click.option('--skip-assets',is_flag=True, default=False, help="Do not build assets")
 @click.option('--verbose',is_flag=True, help="Verbose output during install")
-def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, no_auto_update, clone_from, verbose, skip_redis_config_generation, clone_without_update, ignore_exist=False, skip_assets=False, python='python3'):
+def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, clone_from, verbose, skip_redis_config_generation, clone_without_update, ignore_exist=False, skip_assets=False, python='python3'):
-	'''
-	Create a New Bench Instance.
-	'''
 	from bench.utils import init, log
 
 	try:
@@ -27,7 +25,6 @@ def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, n
 			apps_path=apps_path,
 			no_procfile=no_procfile,
 			no_backups=no_backups,
-			no_auto_update=no_auto_update,
 			frappe_path=frappe_path,
 			frappe_branch=frappe_branch,
 			verbose=verbose,
@@ -41,10 +38,11 @@ def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, n
 		log('Bench {} initialized'.format(path), level=1)
 	except SystemExit:
 		pass
-	except:
+	except Exception as e:
 		import os, shutil, time, six
 		# add a sleep here so that the traceback of other processes doesnt overlap with the prompts
 		time.sleep(1)
+		print(e)
 		log("There was a problem while creating {}".format(path), level=2)
 		if six.moves.input("Do you want to rollback these changes? [Y/n]: ").lower() == "y":
 			print('Rolling back Bench "{}"'.format(path))
@@ -52,44 +50,53 @@ def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, n
 				shutil.rmtree(path)
 
 
-@click.command('get-app')
+@click.command('get-app', help='Clone an app from the internet or filesystem and set it up in your bench')
 @click.argument('name', nargs=-1) # Dummy argument for backward compatibility
 @click.argument('git-url')
 @click.option('--branch', default=None, help="branch to checkout")
+@click.option('--overwrite', is_flag=True, default=False)
 @click.option('--skip-assets', is_flag=True, default=False, help="Do not build assets")
-def get_app(git_url, branch, name=None, skip_assets=False):
+def get_app(git_url, branch, name=None, overwrite=False, skip_assets=False):
 	"clone an app from the internet and set it up in your bench"
 	from bench.app import get_app
-	get_app(git_url, branch=branch, skip_assets=skip_assets)
+	get_app(git_url, branch=branch, skip_assets=skip_assets, overwrite=overwrite)
 
 
-@click.command('new-app')
+@click.command('new-app', help='Create a new Frappe application under apps folder')
 @click.argument('app-name')
 def new_app(app_name):
-	"start a new app"
 	from bench.app import new_app
 	new_app(app_name)
 
 
-@click.command('remove-app')
+@click.command('remove-app', help='Completely remove app from bench and re-build assets if not installed on any site')
 @click.argument('app-name')
 def remove_app(app_name):
-	"completely remove app from bench"
 	from bench.app import remove_app
 	remove_app(app_name)
 
 
-@click.command('exclude-app')
+@click.command('exclude-app', help='Exclude app from updating')
 @click.argument('app_name')
 def exclude_app_for_update(app_name):
-	"Exclude app from updating"
 	from bench.app import add_to_excluded_apps_txt
 	add_to_excluded_apps_txt(app_name)
 
 
-@click.command('include-app')
+@click.command('include-app', help='Include app for updating')
 @click.argument('app_name')
 def include_app_for_update(app_name):
 	"Include app from updating"
 	from bench.app import remove_from_excluded_apps_txt
 	remove_from_excluded_apps_txt(app_name)
+
+
+@click.command('pip', context_settings={"ignore_unknown_options": True}, help="For pip help use `bench pip help [COMMAND]` or `bench pip [COMMAND] -h`")
+@click.argument('args', nargs=-1)
+@click.pass_context
+def pip(ctx, args):
+	"Run pip commands in bench env"
+	import os
+	from bench.utils import get_env_cmd
+	env_pip = get_env_cmd('pip')
+	os.execv(env_pip, (env_pip,) + args)

bench/commands/setup.py

@@ -1,233 +1,216 @@
-from bench.utils import exec_cmd
+# imports - standard imports
-from six import PY3
-import click, sys, json
 import os
+import sys
 
-@click.group()
+# imports - module imports
+from bench.utils import exec_cmd
+
+# imports - third party imports
+from six import PY3
+import click
+
+
+@click.group(help="Setup command group for enabling setting up a Frappe environment")
 def setup():
-	"Setup bench"
 	pass
 
-@click.command('sudoers')
+
-@click.argument('user')
+@click.command("sudoers", help="Add commands to sudoers list for execution without password")
+@click.argument("user")
 def setup_sudoers(user):
-	"Add commands to sudoers list for execution without password"
 	from bench.utils import setup_sudoers
 	setup_sudoers(user)
 
-@click.command('nginx')
+
-@click.option('--yes', help='Yes to regeneration of nginx config file', default=False, is_flag=True)
+@click.command("nginx", help="Generate configuration files for NGINX")
+@click.option("--yes", help="Yes to regeneration of nginx config file", default=False, is_flag=True)
 def setup_nginx(yes=False):
-	"generate config for nginx"
 	from bench.config.nginx import make_nginx_conf
 	make_nginx_conf(bench_path=".", yes=yes)
 
-@click.command('reload-nginx')
+
+@click.command("reload-nginx", help="Checks NGINX config file and reloads service")
 def reload_nginx():
 	from bench.config.production_setup import reload_nginx
 	reload_nginx()
 
-@click.command('supervisor')
+
-@click.option('--user')
+@click.command("supervisor", help="Generate configuration for supervisor")
-@click.option('--yes', help='Yes to regeneration of supervisor config', is_flag=True, default=False)
+@click.option("--user", help="optional user argument")
+@click.option("--yes", help="Yes to regeneration of supervisor config", is_flag=True, default=False)
 def setup_supervisor(user=None, yes=False):
-	"generate config for supervisor with an optional user argument"
 	from bench.config.supervisor import generate_supervisor_config
 	generate_supervisor_config(bench_path=".", user=user, yes=yes)
 
-@click.command('redis')
+
+@click.command("redis", help="Generates configuration for Redis")
 def setup_redis():
-	"generate config for redis cache"
 	from bench.config.redis import generate_config
-	generate_config('.')
+	generate_config(".")
 
 
-@click.command('fonts')
+@click.command("fonts", help="Add Frappe fonts to system")
 def setup_fonts():
-	"Add frappe fonts to system"
 	from bench.utils import setup_fonts
 	setup_fonts()
 
-@click.command('production')
+
-@click.argument('user')
+@click.command("production", help="Setup Frappe production environment for specific user")
-@click.option('--yes', help='Yes to regeneration config', is_flag=True, default=False)
+@click.argument("user")
+@click.option("--yes", help="Yes to regeneration config", is_flag=True, default=False)
 def setup_production(user, yes=False):
-	"setup bench for production"
 	from bench.config.production_setup import setup_production
-	from bench.utils import run_playbook
 	# Install prereqs for production
 	from distutils.spawn import find_executable
-	if not find_executable('ansible'):
+	if not find_executable("ansible"):
-		exec_cmd("sudo {0} install ansible".format("pip3" if PY3 else "pip2"))
+		exec_cmd("sudo -H {0} -m pip install ansible".format(sys.executable))
-	if not find_executable('fail2ban-client'):
+	if not find_executable("fail2ban-client"):
 		exec_cmd("bench setup role fail2ban")
-	if not find_executable('nginx'):
+	if not find_executable("nginx"):
 		exec_cmd("bench setup role nginx")
-	if not find_executable('supervisord'):
+	if not find_executable("supervisord"):
 		exec_cmd("bench setup role supervisor")
 	setup_production(user=user, yes=yes)
 
 
-@click.command('auto-update')
+@click.command("backups", help="Add cronjob for bench backups")
-def setup_auto_update():
-	"Add cronjob for bench auto update"
-	from bench.utils import setup_auto_update
-	setup_auto_update()
-
-
-@click.command('backups')
 def setup_backups():
-	"Add cronjob for bench backups"
 	from bench.utils import setup_backups
 	setup_backups()
 
-@click.command('env')
+
-@click.option('--python', type = str, default = 'python3', help = 'Path to Python Executable.')
+@click.command("env", help="Setup virtualenv for bench")
-def setup_env(python='python3'):
+@click.option("--python", type = str, default = "python3", help = "Path to Python Executable.")
-	"Setup virtualenv for bench"
+def setup_env(python="python3"):
 	from bench.utils import setup_env
 	setup_env(python=python)
 
-@click.command('firewall')
+
-@click.option('--ssh_port')
+@click.command("firewall", help="Setup firewall for system")
-@click.option('--force')
+@click.option("--ssh_port")
+@click.option("--force")
 def setup_firewall(ssh_port=None, force=False):
-	"Setup firewall"
 	from bench.utils import run_playbook
 
 	if not force:
-		click.confirm('Setting up the firewall will block all ports except 80, 443 and 22\n'
+		click.confirm("Setting up the firewall will block all ports except 80, 443 and {0}\nDo you want to continue?".format(ssh_port), abort=True)
-			'Do you want to continue?',
-			abort=True)
 
 	if not ssh_port:
 		ssh_port = 22
 
-	run_playbook('roles/bench/tasks/setup_firewall.yml', {"ssh_port": ssh_port})
+	run_playbook("roles/bench/tasks/setup_firewall.yml", {"ssh_port": ssh_port})
 
-@click.command('ssh-port')
+
-@click.argument('port')
+@click.command("ssh-port", help="Set SSH Port for system")
-@click.option('--force')
+@click.argument("port")
+@click.option("--force")
 def set_ssh_port(port, force=False):
-	"Set SSH Port"
 	from bench.utils import run_playbook
 
 	if not force:
-		click.confirm('This will change your SSH Port to {}\n'
+		click.confirm("This will change your SSH Port to {}\nDo you want to continue?".format(port), abort=True)
-			'Do you want to continue?'.format(port),
-			abort=True)
 
-	run_playbook('roles/bench/tasks/change_ssh_port.yml', {"ssh_port": port})
+	run_playbook("roles/bench/tasks/change_ssh_port.yml", {"ssh_port": port})
 
-@click.command('lets-encrypt')
+
-@click.argument('site')
+@click.command("lets-encrypt", help="Setup lets-encrypt SSL for site")
-@click.option('--custom-domain')
+@click.argument("site")
-@click.option('-n', '--non-interactive', default=False, is_flag=True, help="Run certbot non-interactively. Shouldn't be used on 1'st attempt")
+@click.option("--custom-domain")
+@click.option('-n', '--non-interactive', default=False, is_flag=True, help="Run command non-interactively. This flag restarts nginx and runs certbot non interactively. Shouldn't be used on 1'st attempt")
 def setup_letsencrypt(site, custom_domain, non_interactive):
-	"Setup lets-encrypt for site"
 	from bench.config.lets_encrypt import setup_letsencrypt
-	setup_letsencrypt(site, custom_domain, bench_path='.', interactive=not non_interactive)
+	setup_letsencrypt(site, custom_domain, bench_path=".", interactive=not non_interactive)
 
 
-@click.command('wildcard-ssl')
+@click.command("wildcard-ssl", help="Setup wildcard SSL certificate for multi-tenant bench")
-@click.argument('domain')
+@click.argument("domain")
-@click.option('--email')
+@click.option("--email")
-@click.option('--exclude-base-domain', default=False, is_flag=True, help="SSL Certificate not applicable for base domain")
+@click.option("--exclude-base-domain", default=False, is_flag=True, help="SSL Certificate not applicable for base domain")
 def setup_wildcard_ssl(domain, email, exclude_base_domain):
-	''' Setup wildcard ssl certificate '''
 	from bench.config.lets_encrypt import setup_wildcard_ssl
-	setup_wildcard_ssl(domain, email, bench_path='.', exclude_base_domain=exclude_base_domain)
+	setup_wildcard_ssl(domain, email, bench_path=".", exclude_base_domain=exclude_base_domain)
 
 
-@click.command('procfile')
+@click.command("procfile", help="Generate Procfile for bench start")
 def setup_procfile():
-	"Setup Procfile for bench start"
 	from bench.config.procfile import setup_procfile
-	setup_procfile('.')
+	setup_procfile(".")
 
 
-@click.command('socketio')
+@click.command("socketio", help="Setup node dependencies for socketio server")
 def setup_socketio():
-	"Setup node deps for socketio server"
 	from bench.utils import setup_socketio
 	setup_socketio()
 
-@click.command('requirements', help="Update Python and Node packages")
+
-@click.option('--node', help="Update only Node packages", default=False, is_flag=True)
+@click.command("requirements", help="Setup Python and Node dependencies")
-@click.option('--python', help="Update only Python packages", default=False, is_flag=True)
+@click.option("--node", help="Update only Node packages", default=False, is_flag=True)
+@click.option("--python", help="Update only Python packages", default=False, is_flag=True)
 def setup_requirements(node=False, python=False):
-	"Setup python and node requirements"
-
 	if not node:
-		setup_python_requirements()
+		from bench.utils import update_requirements as setup_python_packages
+		setup_python_packages()
+
 	if not python:
-		setup_node_requirements()
+		from bench.utils import update_node_packages as setup_node_packages
-
+		setup_node_packages()
-def setup_python_requirements():
-	from bench.utils import update_requirements
-	update_requirements()
-
-def setup_node_requirements():
-	from bench.utils import update_node_packages
-	update_node_packages()
 
 
-@click.command('manager')
+@click.command("manager", help="Setup bench-manager.local site with the bench_manager app installed on it")
-@click.option('--yes', help='Yes to regeneration of nginx config file', default=False, is_flag=True)
+@click.option("--yes", help="Yes to regeneration of nginx config file", default=False, is_flag=True)
-@click.option('--port', help='Port on which you want to run bench manager', default=23624)
+@click.option("--port", help="Port on which you want to run bench manager", default=23624)
-@click.option('--domain', help='Domain on which you want to run bench manager')
+@click.option("--domain", help="Domain on which you want to run bench manager")
 def setup_manager(yes=False, port=23624, domain=None):
-	"Setup bench-manager.local site with the bench_manager app installed on it"
 	from six.moves import input
+	from bench.utils import get_sites
+	from bench.config.common_site_config import get_config
+	from bench.config.nginx import make_bench_manager_nginx_conf
+
 	create_new_site = True
-	if 'bench-manager.local' in os.listdir('sites'):
+
-		ans = input('Site already exists. Overwrite existing site? [Y/n]: ').lower()
+	if "bench-manager.local" in os.listdir("sites"):
-		while ans not in ('y', 'n', ''):
+		ans = input("Site already exists. Overwrite existing site? [Y/n]: ").lower()
-			ans = input(
+		while ans not in ("y", "n", ""):
-				'Please enter "y" or "n". Site already exists. Overwrite existing site? [Y/n]: ').lower()
+			ans = input("Please enter 'y' or 'n'. Site already exists. Overwrite existing site? [Y/n]: ").lower()
-		if ans == 'n':
+		if ans == "n":
 			create_new_site = False
+
 	if create_new_site:
 		exec_cmd("bench new-site --force bench-manager.local")
 
-	if 'bench_manager' in os.listdir('apps'):
+	if "bench_manager" in os.listdir("apps"):
-		print('App already exists. Skipping app download.')
+		print("App already exists. Skipping app download.")
 	else:
 		exec_cmd("bench get-app bench_manager")
 
 	exec_cmd("bench --site bench-manager.local install-app bench_manager")
 
-	from bench.config.common_site_config import get_config
+	bench_path = "."
-	bench_path = '.'
 	conf = get_config(bench_path)
-	if conf.get('restart_supervisor_on_update') or conf.get('restart_systemd_on_update'):
+
+	if conf.get("restart_supervisor_on_update") or conf.get("restart_systemd_on_update"):
 		# implicates a production setup or so I presume
 		if not domain:
 			print("Please specify the site name on which you want to host bench-manager using the 'domain' flag")
 			sys.exit(1)
 
-		from bench.utils import get_sites, get_bench_name
-		bench_name = get_bench_name(bench_path)
-
 		if domain not in get_sites(bench_path):
 			raise Exception("No such site")
 
-		from bench.config.nginx import make_bench_manager_nginx_conf
 		make_bench_manager_nginx_conf(bench_path, yes=yes, port=port, domain=domain)
 
 
-@click.command('config')
+@click.command("config", help="Generate or over-write sites/common_site_config.json")
 def setup_config():
-	"overwrite or make config.json"
 	from bench.config.common_site_config import make_config
-	make_config('.')
+	make_config(".")
 
 
-@click.command('add-domain')
+@click.command("add-domain", help="Add a custom domain to a particular site")
-@click.argument('domain')
+@click.argument("domain")
-@click.option('--site', prompt=True)
+@click.option("--site", prompt=True)
-@click.option('--ssl-certificate', help="Absolute path to SSL Certificate")
+@click.option("--ssl-certificate", help="Absolute path to SSL Certificate")
-@click.option('--ssl-certificate-key', help="Absolute path to SSL Certificate Key")
+@click.option("--ssl-certificate-key", help="Absolute path to SSL Certificate Key")
 def add_domain(domain, site=None, ssl_certificate=None, ssl_certificate_key=None):
 	"""Add custom domain to site"""
 	from bench.config.site_config import add_domain
@@ -236,24 +219,25 @@ def add_domain(domain, site=None, ssl_certificate=None, ssl_certificate_key=None
 		print("Please specify site")
 		sys.exit(1)
 
-	add_domain(site, domain, ssl_certificate, ssl_certificate_key, bench_path='.')
+	add_domain(site, domain, ssl_certificate, ssl_certificate_key, bench_path=".")
 
-@click.command('remove-domain')
+
-@click.argument('domain')
+@click.command("remove-domain", help="Remove custom domain from a site")
-@click.option('--site', prompt=True)
+@click.argument("domain")
+@click.option("--site", prompt=True)
 def remove_domain(domain, site=None):
-	"""Remove custom domain from a site"""
 	from bench.config.site_config import remove_domain
 
 	if not site:
 		print("Please specify site")
 		sys.exit(1)
 
-	remove_domain(site, domain, bench_path='.')
+	remove_domain(site, domain, bench_path=".")
 
-@click.command('sync-domains')
+
-@click.option('--domain', multiple=True)
+@click.command("sync-domains", help="Check if there is a change in domains. If yes, updates the domains list.")
-@click.option('--site', prompt=True)
+@click.option("--domain", multiple=True)
+@click.option("--site", prompt=True)
 def sync_domains(domain=None, site=None):
 	from bench.config.site_config import sync_domains
 
@@ -262,53 +246,55 @@ def sync_domains(domain=None, site=None):
 		sys.exit(1)
 
 	try:
-		domains = list(map(str,domain))
+		domains = list(map(str, domain))
 	except Exception:
 		print("Domains should be a json list of strings or dictionaries")
 		sys.exit(1)
 
-	changed = sync_domains(site, domains, bench_path='.')
+	changed = sync_domains(site, domains, bench_path=".")
 
 	# if changed, success, else failure
 	sys.exit(0 if changed else 1)
 
-@click.command('role')
+
-@click.argument('role')
+@click.command("role", help="Install dependencies via ansible roles")
-@click.option('--admin_emails', default='')
+@click.argument("role")
-@click.option('--mysql_root_password')
+@click.option("--admin_emails", default="")
-@click.option('--container', is_flag=True, default=False)
+@click.option("--mysql_root_password")
+@click.option("--container", is_flag=True, default=False)
 def setup_roles(role, **kwargs):
-	"Install dependancies via roles"
 	from bench.utils import run_playbook
 
 	extra_vars = {"production": True}
 	extra_vars.update(kwargs)
 
 	if role:
-		run_playbook('site.yml', extra_vars=extra_vars, tag=role)
+		run_playbook("site.yml", extra_vars=extra_vars, tag=role)
 	else:
-		run_playbook('site.yml', extra_vars=extra_vars)
+		run_playbook("site.yml", extra_vars=extra_vars)
 
-@click.command('fail2ban')
+
-@click.option('--maxretry', default=6, help="Number of matches (i.e. value of the counter) which triggers ban action on the IP. Default is 6 seconds" )
+@click.command("fail2ban", help="Setup fail2ban, an intrusion prevention software framework that protects computer servers from brute-force attacks")
-@click.option('--bantime', default=600, help="The counter is set to zero if no match is found within 'findtime' seconds. Default is 600 seconds")
+@click.option("--maxretry", default=6, help="Number of matches (i.e. value of the counter) which triggers ban action on the IP. Default is 6 seconds" )
-@click.option('--findtime', default=600, help='Duration (in seconds) for IP to be banned for. Negative number for "permanent" ban. Default is 600 seconds')
+@click.option("--bantime", default=600, help="The counter is set to zero if no match is found within 'findtime' seconds. Default is 600 seconds")
+@click.option("--findtime", default=600, help="Duration (in seconds) for IP to be banned for. Negative number for 'permanent' ban. Default is 600 seconds")
 def setup_nginx_proxy_jail(**kwargs):
 	from bench.utils import run_playbook
-	run_playbook('roles/fail2ban/tasks/configure_nginx_jail.yml', extra_vars=kwargs)
+	run_playbook("roles/fail2ban/tasks/configure_nginx_jail.yml", extra_vars=kwargs)
 
-@click.command('systemd')
+
-@click.option('--user')
+@click.command("systemd", help="Generate configuration for systemd")
-@click.option('--yes', help='Yes to regeneration of systemd config files', is_flag=True, default=False)
+@click.option("--user", help="Optional user argument")
-@click.option('--stop', help='Stop bench services', is_flag=True, default=False)
+@click.option("--yes", help="Yes to regeneration of systemd config files", is_flag=True, default=False)
-@click.option('--create-symlinks', help='Create Symlinks', is_flag=True, default=False)
+@click.option("--stop", help="Stop bench services", is_flag=True, default=False)
-@click.option('--delete-symlinks', help='Delete Symlinks', is_flag=True, default=False)
+@click.option("--create-symlinks", help="Create Symlinks", is_flag=True, default=False)
+@click.option("--delete-symlinks", help="Delete Symlinks", is_flag=True, default=False)
 def setup_systemd(user=None, yes=False, stop=False, create_symlinks=False, delete_symlinks=False):
-	"generate configs for systemd with an optional user argument"
 	from bench.config.systemd import generate_systemd_config
 	generate_systemd_config(bench_path=".", user=user, yes=yes,
 		stop=stop, create_symlinks=create_symlinks, delete_symlinks=delete_symlinks)
 
+
 setup.add_command(setup_sudoers)
 setup.add_command(setup_nginx)
 setup.add_command(reload_nginx)
@@ -317,7 +303,6 @@ setup.add_command(setup_redis)
 setup.add_command(setup_letsencrypt)
 setup.add_command(setup_wildcard_ssl)
 setup.add_command(setup_production)
-setup.add_command(setup_auto_update)
 setup.add_command(setup_backups)
 setup.add_command(setup_env)
 setup.add_command(setup_procfile)

bench/commands/update.py

@@ -1,124 +1,31 @@
-import click
+# imports - standard imports
-import sys
 import os
-from bench.config.common_site_config import get_config, update_config
+
-from bench.app import pull_all_apps, is_version_upgrade, validate_branch
+# imports - third party imports
-from bench.utils import (update_bench, validate_upgrade, pre_upgrade, post_upgrade, before_update,
+import click
-	update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets,
-	restart_supervisor_processes, restart_systemd_processes, is_bench_directory)
-from bench import patches
 from six.moves import reload_module
 
+# imports - module imports
+from bench.app import pull_all_apps
+from bench.utils import post_upgrade, patch_sites, build_assets
 
-@click.command('update')
+
-@click.option('--pull', is_flag=True, help="Pull changes in all the apps in bench")
+@click.command('update', help="Updates bench tool and if executed in a bench directory, without any flags will backup, pull, setup requirements, build, run patches and restart bench. Using specific flags will only do certain tasks instead of all")
+@click.option('--pull', is_flag=True, help="Pull updates for all the apps in bench")
 @click.option('--patch', is_flag=True, help="Run migrations for all sites in the bench")
-@click.option('--build', is_flag=True, help="Build JS and CSS artifacts for the bench")
+@click.option('--build', is_flag=True, help="Build JS and CSS assets for the bench")
-@click.option('--bench', is_flag=True, help="Update bench")
+@click.option('--requirements', is_flag=True, help="Update requirements. If run alone, equivalent to `bench setup requirements`")
-@click.option('--requirements', is_flag=True, help="Update requirements")
+@click.option('--restart-supervisor', is_flag=True, help="Restart supervisor processes after update")
-@click.option('--restart-supervisor', is_flag=True, help="restart supervisor processes after update")
+@click.option('--restart-systemd', is_flag=True, help="Restart systemd units after update")
-@click.option('--restart-systemd', is_flag=True, help="restart systemd units after update")
+@click.option('--no-backup', is_flag=True, help="If this flag is set, sites won't be backed up prior to updates. Note: This is not recommended in production.")
-@click.option('--auto', is_flag=True)
+@click.option('--force', is_flag=True, help="Forces major version upgrades")
-@click.option('--no-backup', is_flag=True)
-@click.option('--force', is_flag=True)
 @click.option('--reset', is_flag=True, help="Hard resets git branch's to their new states overriding any changes and overriding rebase on pull")
-def update(pull=False, patch=False, build=False, bench=False, auto=False, restart_supervisor=False, restart_systemd=False, requirements=False, no_backup=False, force=False, reset=False):
+def update(pull, patch, build, requirements, restart_supervisor, restart_systemd, no_backup, force, reset):
-	"Update bench"
+	from bench.utils import update
+	update(pull=pull, patch=patch, build=build, requirements=requirements, restart_supervisor=restart_supervisor, restart_systemd=restart_systemd, backup=not no_backup, force=force, reset=reset)
 
-	if not is_bench_directory():
-		"""Update only bench if bench update called from outside a bench"""
-		update_bench(bench_repo=True, requirements=True)
-		sys.exit()
 
-	if not (pull or patch or build or bench or requirements):
+@click.command('retry-upgrade', help="Retry a failed upgrade")
-		pull, patch, build, bench, requirements = True, True, True, True, True
-
-	if auto:
-		sys.exit(1)
-
-	patches.run(bench_path='.')
-	conf = get_config(".")
-
-	if bench and conf.get('update_bench_on_update'):
-		update_bench(bench_repo=True, requirements=False)
-		restart_update({
-				'pull': pull,
-				'patch': patch,
-				'build': build,
-				'requirements': requirements,
-				'no-backup': no_backup,
-				'restart-supervisor': restart_supervisor,
-				'reset': reset
-		})
-
-	if conf.get('release_bench'):
-		print('Release bench, cannot update')
-		sys.exit(1)
-
-	validate_branch()
-
-	version_upgrade = is_version_upgrade()
-	if version_upgrade[0]:
-		print()
-		print()
-		print("This update will cause a major version change in Frappe/ERPNext from {0} to {1}.".format(*version_upgrade[1:]))
-		print("This would take significant time to migrate and might break custom apps.")
-		click.confirm('Do you want to continue?', abort=True)
-
-	_update(pull, patch, build, bench, auto, restart_supervisor, restart_systemd, requirements, no_backup, force=force, reset=reset)
-
-def _update(pull=False, patch=False, build=False, update_bench=False, auto=False, restart_supervisor=False,
-		restart_systemd=False, requirements=False, no_backup=False, bench_path='.', force=False, reset=False):
-	conf = get_config(bench_path=bench_path)
-	version_upgrade = is_version_upgrade(bench_path=bench_path)
-
-	if version_upgrade[0] or (not version_upgrade[0] and force):
-		validate_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path)
-
-	before_update(bench_path=bench_path, requirements=requirements)
-
-	conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 })
-	update_config(conf, bench_path=bench_path)
-
-	if not no_backup:
-		print('Backing up sites...')
-		backup_all_sites(bench_path=bench_path)
-
-	if pull:
-		pull_all_apps(bench_path=bench_path, reset=reset)
-
-	if requirements:
-		update_requirements(bench_path=bench_path)
-		update_node_packages(bench_path=bench_path)
-
-	if version_upgrade[0] or (not version_upgrade[0] and force):
-		pre_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path)
-		import bench.utils, bench.app
-		print('Reloading bench...')
-		reload_module(bench.utils)
-		reload_module(bench.app)
-
-	if patch:
-		print('Patching sites...')
-		patch_sites(bench_path=bench_path)
-	if build:
-		build_assets(bench_path=bench_path)
-	if version_upgrade[0] or (not version_upgrade[0] and force):
-		post_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path)
-	if restart_supervisor or conf.get('restart_supervisor_on_update'):
-		restart_supervisor_processes(bench_path=bench_path)
-	if restart_systemd or conf.get('restart_systemd_on_update'):
-		restart_systemd_processes(bench_path=bench_path)
-
-	conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 })
-	update_config(conf, bench_path=bench_path)
-
-	print("_"*80)
-	print("Bench: Deployment tool for Frappe and ERPNext (https://erpnext.org).")
-	print("Open source depends on your contributions, so please contribute bug reports, patches, fixes or cash and be a part of the community")
-	print()
-
-@click.command('retry-upgrade')
 @click.option('--version', default=5)
 def retry_upgrade(version):
 	pull_all_apps()
@@ -126,35 +33,24 @@ def retry_upgrade(version):
 	build_assets()
 	post_upgrade(version-1, version)
 
-def restart_update(kwargs):
-	args = ['--'+k for k, v in list(kwargs.items()) if v]
-	os.execv(sys.argv[0], sys.argv[:2] + args)
 
-@click.command('switch-to-branch')
+@click.command('switch-to-branch', help="Switch all apps to specified branch, or specify apps separated by space")
 @click.argument('branch')
 @click.argument('apps', nargs=-1)
 @click.option('--upgrade',is_flag=True)
 def switch_to_branch(branch, apps, upgrade=False):
-	"Switch all apps to specified branch, or specify apps separated by space"
 	from bench.app import switch_to_branch
 	switch_to_branch(branch=branch, apps=list(apps), upgrade=upgrade)
-	print('Switched to ' + branch)
-	print('Please run `bench update --patch` to be safe from any differences in database schema')
 
-@click.command('switch-to-master')
+
+@click.command('switch-to-master', help="[DEPRECATED]: Switch frappe and erpnext to master branch")
 def switch_to_master():
-	"Switch frappe and erpnext to master branch"
+	from bench.utils import log
-	from bench.app import switch_to_master
+	log("`switch-to-master` has been deprecated as master branches were renamed to version-11")
-	switch_to_master(apps=['frappe', 'erpnext'])
+
-	print()
-	print('Switched to master')
-	print('Please run `bench update --patch` to be safe from any differences in database schema')
 
 @click.command('switch-to-develop')
 def switch_to_develop(upgrade=False):
 	"Switch frappe and erpnext to develop branch"
 	from bench.app import switch_to_develop
 	switch_to_develop(apps=['frappe', 'erpnext'])
-	print()
-	print('Switched to develop')
-	print('Please run `bench update --patch` to be safe from any differences in database schema')

bench/commands/utils.py

@@ -1,23 +1,25 @@
+# imports - standard imports
+import os
+import sys
+
+# imports - third party imports
 import click
-import sys, os, copy
 
 
-@click.command('start')
+@click.command('start', help="Start Frappe development processes")
 @click.option('--no-dev', is_flag=True, default=False)
 @click.option('--concurrency', '-c', type=str)
 @click.option('--procfile', '-p', type=str)
 def start(no_dev, concurrency, procfile):
-	"Start Frappe development processes"
 	from bench.utils import start
 	start(no_dev=no_dev, concurrency=concurrency, procfile=procfile)
 
 
-@click.command('restart')
+@click.command('restart', help="Restart supervisor processes or systemd units")
 @click.option('--web', is_flag=True, default=False)
 @click.option('--supervisor', is_flag=True, default=False)
 @click.option('--systemd', is_flag=True, default=False)
 def restart(web, supervisor, systemd):
-	"Restart supervisor processes or systemd units"
 	from bench.utils import restart_supervisor_processes, restart_systemd_processes
 	from bench.config.common_site_config import get_config
 	if get_config('.').get('restart_supervisor_on_update') or supervisor:
@@ -25,134 +27,112 @@ def restart(web, supervisor, systemd):
 	if get_config('.').get('restart_systemd_on_update') or systemd:
 		restart_systemd_processes(bench_path='.', web_workers=web)
 
-@click.command('set-nginx-port')
+
+@click.command('set-nginx-port', help="Set NGINX port for site")
 @click.argument('site')
 @click.argument('port', type=int)
 def set_nginx_port(site, port):
-	"Set nginx port for site"
 	from bench.config.site_config import set_nginx_port
 	set_nginx_port(site, port)
 
 
-@click.command('set-ssl-certificate')
+@click.command('set-ssl-certificate', help="Set SSL certificate path for site")
 @click.argument('site')
 @click.argument('ssl-certificate-path')
 def set_ssl_certificate(site, ssl_certificate_path):
-	"Set ssl certificate path for site"
 	from bench.config.site_config import set_ssl_certificate
 	set_ssl_certificate(site, ssl_certificate_path)
 
 
-@click.command('set-ssl-key')
+@click.command('set-ssl-key', help="Set SSL certificate private key path for site")
 @click.argument('site')
 @click.argument('ssl-certificate-key-path')
 def set_ssl_certificate_key(site, ssl_certificate_key_path):
-	"Set ssl certificate private key path for site"
 	from bench.config.site_config import set_ssl_certificate_key
 	set_ssl_certificate_key(site, ssl_certificate_key_path)
 
 
-@click.command('set-url-root')
+@click.command('set-url-root', help="Set URL root for site")
 @click.argument('site')
 @click.argument('url-root')
 def set_url_root(site, url_root):
-	"Set url root for site"
 	from bench.config.site_config import set_url_root
 	set_url_root(site, url_root)
 
 
-@click.command('set-mariadb-host')
+@click.command('set-mariadb-host', help="Set MariaDB host for bench")
 @click.argument('host')
 def set_mariadb_host(host):
-	"Set MariaDB host for bench"
 	from bench.utils import set_mariadb_host
 	set_mariadb_host(host)
 
-@click.command('set-redis-cache-host')
+
+@click.command('set-redis-cache-host', help="Set Redis cache host for bench")
 @click.argument('host')
 def set_redis_cache_host(host):
 	"""
-	Set Redis cache host for bench
+	Usage: bench set-redis-cache-host localhost:6379/1
-	Eg: bench set-redis-cache-host localhost:6379/1
 	"""
 	from bench.utils import set_redis_cache_host
 	set_redis_cache_host(host)
 
-@click.command('set-redis-queue-host')
+
+@click.command('set-redis-queue-host', help="Set Redis queue host for bench")
 @click.argument('host')
 def set_redis_queue_host(host):
 	"""
-	Set Redis queue host for bench
+	Usage: bench set-redis-queue-host localhost:6379/2
-	Eg: bench set-redis-queue-host localhost:6379/2
 	"""
 	from bench.utils import set_redis_queue_host
 	set_redis_queue_host(host)
 
-@click.command('set-redis-socketio-host')
+
+@click.command('set-redis-socketio-host', help="Set Redis socketio host for bench")
 @click.argument('host')
 def set_redis_socketio_host(host):
 	"""
-	Set Redis socketio host for bench
+	Usage: bench set-redis-socketio-host localhost:6379/3
-	Eg: bench set-redis-socketio-host localhost:6379/3
 	"""
 	from bench.utils import set_redis_socketio_host
 	set_redis_socketio_host(host)
 
 
-@click.command('set-default-site')
+@click.command('set-default-site', help="Set default site for bench")
 @click.argument('site')
 def set_default_site(site):
-	"Set default site for bench"
 	from bench.utils import set_default_site
 	set_default_site(site)
 
 
-@click.command('download-translations')
+@click.command('download-translations', help="Download latest translations")
 def download_translations():
-	"Download latest translations"
 	from bench.utils import download_translations_p
 	download_translations_p()
 
-@click.command('renew-lets-encrypt')
+
+@click.command('renew-lets-encrypt', help="Renew Let's Encrypt certificate")
 def renew_lets_encrypt():
-	"Renew Let's Encrypt certificate"
 	from bench.config.lets_encrypt import renew_certs
 	renew_certs()
 
-@click.command()
-def shell(bench_path='.'):
-	if not os.environ.get('SHELL'):
-		print("Cannot get shell")
-		sys.exit(1)
-	if not os.path.exists('sites'):
-		print("sites dir doesn't exist")
-		sys.exit(1)
-	env = copy.copy(os.environ)
-	env['PS1'] = '(' + os.path.basename(os.path.dirname(os.path.abspath(__file__))) + ')' + env.get('PS1', '')
-	env['PATH'] = os.path.dirname(os.path.abspath(os.path.join('env','bin')) + ':' + env['PATH'])
-	os.chdir('sites')
-	os.execve(env['SHELL'], [env['SHELL']], env)
 
-
+@click.command('backup', help="Backup single site")
-@click.command('backup')
 @click.argument('site')
 def backup_site(site):
-	"backup site"
 	from bench.utils import get_sites, backup_site
 	if site not in get_sites(bench_path='.'):
-		print('site not found')
+		print('Site `{0}` not found'.format(site))
 		sys.exit(1)
 	backup_site(site, bench_path='.')
 
 
-@click.command('backup-all-sites')
+@click.command('backup-all-sites', help="Backup all sites in current bench")
 def backup_all_sites():
-	"backup all sites"
 	from bench.utils import backup_all_sites
 	backup_all_sites(bench_path='.')
 
 
-@click.command('release')
+@click.command('release', help="Release a Frappe app (internal to the Frappe team)")
 @click.argument('app')
 @click.argument('bump-type', type=click.Choice(['major', 'minor', 'patch', 'stable', 'prerelease']))
 @click.option('--from-branch', default='develop')
@@ -162,39 +142,41 @@ def backup_all_sites():
 @click.option('--repo-name')
 @click.option('--dont-frontport', is_flag=True, default=False, help='Front port fixes to new branches, example merging hotfix(v10) into staging-fixes(v11)')
 def release(app, bump_type, from_branch, to_branch, owner, repo_name, remote, dont_frontport):
-	"Release app (internal to the Frappe team)"
 	from bench.release import release
 	frontport = not dont_frontport
-	release(bench_path='.', app=app, bump_type=bump_type, from_branch=from_branch, to_branch=to_branch,
+	release(bench_path='.', app=app, bump_type=bump_type, from_branch=from_branch, to_branch=to_branch, remote=remote, owner=owner, repo_name=repo_name, frontport=frontport)
-		remote=remote, owner=owner, repo_name=repo_name, frontport=frontport)
 
 
-@click.command('prepare-beta-release')
+@click.command('prepare-beta-release', help="Prepare major beta release from develop branch")
 @click.argument('app')
 @click.option('--owner', default='frappe')
 def prepare_beta_release(app, owner):
-	"""Prepare major beta release from develop branch"""
 	from bench.prepare_beta_release import prepare_beta_release
 	prepare_beta_release(bench_path='.', app=app, owner=owner)
 
 
-@click.command('disable-production')
+@click.command('disable-production', help="Disables production environment for the bench.")
 def disable_production():
-	"""Disables production environment for the bench."""
 	from bench.config.production_setup import disable_production
 	disable_production(bench_path='.')
 
 
-@click.command('src')
+@click.command('src', help="Prints bench source folder path, which can be used as: cd `bench src`")
 def bench_src():
-	"""Prints bench source folder path, which can be used as: cd `bench src` """
 	import bench
 	print(os.path.dirname(bench.__path__[0]))
 
 
-@click.command('find')
+@click.command('find', help="Finds benches recursively from location")
 @click.argument('location', default='')
 def find_benches(location):
-	"""Finds benches recursively from location"""
 	from bench.utils import find_benches
 	find_benches(directory=location)
+
+
+@click.command('migrate-env', help="Migrate Virtual Environment to desired Python Version")
+@click.argument('python', type=str)
+@click.option('--no-backup', 'backup', is_flag=True, default=True)
+def migrate_env(python, backup=True):
+	from bench.utils import migrate_env
+	migrate_env(python=python, backup=backup)

bench/config/common_site_config.py

@@ -1,9 +1,12 @@
-import os, multiprocessing, getpass, json
+# imports - standard imports
+import getpass
+import json
+import multiprocessing
+import os
+
+# imports - third party imports
+from six.moves.urllib.parse import urlparse
 
-try:
-	from urllib.parse import urlparse
-except ImportError:
-	from urlparse import urlparse
 
 default_config = {
 	'restart_supervisor_on_update': False,
@@ -11,7 +14,6 @@ default_config = {
 	'auto_update': False,
 	'serve_default_site': True,
 	'rebase_on_pull': False,
-	'update_bench_on_update': True,
 	'frappe_user': getpass.getuser(),
 	'shallow_clone': True,
 	'background_workers': 1

bench/config/lets_encrypt.py

@@ -29,9 +29,10 @@ def setup_letsencrypt(site, custom_domain, bench_path, interactive):
 			print("No custom domain named {0} set for site".format(custom_domain))
 			return
 
-	click.confirm('Running this will stop the nginx service temporarily causing your sites to go offline\n'
+	if interactive:
-		'Do you want to continue?',
+		click.confirm('Running this will stop the nginx service temporarily causing your sites to go offline\n'
-		abort=True)
+			'Do you want to continue?',
+			abort=True)
 
 	if not get_config(bench_path).get("dns_multitenant"):
 		print("You cannot setup SSL without DNS Multitenancy")
@@ -82,11 +83,10 @@ def run_certbot_and_setup_ssl(site, custom_domain, bench_path, interactive=True)
 
 def setup_crontab():
 	job_command = '/opt/certbot-auto renew -a nginx --post-hook "systemctl reload nginx"'
-	system_crontab = CronTab(tabfile='/etc/crontab', user=True)
+	system_crontab = CronTab(user='root')
 	if job_command not in str(system_crontab):
-		job  = system_crontab.new(command=job_command, comment="Renew lets-encrypt every month")
+		job = system_crontab.new(command=job_command, comment="Renew lets-encrypt every month")
-		job.every().month()
+		job.day.on(1)
-		job.enable()
 		system_crontab.write()
 
 

bench/config/nginx.py

@@ -1,8 +1,24 @@
-import os, json, click, random, string, hashlib
+# imports - standard imports
-from bench.utils import get_sites, get_bench_name, exec_cmd
+import hashlib
+import os
+import random
+import string
+
+# imports - third party imports
+import click
 from six import string_types
 
+# imports - module imports
+from bench.utils import get_bench_name, get_sites
+
+
 def make_nginx_conf(bench_path, yes=False):
+	conf_path = os.path.join(bench_path, "config", "nginx.conf")
+
+	if not yes and os.path.exists(conf_path):
+		if not click.confirm('nginx.conf already exists and this will overwrite it. Do you want to continue?'):
+			return
+
 	from bench import env
 	from bench.config.common_site_config import get_config
 
@@ -37,10 +53,6 @@ def make_nginx_conf(bench_path, yes=False):
 
 	nginx_conf = template.render(**template_vars)
 
-	conf_path = os.path.join(bench_path, "config", "nginx.conf")
-	if not yes and os.path.exists(conf_path):
-		click.confirm('nginx.conf already exists and this will overwrite it. Do you want to continue?',
-			abort=True)
 
 	with open(conf_path, "w") as f:
 		f.write(nginx_conf)

bench/config/procfile.py

@@ -3,7 +3,7 @@ from bench.utils import find_executable
 from bench.app import use_rq
 from bench.config.common_site_config import get_config
 
-def setup_procfile(bench_path, yes=False):
+def setup_procfile(bench_path, yes=False, skip_redis=False):
 	config = get_config(bench_path=bench_path)
 	procfile_path = os.path.join(bench_path, 'Procfile')
 	if not yes and os.path.exists(procfile_path):
@@ -14,7 +14,8 @@ def setup_procfile(bench_path, yes=False):
 		node=find_executable("node") or find_executable("nodejs"),
 		use_rq=use_rq(bench_path),
 		webserver_port=config.get('webserver_port'),
-		CI=os.environ.get('CI'))
+		CI=os.environ.get('CI'),
+		skip_redis=skip_redis)
 
 	with open(procfile_path, 'w') as f:
 		f.write(procfile)

bench/config/redis.py

@@ -1,11 +1,16 @@
-from .common_site_config import get_config
+# imports - standard imports
-import re, os, subprocess, semantic_version
+import os
-import bench
+import re
+import subprocess
+
+# imports - third party imports
+import semantic_version
+from six.moves.urllib.parse import urlparse
+
+# imports - module imports
+import bench
+from bench.config.common_site_config import get_config
 
-try:
-	from urllib.parse import urlparse
-except ImportError:
-	from urlparse import urlparse
 
 def generate_config(bench_path):
 	config = get_config(bench_path)

bench/config/templates/Procfile

@@ -1,6 +1,8 @@
+{% if not skip_redis %}
 redis_cache: redis-server config/redis_cache.conf
 redis_socketio: redis-server config/redis_socketio.conf
 redis_queue: redis-server config/redis_queue.conf
+{% endif %}
 web: bench serve {% if webserver_port -%} --port {{ webserver_port }} {%- endif %}
 
 socketio: {{ node }} apps/frappe/socketio.js

bench/prepare_staging.py

@@ -19,9 +19,7 @@ def prepare_staging(bench_path, app, remote='upstream'):
 		print('No commits to release')
 		return
 
-	print()
 	print(message)
-	print()
 
 	click.confirm('Do you want to continue?', abort=True)
 

bench/release.py

@@ -83,9 +83,7 @@ def bump(bench_path, app, bump_type, from_branch, to_branch, remote, owner, repo
 		print('No commits to release')
 		return
 
-	print()
 	print(message)
-	print()
 
 	click.confirm('Do you want to continue?', abort=True)
 

bench/tests/test_base.py

@@ -0,0 +1,96 @@
+# imports - standard imports
+import json
+import os
+import shutil
+import subprocess
+import sys
+import unittest
+import getpass
+
+# imports - module imports
+import bench
+import bench.utils
+
+
+class TestBenchBase(unittest.TestCase):
+	def setUp(self):
+		self.benches_path = "."
+		self.benches = []
+
+	def tearDown(self):
+		for bench_name in self.benches:
+			bench_path = os.path.join(self.benches_path, bench_name)
+			mariadb_password = "travis" if os.environ.get("CI") else getpass.getpass(prompt="Enter MariaDB root Password: ")
+			if os.path.exists(bench_path):
+				sites = bench.utils.get_sites(bench_path=bench_path)
+				for site in sites:
+					subprocess.call(["bench", "drop-site", site, "--force", "--no-backup", "--root-password", mariadb_password], cwd=bench_path)
+				shutil.rmtree(bench_path, ignore_errors=True)
+
+	def assert_folders(self, bench_name):
+		for folder in bench.utils.folders_in_bench:
+			self.assert_exists(bench_name, folder)
+		self.assert_exists(bench_name, "apps", "frappe")
+
+	def assert_virtual_env(self, bench_name):
+		bench_path = os.path.abspath(bench_name)
+		python_path = os.path.abspath(os.path.join(bench_path, "env", "bin", "python"))
+		self.assertTrue(python_path.startswith(bench_path))
+		for subdir in ("bin", "include", "lib", "share"):
+			self.assert_exists(bench_name, "env", subdir)
+
+	def assert_config(self, bench_name):
+		for config, search_key in (
+			("redis_queue.conf", "redis_queue.rdb"),
+			("redis_socketio.conf", "redis_socketio.rdb"),
+			("redis_cache.conf", "redis_cache.rdb")):
+
+			self.assert_exists(bench_name, "config", config)
+
+			with open(os.path.join(bench_name, "config", config), "r") as f:
+				self.assertTrue(search_key in f.read())
+
+	def assert_common_site_config(self, bench_name, expected_config):
+		common_site_config_path = os.path.join(self.benches_path, bench_name, 'sites', 'common_site_config.json')
+		self.assertTrue(os.path.exists(common_site_config_path))
+
+		with open(common_site_config_path, "r") as f:
+			config = json.load(f)
+
+		for key, value in list(expected_config.items()):
+			self.assertEqual(config.get(key), value)
+
+	def assert_exists(self, *args):
+		self.assertTrue(os.path.exists(os.path.join(*args)))
+
+	def new_site(self, site_name, bench_name):
+		new_site_cmd = ["bench", "new-site", site_name, "--admin-password", "admin"]
+
+		if os.environ.get('CI'):
+			new_site_cmd.extend(["--mariadb-root-password", "travis"])
+
+		subprocess.call(new_site_cmd, cwd=os.path.join(self.benches_path, bench_name))
+
+	def init_bench(self, bench_name, **kwargs):
+		self.benches.append(bench_name)
+		frappe_tmp_path = "/tmp/frappe"
+
+		if not os.path.exists(frappe_tmp_path):
+			bench.utils.exec_cmd("git clone https://github.com/frappe/frappe --depth 1 --origin upstream {location}".format(location=frappe_tmp_path))
+
+		kwargs.update(dict(
+			python=sys.executable,
+			no_procfile=True,
+			no_backups=True,
+			skip_assets=True,
+			frappe_path=frappe_tmp_path
+		))
+
+		if not os.path.exists(os.path.join(self.benches_path, bench_name)):
+			bench.utils.init(bench_name, **kwargs)
+			bench.utils.exec_cmd("git remote set-url upstream https://github.com/frappe/frappe", cwd=os.path.join(self.benches_path, bench_name, "apps", "frappe"))
+
+	def file_exists(self, path):
+		if os.environ.get("CI"):
+			return not subprocess.call(["sudo", "test", "-f", path])
+		return os.path.isfile(path)

bench/tests/test_init.py

@@ -1,26 +1,20 @@
-
+# imports - standard imports
+import json
+import os
+import subprocess
 import unittest
-import json, os, shutil, subprocess
+
+# imports - third paty imports
+import git
+
+# imports - module imports
 import bench
 import bench.utils
-import bench.app
-import bench.config.common_site_config
-import bench.cli
 from bench.release import get_bumped_version
+from bench.tests.test_base import TestBenchBase
 
-bench.cli.from_command_line = True
-
-class TestBenchInit(unittest.TestCase):
-	def setUp(self):
-		self.benches_path = "."
-		self.benches = []
-
-	def tearDown(self):
-		for bench_name in self.benches:
-			bench_path = os.path.join(self.benches_path, bench_name)
-			if os.path.exists(bench_path):
-				shutil.rmtree(bench_path, ignore_errors=True)
 
+class TestBenchInit(TestBenchBase):
 	def test_semantic_version(self):
 		self.assertEqual( get_bumped_version('11.0.4', 'major'), '12.0.0' )
 		self.assertEqual( get_bumped_version('11.0.4', 'minor'), '11.1.0' )
@@ -35,20 +29,14 @@ class TestBenchInit(unittest.TestCase):
 
 	def test_init(self, bench_name="test-bench", **kwargs):
 		self.init_bench(bench_name, **kwargs)
-
 		self.assert_folders(bench_name)
-
 		self.assert_virtual_env(bench_name)
-
-		self.assert_common_site_config(bench_name, bench.config.common_site_config.default_config)
-
 		self.assert_config(bench_name)
 
-		self.assert_socketio(bench_name)
 
 	def test_multiple_benches(self):
-		# 1st bench
+		for bench_name in ("test-bench-1", "test-bench-2"):
-		self.test_init("test-bench-1")
+			self.init_bench(bench_name)
 
 		self.assert_common_site_config("test-bench-1", {
 			"webserver_port": 8000,
@@ -59,9 +47,6 @@ class TestBenchInit(unittest.TestCase):
 			"redis_cache": "redis://localhost:13000"
 		})
 
-		# 2nd bench
-		self.test_init("test-bench-2")
-
 		self.assert_common_site_config("test-bench-2", {
 			"webserver_port": 8001,
 			"socketio_port": 9001,
@@ -71,196 +56,92 @@ class TestBenchInit(unittest.TestCase):
 			"redis_cache": "redis://localhost:13001"
 		})
 
+
+
 	def test_new_site(self):
-		self.init_bench('test-bench')
+		bench_name = "test-bench"
-		self.new_site("test-site-1.dev")
+		site_name = "test-site.local"
+		bench_path = os.path.join(self.benches_path, bench_name)
+		site_path = os.path.join(bench_path, "sites", site_name)
+		site_config_path = os.path.join(site_path, "site_config.json")
 
-	def new_site(self, site_name):
+		self.init_bench(bench_name)
-		new_site_cmd = ["bench", "new-site", site_name, "--admin-password", "admin"]
+		bench.utils.exec_cmd("bench setup requirements --node", cwd=bench_path)
-
+		self.new_site(site_name, bench_name)
-		# set in CI
-		if os.environ.get('CI'):
-			new_site_cmd.extend(["--mariadb-root-password", "travis"])
-
-		subprocess.check_output(new_site_cmd, cwd=os.path.join(self.benches_path, "test-bench"))
-
-		site_path = os.path.join(self.benches_path, "test-bench", "sites", site_name)
 
 		self.assertTrue(os.path.exists(site_path))
 		self.assertTrue(os.path.exists(os.path.join(site_path, "private", "backups")))
 		self.assertTrue(os.path.exists(os.path.join(site_path, "private", "files")))
 		self.assertTrue(os.path.exists(os.path.join(site_path, "public", "files")))
-
-		site_config_path = os.path.join(site_path, "site_config.json")
 		self.assertTrue(os.path.exists(site_config_path))
+
 		with open(site_config_path, "r") as f:
 			site_config = json.loads(f.read())
 
-		for key in ("db_name", "db_password"):
+			for key in ("db_name", "db_password"):
-			self.assertTrue(key in site_config)
+				self.assertTrue(key in site_config)
-			self.assertTrue(site_config[key])
+				self.assertTrue(site_config[key])
 
 	def test_get_app(self):
-		site_name = "test-site-2.dev"
+		self.init_bench("test-bench")
-		self.init_bench('test-bench')
-
-		self.new_site(site_name)
 		bench_path = os.path.join(self.benches_path, "test-bench")
+		bench.utils.exec_cmd("bench get-app frappe_theme --skip-assets", cwd=bench_path)
+		self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "frappe_theme")))
+		app_installed_in_env = "frappe_theme" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8')
+		self.assertTrue(app_installed_in_env)
 
-		bench.app.get_app("https://github.com/frappe/frappe-client", bench_path=bench_path)
-		self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "frappeclient")))
 
 	def test_install_app(self):
-		site_name = "test-site-3.dev"
+		bench_name = "test-bench"
-		self.init_bench('test-bench')
+		site_name = "install-app.test"
-
-		self.new_site(site_name)
 		bench_path = os.path.join(self.benches_path, "test-bench")
 
-		# get app
+		self.init_bench(bench_name)
-		bench.app.get_app("https://github.com/frappe/erpnext", "develop", bench_path=bench_path)
+		bench.utils.exec_cmd("bench setup requirements --node", cwd=bench_path)
+		bench.utils.exec_cmd("bench build", cwd=bench_path)
+		bench.utils.exec_cmd("bench get-app erpnext", cwd=bench_path)
 
 		self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "erpnext")))
 
-		# install app
+		# check if app is installed
-		bench.app.install_app("erpnext", bench_path=bench_path)
+		app_installed_in_env = "erpnext" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8')
+		self.assertTrue(app_installed_in_env)
 
-		# install it to site
+		# create and install app on site
-		subprocess.check_output(["bench", "--site", site_name, "install-app", "erpnext"], cwd=bench_path)
+		self.new_site(site_name, bench_name)
+		bench.utils.exec_cmd("bench --site {0} install-app erpnext".format(site_name), cwd=bench_path)
 
-		out = subprocess.check_output(["bench", "--site", site_name, "list-apps"], cwd=bench_path)
+		app_installed_on_site = subprocess.check_output(["bench", "--site", site_name, "list-apps"], cwd=bench_path).decode('utf8')
-		self.assertTrue("erpnext" in out)
+		self.assertTrue("erpnext" in app_installed_on_site)
 
 
 	def test_remove_app(self):
-		self.init_bench('test-bench')
+		self.init_bench("test-bench")
-
 		bench_path = os.path.join(self.benches_path, "test-bench")
 
-		# get app
+		bench.utils.exec_cmd("bench setup requirements --node", cwd=bench_path)
-		bench.app.get_app("https://github.com/frappe/erpnext", "develop", bench_path=bench_path)
+		bench.utils.exec_cmd("bench get-app erpnext --branch version-12 --skip-assets --overwrite", cwd=bench_path)
-
+		bench.utils.exec_cmd("bench remove-app erpnext", cwd=bench_path)
-		self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "erpnext")))
-
-		# remove it
-		bench.app.remove_app("erpnext", bench_path=bench_path)
 
+		with open(os.path.join(bench_path, "sites", "apps.txt")) as f:
+			self.assertFalse("erpnext" in f.read())
+		self.assertFalse("erpnext" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8'))
 		self.assertFalse(os.path.exists(os.path.join(bench_path, "apps", "erpnext")))
 
 
 	def test_switch_to_branch(self):
-		self.init_bench('test-bench')
+		self.init_bench("test-bench")
-
 		bench_path = os.path.join(self.benches_path, "test-bench")
 		app_path = os.path.join(bench_path, "apps", "frappe")
 
-		bench.app.switch_branch(branch="master", apps=["frappe"], bench_path=bench_path, check_upgrade=False)
+		bench.utils.exec_cmd("bench switch-to-branch version-12 frappe", cwd=bench_path)
-		out = subprocess.check_output(['git', 'status'], cwd=app_path)
+		app_branch_after_switch = str(git.Repo(path=app_path).active_branch)
-		self.assertTrue("master" in out)
+		self.assertEqual("version-12", app_branch_after_switch)
 
-		# bring it back to develop!
+		bench.utils.exec_cmd("bench switch-to-branch develop frappe", cwd=bench_path)
-		bench.app.switch_branch(branch="develop", apps=["frappe"], bench_path=bench_path, check_upgrade=False)
+		app_branch_after_second_switch = str(git.Repo(path=app_path).active_branch)
-		out = subprocess.check_output(['git', 'status'], cwd=app_path)
+		self.assertEqual("develop", app_branch_after_second_switch)
-		self.assertTrue("develop" in out)
 
-	def init_bench(self, bench_name, **kwargs):
-		self.benches.append(bench_name)
-		bench.utils.init(bench_name, **kwargs)
 
-	def test_drop_site(self):
+if __name__ == '__main__':
-		self.init_bench('test-bench')
+	unittest.main()
-		# Check without archive_path given to drop-site command
-		self.drop_site("test-drop-without-archive-path")
-
-		# Check with archive_path given to drop-site command
-		home = os.path.abspath(os.path.expanduser('~'))
-		archived_sites_path = os.path.join(home, 'archived_sites')
-
-		self.drop_site("test-drop-with-archive-path", archived_sites_path=archived_sites_path)
-
-	def drop_site(self, site_name, archived_sites_path=None):
-		self.new_site(site_name)
-
-		drop_site_cmd = ['bench', 'drop-site', site_name]
-
-		if archived_sites_path:
-			drop_site_cmd.extend(['--archived-sites-path', archived_sites_path])
-
-		if os.environ.get('CI'):
-			drop_site_cmd.extend(['--root-password', 'travis'])
-
-		bench_path = os.path.join(self.benches_path, 'test-bench')
-		try:
-			subprocess.check_output(drop_site_cmd, cwd=bench_path)
-		except subprocess.CalledProcessError as err:
-			print(err.output)
-
-		if not archived_sites_path:
-			archived_sites_path = os.path.join(bench_path, 'archived_sites')
-			self.assertTrue(os.path.exists(archived_sites_path))
-			self.assertTrue(os.path.exists(os.path.join(archived_sites_path, site_name)))
-
-		else:
-			self.assertTrue(os.path.exists(archived_sites_path))
-			self.assertTrue(os.path.exists(os.path.join(archived_sites_path, site_name)))
-
-	def assert_folders(self, bench_name):
-		for folder in bench.utils.folders_in_bench:
-			self.assert_exists(bench_name, folder)
-
-		self.assert_exists(bench_name, "sites", "assets")
-		self.assert_exists(bench_name, "apps", "frappe")
-		self.assert_exists(bench_name, "apps", "frappe", "setup.py")
-
-	def assert_virtual_env(self, bench_name):
-		bench_path = os.path.abspath(bench_name)
-		python = os.path.join(bench_path, "env", "bin", "python")
-		python_path = bench.utils.get_cmd_output('{python} -c "import os; print os.path.dirname(os.__file__)"'.format(python=python))
-
-		# part of bench's virtualenv
-		self.assertTrue(python_path.startswith(bench_path))
-		self.assert_exists(python_path)
-		self.assert_exists(python_path, "site-packages")
-		self.assert_exists(python_path, "site-packages", "IPython")
-		self.assert_exists(python_path, "site-packages", "pip")
-
-		site_packages = os.listdir(os.path.join(python_path, "site-packages"))
-		# removing test case temporarily
-		# as develop and master branch havin differnt version of mysqlclient
-		#self.assertTrue(any(package.startswith("mysqlclient-1.3.12") for package in site_packages))
-
-	def assert_config(self, bench_name):
-		for config, search_key in (
-			("redis_queue.conf", "redis_queue.rdb"),
-			("redis_socketio.conf", "redis_socketio.rdb"),
-			("redis_cache.conf", "redis_cache.rdb")):
-
-			self.assert_exists(bench_name, "config", config)
-
-			with open(os.path.join(bench_name, "config", config), "r") as f:
-				f = f.read().decode("utf-8")
-				self.assertTrue(search_key in f)
-
-	def assert_socketio(self, bench_name):
-		try: # for v10 and under
-			self.assert_exists(bench_name, "node_modules")
-			self.assert_exists(bench_name, "node_modules", "socket.io")
-		except: # for v11 and above
-			self.assert_exists(bench_name, "apps", "frappe", "node_modules")
-			self.assert_exists(bench_name, "apps", "frappe", "node_modules", "socket.io")
-
-	def assert_common_site_config(self, bench_name, expected_config):
-		common_site_config_path = os.path.join(bench_name, 'sites', 'common_site_config.json')
-		self.assertTrue(os.path.exists(common_site_config_path))
-
-		config = self.load_json(common_site_config_path)
-
-		for key, value in list(expected_config.items()):
-			self.assertEqual(config.get(key), value)
-
-	def assert_exists(self, *args):
-		self.assertTrue(os.path.exists(os.path.join(*args)))
-
-	def load_json(self, path):
-		with open(path, "r") as f:
-			return json.loads(f.read().decode("utf-8"))

bench/tests/test_setup_production.py

@@ -1,69 +1,51 @@
-
+# imports - standard imports
-from bench.tests import test_init
-from bench.config.production_setup import setup_production, get_supervisor_confdir, disable_production
-import bench.utils
-import os
 import getpass
+import os
 import re
-import unittest
+import subprocess
 import time
+import unittest
 
-class TestSetupProduction(test_init.TestBenchInit):
+# imports - module imports
-	# setUp, tearDown and other tests are defiend in TestBenchInit
+import bench.utils
+from bench.config.production_setup import get_supervisor_confdir
+from bench.tests.test_base import TestBenchBase
 
+
+class TestSetupProduction(TestBenchBase):
 	def test_setup_production(self):
-		self.test_multiple_benches()
-
 		user = getpass.getuser()
 
 		for bench_name in ("test-bench-1", "test-bench-2"):
 			bench_path = os.path.join(os.path.abspath(self.benches_path), bench_name)
-			setup_production(user, bench_path)
+			self.init_bench(bench_name)
+			bench.utils.exec_cmd("sudo bench setup production {0}".format(user), cwd=bench_path)
 			self.assert_nginx_config(bench_name)
 			self.assert_supervisor_config(bench_name)
-
-		# test after start of both benches
-		for bench_name in ("test-bench-1", "test-bench-2"):
 			self.assert_supervisor_process(bench_name)
 
 		self.assert_nginx_process()
-
+		bench.utils.exec_cmd("sudo bench setup sudoers {0}".format(user))
-		# sudoers
-		bench.utils.setup_sudoers(user)
 		self.assert_sudoers(user)
 
-		for bench_name in ("test-bench-1", "test-bench-2"):
+		for bench_name in self.benches:
 			bench_path = os.path.join(os.path.abspath(self.benches_path), bench_name)
-			disable_production(bench_path)
+			bench.utils.exec_cmd("sudo bench disable-production", cwd=bench_path)
 
-	def test_disable_production(self):
-		bench_name = 'test-disable-prod'
-		self.test_init(bench_name, frappe_branch='master')
-
-		user = getpass.getuser()
-
-		bench_path = os.path.join(os.path.abspath(self.benches_path), bench_name)
-		setup_production(user, bench_path)
-
-		disable_production(bench_path)
-
-		self.assert_nginx_link(bench_name)
-		self.assert_supervisor_link(bench_name)
-		self.assert_supervisor_process(bench_name=bench_name, disable_production=True)
 
 	def assert_nginx_config(self, bench_name):
 		conf_src = os.path.join(os.path.abspath(self.benches_path), bench_name, 'config', 'nginx.conf')
 		conf_dest = "/etc/nginx/conf.d/{bench_name}.conf".format(bench_name=bench_name)
 
-		self.assertTrue(os.path.exists(conf_src))
+		self.assertTrue(self.file_exists(conf_src))
-		self.assertTrue(os.path.exists(conf_dest))
+		self.assertTrue(self.file_exists(conf_dest))
 
 		# symlink matches
 		self.assertEqual(os.path.realpath(conf_dest), conf_src)
 
 		# file content
 		with open(conf_src, "r") as f:
-			f = f.read().decode("utf-8")
+			f = f.read()
 
 			for key in (
 					"upstream {bench_name}-frappe",
@@ -71,48 +53,56 @@ class TestSetupProduction(test_init.TestBenchInit):
 				):
 				self.assertTrue(key.format(bench_name=bench_name) in f)
 
+
 	def assert_nginx_process(self):
 		out = bench.utils.get_cmd_output("sudo nginx -t 2>&1")
 		self.assertTrue("nginx: configuration file /etc/nginx/nginx.conf test is successful" in out)
 
+
 	def assert_sudoers(self, user):
 		sudoers_file = '/etc/sudoers.d/frappe'
-		self.assertTrue(os.path.exists(sudoers_file))
+		self.assertTrue(self.file_exists(sudoers_file))
 
-		with open(sudoers_file, 'r') as f:
+		if os.environ.get("CI"):
-			sudoers = f.read().decode('utf-8')
+			sudoers = subprocess.check_output(["sudo", "cat", sudoers_file]).decode("utf-8")
+		else:
+			with open(sudoers_file, 'r') as f:
+				sudoers = f.read()
 
 		self.assertTrue('{user} ALL = (root) NOPASSWD: /usr/sbin/service nginx *'.format(user=user) in sudoers)
 		self.assertTrue('{user} ALL = (root) NOPASSWD: /usr/bin/supervisorctl'.format(user=user) in sudoers)
 		self.assertTrue('{user} ALL = (root) NOPASSWD: /usr/sbin/nginx'.format(user=user) in sudoers)
 
+
 	def assert_supervisor_config(self, bench_name, use_rq=True):
 		conf_src = os.path.join(os.path.abspath(self.benches_path), bench_name, 'config', 'supervisor.conf')
 
 		supervisor_conf_dir = get_supervisor_confdir()
 		conf_dest = "{supervisor_conf_dir}/{bench_name}.conf".format(supervisor_conf_dir=supervisor_conf_dir, bench_name=bench_name)
 
-		self.assertTrue(os.path.exists(conf_src))
+		self.assertTrue(self.file_exists(conf_src))
-		self.assertTrue(os.path.exists(conf_dest))
+		self.assertTrue(self.file_exists(conf_dest))
 
 		# symlink matches
 		self.assertEqual(os.path.realpath(conf_dest), conf_src)
 
 		# file content
 		with open(conf_src, "r") as f:
-			f = f.read().decode("utf-8")
+			f = f.read()
 
 			tests = [
 				"program:{bench_name}-frappe-web",
 				"program:{bench_name}-redis-cache",
 				"program:{bench_name}-redis-queue",
 				"program:{bench_name}-redis-socketio",
-				"program:{bench_name}-node-socketio",
 				"group:{bench_name}-web",
 				"group:{bench_name}-workers",
 				"group:{bench_name}-redis"
 			]
 
+			if not os.environ.get("CI"):
+				tests.append("program:{bench_name}-node-socketio")
+
 			if use_rq:
 				tests.extend([
 					"program:{bench_name}-frappe-schedule",
@@ -130,8 +120,11 @@ class TestSetupProduction(test_init.TestBenchInit):
 				])
 
 			for key in tests:
+				if key.format(bench_name=bench_name) not in f:
+					print(key.format(bench_name=bench_name))
 				self.assertTrue(key.format(bench_name=bench_name) in f)
 
+
 	def assert_supervisor_process(self, bench_name, use_rq=True, disable_production=False):
 		out = bench.utils.get_cmd_output("sudo supervisorctl status")
 
@@ -172,15 +165,6 @@ class TestSetupProduction(test_init.TestBenchInit):
 			else:
 				self.assertTrue(re.search(key.format(bench_name=bench_name), out))
 
-	def assert_nginx_link(self, bench_name):
-		nginx_conf_name = '{bench_name}.conf'.format(bench_name=bench_name)
-		nginx_conf_path = os.path.join('/etc/nginx/conf.d', nginx_conf_name)
 
-		self.assertFalse(os.path.islink(nginx_conf_path))
+if __name__ == '__main__':
-
+	unittest.main()
-	def assert_supervisor_link(self, bench_name):
-		supervisor_conf_dir = get_supervisor_confdir()
-		supervisor_conf_name = '{bench_name}.conf'.format(bench_name=bench_name)
-		supervisor_conf_path = os.path.join(supervisor_conf_dir, supervisor_conf_name)
-
-		self.assertFalse(os.path.islink(supervisor_conf_path))

bench/utils.py

@@ -1,12 +1,31 @@
-import errno, glob, grp, itertools, json, logging, multiprocessing, os, platform, pwd, re, select, shutil, site, subprocess, sys
+# imports - standard imports
+import errno
+import glob
+import grp
+import itertools
+import json
+import logging
+import multiprocessing
+import os
+import platform
+import pwd
+import re
+import select
+import shutil
+import site
+import subprocess
+import sys
+from datetime import datetime
 from distutils.spawn import find_executable
 
+# imports - third party imports
+import click
 import requests
-import semantic_version
 from six import iteritems
+from six.moves.urllib.parse import urlparse
 
+# imports - module imports
 import bench
-from bench import env
 
 
 class PatchError(Exception):
@@ -26,6 +45,7 @@ class color:
 	green = '\033[92m'
 	yellow = '\033[93m'
 	red = '\033[91m'
+	silver = '\033[90m'
 
 
 def is_bench_directory(directory=os.path.curdir):
@@ -58,6 +78,7 @@ def safe_decode(string, encoding = 'utf-8'):
 		pass
 	return string
 
+
 def get_frappe(bench_path='.'):
 	frappe = get_env_cmd('frappe', bench_path=bench_path)
 	if not os.path.exists(frappe):
@@ -65,12 +86,16 @@ def get_frappe(bench_path='.'):
 		print('bench get-app https://github.com/frappe/frappe.git')
 	return frappe
 
+
 def get_env_cmd(cmd, bench_path='.'):
 	return os.path.abspath(os.path.join(bench_path, 'env', 'bin', cmd))
 
-def init(path, apps_path=None, no_procfile=False, no_backups=False, no_auto_update=False,
+
-		frappe_path=None, frappe_branch=None, wheel_cache_dir=None, verbose=False, clone_from=None,
+def init(path, apps_path=None, no_procfile=False, no_backups=False,
-		skip_redis_config_generation=False, clone_without_update=False, ignore_exist = False, skip_assets=False, python='python3'):
+		frappe_path=None, frappe_branch=None, verbose=False, clone_from=None,
+		skip_redis_config_generation=False, clone_without_update=False, ignore_exist=False, skip_assets=False,
+		python='python3'):
+	"""Initialize a new bench directory"""
 	from bench.app import get_app, install_apps_from_path
 	from bench.config import redis
 	from bench.config.common_site_config import make_config
@@ -122,17 +147,85 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, no_auto_upda
 		redis.generate_config(path)
 
 	if not no_procfile:
-		setup_procfile(path)
+		setup_procfile(path, skip_redis=skip_redis_config_generation)
 	if not no_backups:
 		setup_backups(bench_path=path)
-	if not no_auto_update:
+
-		setup_auto_update(bench_path=path)
 	copy_patches_txt(path)
 
+
+def update(pull=False, patch=False, build=False, requirements=False, backup=True, force=False, reset=False,
+	restart_supervisor=False, restart_systemd=False):
+	"""command: bench update"""
+	from bench import patches
+	from bench.app import is_version_upgrade, pull_all_apps, validate_branch
+	from bench.config.common_site_config import get_config, update_config
+
+	bench_path = os.path.abspath(".")
+	patches.run(bench_path=bench_path)
+	conf = get_config(bench_path)
+
+	if conf.get('release_bench'):
+		print('Release bench detected, cannot update!')
+		sys.exit(1)
+
+	if not (pull or patch or build or requirements):
+		pull, patch, build, requirements = True, True, True, True
+
+	validate_branch()
+	version_upgrade = is_version_upgrade()
+
+	if version_upgrade[0]:
+		if force:
+			print("Force flag has been used for a major version change in Frappe and it's apps. \nThis will take significant time to migrate and might break custom apps.")
+		else:
+			print("This update will cause a major version change in Frappe/ERPNext from {0} to {1}. \nThis would take significant time to migrate and might break custom apps.".format(*version_upgrade[1:]))
+			click.confirm('Do you want to continue?', abort=True)
+
+	if version_upgrade[0] or (not version_upgrade[0] and force):
+		validate_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path)
+
+	conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 })
+	update_config(conf, bench_path=bench_path)
+
+	if backup:
+		print('Backing up sites...')
+		backup_all_sites(bench_path=bench_path)
+
+	if pull:
+		pull_all_apps(bench_path=bench_path, reset=reset)
+
+	if requirements:
+		update_requirements(bench_path=bench_path)
+		update_node_packages(bench_path=bench_path)
+
+	if patch:
+		print('Patching sites...')
+		patch_sites(bench_path=bench_path)
+
+	if build:
+		build_assets(bench_path=bench_path)
+
+	if version_upgrade[0] or (not version_upgrade[0] and force):
+		post_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path)
+
+	if restart_supervisor or conf.get('restart_supervisor_on_update'):
+		restart_supervisor_processes(bench_path=bench_path)
+
+	if restart_systemd or conf.get('restart_systemd_on_update'):
+		restart_systemd_processes(bench_path=bench_path)
+
+	conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 })
+	update_config(conf, bench_path=bench_path)
+
+	print("_" * 80 + "\nBench: Deployment tool for Frappe and Frappe Applications (https://frappe.io/bench).\nOpen source depends on your contributions, so please contribute bug reports, patches, fixes or cash and be a part of the community")
+
+
 def copy_patches_txt(bench_path):
 	shutil.copy(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'patches', 'patches.txt'),
 		os.path.join(bench_path, 'patches.txt'))
 
+
 def clone_apps_from(bench_path, clone_from, update_app=True):
 	from .app import install_app
 	print('Copying apps from {0}...'.format(clone_from))
@@ -169,30 +262,15 @@ def clone_apps_from(bench_path, clone_from, update_app=True):
 	for app in apps:
 		setup_app(app)
 
+
 def exec_cmd(cmd, cwd='.'):
-	from .cli import from_command_line
+	import shlex
+	print("{0}$ {1}{2}".format(color.silver, cmd, color.nc))
+	cmd = shlex.split(cmd)
+	subprocess.call(cmd, cwd=cwd, universal_newlines=True)
 
-	is_async = False if from_command_line else True
-	if is_async:
-		stderr = stdout = subprocess.PIPE
-	else:
-		stderr = stdout = None
-
-	logger.info(cmd)
-
-	p = subprocess.Popen(cmd, cwd=cwd, shell=True, stdout=stdout, stderr=stderr,
-		universal_newlines=True)
-
-	if is_async:
-		return_code = print_output(p)
-	else:
-		return_code = p.wait()
-
-	if return_code > 0:
-		raise CommandFailedError(cmd)
 
 def which(executable, raise_err = False):
-	from distutils.spawn import find_executable
 	exec_ = find_executable(executable)
 
 	if not exec_ and raise_err:
@@ -202,18 +280,33 @@ def which(executable, raise_err = False):
 
 	return exec_
 
-def setup_env(bench_path='.', python = 'python3'):
-	python = which(python, raise_err = True)
-	pip    = os.path.join('env', 'bin', 'pip')
 
-	exec_cmd('virtualenv -q {} -p {}'.format('env', python), cwd=bench_path)
+def get_venv_path():
-	exec_cmd('{} -q install -U pip==19.3.1 wheel six'.format(pip), cwd=bench_path)
+	venv = which('virtualenv')
-	exec_cmd('{} -q install -e git+https://github.com/frappe/python-pdfkit.git#egg=pdfkit'.format(pip), cwd=bench_path)
+
+	if not venv:
+		current_python = sys.executable
+		with open(os.devnull, "wb") as devnull:
+			is_venv_installed = not subprocess.call([current_python, "-m", "venv", "--help"], stdout=devnull)
+		if is_venv_installed:
+			venv = "{} -m venv".format(current_python)
+
+	return venv or log("virtualenv cannot be found", level=2)
+
+def setup_env(bench_path='.', python='python3'):
+	frappe = os.path.join(bench_path, "apps", "frappe")
+	pip = os.path.join(".", "env", "bin", "pip")
+	virtualenv = get_venv_path()
+
+	exec_cmd('{} -q env -p {}'.format(virtualenv, python), cwd=bench_path)
+	exec_cmd('{} install -q -U -e {}'.format(pip, frappe), cwd=bench_path)
+
 
 def setup_socketio(bench_path='.'):
 	exec_cmd("npm install socket.io redis express superagent cookie babel-core less chokidar \
 		babel-cli babel-preset-es2015 babel-preset-es2016 babel-preset-es2017 babel-preset-babili", cwd=bench_path)
 
+
 def patch_sites(bench_path='.'):
 	bench.set_frappe_version(bench_path=bench_path)
 
@@ -225,6 +318,7 @@ def patch_sites(bench_path='.'):
 	except subprocess.CalledProcessError:
 		raise PatchError
 
+
 def build_assets(bench_path='.', app=None):
 	bench.set_frappe_version(bench_path=bench_path)
 
@@ -236,19 +330,16 @@ def build_assets(bench_path='.', app=None):
 			command += ' --app {}'.format(app)
 		exec_cmd(command, cwd=bench_path)
 
+
 def get_sites(bench_path='.'):
 	sites_path = os.path.join(bench_path, 'sites')
 	sites = (site for site in os.listdir(sites_path) if os.path.exists(os.path.join(sites_path, site, 'site_config.json')))
 	return sites
 
+
 def get_bench_dir(bench_path='.'):
 	return os.path.abspath(bench_path)
 
-def setup_auto_update(bench_path='.'):
-	logger.info('setting up auto update')
-	add_to_crontab('0 10 * * * cd {bench_dir} &&  {bench} update --auto >> {logfile} 2>&1'.format(bench_dir=get_bench_dir(bench_path=bench_path),
-		bench=os.path.join(get_bench_dir(bench_path=bench_path), 'env', 'bin', 'bench'),
-		logfile=os.path.join(get_bench_dir(bench_path=bench_path), 'logs', 'auto_update_log.log')))
 
 def setup_backups(bench_path='.'):
 	logger.info('setting up backups')
@@ -263,43 +354,30 @@ def setup_backups(bench_path='.'):
 	add_to_crontab('0 */6 * * *  {backup_command} >> {logfile} 2>&1'.format(backup_command=backup_command,
 		logfile=os.path.join(get_bench_dir(bench_path=bench_path), 'logs', 'backup.log')))
 
+
 def add_to_crontab(line):
 	current_crontab = read_crontab()
 	line = str.encode(line)
 	if not line in current_crontab:
 		cmd = ["crontab"]
-		if platform.system() == 'FreeBSD' or platform.linux_distribution()[0]=="arch":
+		if platform.system() == 'FreeBSD':
 			cmd = ["crontab", "-"]
 		s = subprocess.Popen(cmd, stdin=subprocess.PIPE)
 		s.stdin.write(current_crontab)
 		s.stdin.write(line + b'\n')
 		s.stdin.close()
 
+
 def read_crontab():
 	s = subprocess.Popen(["crontab", "-l"], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
 	out = s.stdout.read()
 	s.stdout.close()
 	return out
 
-def update_bench(bench_repo=True, requirements=True):
-	logger.info("Updating bench")
-
-	# bench-repo folder
-	cwd = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-
-	if bench_repo:
-		try:
-			exec_cmd("git pull", cwd=cwd)
-		except bench.utils.CommandFailedError:
-			exec_cmd("git -c user.name=bench -c user.email=developers@frappe.io stash", cwd=cwd)
-			logger.info("Stashing changes made at {}\nUse git stash apply to recover changes after the successful update!".format(cwd))
-
-	if requirements:
-		update_bench_requirements()
-
-	logger.info("Bench Updated!")
 
 def setup_sudoers(user):
+	from bench import env
+
 	if not os.path.exists('/etc/sudoers.d'):
 		os.makedirs('/etc/sudoers.d')
 
@@ -331,6 +409,7 @@ def setup_sudoers(user):
 
 	os.chmod(sudoers_file, 0o440)
 
+
 def setup_logging(bench_path='.'):
 	if os.path.exists(os.path.join(bench_path, 'logs')):
 		logger = logging.getLogger('bench')
@@ -341,6 +420,7 @@ def setup_logging(bench_path='.'):
 		logger.addHandler(hdlr)
 		logger.setLevel(logging.DEBUG)
 
+
 def get_program(programs):
 	program = None
 	for p in programs:
@@ -349,9 +429,11 @@ def get_program(programs):
 			break
 	return program
 
+
 def get_process_manager():
 	return get_program(['foreman', 'forego', 'honcho'])
 
+
 def start(no_dev=False, concurrency=None, procfile=None):
 	program = get_process_manager()
 	if not program:
@@ -369,6 +451,7 @@ def start(no_dev=False, concurrency=None, procfile=None):
 
 	os.execv(program, command)
 
+
 def check_cmd(cmd, cwd='.'):
 	try:
 		subprocess.check_call(cmd, cwd=cwd, shell=True)
@@ -376,6 +459,7 @@ def check_cmd(cmd, cwd='.'):
 	except subprocess.CalledProcessError:
 		return False
 
+
 def get_git_version():
 	'''returns git version from `git --version`
 	extracts version number from string `get version 1.9.1` etc'''
@@ -385,6 +469,7 @@ def get_git_version():
 	version = '.'.join(version.split('.')[0:2])
 	return float(version)
 
+
 def check_git_for_shallow_clone():
 	from .config.common_site_config import get_config
 	config = get_config('.')
@@ -400,6 +485,7 @@ def check_git_for_shallow_clone():
 	if git_version > 1.9:
 		return True
 
+
 def get_cmd_output(cmd, cwd='.'):
 	try:
 		output = subprocess.check_output(cmd, cwd=cwd, shell=True, stderr=subprocess.PIPE).strip()
@@ -410,6 +496,7 @@ def get_cmd_output(cmd, cwd='.'):
 			print(e.output)
 		raise
 
+
 def safe_encode(what, encoding = 'utf-8'):
 	try:
 		what = what.encode(encoding)
@@ -418,6 +505,7 @@ def safe_encode(what, encoding = 'utf-8'):
 
 	return what
 
+
 def restart_supervisor_processes(bench_path='.', web_workers=False):
 	from .config.common_site_config import get_config
 	conf = get_config(bench_path=bench_path)
@@ -447,26 +535,26 @@ def restart_supervisor_processes(bench_path='.', web_workers=False):
 
 		exec_cmd('sudo supervisorctl restart {group}'.format(group=group), cwd=bench_path)
 
+
 def restart_systemd_processes(bench_path='.', web_workers=False):
 	from .config.common_site_config import get_config
 	bench_name = get_bench_name(bench_path)
 	exec_cmd('sudo systemctl stop -- $(systemctl show -p Requires {bench_name}.target | cut -d= -f2)'.format(bench_name=bench_name))
 	exec_cmd('sudo systemctl start -- $(systemctl show -p Requires {bench_name}.target | cut -d= -f2)'.format(bench_name=bench_name))
 
+
 def set_default_site(site, bench_path='.'):
 	if site not in get_sites(bench_path=bench_path):
 		raise Exception("Site not in bench")
 	exec_cmd("{frappe} --use {site}".format(frappe=get_frappe(bench_path=bench_path), site=site),
 			cwd=os.path.join(bench_path, 'sites'))
 
-def update_bench_requirements():
-	bench_req_file = os.path.join(os.path.dirname(bench.__path__[0]), 'requirements.txt')
-	install_requirements(bench_req_file, user=True)
 
 def update_env_pip(bench_path):
 	env_pip = os.path.join(bench_path, 'env', 'bin', 'pip')
 	exec_cmd("{pip} install -q -U pip".format(pip=env_pip))
 
+
 def update_requirements(bench_path='.'):
 	from bench.app import get_apps, install_app
 	print('Updating Python libraries...')
@@ -474,11 +562,9 @@ def update_requirements(bench_path='.'):
 	# update env pip
 	update_env_pip(bench_path)
 
-	# Update bench requirements (at user level)
-	update_bench_requirements()
-
 	for app in get_apps():
-		install_app(app, bench_path=bench_path)
+		install_app(app, bench_path=bench_path, skip_assets=True)
+
 
 def update_node_packages(bench_path='.'):
 	print('Updating node packages...')
@@ -486,7 +572,6 @@ def update_node_packages(bench_path='.'):
 	from distutils.version import LooseVersion
 	v = LooseVersion(get_develop_version('frappe', bench_path = bench_path))
 
-
 	# After rollup was merged, frappe_version = 10.1
 	# if develop_verion is 11 and up, only then install yarn
 	if v < LooseVersion('11.x.x-develop'):
@@ -494,6 +579,7 @@ def update_node_packages(bench_path='.'):
 	else:
 		update_yarn_packages(bench_path)
 
+
 def update_yarn_packages(bench_path='.'):
 	apps_dir = os.path.join(bench_path, 'apps')
 
@@ -555,6 +641,7 @@ def install_requirements(req_file, user=False):
 
 		exec_cmd("{python} -m pip install {user_flag} -q -U -r {req_file}".format(python=python, user_flag=user_flag, req_file=req_file))
 
+
 def backup_site(site, bench_path='.'):
 	bench.set_frappe_version(bench_path=bench_path)
 
@@ -564,30 +651,38 @@ def backup_site(site, bench_path='.'):
 	else:
 		run_frappe_cmd('--site', site, 'backup', bench_path=bench_path)
 
+
 def backup_all_sites(bench_path='.'):
 	for site in get_sites(bench_path=bench_path):
 		backup_site(site, bench_path=bench_path)
 
+
 def is_root():
 	if os.getuid() == 0:
 		return True
 	return False
 
+
 def set_mariadb_host(host, bench_path='.'):
 	update_common_site_config({'db_host': host}, bench_path=bench_path)
 
+
 def set_redis_cache_host(host, bench_path='.'):
 	update_common_site_config({'redis_cache': "redis://{}".format(host)}, bench_path=bench_path)
 
+
 def set_redis_queue_host(host, bench_path='.'):
 	update_common_site_config({'redis_queue': "redis://{}".format(host)}, bench_path=bench_path)
 
+
 def set_redis_socketio_host(host, bench_path='.'):
 	update_common_site_config({'redis_socketio': "redis://{}".format(host)}, bench_path=bench_path)
 
+
 def update_common_site_config(ddict, bench_path='.'):
 	update_json_file(os.path.join(bench_path, 'sites', 'common_site_config.json'), ddict)
 
+
 def update_json_file(filename, ddict):
 	if os.path.exists(filename):
 		with open(filename, 'r') as f:
@@ -600,6 +695,7 @@ def update_json_file(filename, ddict):
 	with open(filename, 'w') as f:
 		json.dump(content, f, indent=1, sort_keys=True)
 
+
 def drop_privileges(uid_name='nobody', gid_name='nogroup'):
 	# from http://stackoverflow.com/a/2699996
 	if os.getuid() != 0:
@@ -620,6 +716,7 @@ def drop_privileges(uid_name='nobody', gid_name='nogroup'):
 	# Ensure a very conservative umask
 	os.umask(0o22)
 
+
 def fix_prod_setup_perms(bench_path='.', frappe_user=None):
 	from .config.common_site_config import get_config
 
@@ -637,6 +734,7 @@ def fix_prod_setup_perms(bench_path='.', frappe_user=None):
 			gid = grp.getgrnam(frappe_user).gr_gid
 			os.chown(path, uid, gid)
 
+
 def fix_file_perms():
 	for dir_path, dirs, files in os.walk('.'):
 		for _dir in dirs:
@@ -649,10 +747,12 @@ def fix_file_perms():
 			if not _file.startswith('activate'):
 				os.chmod(os.path.join(bin_dir, _file), 0o755)
 
+
 def get_current_frappe_version(bench_path='.'):
 	from .app import get_current_frappe_version as fv
 	return fv(bench_path=bench_path)
 
+
 def run_frappe_cmd(*args, **kwargs):
 	from .cli import from_command_line
 
@@ -676,7 +776,7 @@ def run_frappe_cmd(*args, **kwargs):
 
 	if return_code > 0:
 		sys.exit(return_code)
-		#raise CommandFailedError(args)
+
 
 def get_frappe_cmd_output(*args, **kwargs):
 	bench_path = kwargs.get('bench_path', '.')
@@ -684,24 +784,12 @@ def get_frappe_cmd_output(*args, **kwargs):
 	sites_dir = os.path.join(bench_path, 'sites')
 	return subprocess.check_output((f, '-m', 'frappe.utils.bench_helper', 'frappe') + args, cwd=sites_dir)
 
+
 def validate_upgrade(from_ver, to_ver, bench_path='.'):
 	if to_ver >= 6:
 		if not find_executable('npm') and not (find_executable('node') or find_executable('nodejs')):
 			raise Exception("Please install nodejs and npm")
 
-def pre_upgrade(from_ver, to_ver, bench_path='.'):
-	pip = os.path.join(bench_path, 'env', 'bin', 'pip')
-
-	if from_ver <= 4 and to_ver >= 5:
-		from .migrate_to_v5 import remove_shopping_cart
-		apps = ('frappe', 'erpnext')
-		remove_shopping_cart(bench_path=bench_path)
-
-		for app in apps:
-			cwd = os.path.abspath(os.path.join(bench_path, 'apps', app))
-			if os.path.exists(cwd):
-				exec_cmd("git clean -dxf", cwd=cwd)
-				exec_cmd("{pip} install --upgrade -e {app}".format(pip=pip, app=cwd))
 
 def post_upgrade(from_ver, to_ver, bench_path='.'):
 	from .config.common_site_config import get_config
@@ -709,8 +797,7 @@ def post_upgrade(from_ver, to_ver, bench_path='.'):
 	from .config.supervisor import generate_supervisor_config
 	from .config.nginx import make_nginx_conf
 	conf = get_config(bench_path=bench_path)
-	print("-"*80)
+	print("-" * 80 + "Your bench was upgraded to version {0}".format(to_ver))
-	print("Your bench was upgraded to version {0}".format(to_ver))
 
 	if conf.get('restart_supervisor_on_update'):
 		redis.generate_config(bench_path=bench_path)
@@ -814,38 +901,6 @@ def get_output(*cmd):
 	return out
 
 
-def before_update(bench_path, requirements):
-	validate_pillow_dependencies(bench_path, requirements)
-
-
-def validate_pillow_dependencies(bench_path, requirements):
-	if not requirements:
-		return
-
-	try:
-		pip = os.path.join(bench_path, 'env', 'bin', 'pip')
-		exec_cmd("{pip} install Pillow".format(pip=pip))
-
-	except CommandFailedError:
-		distro = platform.linux_distribution()
-		distro_name = distro[0].lower()
-		if "centos" in distro_name or "fedora" in distro_name:
-			print("Please install these dependencies using the command:")
-			print("sudo yum install libtiff-devel libjpeg-devel libzip-devel freetype-devel lcms2-devel libwebp-devel tcl-devel tk-devel")
-
-			raise
-
-		elif "ubuntu" in distro_name or "elementary os" in distro_name or "debian" in distro_name:
-			print("Please install these dependencies using the command:")
-
-			if "ubuntu" in distro_name and distro[1]=="12.04":
-				print("sudo apt-get install -y libtiff4-dev libjpeg8-dev zlib1g-dev libfreetype6-dev liblcms2-dev libwebp-dev tcl8.5-dev tk8.5-dev python-tk")
-			else:
-				print("sudo apt-get install -y libtiff5-dev libjpeg8-dev zlib1g-dev libfreetype6-dev liblcms2-dev libwebp-dev tcl8.6-dev tk8.6-dev python-tk")
-
-			raise
-
-
 def get_bench_name(bench_path):
 	return os.path.basename(os.path.abspath(bench_path))
 
@@ -985,3 +1040,75 @@ def in_virtual_env():
 		return _no_global_under_venv()
 
 	return False
+
+
+def migrate_env(python, backup=False):
+	from bench.config.common_site_config import get_config
+	from bench.app import get_apps
+
+	log = logging.getLogger(__name__)
+	log.setLevel(logging.DEBUG)
+
+	nvenv = 'env'
+	path = os.getcwd()
+	python = which(python)
+	virtualenv = which('virtualenv')
+	pvenv = os.path.join(path, nvenv)
+	pip = os.path.join(pvenv, 'bin', 'pip')
+
+	# Clear Cache before Bench Dies.
+	try:
+		config = get_config(bench_path=os.getcwd())
+		rredis = urlparse(config['redis_cache'])
+
+		redis  = '{redis} -p {port}'.format(redis=which('redis-cli'), port=rredis.port)
+
+		log.debug('Clearing Redis Cache...')
+		exec_cmd('{redis} FLUSHALL'.format(redis = redis))
+		log.debug('Clearing Redis DataBase...')
+		exec_cmd('{redis} FLUSHDB'.format(redis = redis))
+	except:
+		log.warn('Please ensure Redis Connections are running or Daemonized.')
+
+	# Backup venv: restore using `virtualenv --relocatable` if needed
+	if backup:
+		parch = os.path.join(path, 'archived_envs')
+		if not os.path.exists(parch):
+			os.mkdir(parch)
+
+		source = os.path.join(path, 'env')
+		target = parch
+
+		log.debug('Backing up Virtual Environment')
+		stamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+		dest = os.path.join(path, str(stamp))
+
+		os.rename(source, dest)
+		shutil.move(dest, target)
+
+	# Create virtualenv using specified python
+	try:
+		log.debug('Setting up a New Virtual {} Environment'.format(python))
+		exec_cmd('{virtualenv} --python {python} {pvenv}'.format(virtualenv=virtualenv, python=python, pvenv=pvenv))
+
+		apps = ' '.join(["-e {}".format(os.path.join("apps", app)) for app in get_apps()])
+		exec_cmd('{0} install -q -U {1}'.format(pip, apps))
+
+		log.debug('Migration Successful to {}'.format(python))
+	except:
+		log.debug('Migration Error')
+		raise
+
+
+def find_parent_bench(path):
+	"""Checks if parent directories are benches"""
+	if is_bench_directory(directory=path):
+		return path
+
+	home_path = os.path.expanduser("~")
+	root_path = os.path.abspath(os.sep)
+
+	if path not in {home_path, root_path}:
+		# NOTE: the os.path.split assumes that given path is absolute
+		parent_dir = os.path.split(path)[0]
+		return find_parent_bench(parent_dir)

docs/bench_custom_cmd.md

@@ -0,0 +1,48 @@
+## How are Frappe Framework commands available via bench?
+
+bench utilizes `frappe.utils.bench_manager` to get the framework's as well as those of any custom commands written in application installed in the Frappe environment. Currently, with *version 12* there are commands related to the scheduler, sites, translations and other utils in Frappe inherited by bench.
+
+
+## Can I add CLI commands in my custom app and call them via bench?
+
+Along with the framework commands, Frappe's `bench_manager` module also searches for any commands in your custom applications. Thereby, bench communicates with the respective bench's Frappe which in turn checks for available commands in all of the applications.
+
+To make your custom command available to bench, just create a `commands` module under your parent module and write the command with a click wrapper and a variable commands which contains a list of click functions, which are your own commands. The directory strcuture may be visualized as:
+
+```
+frappe-bench
+|──apps
+    |── frappe
+    ├── custom_app
+    │   ├── README.md
+    │   ├── custom_app
+    │   │   ├── commands    <------ commands module
+    │   ├── license.txt
+    │   ├── requirements.txt
+    │   └── setup.py
+```
+
+The commands module maybe a single file such as `commands.py` or a directory with an `__init__.py` file. For a custom application of name 'flags', example may be given as
+
+```python
+# file_path: frappe-bench/apps/flags/flags/commands.py
+import click
+
+@click.command('set-flags')
+@click.argument('state', type=click.Choice(['on', 'off']))
+def set_flags(state):
+    from flags.utils import set_flags
+    set_flags(state=state)
+
+commands = [
+    set_flags
+]
+```
+
+and with context of the current bench, this command maybe executed simply as
+
+```zsh
+➜ bench set-flags
+Flags are set to state: 'on'
+```
+

docs/bench_usage.md

@@ -0,0 +1,201 @@
+# bench CLI Usage
+
+This may not be known to a lot of people but half the bench commands we're used to, exist in the Frappe Framework and not in bench directly. Those commands generally are the `--site` commands. This page is concerned only with the commands in the bench project. Any framework commands won't be a part of this consolidation.
+
+
+# bench CLI Commands
+
+Under Click's structure, `bench` is the main command group, under which there are three main groups of commands in bench currently, namely
+
+ - **install**: The install command group deals with commands used to install system dependencies for setting up Frappe environment
+
+ - **setup**: This command group for consists of commands used to maipulate the requirements and environments required by your Frappe environment
+
+ - **config**: The config command group deals with making changes in the current bench (not the CLI tool) configuration
+
+
+## Using the bench command line
+
+```zsh
+➜ bench
+Usage: bench [OPTIONS] COMMAND [ARGS]...
+
+  Bench manager for Frappe
+
+Options:
+  --version
+  --help     Show this message and exit.
+
+Commands:
+  backup                   Backup single site
+  backup-all-sites         Backup all sites in current bench
+  config                   Change bench configuration
+  disable-production       Disables production environment for the bench.
+  download-translations    Download latest translations
+  exclude-app              Exclude app from updating
+  find                     Finds benches recursively from location
+  get-app                  Clone an app from the internet or filesystem and...
+```
+
+Similarly, all available flags and options can be checked for commands individually by executing them with the `--help` flag. The `init` command for instance:
+
+```zsh
+➜ bench init --help
+Usage: bench init [OPTIONS] PATH
+
+  Initialize a new bench instance in the specified path
+
+Options:
+  --python TEXT                   Path to Python Executable.
+  --ignore-exist                  Ignore if Bench instance exists.
+  --apps_path TEXT                path to json files with apps to install
+                                  after init
+```
+
+
+
+## bench and sudo
+
+Some bench commands may require sudo, such as some `setup` commands and everything else under the `install` commands group. For these commands, you may not be asked for your root password if sudoers setup has been done. The security implications, well we'll talk about those soon.
+
+
+
+## General Commands
+
+These commands belong directly to the bench group so they can be invoked directly prefixing each with `bench` in your shell. Therefore, the usage for these commands is as
+
+```zsh
+    bench COMMAND [ARGS]...
+```
+
+### The usual commands
+
+ - **init**: Initialize a new bench instance in the specified path. This sets up a complete bench folder with an `apps` folder which contains all the Frappe apps available in the current bench, `sites` folder that stores all site data seperated by individual site folders, `config` folder that contains your redis, NGINX and supervisor configuration files. The `env` folder consists of all python dependencies the current bench and installed Frappe applications have.
+ - **restart**: Restart web, supervisor, systemd processes units. Used in production setup.
+ - **update**: Updates bench tool and if executed in a bench directory, without any flags will backup, pull, setup requirements, build, run patches and restart bench. Using specific flags will only do certain tasks instead of all.
+ - **migrate-env**: Migrate Virtual Environment to desired Python version. This regenerates the `env` folder with the specified Python version.
+ - **retry-upgrade**: Retry a failed upgrade
+ - **disable-production**: Disables production environment for the bench.
+ - **renew-lets-encrypt**: Renew Let's Encrypt certificate for site SSL.
+ - **backup**: Backup single site data. Can be used to backup files as well.
+ - **backup-all-sites**: Backup all sites in current bench.
+
+ - **get-app**: Download an app from the internet or filesystem and set it up in your bench. This clones the git repo of the Frappe project and installs it in the bench environment.
+ - **remove-app**: Completely remove app from bench and re-build assets if not installed on any site.
+ - **exclude-app**: Exclude app from updating during a `bench update`
+ - **include-app**: Include app for updating. All Frappe applications are included by default when installed.
+ - **remote-set-url**: Set app remote url
+ - **remote-reset-url**: Reset app remote url to frappe official
+ - **remote-urls**: Show apps remote url
+ - **switch-to-branch**: Switch all apps to specified branch, or specify apps separated by space
+ - **switch-to-develop**: Switch Frappe and ERPNext to develop branch
+
+
+### A little advanced
+
+ - **set-nginx-port**: Set NGINX port for site
+ - **set-ssl-certificate**: Set SSL certificate path for site
+ - **set-ssl-key**: Set SSL certificate private key path for site
+ - **set-url-root**: Set URL root for site
+ - **set-mariadb-host**: Set MariaDB host for bench
+ - **set-redis-cache-host**: Set Redis cache host for bench
+ - **set-redis-queue-host**: Set Redis queue host for bench
+ - **set-redis-socketio-host**: Set Redis socketio host for bench
+ - **set-default-site**: Set default site for bench
+ - **download-translations**: Download latest translations
+
+
+### Developer's commands
+
+ - **start**: Start Frappe development processes. Uses the Procfile to start the Frappe development environment.
+ - **src**: Prints bench source folder path, which can be used to cd into the bench installation repository by `cd $(bench src)`.
+ - **find**: Finds benches recursively from location or specified path.
+ - **pip**: Use the current bench's pip to manage Python packages. For help about pip usage: `bench pip help [COMMAND]` or `bench pip [COMMAND] -h`.
+ - **new-app**: Create a new Frappe application under apps folder.
+
+
+### Release bench
+ - **release**: Create a release of a Frappe application
+ - **prepare-beta-release**: Prepare major beta release from develop branch
+
+
+
+## Setup commands
+
+The setup commands used for setting up the Frappe environment in context of the current bench need to be executed using `bench setup` as the prefix. So, the general usage of these commands is as
+
+```zsh
+    bench setup COMMAND [ARGS]...
+```
+
+ - **sudoers**: Add commands to sudoers list for allowing bench commands execution without root password
+
+ - **env**: Setup virtualenv for bench. This sets up a `env` folder under the root of the bench directory.
+ - **redis**: Generates configuration for Redis
+ - **fonts**: Add Frappe fonts to system
+ - **config**: Generate or over-write sites/common_site_config.json
+ - **backups**: Add cronjob for bench backups
+ - **socketio**: Setup node dependencies for socketio server
+ - **requirements**: Setup Python and Node dependencies
+
+ - **manager**: Setup `bench-manager.local` site with the [Bench Manager](https://github.com/frappe/bench_manager) app, a GUI for bench installed on it.
+
+ - **procfile**: Generate Procfile for bench start
+
+ - **production**: Setup Frappe production environment for specific user. This installs ansible, NGINX, supervisor, fail2ban and generates the respective configuration files.
+ - **nginx**: Generate configuration files for NGINX
+ - **fail2ban**: Setup fail2ban, an intrusion prevention software framework that protects computer servers from brute-force attacks
+ - **systemd**: Generate configuration for systemd
+ - **firewall**: Setup firewall for system
+ - **ssh-port**: Set SSH Port for system
+ - **reload-nginx**: Checks NGINX config file and reloads service
+ - **supervisor**: Generate configuration for supervisor
+ - **lets-encrypt**: Setup lets-encrypt SSL for site
+ - **wildcard-ssl**: Setup wildcard SSL certificate for multi-tenant bench
+
+ - **add-domain**: Add a custom domain to a particular site
+ - **remove-domain**: Remove custom domain from a site
+ - **sync-domains**: Check if there is a change in domains. If yes, updates the domains list.
+
+ - **role**: Install dependencies via ansible roles
+
+
+
+## Config commands
+
+The config group commands are used for manipulating configurations in the current bench context. The usage for these commands is as
+
+```zsh
+    bench config COMMAND [ARGS]...
+```
+
+ - **set-common-config**: Set value in common config
+ - **remove-common-config**: Remove specific keys from current bench's common config
+
+ - **update_bench_on_update**: Enable/Disable bench updates on running bench update
+ - **restart_supervisor_on_update**: Enable/Disable auto restart of supervisor processes
+ - **restart_systemd_on_update**: Enable/Disable auto restart of systemd units
+ - **dns_multitenant**: Enable/Disable bench multitenancy on running bench update
+ - **serve_default_site**: Configure nginx to serve the default site on port 80
+ - **http_timeout**: Set HTTP timeout
+
+
+
+## Install commands
+
+The install group commands are used for manipulating system level dependencies. The usage for these commands is as
+
+```zsh
+    bench install COMMAND [ARGS]...
+```
+
+ - **prerequisites**: Installs pre-requisite libraries, essential tools like b2zip, htop, screen, vim, x11-fonts, python libs, cups and Redis
+ - **nodejs**: Installs Node.js v8
+ - **nginx**: Installs NGINX. If user is specified, sudoers is setup for that user
+ - **packer**: Installs Oracle virtualbox and packer 1.2.1
+ - **psutil**: Installs psutil via pip
+ - **mariadb**: Install and setup MariaDB of specified version and root password
+ - **wkhtmltopdf**: Installs wkhtmltopdf v0.12.3 for linux
+ - **supervisor**: Installs supervisor. If user is specified, sudoers is setup for that user
+ - **fail2ban**: Install fail2ban, an intrusion prevention software framework that protects computer servers from brute-force attacks
+ - **virtualbox**: Installs supervisor

docs/easy_install.md

@@ -1,7 +1,7 @@
 # Easy Install Script
 
 - This script will install the pre-requisites, install bench and setup an ERPNext site `(site1.local under frappe-bench)`
-- Passwords for Frappe Administrator and MariaDB (root) will be asked and saved under `~/passwoords.txt`
+- Passwords for Frappe Administrator and MariaDB (root) will be asked and saved under `~/passwords.txt`
 - MariaDB (root) password may be `password` on a fresh server
 - You can then login as **Administrator** with the Administrator password
 - The log file is saved under `/tmp/logs/install_bench.log` in case you run into any issues during the install.
@@ -45,7 +45,7 @@ Switch to `[frappe-user]` (using `su [frappe-user]`) and start the setup
 
 *Note: `user` flag to create a user and install using that user (By default, the script will create a user with the username `frappe` if the --user flag is not used)*
 
-For production or development, append teh `--production` or `--develop` flag to the command respectively.
+For production or development, append the `--production` or `--develop` flag to the command respectively.
 
 	sudo python3 install.py --production --user [frappe-user]
 

playbooks/install.py

@@ -1,5 +1,16 @@
 #!/usr/bin/env python3
-import os, sys, subprocess, getpass, json, multiprocessing, shutil, platform, warnings, datetime
+from __future__ import print_function
+import os
+import sys
+import subprocess
+import getpass
+import json
+import multiprocessing
+import shutil
+import platform
+import warnings
+import datetime
+
 
 tmp_bench_repo = os.path.join('/', 'tmp', '.bench')
 tmp_log_folder = os.path.join('/', 'tmp', 'logs')
@@ -395,8 +406,20 @@ def parse_commandline_args():
 
 if __name__ == '__main__':
 	if sys.version[0] == '2':
-		if not raw_input("It is recommended to run this script with Python 3\nDo you still wish to continue? [Y/n]: ").lower() == "y":
+		if not os.environ.get('CI'):
-			sys.exit()
+			if not raw_input("It is recommended to run this script with Python 3\nDo you still wish to continue? [Y/n]: ").lower() == "y":
+				sys.exit()
+
+		try:
+			from distutils.spawn import find_executable
+		except ImportError:
+			try:
+				subprocess.check_call('pip install --upgrade setuptools')
+			except subprocess.CalledProcessError:
+				print("Install distutils or use Python3 to run the script")
+				sys.exit(1)
+
+		shutil.which = find_executable
 
 	if not is_sudo_user():
 		log("Please run this script as a non-root user with sudo privileges", level=3)

playbooks/roles/wkhtmltopdf/tasks/main.yml

@@ -20,24 +20,76 @@
     force: yes
   when: ansible_os_family == 'Debian'
 
-# wkhtmltopdf has been locked down to 0.12.3 intentionally since 0.12.4 has problems.
+- name: download wkthmltox Ubuntu 18
-# I you want to upgrade try it on multiple platforms and then decide
+  get_url:
-- name: download wkthmltox linux
+    url: https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.deb
-  get_url: url=https://github.com/frappe/wkhtmltopdf/raw/master/wkhtmltox-0.12.3_linux-generic-{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.tar.xz dest=/tmp/wkhtmltox.tar.xz
+    dest: /tmp/wkhtmltox.deb
+    checksum: "sha256:{{ 'db48fa1a043309c4bfe8c8e0e38dc06c183f821599dd88d4e3cea47c5a5d4cd3' if ansible_architecture == 'x86_64' else '1f5ac84c1cb25e385b49b94a04807d60bf73da217bc6c9fe2cbd1f0a61d33f63' }}"
+  when: ansible_distribution == 'Ubuntu' and ansible_distribution_major_version == '18'
 
-- name: Creates directory
+- name: download wkthmltox Ubuntu 16
-  file: path=/tmp/wkhtmltox state=directory
+  get_url:
+    url: https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.xenial_{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.deb
+    dest: /tmp/wkhtmltox.deb
+    checksum: "sha256:{{ 'df203cee4dc9b3efb8d0cd6fc25fa819883224f50c75b76bd9c856903711dc14' if ansible_architecture == 'x86_64' else '27b6edafee099b87b2911cc68b780e79cffed3948bb5a074e8ea1cf8820da156' }}"
+  when: ansible_distribution == 'Ubuntu' and ansible_distribution_major_version == '16'
 
-- name: unarchive wkhtmltopdf
+- name: download wkthmltox Ubuntu 14
-  unarchive: src=/tmp/wkhtmltox.tar.xz dest=/tmp
+  get_url:
+    url: https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.trusty_{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.deb
+    dest: /tmp/wkhtmltox.deb
+    checksum: "sha256:{{ '2a3d1fe80da0dbc69da56cf90a3d0ec2786d1b919be29527630d609fea4a6b7c' if ansible_architecture == 'x86_64' else '582e02881e4bc6be9aaa634da1fe8c02d3233fb57f6daab9efa137edb812dd3b' }}"
+  when: ansible_distribution == 'Ubuntu' and ansible_distribution_major_version == '14'
 
-- name: copy to /usr/local/bin
+- name: download wkthmltox CentOS 6
-  copy: src="/tmp/wkhtmltox/bin/wkhtmltopdf" dest="/usr/local/bin/wkhtmltopdf"
+  get_url:
-  become: true
+    url: https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox-0.12.5-1.centos6.{{ "x86_64" if ansible_architecture == "x86_64" else "i686"}}.rpm
-  become_user: root
+    dest: /tmp/wkhtmltox.rpm
+    checksum: "sha256:{{ '17bff4966143d240a126b6cc414c6f79aa2106c0c97c772228e84d685221c25f' if ansible_architecture == 'x86_64' else 'c60e75fef5bfa1e79983919ffb47b40dcfbb49d121a510f11ca4b2a2603c00f1' }}"
+  when: ansible_distribution == 'CentOS' and ansible_distribution_major_version == '6'
 
-- name: make wkhtmltopdf executable
+- name: download wkthmltox CentOS 7
-  file: path=/usr/local/bin/wkhtmltopdf mode="o+x"
+  get_url:
-  become: true
+    url: https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox-0.12.5-1.centos7.{{ "x86_64" if ansible_architecture == "x86_64" else "i686"}}.rpm
-  become_user: root
+    dest: /tmp/wkhtmltox.rpm
+    checksum: "sha256:{{ 'ac4f909b836fa1fc0188d19a1ab844910f91612e9ccefcb5298aa955a058ffe4' if ansible_architecture == 'x86_64' else '1030279ac4b5b15dda04de2587b2a1942bde1c78aa1837dfec4ddcbea426721f' }}"
+  when: ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7'
+
+- name: download wkthmltox CentOS 8
+  get_url:
+    url: https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox-0.12.5-1.centos8.x86_64.rpm
+    dest: /tmp/wkhtmltox.rpm
+  when: ansible_distribution == 'CentOS' and ansible_distribution_major_version == '8'
+
+- name: download wkthmltox Debian 8
+  get_url:
+    url: https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.jessie_{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.deb
+    dest: /tmp/wkhtmltox.deb
+    checksum: "sha256:{{ '2583399a865d7604726da166ee7cec656b87ae0a6016e6bce7571dcd3045f98b' if ansible_architecture == 'x86_64' else '3a6969f3ed207a805092e05794644eb9e152aaa6518e9204c819fa318947a8a8' }}"
+  when: ansible_distribution == 'Debian' and ansible_distribution_major_version == '8'
+
+- name: download wkthmltox Debian 9
+  get_url:
+    url: https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.stretch_{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.deb
+    dest: /tmp/wkhtmltox.deb
+    checksum: '{{ 1140b0ab02aa6e17346af2f14ed0de807376de475ba90e1db3975f112fbd20bb if ansible_architecture == "x86_64" else 5b2d15e738ac479e7a8ca6fd765f406c3684a48091813520f87878278d6dd22a }}'
+  when: ansible_distribution == 'Debian' and ansible_distribution_major_version == '9'
+
+- name: download wkthmltox Debian 10
+  get_url:
+    url: https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.buster_{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.deb
+    dest: /tmp/wkhtmltox.deb
+  when: ansible_distribution == 'Debian' and ansible_distribution_major_version == '10'
+
+- name: Install wkhtmltox rpm
+  yum:
+     name: /tmp/wkhtmltox.rpm
+     state: present
+  when: ansible_os_family == 'RedHat'
+
+- name: Install wkhtmltox deb
+  apt:
+     deb: /tmp/wkhtmltox.deb
+     state: present
+  when: ansible_os_family == 'Debian'
 ...

requirements.txt

@@ -1,10 +1,10 @@
 Click==7.0
-GitPython==2.1.11
+GitPython==2.1.15
 honcho==1.0.1
 Jinja2==2.10.3
-python_crontab==2.4.0
+python-crontab==2.4.0
 requests==2.22.0
-semantic_version==2.8.2
+semantic-version==2.8.2
 setuptools==40.8.0
 six==1.12.0
 virtualenv==16.6.0