Merge pull request #1208 from gavindsouza/get-app-on-steroids

feat: get-app on steroids

.travis.yml

@@ -76,18 +76,20 @@ matrix:
     script: sudo python $TRAVIS_BUILD_DIR/install.py --user travis --run-travis --production --verbose
 
 install:
-  - pip install urllib3 pyOpenSSL ndg-httpsclient pyasn1
+  - pip3 install urllib3 pyOpenSSL ndg-httpsclient pyasn1
 
   - if [ $TEST == "bench" ];then
-       wget -q -O /tmp/wkhtmltox.tar.xz https://github.com/frappe/wkhtmltopdf/raw/master/wkhtmltox-0.12.3_linux-generic-amd64.tar.xz;
-       tar -xf /tmp/wkhtmltox.tar.xz -C /tmp;
-       sudo mv /tmp/wkhtmltox/bin/wkhtmltopdf /usr/local/bin/wkhtmltopdf;
-       sudo chmod o+x /usr/local/bin/wkhtmltopdf;
+        wget -q -O /tmp/wkhtmltox.tar.xz https://github.com/frappe/wkhtmltopdf/raw/master/wkhtmltox-0.12.3_linux-generic-amd64.tar.xz;
+        tar -xf /tmp/wkhtmltox.tar.xz -C /tmp;
+        sudo mv /tmp/wkhtmltox/bin/wkhtmltopdf /usr/local/bin/wkhtmltopdf;
+        sudo chmod o+x /usr/local/bin/wkhtmltopdf;
+
+      nvm install 14;
+      nvm use 14;
 
       mkdir -p ~/.bench;
       cp -r $TRAVIS_BUILD_DIR/* ~/.bench;
-      pip install -q -U -e ~/.bench;
-      sudo pip install -q -U -e ~/.bench;
+      pip3 install -q -U -e ~/.bench;
 
       mysql -u root -e "SET GLOBAL character_set_server = 'utf8mb4'";
       mysql -u root -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'";

README.md

@@ -6,10 +6,21 @@
 Bench is a command-line utility that helps you to install, update, and manage multiple sites for Frappe/ERPNext applications on [*nix systems](https://en.wikipedia.org/wiki/Unix-like) for development and production.
 
 <div align="center">
-
-[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Ffrappe%2Fbench.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2Ffrappe%2Fbench?ref=badge_shield)
-[![CI Status](https://app.travis-ci.com/frappe/bench.svg?branch=develop)](https://app.travis-ci.com/github/frappe/bench)
-
+	<a target="_blank" href="https://www.python.org/downloads/" title="Python version">
+		<img src="https://img.shields.io/badge/python-%3E=_3.6-green.svg">
+	</a>
+	<a target="_blank" href="https://app.travis-ci.com/github/frappe/bench" title="CI Status">
+		<img src="https://app.travis-ci.com/frappe/bench.svg?branch=develop">
+	</a>
+	<a target="_blank">
+		<img src="https://img.shields.io/badge/platform-linux%20%7C%20osx-blue">
+	</a>
+	<a target="_blank" href="https://app.fossa.com/projects/git%2Bgithub.com%2Ffrappe%2Fbench?ref=badge_shield" title="FOSSA Status">
+		<img src="https://app.fossa.com/api/projects/git%2Bgithub.com%2Ffrappe%2Fbench.svg?type=shield">
+	</a>
+	<a target="_blank" href="#LICENSE" title="License: GPLv3">
+		<img src="https://img.shields.io/badge/License-GPLv3-blue.svg">
+	</a>
 </div>
 
 ## Table of Contents

bench/__init__.py

@@ -1,10 +1,14 @@
 VERSION = "5.0.0-dev"
 PROJECT_NAME = "frappe-bench"
 FRAPPE_VERSION = None
+current_path = None
+updated_path = None
+LOG_BUFFER = []
 
 
-def set_frappe_version(bench_path='.'):
-	from .app import get_current_frappe_version
+def set_frappe_version(bench_path="."):
+	from .utils.app import get_current_frappe_version
+
 	global FRAPPE_VERSION
 	if not FRAPPE_VERSION:
-		FRAPPE_VERSION = get_current_frappe_version(bench_path=bench_path)
+		FRAPPE_VERSION = get_current_frappe_version(bench_path=bench_path)

bench/app.py

@@ -1,292 +1,436 @@
 # imports - standard imports
+import functools
 import json
-from json.decoder import JSONDecodeError
 import logging
 import os
 import re
+import shutil
 import subprocess
 import sys
+import typing
+from datetime import date
 
 # imports - third party imports
 import click
-from setuptools.config import read_configuration
 
 # imports - module imports
 import bench
-from bench.utils import color, CommandFailedError, build_assets, check_git_for_shallow_clone, exec_cmd, get_cmd_output, get_frappe, restart_supervisor_processes, restart_systemd_processes, run_frappe_cmd
+from bench.exceptions import NotInBenchDirectoryError
+from bench.utils import (
+	fetch_details_from_tag,
+	get_available_folder_name,
+	is_bench_directory,
+	is_git_url,
+	log,
+	run_frappe_cmd,
+)
+from bench.utils.bench import (
+	build_assets,
+	get_env_cmd,
+	install_python_dev_dependencies,
+	restart_supervisor_processes,
+	restart_systemd_processes,
+)
+from bench.utils.render import step
 
 
 logger = logging.getLogger(bench.PROJECT_NAME)
 
 
-class InvalidBranchException(Exception): pass
-class InvalidRemoteException(Exception): pass
+if typing.TYPE_CHECKING:
+	from bench.bench import Bench
 
-class MajorVersionUpgradeException(Exception):
-	def __init__(self, message, upstream_version, local_version):
-		super(MajorVersionUpgradeException, self).__init__(message)
-		self.upstream_version = upstream_version
-		self.local_version = local_version
 
-def get_apps(bench_path='.'):
-	try:
-		with open(os.path.join(bench_path, 'sites', 'apps.txt')) as f:
-			return f.read().strip().split('\n')
-	except IOError:
-		return []
+class AppMeta:
+	def __init__(self, name: str, branch: str = None, to_clone: bool = True):
+		"""
+		name (str): This could look something like
+			1. https://github.com/frappe/healthcare.git
+			2. git@github.com:frappe/healthcare.git
+			3. frappe/healthcare@develop
+			4. healthcare
+			5. healthcare@develop, healthcare@v13.12.1
+
+		References for Version Identifiers:
+		 * https://www.python.org/dev/peps/pep-0440/#version-specifiers
+		 * https://docs.npmjs.com/about-semantic-versioning
+
+		class Healthcare(AppConfig):
+			dependencies = [{"frappe/erpnext": "~13.17.0"}]
+		"""
+		self.name = name
+		self.remote_server = "github.com"
+		self.to_clone = to_clone
+		self.on_disk = False
+		self.use_ssh = False
+		self.from_apps = False
+		self.branch = branch
+		self.setup_details()
+
+	def setup_details(self):
+		# fetch meta from installed apps
+		if (
+			not self.to_clone
+			and hasattr(self, "bench")
+			and os.path.exists(os.path.join(self.bench.name, "apps", self.name))
+		):
+			self.from_apps = True
+			self._setup_details_from_installed_apps()
+
+		# fetch meta for repo on mounted disk
+		elif os.path.exists(self.name):
+			self.on_disk = True
+			self._setup_details_from_mounted_disk()
+
+		# fetch meta for repo from remote git server - traditional get-app url
+		elif is_git_url(self.name):
+			self._setup_details_from_git_url()
+
+		# fetch meta from new styled name tags & first party apps on github
+		else:
+			self._setup_details_from_name_tag()
+
+	def _setup_details_from_mounted_disk(self):
+		self.org, self.repo, self.tag = os.path.split(self.name)[-2:] + (self.branch,)
+
+	def _setup_details_from_name_tag(self):
+		self.org, self.repo, self.tag = fetch_details_from_tag(self.name)
+
+	def _setup_details_from_installed_apps(self):
+		self.org, self.repo, self.tag = os.path.split(
+			os.path.join(self.bench.name, "apps", self.name)
+		)[-2:] + (self.branch,)
+
+	def _setup_details_from_git_url(self):
+		return self.__setup_details_from_git()
+
+	def __setup_details_from_git(self):
+		if self.use_ssh:
+			self.org, _repo = self.name.split(":")[1].split("/")
+		else:
+			self.org, _repo = self.name.split("/")[-2:]
+
+		self.tag = self.branch
+		self.repo = _repo.split(".")[0]
+
+	@property
+	def url(self):
+		if self.from_apps:
+			return os.path.abspath(os.path.join("apps", self.name))
+
+		if self.on_disk:
+			return os.path.abspath(self.name)
+
+		if self.use_ssh:
+			return self.get_ssh_url()
+
+		return self.get_http_url()
+
+	def get_http_url(self):
+		return f"https://{self.remote_server}/{self.org}/{self.repo}.git"
+
+	def get_ssh_url(self):
+		return f"git@{self.remote_server}:{self.org}/{self.repo}.git"
+
+
+@functools.lru_cache(maxsize=None)
+class App(AppMeta):
+	def __init__(
+		self, name: str, branch: str = None, bench: "Bench" = None, *args, **kwargs
+	):
+		self.bench = bench
+		super().__init__(name, branch, *args, **kwargs)
+
+	@step(title="Fetching App {repo}", success="App {repo} Fetched")
+	def get(self):
+		branch = f"--branch {self.tag}" if self.tag else ""
+		shallow = "--depth 1" if self.bench.shallow_clone else ""
+
+		fetch_txt = f"Getting {self.repo}"
+		click.secho(fetch_txt, fg="yellow")
+		logger.log(fetch_txt)
+
+		self.bench.run(
+			f"git clone {self.url} {branch} {shallow} --origin upstream",
+			cwd=os.path.join(self.bench.name, "apps"),
+		)
+
+	@step(title="Archiving App {repo}", success="App {repo} Archived")
+	def remove(self):
+		active_app_path = os.path.join("apps", self.repo)
+		archived_path = os.path.join("archived", "apps")
+		archived_name = get_available_folder_name(
+			f"{self.repo}-{date.today()}", archived_path
+		)
+		archived_app_path = os.path.join(archived_path, archived_name)
+		log(f"App moved from {active_app_path} to {archived_app_path}")
+		shutil.move(active_app_path, archived_app_path)
+
+	@step(title="Installing App {repo}", success="App {repo} Installed")
+	def install(self, skip_assets=False, verbose=False):
+		import bench.cli
+		from bench.utils.app import get_app_name
+
+		verbose = bench.cli.verbose or verbose
+		app_name = get_app_name(self.bench.name, self.repo)
+
+		# TODO: this should go inside install_app only tho - issue: default/resolved branch
+		setup_app_dependencies(
+			repo_name=self.repo,
+			bench_path=self.bench.name,
+			branch=self.tag,
+			verbose=verbose,
+			skip_assets=skip_assets,
+		)
+
+		install_app(
+			app=app_name, bench_path=self.bench.name, verbose=verbose, skip_assets=skip_assets,
+		)
+
+	@step(title="Uninstalling App {repo}", success="App {repo} Uninstalled")
+	def uninstall(self):
+		self.bench.run(f"{self.bench.python} -m pip uninstall -y {self.repo}")
+
+
+def add_to_appstxt(app, bench_path="."):
+	from bench.bench import Bench
+
+	apps = Bench(bench_path).apps
 
-def add_to_appstxt(app, bench_path='.'):
-	apps = get_apps(bench_path=bench_path)
 	if app not in apps:
 		apps.append(app)
 		return write_appstxt(apps, bench_path=bench_path)
 
-def remove_from_appstxt(app, bench_path='.'):
-	apps = get_apps(bench_path=bench_path)
+
+def remove_from_appstxt(app, bench_path="."):
+	from bench.bench import Bench
+
+	apps = Bench(bench_path).apps
+
 	if app in apps:
 		apps.remove(app)
 		return write_appstxt(apps, bench_path=bench_path)
 
-def write_appstxt(apps, bench_path='.'):
-	with open(os.path.join(bench_path, 'sites', 'apps.txt'), 'w') as f:
-		return f.write('\n'.join(apps))
 
-def is_git_url(url):
-	# modified to allow without the tailing .git from https://github.com/jonschlinkert/is-git-url.git
-	pattern = r"(?:git|ssh|https?|\w*@[-\w.]+):(\/\/)?(.*?)(\.git)?(\/?|\#[-\d\w._]+?)$"
-	return bool(re.match(pattern, url))
+def write_appstxt(apps, bench_path="."):
+	with open(os.path.join(bench_path, "sites", "apps.txt"), "w") as f:
+		return f.write("\n".join(apps))
 
-def get_excluded_apps(bench_path='.'):
+
+def get_excluded_apps(bench_path="."):
 	try:
-		with open(os.path.join(bench_path, 'sites', 'excluded_apps.txt')) as f:
-			return f.read().strip().split('\n')
+		with open(os.path.join(bench_path, "sites", "excluded_apps.txt")) as f:
+			return f.read().strip().split("\n")
 	except IOError:
 		return []
 
-def add_to_excluded_apps_txt(app, bench_path='.'):
-	if app == 'frappe':
-		raise ValueError('Frappe app cannot be excludeed from update')
-	if app not in os.listdir('apps'):
-		raise ValueError(f'The app {app} does not exist')
+
+def add_to_excluded_apps_txt(app, bench_path="."):
+	if app == "frappe":
+		raise ValueError("Frappe app cannot be excludeed from update")
+	if app not in os.listdir("apps"):
+		raise ValueError(f"The app {app} does not exist")
 	apps = get_excluded_apps(bench_path=bench_path)
 	if app not in apps:
 		apps.append(app)
 		return write_excluded_apps_txt(apps, bench_path=bench_path)
 
-def write_excluded_apps_txt(apps, bench_path='.'):
-	with open(os.path.join(bench_path, 'sites', 'excluded_apps.txt'), 'w') as f:
-		return f.write('\n'.join(apps))
 
-def remove_from_excluded_apps_txt(app, bench_path='.'):
+def write_excluded_apps_txt(apps, bench_path="."):
+	with open(os.path.join(bench_path, "sites", "excluded_apps.txt"), "w") as f:
+		return f.write("\n".join(apps))
+
+
+def remove_from_excluded_apps_txt(app, bench_path="."):
 	apps = get_excluded_apps(bench_path=bench_path)
 	if app in apps:
 		apps.remove(app)
 		return write_excluded_apps_txt(apps, bench_path=bench_path)
 
-def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=False, restart_bench=True, overwrite=False):
-	import requests
-	import shutil
 
-	if not os.path.exists(git_url):
-		if not is_git_url(git_url):
-			orgs = ['frappe', 'erpnext']
-			for org in orgs:
-				url = f'https://api.github.com/repos/{org}/{git_url}'
-				res = requests.get(url)
-				if res.ok:
-					data = res.json()
-					if 'name' in data:
-						if git_url == data['name']:
-							git_url = f'https://github.com/{org}/{git_url}'
-							break
-				else:
-					bench.utils.log(f"App {git_url} not found", level=2)
-					sys.exit(1)
+def setup_app_dependencies(
+	repo_name, bench_path=".", branch=None, skip_assets=False, verbose=False
+):
+	# branch kwarg is somewhat of a hack here; since we're assuming the same branches for all apps
+	# for eg: if you're installing erpnext@develop, you'll want frappe@develop and healthcare@develop too
+	import glob
+	import bench.cli
+	from bench.bench import Bench
 
-		# Gets repo name from URL
-		repo_name = git_url.rstrip('/').rsplit('/', 1)[1].rsplit('.', 1)[0]
-		shallow_clone = '--depth 1' if check_git_for_shallow_clone() else ''
-		branch = f'--branch {branch}' if branch else ''
-	else:
-		git_url = os.path.abspath(git_url)
-		_, repo_name = os.path.split(git_url)
-		shallow_clone = ''
-		branch = f'--branch {branch}' if branch else ''
+	verbose = bench.cli.verbose or verbose
+	apps_path = os.path.join(os.path.abspath(bench_path), "apps")
+	files = glob.glob(os.path.join(apps_path, repo_name, "**", "hooks.py"))
 
-	if os.path.isdir(os.path.join(bench_path, 'apps', repo_name)):
-		# application directory already exists
-		# prompt user to overwrite it
-		if overwrite or click.confirm(f'''A directory for the application "{repo_name}" already exists.
-Do you want to continue and overwrite it?'''):
-			shutil.rmtree(os.path.join(bench_path, 'apps', repo_name))
-		elif click.confirm('''Do you want to reinstall the existing application?''', abort=True):
-			app_name = get_app_name(bench_path, repo_name)
-			install_app(app=app_name, bench_path=bench_path, verbose=verbose, skip_assets=skip_assets)
-			sys.exit()
-
-	print(f'\n{color.yellow}Getting {repo_name}{color.nc}')
-	logger.log(f'Getting app {repo_name}')
-	exec_cmd(f"git clone {git_url} {branch} {shallow_clone} --origin upstream",
-		cwd=os.path.join(bench_path, 'apps'))
-
-	app_name = get_app_name(bench_path, repo_name)
-	install_app(app=app_name, bench_path=bench_path, verbose=verbose, skip_assets=skip_assets)
+	if files:
+		with open(files[0]) as f:
+			lines = [x for x in f.read().split("\n") if x.strip().startswith("required_apps")]
+		if lines:
+			required_apps = eval(lines[0].strip("required_apps").strip().lstrip("=").strip())
+			# TODO: when the time comes, add version check here
+			for app in required_apps:
+				if app not in Bench(bench_path).apps:
+					get_app(
+						app,
+						bench_path=bench_path,
+						branch=branch,
+						skip_assets=skip_assets,
+						verbose=verbose,
+					)
 
 
-def get_app_name(bench_path, repo_name):
-	app_name = None
-	apps_path = os.path.join(os.path.abspath(bench_path), 'apps')
-	config_path = os.path.join(apps_path, repo_name, 'setup.cfg')
-	if os.path.exists(config_path):
-		config = read_configuration(config_path)
-		app_name = config.get('metadata', {}).get('name')
+def get_app(
+	git_url,
+	branch=None,
+	bench_path=".",
+	skip_assets=False,
+	verbose=False,
+	overwrite=False,
+	init_bench=False,
+):
+	"""bench get-app clones a Frappe App from remote (GitHub or any other git server),
+	and installs it on the current bench. This also resolves dependencies based on the
+	apps' required_apps defined in the hooks.py file.
 
-	if not app_name:
-		# retrieve app name from setup.py as fallback
-		app_path = os.path.join(apps_path, repo_name, 'setup.py')
-		with open(app_path, 'rb') as f:
-			app_name = re.search(r'name\s*=\s*[\'"](.*)[\'"]', f.read().decode('utf-8')).group(1)
+	If the bench_path is not a bench directory, a new bench is created named using the
+	git_url parameter.
+	"""
+	from bench.bench import Bench
+	import bench as _bench
+	import bench.cli as bench_cli
 
-	if app_name and repo_name != app_name:
-		os.rename(os.path.join(apps_path, repo_name), os.path.join(apps_path, app_name))
-		return app_name
+	bench = Bench(bench_path)
+	app = App(git_url, branch=branch, bench=bench)
+	git_url = app.url
+	repo_name = app.repo
+	branch = app.tag
+	bench_setup = False
 
-	return repo_name
+	if not is_bench_directory(bench_path):
+		if not init_bench:
+			raise NotInBenchDirectoryError(
+				f"{os.path.realpath(bench_path)} is not a valid bench directory. "
+				"Run with --init-bench if you'd like to create a Bench too."
+			)
+
+		from bench.utils.system import init
+
+		bench_path = get_available_folder_name(f"{app.repo}-bench", bench_path)
+		init(path=bench_path, frappe_branch=branch)
+		os.chdir(bench_path)
+		bench_setup = True
+
+	if bench_setup and bench_cli.from_command_line and bench_cli.dynamic_feed:
+		_bench.LOG_BUFFER.append({
+			"message": f"Fetching App {repo_name}",
+			"prefix": click.style('⏼', fg='bright_yellow'),
+			"is_parent": True,
+			"color": None,
+		})
 
 
-def new_app(app, bench_path='.'):
+	cloned_path = os.path.join(bench_path, "apps", repo_name)
+	dir_already_exists = os.path.isdir(cloned_path)
+	to_clone = not dir_already_exists
+
+	# application directory already exists
+	# prompt user to overwrite it
+	if dir_already_exists and (
+		overwrite
+		or click.confirm(
+			f"A directory for the application '{repo_name}' already exists. "
+			"Do you want to continue and overwrite it?"
+		)
+	):
+		shutil.rmtree(cloned_path)
+		to_clone = True
+
+	if to_clone:
+		app.get()
+
+	if (
+		to_clone
+		or overwrite
+		or click.confirm("Do you want to reinstall the existing application?")
+	):
+		app.install(verbose=verbose, skip_assets=skip_assets)
+
+
+def new_app(app, bench_path="."):
 	# For backwards compatibility
 	app = app.lower().replace(" ", "_").replace("-", "_")
-	logger.log(f'creating new app {app}')
-	apps = os.path.abspath(os.path.join(bench_path, 'apps'))
-	run_frappe_cmd('make-app', apps, app, bench_path=bench_path)
+	logger.log(f"creating new app {app}")
+	apps = os.path.abspath(os.path.join(bench_path, "apps"))
+	run_frappe_cmd("make-app", apps, app, bench_path=bench_path)
 	install_app(app, bench_path=bench_path)
 
 
-def install_app(app, bench_path=".", verbose=False, no_cache=False, restart_bench=True, skip_assets=False):
-	from bench.config.common_site_config import get_config
+def install_app(
+	app,
+	bench_path=".",
+	verbose=False,
+	no_cache=False,
+	restart_bench=True,
+	skip_assets=False,
+):
+	import bench.cli as bench_cli
+	from bench.bench import Bench
 
-	print(f'\n{color.yellow}Installing {app}{color.nc}')
-	logger.log(f"installing {app}")
+	install_text = f"Installing {app}"
+	click.secho(install_text, fg="yellow")
+	logger.log(install_text)
 
-	python_path = os.path.join(bench_path, "env", "bin", "python")
-	quiet_flag = "-q" if not verbose else ""
-	app_path = os.path.join(bench_path, "apps", app)
+	bench = Bench(bench_path)
+	conf = bench.conf
+
+	verbose = bench_cli.verbose or verbose
+	quiet_flag = "" if verbose else "--quiet"
 	cache_flag = "--no-cache-dir" if no_cache else ""
 
-	exec_cmd(f"{python_path} -m pip install {quiet_flag} -U -e {app_path} {cache_flag}")
+	app_path = os.path.realpath(os.path.join(bench_path, "apps", app))
 
-	if os.path.exists(os.path.join(app_path, 'package.json')):
-		exec_cmd("yarn install", cwd=app_path)
-
-	add_to_appstxt(app, bench_path=bench_path)
-
-	conf = get_config(bench_path=bench_path)
+	bench.run(f"{bench.python} -m pip install {quiet_flag} --upgrade -e {app_path} {cache_flag}")
 
 	if conf.get("developer_mode"):
-		from bench.utils import install_python_dev_dependencies
-		install_python_dev_dependencies(apps=app)
+		install_python_dev_dependencies(apps=app, bench_path=bench_path, verbose=verbose)
+
+	if os.path.exists(os.path.join(app_path, "package.json")):
+		bench.run("yarn install", cwd=app_path)
+
+	bench.apps.sync()
 
 	if not skip_assets:
 		build_assets(bench_path=bench_path, app=app)
 
 	if restart_bench:
-		if conf.get('restart_supervisor_on_update'):
+		if conf.get("restart_supervisor_on_update"):
 			restart_supervisor_processes(bench_path=bench_path)
-		if conf.get('restart_systemd_on_update'):
+		if conf.get("restart_systemd_on_update"):
 			restart_systemd_processes(bench_path=bench_path)
 
 
-def remove_app(app, bench_path='.'):
-	import shutil
-	from bench.config.common_site_config import get_config
+def pull_apps(apps=None, bench_path=".", reset=False):
+	"""Check all apps if there no local changes, pull"""
+	from bench.bench import Bench
+	from bench.utils.app import get_current_branch, get_remote
 
-	app_path = os.path.join(bench_path, 'apps', app)
-	py = os.path.join(bench_path, 'env', 'bin', 'python')
+	bench = Bench(bench_path)
+	rebase = "--rebase" if bench.conf.get("rebase_on_pull") else ""
+	apps = apps or bench.apps
+	excluded_apps = bench.excluded_apps
 
-	# validate app removal
-	if app not in get_apps(bench_path):
-		print(f"No app named {app}")
-		sys.exit(1)
-
-	validate_app_installed_on_sites(app, bench_path=bench_path)
-
-	# remove app from bench
-	exec_cmd("{0} -m pip uninstall -y {1}".format(py, app), cwd=bench_path)
-	remove_from_appstxt(app, bench_path)
-	shutil.rmtree(app_path)
-
-	# re-build assets and restart processes
-	run_frappe_cmd("build", bench_path=bench_path)
-	if get_config(bench_path).get('restart_supervisor_on_update'):
-		restart_supervisor_processes(bench_path=bench_path)
-	if get_config(bench_path).get('restart_systemd_on_update'):
-		restart_systemd_processes(bench_path=bench_path)
-
-
-def validate_app_installed_on_sites(app, bench_path="."):
-	print("Checking if app installed on active sites...")
-	ret = check_app_installed(app, bench_path=bench_path)
-
-	if ret is None:
-		check_app_installed_legacy(app, bench_path=bench_path)
-	else:
-		return ret
-
-
-def check_app_installed(app, bench_path="."):
-	try:
-		out = subprocess.check_output(
-			["bench", "--site", "all", "list-apps", "--format", "json"],
-			stderr=open(os.devnull, "wb"),
-			cwd=bench_path,
-		).decode('utf-8')
-	except subprocess.CalledProcessError:
-		return None
-
-	try:
-		apps_sites_dict = json.loads(out)
-	except JSONDecodeError:
-		return None
-
-	for site, apps in apps_sites_dict.items():
-		if app in apps:
-			print("Cannot remove, app is installed on site: {0}".format(site))
-			sys.exit(1)
-
-
-def check_app_installed_legacy(app, bench_path="."):
-	site_path = os.path.join(bench_path, 'sites')
-
-	for site in os.listdir(site_path):
-		req_file = os.path.join(site_path, site, 'site_config.json')
-		if os.path.exists(req_file):
-			out = subprocess.check_output(["bench", "--site", site, "list-apps"], cwd=bench_path).decode('utf-8')
-			if re.search(r'\b' + app + r'\b', out):
-				print(f"Cannot remove, app is installed on site: {site}")
-				sys.exit(1)
-
-
-def pull_apps(apps=None, bench_path='.', reset=False):
-	'''Check all apps if there no local changes, pull'''
-	from bench.config.common_site_config import get_config
-
-	rebase = '--rebase' if get_config(bench_path).get('rebase_on_pull') else ''
-
-	apps = apps or get_apps(bench_path=bench_path)
 	# check for local changes
 	if not reset:
 		for app in apps:
-			excluded_apps = get_excluded_apps()
 			if app in excluded_apps:
 				print(f"Skipping reset for app {app}")
 				continue
 			app_dir = get_repo_dir(app, bench_path=bench_path)
-			if os.path.exists(os.path.join(app_dir, '.git')):
-				out = subprocess.check_output('git status', shell=True, cwd=app_dir)
-				out = out.decode('utf-8')
-				if not re.search(r'nothing to commit, working (directory|tree) clean', out):
-					print(f'''
+			if os.path.exists(os.path.join(app_dir, ".git")):
+				out = subprocess.check_output("git status", shell=True, cwd=app_dir)
+				out = out.decode("utf-8")
+				if not re.search(r"nothing to commit, working (directory|tree) clean", out):
+					print(
+						f"""
 
 Cannot proceed with update: You have local changes in app "{app}" that are not committed.
 
@@ -296,246 +440,74 @@ Here are your choices:
 1. Temporarily remove your changes with "git stash" or discard them completely
 	with "bench update --reset" or for individual repositries "git reset --hard"
 2. If your changes are helpful for others, send in a pull request via GitHub and
-	wait for them to be merged in the core.''')
+	wait for them to be merged in the core."""
+					)
 					sys.exit(1)
 
-	excluded_apps = get_excluded_apps()
 	for app in apps:
 		if app in excluded_apps:
 			print(f"Skipping pull for app {app}")
 			continue
 		app_dir = get_repo_dir(app, bench_path=bench_path)
-		if os.path.exists(os.path.join(app_dir, '.git')):
+		if os.path.exists(os.path.join(app_dir, ".git")):
 			remote = get_remote(app)
 			if not remote:
 				# remote is False, i.e. remote doesn't exist, add the app to excluded_apps.txt
 				add_to_excluded_apps_txt(app, bench_path=bench_path)
-				print(f"Skipping pull for app {app}, since remote doesn't exist, and adding it to excluded apps")
+				print(
+					f"Skipping pull for app {app}, since remote doesn't exist, and"
+					" adding it to excluded apps"
+				)
 				continue
 
-			if not get_config(bench_path).get('shallow_clone') or not reset:
+			if not bench.conf.get("shallow_clone") or not reset:
 				is_shallow = os.path.exists(os.path.join(app_dir, ".git", "shallow"))
 				if is_shallow:
 					s = " to safely pull remote changes." if not reset else ""
 					print(f"Unshallowing {app}{s}")
-					exec_cmd(f"git fetch {remote} --unshallow", cwd=app_dir)
+					bench.run(f"git fetch {remote} --unshallow", cwd=app_dir)
 
 			branch = get_current_branch(app, bench_path=bench_path)
-			logger.log(f'pulling {app}')
+			logger.log(f"pulling {app}")
 			if reset:
 				reset_cmd = f"git reset --hard {remote}/{branch}"
-				if get_config(bench_path).get('shallow_clone'):
-					exec_cmd(f"git fetch --depth=1 --no-tags {remote} {branch}",
-						cwd=app_dir)
-					exec_cmd(reset_cmd, cwd=app_dir)
-					exec_cmd("git reflog expire --all", cwd=app_dir)
-					exec_cmd("git gc --prune=all", cwd=app_dir)
+				if bench.conf.get("shallow_clone"):
+					bench.run(f"git fetch --depth=1 --no-tags {remote} {branch}", cwd=app_dir)
+					bench.run(reset_cmd, cwd=app_dir)
+					bench.run("git reflog expire --all", cwd=app_dir)
+					bench.run("git gc --prune=all", cwd=app_dir)
 				else:
-					exec_cmd("git fetch --all", cwd=app_dir)
-					exec_cmd(reset_cmd, cwd=app_dir)
+					bench.run("git fetch --all", cwd=app_dir)
+					bench.run(reset_cmd, cwd=app_dir)
 			else:
-				exec_cmd(f"git pull {rebase} {remote} {branch}", cwd=app_dir)
-			exec_cmd('find . -name "*.pyc" -delete', cwd=app_dir)
+				bench.run(f"git pull {rebase} {remote} {branch}", cwd=app_dir)
+			bench.run('find . -name "*.pyc" -delete', cwd=app_dir)
 
 
-def is_version_upgrade(app='frappe', bench_path='.', branch=None):
-	upstream_version = get_upstream_version(app=app, branch=branch, bench_path=bench_path)
-
-	if not upstream_version:
-		raise InvalidBranchException(f'Specified branch of app {app} is not in upstream remote')
-
-	local_version = get_major_version(get_current_version(app, bench_path=bench_path))
-	upstream_version = get_major_version(upstream_version)
-
-	if upstream_version > local_version:
-		return (True, local_version, upstream_version)
-
-	return (False, local_version, upstream_version)
-
-def get_current_frappe_version(bench_path='.'):
-	try:
-		return get_major_version(get_current_version('frappe', bench_path=bench_path))
-	except IOError:
-		return 0
-
-def get_current_branch(app, bench_path='.'):
-	repo_dir = get_repo_dir(app, bench_path=bench_path)
-	return get_cmd_output("basename $(git symbolic-ref -q HEAD)", cwd=repo_dir)
-
-def get_remote(app, bench_path='.'):
-	repo_dir = get_repo_dir(app, bench_path=bench_path)
-	contents = subprocess.check_output(['git', 'remote', '-v'], cwd=repo_dir, stderr=subprocess.STDOUT)
-	contents = contents.decode('utf-8')
-	if re.findall('upstream[\s]+', contents):
-		return 'upstream'
-	elif not contents:
-		# if contents is an empty string => remote doesn't exist
-		return False
-	else:
-		# get the first remote
-		return contents.splitlines()[0].split()[0]
-
 def use_rq(bench_path):
 	bench_path = os.path.abspath(bench_path)
-	celery_app = os.path.join(bench_path, 'apps', 'frappe', 'frappe', 'celery_app.py')
+	celery_app = os.path.join(bench_path, "apps", "frappe", "frappe", "celery_app.py")
 	return not os.path.exists(celery_app)
 
-def get_current_version(app, bench_path='.'):
-	current_version = None
-	repo_dir = get_repo_dir(app, bench_path=bench_path)
-	config_path = os.path.join(repo_dir, "setup.cfg")
-	init_path = os.path.join(repo_dir, os.path.basename(repo_dir), '__init__.py')
-	setup_path = os.path.join(repo_dir, 'setup.py')
 
-	try:
-		if os.path.exists(config_path):
-			config = read_configuration(config_path)
-			current_version = config.get("metadata", {}).get("version")
-		if not current_version:
-			with open(init_path) as f:
-				current_version = get_version_from_string(f.read())
-
-	except AttributeError:
-		# backward compatibility
-		with open(setup_path) as f:
-			current_version = get_version_from_string(f.read(), field='version')
-
-	return current_version
-
-def get_develop_version(app, bench_path='.'):
-	repo_dir = get_repo_dir(app, bench_path=bench_path)
-	with open(os.path.join(repo_dir, os.path.basename(repo_dir), 'hooks.py')) as f:
-		return get_version_from_string(f.read(), field='develop_version')
-
-def get_upstream_version(app, branch=None, bench_path='.'):
-	repo_dir = get_repo_dir(app, bench_path=bench_path)
-	if not branch:
-		branch = get_current_branch(app, bench_path=bench_path)
-
-	try:
-		subprocess.call(f'git fetch --depth=1 --no-tags upstream {branch}', shell=True, cwd=repo_dir)
-	except CommandFailedError:
-		raise InvalidRemoteException(f'Failed to fetch from remote named upstream for {app}')
-
-	try:
-		contents = subprocess.check_output(f'git show upstream/{branch}:{app}/__init__.py',
-			shell=True, cwd=repo_dir, stderr=subprocess.STDOUT)
-		contents = contents.decode('utf-8')
-	except subprocess.CalledProcessError as e:
-		if b"Invalid object" in e.output:
-			return None
-		else:
-			raise
-	return get_version_from_string(contents)
-
-def get_repo_dir(app, bench_path='.'):
-	return os.path.join(bench_path, 'apps', app)
-
-def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrade=True):
-	import git
-	import importlib
-	from bench.utils import update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets, post_upgrade
-
-	apps_dir = os.path.join(bench_path, 'apps')
-	version_upgrade = (False,)
-	switched_apps = []
-
-	if not apps:
-		apps = [name for name in os.listdir(apps_dir)
-			if os.path.isdir(os.path.join(apps_dir, name))]
-		if branch=="v4.x.x":
-			apps.append('shopping_cart')
-
-	for app in apps:
-		app_dir = os.path.join(apps_dir, app)
-
-		if not os.path.exists(app_dir):
-			bench.utils.log(f"{app} does not exist!", level=2)
-			continue
-
-		repo = git.Repo(app_dir)
-		unshallow_flag = os.path.exists(os.path.join(app_dir, ".git", "shallow"))
-		bench.utils.log(f"Fetching upstream {'unshallow ' if unshallow_flag else ''}for {app}")
-
-		bench.utils.exec_cmd("git remote set-branches upstream  '*'", cwd=app_dir)
-		bench.utils.exec_cmd(f"git fetch --all{' --unshallow' if unshallow_flag else ''} --quiet", cwd=app_dir)
-
-		if check_upgrade:
-			version_upgrade = is_version_upgrade(app=app, bench_path=bench_path, branch=branch)
-			if version_upgrade[0] and not upgrade:
-				bench.utils.log(f"Switching to {branch} will cause upgrade from {version_upgrade[1]} to {version_upgrade[2]}. Pass --upgrade to confirm", level=2)
-				sys.exit(1)
-
-		print("Switching for "+app)
-		bench.utils.exec_cmd(f"git checkout -f {branch}", cwd=app_dir)
-
-		if str(repo.active_branch) == branch:
-			switched_apps.append(app)
-		else:
-			bench.utils.log(f"Switching branches failed for: {app}", level=2)
-
-	if switched_apps:
-		bench.utils.log("Successfully switched branches for: " + ", ".join(switched_apps), level=1)
-		print('Please run `bench update --patch` to be safe from any differences in database schema')
-
-	if version_upgrade[0] and upgrade:
-		update_requirements()
-		update_node_packages()
-		importlib.reload(bench.utils)
-		backup_all_sites()
-		patch_sites()
-		build_assets()
-		post_upgrade(version_upgrade[1], version_upgrade[2])
+def get_repo_dir(app, bench_path="."):
+	return os.path.join(bench_path, "apps", app)
 
 
-def switch_to_branch(branch=None, apps=None, bench_path='.', upgrade=False):
-	switch_branch(branch, apps=apps, bench_path=bench_path, upgrade=upgrade)
-
-def switch_to_develop(apps=None, bench_path='.', upgrade=True):
-	switch_branch('develop', apps=apps, bench_path=bench_path, upgrade=upgrade)
-
-def get_version_from_string(contents, field='__version__'):
-	match = re.search(r"^(\s*%s\s*=\s*['\\\"])(.+?)(['\"])(?sm)" % field, contents)
-	return match.group(2)
-
-def get_major_version(version):
-	import semantic_version
-
-	return semantic_version.Version(version).major
-
-def install_apps_from_path(path, bench_path='.'):
+def install_apps_from_path(path, bench_path="."):
 	apps = get_apps_json(path)
 	for app in apps:
-		get_app(app['url'], branch=app.get('branch'), bench_path=bench_path, skip_assets=True)
+		get_app(
+			app["url"], branch=app.get("branch"), bench_path=bench_path, skip_assets=True,
+		)
+
 
 def get_apps_json(path):
 	import requests
 
-	if path.startswith('http'):
+	if path.startswith("http"):
 		r = requests.get(path)
 		return r.json()
 
 	with open(path) as f:
 		return json.load(f)
-
-def validate_branch():
-	installed_apps = set(get_apps())
-	check_apps = set(['frappe', 'erpnext'])
-	intersection_apps = installed_apps.intersection(check_apps)
-
-	for app in intersection_apps:
-		branch = get_current_branch(app)
-
-		if branch == "master":
-			print("""'master' branch is renamed to 'version-11' since 'version-12' release.
-As of January 2020, the following branches are
-version		Frappe			ERPNext
-11		version-11		version-11
-12		version-12		version-12
-13		version-13		version-13
-14		develop			develop
-
-Please switch to new branches to get future updates.
-To switch to your required branch, run the following commands: bench switch-to-branch [branch-name]""")
-
-			sys.exit(1)

bench/bench.py

@@ -0,0 +1,371 @@
+# imports - standard imports
+import functools
+import os
+import shutil
+import sys
+import logging
+from typing import List, MutableSequence, TYPE_CHECKING
+
+# imports - module imports
+import bench
+from bench.exceptions import ValidationError
+from bench.config.common_site_config import setup_config
+from bench.utils import (
+	paths_in_bench,
+	exec_cmd,
+	is_bench_directory,
+	is_frappe_app,
+	get_cmd_output,
+	get_git_version,
+	log,
+	run_frappe_cmd,
+)
+from bench.utils.bench import (
+	validate_app_installed_on_sites,
+	restart_supervisor_processes,
+	restart_systemd_processes,
+	restart_process_manager,
+	remove_backups_crontab,
+	get_venv_path,
+	get_env_cmd,
+)
+from bench.utils.render import job, step
+
+
+if TYPE_CHECKING:
+	from bench.app import App
+
+logger = logging.getLogger(bench.PROJECT_NAME)
+
+
+class Base:
+	def run(self, cmd, cwd=None):
+		return exec_cmd(cmd, cwd=cwd or self.cwd)
+
+
+class Validator:
+	def validate_app_uninstall(self, app):
+		if app not in self.apps:
+			raise ValidationError(f"No app named {app}")
+		validate_app_installed_on_sites(app, bench_path=self.name)
+
+
+@functools.lru_cache(maxsize=None)
+class Bench(Base, Validator):
+	def __init__(self, path):
+		self.name = path
+		self.cwd = os.path.abspath(path)
+		self.exists = is_bench_directory(self.name)
+
+		self.setup = BenchSetup(self)
+		self.teardown = BenchTearDown(self)
+		self.apps = BenchApps(self)
+
+		self.apps_txt = os.path.join(self.name, "sites", "apps.txt")
+		self.excluded_apps_txt = os.path.join(self.name, "sites", "excluded_apps.txt")
+
+	@property
+	def python(self) -> str:
+		return get_env_cmd("python", bench_path=self.name)
+
+	@property
+	def shallow_clone(self) -> bool:
+		config = self.conf
+
+		if config:
+			if config.get("release_bench") or not config.get("shallow_clone"):
+				return False
+
+		return get_git_version() > 1.9
+
+	@property
+	def excluded_apps(self) -> List:
+		try:
+			with open(self.excluded_apps_txt) as f:
+				return f.read().strip().split("\n")
+		except Exception:
+			return []
+
+	@property
+	def sites(self) -> List:
+		return [
+			path
+			for path in os.listdir(os.path.join(self.name, "sites"))
+			if os.path.exists(os.path.join("sites", path, "site_config.json"))
+		]
+
+	@property
+	def conf(self):
+		from bench.config.common_site_config import get_config
+
+		return get_config(self.name)
+
+	def init(self):
+		self.setup.dirs()
+		self.setup.env()
+		self.setup.backups()
+
+	def drop(self):
+		self.teardown.backups()
+		self.teardown.dirs()
+
+	def install(self, app, branch=None):
+		from bench.app import App
+
+		app = App(app, branch=branch)
+		self.apps.append(app)
+		self.apps.sync()
+
+	def uninstall(self, app):
+		from bench.app import App
+
+		self.validate_app_uninstall(app)
+		self.apps.remove(App(app, bench=self, to_clone=False))
+		self.apps.sync()
+		# self.build() - removed because it seems unnecessary
+		self.reload()
+
+	@step(title="Building Bench Assets", success="Bench Assets Built")
+	def build(self):
+		# build assets & stuff
+		run_frappe_cmd("build", bench_path=self.name)
+
+	@step(title="Reloading Bench Processes", success="Bench Processes Reloaded")
+	def reload(self):
+		conf = self.conf
+		if conf.get("restart_supervisor_on_update"):
+			restart_supervisor_processes(bench_path=self.name)
+		if conf.get("restart_systemd_on_update"):
+			restart_systemd_processes(bench_path=self.name)
+		if conf.get("developer_mode"):
+			restart_process_manager(bench_path=self.name)
+
+class BenchApps(MutableSequence):
+	def __init__(self, bench: Bench):
+		self.bench = bench
+		self.initialize_apps()
+
+	def sync(self):
+		self.initialize_apps()
+		with open(self.bench.apps_txt, "w") as f:
+			return f.write("\n".join(self.apps))
+
+	def initialize_apps(self):
+		is_installed = lambda app: app in installed_packages
+
+		try:
+			installed_packages = get_cmd_output(f"{self.bench.python} -m pip freeze", cwd=self.bench.name)
+		except Exception:
+			self.apps = []
+			return
+
+		try:
+			self.apps = [
+				x
+				for x in os.listdir(os.path.join(self.bench.name, "apps"))
+				if (
+					is_frappe_app(os.path.join(self.bench.name, "apps", x))
+					and is_installed(x)
+				)
+			]
+			self.apps.sort()
+		except FileNotFoundError:
+			self.apps = []
+
+	def __getitem__(self, key):
+		""" retrieves an item by its index, key"""
+		return self.apps[key]
+
+	def __setitem__(self, key, value):
+		""" set the item at index, key, to value """
+		# should probably not be allowed
+		# self.apps[key] = value
+		raise NotImplementedError
+
+	def __delitem__(self, key):
+		""" removes the item at index, key """
+		# TODO: uninstall and delete app from bench
+		del self.apps[key]
+
+	def __len__(self):
+		return len(self.apps)
+
+	def insert(self, key, value):
+		""" add an item, value, at index, key. """
+		# TODO: fetch and install app to bench
+		self.apps.insert(key, value)
+
+	def add(self, app: "App"):
+		app.get()
+		app.install()
+		super().append(app.repo)
+		self.apps.sort()
+
+	def remove(self, app: "App"):
+		app.uninstall()
+		app.remove()
+		super().remove(app.repo)
+
+	def append(self, app: "App"):
+		return self.add(app)
+
+	def __repr__(self):
+		return self.__str__()
+
+	def __str__(self):
+		return str([x for x in self.apps])
+
+
+class BenchSetup(Base):
+	def __init__(self, bench: Bench):
+		self.bench = bench
+		self.cwd = self.bench.cwd
+
+	@step(title="Setting Up Directories", success="Directories Set Up")
+	def dirs(self):
+		os.makedirs(self.bench.name, exist_ok=True)
+
+		for dirname in paths_in_bench:
+			os.makedirs(os.path.join(self.bench.name, dirname), exist_ok=True)
+
+	@step(title="Setting Up Environment", success="Environment Set Up")
+	def env(self, python="python3"):
+		"""Setup env folder
+		- create env if not exists
+		- upgrade env pip
+		- install frappe python dependencies
+		"""
+		import bench.cli
+
+		frappe = os.path.join(self.bench.name, "apps", "frappe")
+		virtualenv = get_venv_path()
+		quiet_flag = "" if bench.cli.verbose else "--quiet"
+
+		if not os.path.exists(self.bench.python):
+			self.run(f"{virtualenv} {quiet_flag} env -p {python}")
+
+		self.pip()
+
+		if os.path.exists(frappe):
+			self.run(f"{self.bench.python} -m pip install {quiet_flag} --upgrade -e {frappe}")
+
+	@step(title="Setting Up Bench Config", success="Bench Config Set Up")
+	def config(self, redis=True, procfile=True):
+		"""Setup config folder
+		- create pids folder
+		- generate sites/common_site_config.json
+		"""
+		setup_config(self.bench.name)
+
+		if redis:
+			from bench.config.redis import generate_config
+
+			generate_config(self.bench.name)
+
+		if procfile:
+			from bench.config.procfile import setup_procfile
+
+			setup_procfile(self.bench.name, skip_redis=not redis)
+
+	@step(title="Updating pip", success="Updated pip")
+	def pip(self, verbose=False):
+		"""Updates env pip; assumes that env is setup
+		"""
+		import bench.cli
+
+		verbose = bench.cli.verbose or verbose
+		quiet_flag = "" if verbose else "--quiet"
+
+		return self.run(f"{self.bench.python} -m pip install {quiet_flag} --upgrade pip")
+
+	def logging(self):
+		from bench.utils import setup_logging
+
+		return setup_logging(bench_path=self.bench.name)
+
+	@step(title="Setting Up Bench Patches", success="Bench Patches Set Up")
+	def patches(self):
+		shutil.copy(
+			os.path.join(os.path.dirname(os.path.abspath(__file__)), "patches", "patches.txt"),
+			os.path.join(self.bench.name, "patches.txt"),
+		)
+
+	@step(title="Setting Up Backups Cronjob", success="Backups Cronjob Set Up")
+	def backups(self):
+		# TODO: to something better for logging data? - maybe a wrapper that auto-logs with more context
+		logger.log("setting up backups")
+
+		from crontab import CronTab
+
+		bench_dir = os.path.abspath(self.bench.name)
+		user = self.bench.conf.get("frappe_user")
+		logfile = os.path.join(bench_dir, "logs", "backup.log")
+		system_crontab = CronTab(user=user)
+		backup_command = f"cd {bench_dir} && {sys.argv[0]} --verbose --site all backup"
+		job_command = f"{backup_command} >> {logfile} 2>&1"
+
+		if job_command not in str(system_crontab):
+			job = system_crontab.new(
+				command=job_command, comment="bench auto backups set for every 6 hours"
+			)
+			job.every(6).hours()
+			system_crontab.write()
+
+		logger.log("backups were set up")
+
+	def __get_installed_apps(self) -> List:
+		"""Returns list of installed apps on bench, not in excluded_apps.txt
+		"""
+		apps = [app for app in self.bench.apps if app not in self.bench.excluded_apps]
+		apps.remove("frappe")
+		apps.insert(0, "frappe")
+		return apps
+
+	@job(title="Setting Up Bench Dependencies", success="Bench Dependencies Set Up")
+	def requirements(self):
+		"""Install and upgrade all installed apps on given Bench
+		"""
+		from bench.app import App
+
+		apps = self.__get_installed_apps()
+
+		self.pip()
+
+		print(f"Installing {len(apps)} applications...")
+
+		for app in apps:
+			App(app, bench=self.bench, to_clone=False).install()
+
+	def python(self):
+		"""Install and upgrade Python dependencies for installed apps on given Bench
+		"""
+		import bench.cli
+
+		apps = self.__get_installed_apps()
+
+		quiet_flag = "" if bench.cli.verbose else "--quiet"
+
+		self.pip()
+
+		for app in apps:
+			app_path = os.path.join(self.bench.name, "apps", app)
+			log(f"\nInstalling python dependencies for {app}", level=3, no_log=True)
+			self.run(f"{self.bench.python} -m pip install {quiet_flag} --upgrade -e {app_path}")
+
+	def node(self):
+		"""Install and upgrade Node dependencies for all apps on given Bench
+		"""
+		from bench.utils.bench import update_node_packages
+
+		return update_node_packages(bench_path=self.bench.name)
+
+
+class BenchTearDown:
+	def __init__(self, bench):
+		self.bench = bench
+
+	def backups(self):
+		remove_backups_crontab(self.bench.name)
+
+	def dirs(self):
+		shutil.rmtree(self.bench.name)

bench/cli.py

@@ -10,7 +10,7 @@ import click
 
 # imports - module imports
 import bench
-from bench.app import get_apps
+from bench.bench import Bench
 from bench.commands import bench_command
 from bench.config.common_site_config import get_config
 from bench.utils import (
@@ -20,39 +20,50 @@ from bench.utils import (
 	find_parent_bench,
 	generate_command_cache,
 	get_cmd_output,
-	get_env_cmd,
-	get_frappe,
 	is_bench_directory,
 	is_dist_editable,
 	is_root,
 	log,
 	setup_logging,
 )
+from bench.utils.bench import get_env_cmd
+
+# these variables are used to show dynamic outputs on the terminal
+dynamic_feed = False
+verbose = False
+is_envvar_warn_set = None
+from_command_line = False # set when commands are executed via the CLI
+bench.LOG_BUFFER = []
 
-from_command_line = False
 change_uid_msg = "You should not run this command as root"
 src = os.path.dirname(__file__)
 
 
 def cli():
-	global from_command_line
+	global from_command_line, bench_config, is_envvar_warn_set
+
 	from_command_line = True
 	command = " ".join(sys.argv)
+	argv = set(sys.argv)
+	is_envvar_warn_set = not (os.environ.get("BENCH_DEVELOPER") or os.environ.get("CI"))
+	is_cli_command = len(sys.argv) > 1 and not argv.intersection({"src", "--version"})
 
 	change_working_directory()
 	logger = setup_logging()
 	logger.info(command)
 
-	if len(sys.argv) > 1 and sys.argv[1] not in ("src",):
+	bench_config = get_config(".")
+
+	if is_cli_command:
 		check_uid()
 		change_uid()
 		change_dir()
 
 	if (
-		is_dist_editable(bench.PROJECT_NAME)
-		and len(sys.argv) > 1
-		and sys.argv[1] != "src"
-		and not get_config(".").get("developer_mode")
+		is_envvar_warn_set
+		and is_cli_command
+		and is_dist_editable(bench.PROJECT_NAME)
+		and not bench_config.get("developer_mode")
 	):
 		log(
 			"bench is installed in editable mode!\n\nThis is not the recommended mode"
@@ -61,24 +72,23 @@ def cli():
 			level=3,
 		)
 
+	in_bench = is_bench_directory()
+
 	if (
-		not is_bench_directory()
-		and not cmd_requires_root()
+		not in_bench
 		and len(sys.argv) > 1
-		and sys.argv[1] not in ("init", "find", "src")
+		and not argv.intersection({"init", "find", "src", "drop", "get", "get-app", "--version"})
+		and not cmd_requires_root()
 	):
 		log("Command not being executed in bench directory", level=3)
 
-	if len(sys.argv) > 2 and sys.argv[1] == "frappe":
-		old_frappe_cli()
-
-	elif len(sys.argv) > 1:
+	if in_bench and len(sys.argv) > 1:
 		if sys.argv[1] == "--help":
 			print(click.Context(bench_command).get_help())
 			print(get_frappe_help())
 			return
 
-		if sys.argv[1] in ["--site", "--verbose", "--force", "--profile"]:
+		if sys.argv[1] in ["--site", "--force", "--profile"]:
 			frappe_cmd()
 
 		if sys.argv[1] in get_cached_frappe_commands():
@@ -87,26 +97,24 @@ def cli():
 		if sys.argv[1] in get_frappe_commands():
 			frappe_cmd()
 
-		if sys.argv[1] in get_apps():
+		if sys.argv[1] in Bench(".").apps:
 			app_cmd()
 
-	if not (len(sys.argv) > 1 and sys.argv[1] == "src"):
+	if not is_cli_command:
 		atexit.register(check_latest_version)
 
 	try:
 		bench_command()
 	except BaseException as e:
-		return_code = getattr(e, "code", 0)
+		return_code = getattr(e, "code", 1)
+
+		if isinstance(e, Exception):
+			click.secho(f"ERROR: {e}", fg="red")
+
 		if return_code:
 			logger.warning(f"{command} executed with exit code {return_code}")
-		if isinstance(e, Exception):
-			raise e
-	finally:
-		try:
-			return_code
-		except NameError:
-			return_code = 0
-		sys.exit(return_code)
+
+		raise e
 
 
 def check_uid():
@@ -152,7 +160,7 @@ def change_dir():
 
 def change_uid():
 	if is_root() and not cmd_requires_root():
-		frappe_user = get_config(".").get("frappe_user")
+		frappe_user = bench_config.get("frappe_user")
 		if frappe_user:
 			drop_privileges(uid_name=frappe_user, gid_name=frappe_user)
 			os.environ["HOME"] = pwd.getpwnam(frappe_user).pw_dir
@@ -161,12 +169,6 @@ def change_uid():
 			sys.exit(1)
 
 
-def old_frappe_cli(bench_path="."):
-	f = get_frappe(bench_path=bench_path)
-	os.chdir(os.path.join(bench_path, "sites"))
-	os.execv(f, [f] + sys.argv[2:])
-
-
 def app_cmd(bench_path="."):
 	f = get_env_cmd("python", bench_path=bench_path)
 	os.chdir(os.path.join(bench_path, "sites"))
@@ -209,6 +211,8 @@ def change_working_directory():
 	"""Allows bench commands to be run from anywhere inside a bench directory"""
 	cur_dir = os.path.abspath(".")
 	bench_path = find_parent_bench(cur_dir)
+	bench.current_path = os.getcwd()
+	bench.updated_path = bench_path
 
 	if bench_path:
 		os.chdir(bench_path)

bench/commands/__init__.py

@@ -1,24 +1,48 @@
+# imports - third party imports
 import click
 
+# imports - module imports
+from bench.utils.cli import (
+	MultiCommandGroup,
+	print_bench_version,
+	use_experimental_feature,
+	setup_verbosity,
+)
 
-def print_bench_version(ctx, param, value):
-	"""Prints current bench version"""
-	if not value or ctx.resilient_parsing:
-		return
 
+@click.group(cls=MultiCommandGroup)
+@click.option(
+	"--version",
+	is_flag=True,
+	is_eager=True,
+	callback=print_bench_version,
+	expose_value=False,
+)
+@click.option(
+	"--use-feature", is_eager=True, callback=use_experimental_feature, expose_value=False,
+)
+@click.option(
+	"-v", "--verbose", is_flag=True, callback=setup_verbosity, expose_value=False,
+)
+def bench_command(bench_path="."):
 	import bench
-	click.echo(bench.VERSION)
-	ctx.exit()
 
-@click.group()
-@click.option('--version', is_flag=True, is_eager=True, callback=print_bench_version, expose_value=False)
-def bench_command(bench_path='.'):
-	import bench
 	bench.set_frappe_version(bench_path=bench_path)
 
 
-from bench.commands.make import init, get_app, new_app, remove_app, exclude_app_for_update, include_app_for_update, pip
+from bench.commands.make import (
+	drop,
+	exclude_app_for_update,
+	get_app,
+	include_app_for_update,
+	init,
+	new_app,
+	pip,
+	remove_app,
+)
+
 bench_command.add_command(init)
+bench_command.add_command(drop)
 bench_command.add_command(get_app)
 bench_command.add_command(new_app)
 bench_command.add_command(remove_app)
@@ -27,17 +51,44 @@ bench_command.add_command(include_app_for_update)
 bench_command.add_command(pip)
 
 
-from bench.commands.update import update, retry_upgrade, switch_to_branch, switch_to_develop
+from bench.commands.update import (
+	retry_upgrade,
+	switch_to_branch,
+	switch_to_develop,
+	update,
+)
+
 bench_command.add_command(update)
 bench_command.add_command(retry_upgrade)
 bench_command.add_command(switch_to_branch)
 bench_command.add_command(switch_to_develop)
 
 
-from bench.commands.utils import (start, restart, set_nginx_port, set_ssl_certificate, set_ssl_certificate_key, set_url_root,
-	set_mariadb_host, download_translations, backup_site, backup_all_sites, release, renew_lets_encrypt,
-	disable_production, bench_src, prepare_beta_release, set_redis_cache_host, set_redis_queue_host, set_redis_socketio_host, find_benches, migrate_env,
-	generate_command_cache, clear_command_cache)
+from bench.commands.utils import (
+	backup_all_sites,
+	backup_site,
+	bench_src,
+	clear_command_cache,
+	disable_production,
+	download_translations,
+	find_benches,
+	generate_command_cache,
+	migrate_env,
+	prepare_beta_release,
+	release,
+	renew_lets_encrypt,
+	restart,
+	set_mariadb_host,
+	set_nginx_port,
+	set_redis_cache_host,
+	set_redis_queue_host,
+	set_redis_socketio_host,
+	set_ssl_certificate,
+	set_ssl_certificate_key,
+	set_url_root,
+	start,
+)
+
 bench_command.add_command(start)
 bench_command.add_command(restart)
 bench_command.add_command(set_nginx_port)
@@ -62,16 +113,20 @@ bench_command.add_command(generate_command_cache)
 bench_command.add_command(clear_command_cache)
 
 from bench.commands.setup import setup
+
 bench_command.add_command(setup)
 
 
 from bench.commands.config import config
+
 bench_command.add_command(config)
 
-from bench.commands.git import remote_set_url, remote_reset_url, remote_urls
+from bench.commands.git import remote_reset_url, remote_set_url, remote_urls
+
 bench_command.add_command(remote_set_url)
 bench_command.add_command(remote_reset_url)
 bench_command.add_command(remote_urls)
 
 from bench.commands.install import install
+
 bench_command.add_command(install)

bench/commands/config.py

@@ -1,5 +1,5 @@
 # imports - module imports
-from bench.config.common_site_config import update_config, get_config, put_config
+from bench.config.common_site_config import update_config, put_config
 
 # imports - third party imports
 import click
@@ -68,7 +68,8 @@ def set_common_config(configs):
 @click.command('remove-common-config', help='Remove specific keys from current bench\'s common config')
 @click.argument('keys', nargs=-1)
 def remove_common_config(keys):
-	common_site_config = get_config('.')
+	from bench.bench import Bench
+	common_site_config = Bench('.').conf
 	for key in keys:
 		if key in common_site_config:
 			del common_site_config[key]

bench/commands/git.py

@@ -3,8 +3,10 @@ import os
 import subprocess
 
 # imports - module imports
-from bench.app import get_repo_dir, get_apps, get_remote
+from bench.bench import Bench
+from bench.app import get_repo_dir
 from bench.utils import set_git_remote_url
+from bench.utils.app import get_remote
 
 # imports - third party imports
 import click
@@ -25,7 +27,7 @@ def remote_reset_url(app):
 
 @click.command('remote-urls', help="Show apps remote url")
 def remote_urls():
-	for app in get_apps():
+	for app in Bench(".").apps:
 		repo_dir = get_repo_dir(app)
 
 		if os.path.exists(os.path.join(repo_dir, '.git')):

bench/commands/install.py

@@ -1,5 +1,6 @@
 # imports - module imports
-from bench.utils import run_playbook, setup_sudoers
+from bench.utils import run_playbook
+from bench.utils.system import setup_sudoers
 
 # imports - third party imports
 import click

bench/commands/make.py

@@ -2,101 +2,203 @@
 import click
 
 
-@click.command('init', help='Initialize a new bench instance in the specified path')
-@click.argument('path')
-@click.option('--python', type = str, default = 'python3', help = 'Path to Python Executable.')
-@click.option('--ignore-exist', is_flag = True, default = False, help = "Ignore if Bench instance exists.")
-@click.option('--apps_path', default=None, help="path to json files with apps to install after init")
-@click.option('--frappe-path', default=None, help="path to frappe repo")
-@click.option('--frappe-branch', default=None, help="Clone a particular branch of frappe")
-@click.option('--clone-from', default=None, help="copy repos from path")
-@click.option('--clone-without-update', is_flag=True, help="copy repos from path without update")
-@click.option('--no-procfile', is_flag=True, help="Do not create a Procfile")
-@click.option('--no-backups',is_flag=True, help="Do not set up automatic periodic backups for all sites on this bench")
-@click.option('--skip-redis-config-generation', is_flag=True, help="Skip redis config generation if already specifying the common-site-config file")
-@click.option('--skip-assets',is_flag=True, default=False, help="Do not build assets")
-@click.option('--verbose',is_flag=True, help="Verbose output during install")
-def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, clone_from, verbose, skip_redis_config_generation, clone_without_update, ignore_exist=False, skip_assets=False, python='python3'):
-	from bench.utils import init, log
+@click.command("init", help="Initialize a new bench instance in the specified path")
+@click.argument("path")
+@click.option(
+	"--version",
+	"--frappe-branch",
+	"frappe_branch",
+	default=None,
+	help="Clone a particular branch of frappe",
+)
+@click.option(
+	"--ignore-exist", is_flag=True, default=False, help="Ignore if Bench instance exists."
+)
+@click.option(
+	"--python", type=str, default="python3", help="Path to Python Executable."
+)
+@click.option(
+	"--apps_path", default=None, help="path to json files with apps to install after init"
+)
+@click.option("--frappe-path", default=None, help="path to frappe repo")
+@click.option("--clone-from", default=None, help="copy repos from path")
+@click.option(
+	"--clone-without-update", is_flag=True, help="copy repos from path without update"
+)
+@click.option("--no-procfile", is_flag=True, help="Do not create a Procfile")
+@click.option(
+	"--no-backups",
+	is_flag=True,
+	help="Do not set up automatic periodic backups for all sites on this bench",
+)
+@click.option(
+	"--skip-redis-config-generation",
+	is_flag=True,
+	help="Skip redis config generation if already specifying the common-site-config file",
+)
+@click.option("--skip-assets", is_flag=True, default=False, help="Do not build assets")
+@click.option(
+	"--install-app", help="Install particular app after initialization"
+)
+@click.option("--verbose", is_flag=True, help="Verbose output during install")
+def init(
+	path,
+	apps_path,
+	frappe_path,
+	frappe_branch,
+	no_procfile,
+	no_backups,
+	clone_from,
+	verbose,
+	skip_redis_config_generation,
+	clone_without_update,
+	ignore_exist=False,
+	skip_assets=False,
+	python="python3",
+	install_app=None,
+):
+	import os
+
+	from bench.utils import log
+	from bench.utils.system import init
+
+	if not ignore_exist and os.path.exists(path):
+		log(f"Bench instance already exists at {path}", level=2)
+		return
 
 	try:
 		init(
 			path,
-			apps_path=apps_path,
+			apps_path=apps_path, # can be used from --config flag? Maybe config file could have more info?
 			no_procfile=no_procfile,
 			no_backups=no_backups,
 			frappe_path=frappe_path,
 			frappe_branch=frappe_branch,
-			verbose=verbose,
+			install_app=install_app,
 			clone_from=clone_from,
 			skip_redis_config_generation=skip_redis_config_generation,
 			clone_without_update=clone_without_update,
-			ignore_exist=ignore_exist,
 			skip_assets=skip_assets,
 			python=python,
+			verbose=verbose,
 		)
-		log(f'Bench {path} initialized', level=1)
+		log(f"Bench {path} initialized", level=1)
 	except SystemExit:
-		pass
-	except Exception as e:
-		import os, shutil, time
+		raise
+	except Exception:
+		import shutil
+		import time
+
+		from bench.utils import get_traceback
+
 		# add a sleep here so that the traceback of other processes doesnt overlap with the prompts
 		time.sleep(1)
-		print(e)
+		print(get_traceback())
+
 		log(f"There was a problem while creating {path}", level=2)
-		if click.confirm("Do you want to rollback these changes?"):
-			print(f'Rolling back Bench "{path}"')
+		if click.confirm("Do you want to rollback these changes?", abort=True):
+			log(f'Rolling back Bench "{path}"')
 			if os.path.exists(path):
 				shutil.rmtree(path)
 
 
-@click.command('get-app', help='Clone an app from the internet or filesystem and set it up in your bench')
-@click.argument('name', nargs=-1) # Dummy argument for backward compatibility
-@click.argument('git-url')
-@click.option('--branch', default=None, help="branch to checkout")
-@click.option('--overwrite', is_flag=True, default=False)
-@click.option('--skip-assets', is_flag=True, default=False, help="Do not build assets")
-def get_app(git_url, branch, name=None, overwrite=False, skip_assets=False):
+@click.command("drop")
+@click.argument("path")
+def drop(path):
+	from bench.bench import Bench
+	from bench.exceptions import BenchNotFoundError, ValidationError
+
+	bench = Bench(path)
+
+	if not bench.exists:
+		raise BenchNotFoundError(f"Bench {bench.name} does not exist")
+
+	if bench.sites:
+		raise ValidationError("Cannot remove non-empty bench directory")
+
+	bench.drop()
+
+	print("Bench dropped")
+
+
+@click.command(
+	["get", "get-app"],
+	help="Clone an app from the internet or filesystem and set it up in your bench",
+)
+@click.argument("name", nargs=-1)  # Dummy argument for backward compatibility
+@click.argument("git-url")
+@click.option("--branch", default=None, help="branch to checkout")
+@click.option("--overwrite", is_flag=True, default=False)
+@click.option("--skip-assets", is_flag=True, default=False, help="Do not build assets")
+@click.option(
+	"--init-bench", is_flag=True, default=False, help="Initialize Bench if not in one"
+)
+def get_app(
+	git_url, branch, name=None, overwrite=False, skip_assets=False, init_bench=False
+):
 	"clone an app from the internet and set it up in your bench"
 	from bench.app import get_app
-	get_app(git_url, branch=branch, skip_assets=skip_assets, overwrite=overwrite)
+
+	get_app(
+		git_url,
+		branch=branch,
+		skip_assets=skip_assets,
+		overwrite=overwrite,
+		init_bench=init_bench,
+	)
 
 
-@click.command('new-app', help='Create a new Frappe application under apps folder')
-@click.argument('app-name')
+@click.command("new-app", help="Create a new Frappe application under apps folder")
+@click.argument("app-name")
 def new_app(app_name):
 	from bench.app import new_app
+
 	new_app(app_name)
 
 
-@click.command('remove-app', help='Completely remove app from bench and re-build assets if not installed on any site')
-@click.argument('app-name')
+@click.command(
+	["remove", "rm", "remove-app"],
+	help=(
+		"Completely remove app from bench and re-build assets if not installed on any site"
+	),
+)
+@click.argument("app-name")
 def remove_app(app_name):
-	from bench.app import remove_app
-	remove_app(app_name)
+	from bench.bench import Bench
+
+	bench = Bench(".")
+	bench.uninstall(app_name)
 
 
-@click.command('exclude-app', help='Exclude app from updating')
-@click.argument('app_name')
+@click.command("exclude-app", help="Exclude app from updating")
+@click.argument("app_name")
 def exclude_app_for_update(app_name):
 	from bench.app import add_to_excluded_apps_txt
+
 	add_to_excluded_apps_txt(app_name)
 
 
-@click.command('include-app', help='Include app for updating')
-@click.argument('app_name')
+@click.command("include-app", help="Include app for updating")
+@click.argument("app_name")
 def include_app_for_update(app_name):
 	"Include app from updating"
 	from bench.app import remove_from_excluded_apps_txt
+
 	remove_from_excluded_apps_txt(app_name)
 
 
-@click.command('pip', context_settings={"ignore_unknown_options": True}, help="For pip help use `bench pip help [COMMAND]` or `bench pip [COMMAND] -h`")
-@click.argument('args', nargs=-1)
+@click.command(
+	"pip",
+	context_settings={"ignore_unknown_options": True, "help_option_names": []},
+	help="For pip help use `bench pip help [COMMAND]` or `bench pip [COMMAND] -h`",
+)
+@click.argument("args", nargs=-1)
 @click.pass_context
 def pip(ctx, args):
 	"Run pip commands in bench env"
 	import os
-	from bench.utils import get_env_cmd
-	env_py = get_env_cmd('python')
-	os.execv(env_py, (env_py, '-m', 'pip') + args)
+
+	from bench.utils.bench import get_env_cmd
+
+	env_py = get_env_cmd("python")
+	os.execv(env_py, (env_py, "-m", "pip") + args)

bench/commands/setup.py

@@ -6,11 +6,6 @@ import sys
 import click
 
 # imports - module imports
-import bench.config.procfile
-import bench.config.redis
-import bench.config.site_config
-import bench.config.supervisor
-import bench.utils
 from bench.utils import exec_cmd, run_playbook
 
 
@@ -22,22 +17,23 @@ def setup():
 @click.command("sudoers", help="Add commands to sudoers list for execution without password")
 @click.argument("user")
 def setup_sudoers(user):
-	bench.utils.setup_sudoers(user)
+	from bench.utils.system import setup_sudoers
+	setup_sudoers(user)
 
 
 @click.command("nginx", help="Generate configuration files for NGINX")
 @click.option("--yes", help="Yes to regeneration of nginx config file", default=False, is_flag=True)
 def setup_nginx(yes=False):
-	import bench.config.nginx
+	from bench.config.nginx import make_nginx_conf
 
-	bench.config.nginx.make_nginx_conf(bench_path=".", yes=yes)
+	make_nginx_conf(bench_path=".", yes=yes)
 
 
 @click.command("reload-nginx", help="Checks NGINX config file and reloads service")
 def reload_nginx():
-	import bench.config.production_setup
+	from bench.config.production_setup import reload_nginx
 
-	bench.config.production_setup.reload_nginx()
+	reload_nginx()
 
 
 @click.command("supervisor", help="Generate configuration for supervisor")
@@ -45,38 +41,43 @@ def reload_nginx():
 @click.option("--yes", help="Yes to regeneration of supervisor config", is_flag=True, default=False)
 @click.option("--skip-redis", help="Skip redis configuration", is_flag=True, default=False)
 def setup_supervisor(user=None, yes=False, skip_redis=False):
-	bench.config.supervisor.update_supervisord_config(user=user, yes=yes)
-	bench.config.supervisor.generate_supervisor_config(bench_path=".", user=user, yes=yes, skip_redis=skip_redis)
+	from bench.config.supervisor import update_supervisord_config, generate_supervisor_config
+
+	update_supervisord_config(user=user, yes=yes)
+	generate_supervisor_config(bench_path=".", user=user, yes=yes, skip_redis=skip_redis)
 
 
 @click.command("redis", help="Generates configuration for Redis")
 def setup_redis():
-	bench.config.redis.generate_config(".")
+	from bench.config.redis import generate_config
+	generate_config(".")
 
 
 @click.command("fonts", help="Add Frappe fonts to system")
 def setup_fonts():
-	bench.utils.setup_fonts()
+	from bench.utils.system import setup_fonts
+	setup_fonts()
 
 
 @click.command("production", help="Setup Frappe production environment for specific user")
 @click.argument("user")
 @click.option("--yes", help="Yes to regeneration config", is_flag=True, default=False)
 def setup_production(user, yes=False):
-	import bench.config.production_setup
-
-	bench.config.production_setup.setup_production(user=user, yes=yes)
+	from bench.config.production_setup import setup_production
+	setup_production(user=user, yes=yes)
 
 
 @click.command("backups", help="Add cronjob for bench backups")
 def setup_backups():
-	bench.utils.setup_backups()
+	from bench.bench import Bench
+	Bench(".").setup.backups()
 
 
 @click.command("env", help="Setup virtualenv for bench")
 @click.option("--python", type = str, default = "python3", help = "Path to Python Executable.")
 def setup_env(python="python3"):
-	bench.utils.setup_env(python=python)
+	from bench.bench import Bench
+	return Bench(".").setup.env(python=python)
 
 
 @click.command("firewall", help="Setup firewall for system")
@@ -107,9 +108,8 @@ def set_ssh_port(port, force=False):
 @click.option("--custom-domain")
 @click.option('-n', '--non-interactive', default=False, is_flag=True, help="Run command non-interactively. This flag restarts nginx and runs certbot non interactively. Shouldn't be used on 1'st attempt")
 def setup_letsencrypt(site, custom_domain, non_interactive):
-	import bench.config.lets_encrypt
-
-	bench.config.lets_encrypt.setup_letsencrypt(site, custom_domain, bench_path=".", interactive=not non_interactive)
+	from bench.config.lets_encrypt import setup_letsencrypt
+	setup_letsencrypt(site, custom_domain, bench_path=".", interactive=not non_interactive)
 
 
 @click.command("wildcard-ssl", help="Setup wildcard SSL certificate for multi-tenant bench")
@@ -117,40 +117,40 @@ def setup_letsencrypt(site, custom_domain, non_interactive):
 @click.option("--email")
 @click.option("--exclude-base-domain", default=False, is_flag=True, help="SSL Certificate not applicable for base domain")
 def setup_wildcard_ssl(domain, email, exclude_base_domain):
-	import bench.config.lets_encrypt
-
-	bench.config.lets_encrypt.setup_wildcard_ssl(domain, email, bench_path=".", exclude_base_domain=exclude_base_domain)
+	from bench.config.lets_encrypt import setup_wildcard_ssl
+	setup_wildcard_ssl(domain, email, bench_path=".", exclude_base_domain=exclude_base_domain)
 
 
 @click.command("procfile", help="Generate Procfile for bench start")
 def setup_procfile():
-	bench.config.procfile.setup_procfile(".")
+	from bench.config.procfile import setup_procfile
+	setup_procfile(".")
 
 
-@click.command("socketio", help="Setup node dependencies for socketio server")
+@click.command("socketio", help="[DEPRECATED] Setup node dependencies for socketio server")
 def setup_socketio():
-	bench.utils.setup_socketio()
-
+	return
 
 @click.command("requirements", help="Setup Python and Node dependencies")
 @click.option("--node", help="Update only Node packages", default=False, is_flag=True)
 @click.option("--python", help="Update only Python packages", default=False, is_flag=True)
 @click.option("--dev", help="Install optional python development dependencies", default=False, is_flag=True)
 def setup_requirements(node=False, python=False, dev=False):
-	if not (node or python):
-		from bench.utils import update_requirements
-		update_requirements()
+	from bench.bench import Bench
 
-	elif not node:
-		from bench.utils import update_python_packages
-		update_python_packages()
+	bench = Bench(".")
 
-	elif not python:
-		from bench.utils import update_node_packages
-		update_node_packages()
+	if not (node or python or dev):
+		bench.setup.requirements()
 
-	if dev:
-		from bench.utils import install_python_dev_dependencies
+	elif not node and not dev:
+		bench.setup.python()
+
+	elif not python and not dev:
+		bench.setup.node()
+
+	else:
+		from bench.utils.bench import install_python_dev_dependencies
 		install_python_dev_dependencies()
 
 		if node:
@@ -162,8 +162,7 @@ def setup_requirements(node=False, python=False, dev=False):
 @click.option("--port", help="Port on which you want to run bench manager", default=23624)
 @click.option("--domain", help="Domain on which you want to run bench manager")
 def setup_manager(yes=False, port=23624, domain=None):
-	from bench.utils import get_sites
-	from bench.config.common_site_config import get_config
+	from bench.bench import Bench
 	from bench.config.nginx import make_bench_manager_nginx_conf
 
 	create_new_site = True
@@ -182,15 +181,15 @@ def setup_manager(yes=False, port=23624, domain=None):
 	exec_cmd("bench --site bench-manager.local install-app bench_manager")
 
 	bench_path = "."
-	conf = get_config(bench_path)
+	bench = Bench(bench_path)
 
-	if conf.get("restart_supervisor_on_update") or conf.get("restart_systemd_on_update"):
+	if bench.conf.get("restart_supervisor_on_update") or bench.conf.get("restart_systemd_on_update"):
 		# implicates a production setup or so I presume
 		if not domain:
 			print("Please specify the site name on which you want to host bench-manager using the 'domain' flag")
 			sys.exit(1)
 
-		if domain not in get_sites(bench_path):
+		if domain not in bench.sites:
 			raise Exception("No such site")
 
 		make_bench_manager_nginx_conf(bench_path, yes=yes, port=port, domain=domain)
@@ -198,8 +197,8 @@ def setup_manager(yes=False, port=23624, domain=None):
 
 @click.command("config", help="Generate or over-write sites/common_site_config.json")
 def setup_config():
-	from bench.config.common_site_config import make_config
-	make_config(".")
+	from bench.config.common_site_config import setup_config
+	setup_config(".")
 
 
 @click.command("add-domain", help="Add a custom domain to a particular site")
@@ -213,7 +212,8 @@ def add_domain(domain, site=None, ssl_certificate=None, ssl_certificate_key=None
 		print("Please specify site")
 		sys.exit(1)
 
-	bench.config.site_config.add_domain(site, domain, ssl_certificate, ssl_certificate_key, bench_path=".")
+	from bench.config.site_config import add_domain
+	add_domain(site, domain, ssl_certificate, ssl_certificate_key, bench_path=".")
 
 
 @click.command("remove-domain", help="Remove custom domain from a site")
@@ -224,7 +224,8 @@ def remove_domain(domain, site=None):
 		print("Please specify site")
 		sys.exit(1)
 
-	bench.config.site_config.remove_domain(site, domain, bench_path=".")
+	from bench.config.site_config import remove_domain
+	remove_domain(site, domain, bench_path=".")
 
 
 @click.command("sync-domains", help="Check if there is a change in domains. If yes, updates the domains list.")
@@ -241,7 +242,8 @@ def sync_domains(domain=None, site=None):
 		print("Domains should be a json list of strings or dictionaries")
 		sys.exit(1)
 
-	changed = bench.config.site_config.sync_domains(site, domains, bench_path=".")
+	from bench.config.site_config import sync_domains
+	changed = sync_domains(site, domains, bench_path=".")
 
 	# if changed, success, else failure
 	sys.exit(0 if changed else 1)

bench/commands/update.py

@@ -3,7 +3,7 @@ import click
 
 # imports - module imports
 from bench.app import pull_apps
-from bench.utils import post_upgrade, patch_sites, build_assets
+from bench.utils.bench import post_upgrade, patch_sites, build_assets
 
 
 @click.command('update', help="Performs an update operation on current bench. Without any flags will backup, pull, setup requirements, build, run patches and restart bench. Using specific flags will only do certain tasks instead of all")
@@ -19,7 +19,7 @@ from bench.utils import post_upgrade, patch_sites, build_assets
 @click.option('--force', is_flag=True, help="Forces major version upgrades")
 @click.option('--reset', is_flag=True, help="Hard resets git branch's to their new states overriding any changes and overriding rebase on pull")
 def update(pull, apps, patch, build, requirements, restart_supervisor, restart_systemd, no_backup, no_compile, force, reset):
-	from bench.utils import update
+	from bench.utils.bench import update
 	update(pull=pull, apps=apps, patch=patch, build=build, requirements=requirements, restart_supervisor=restart_supervisor, restart_systemd=restart_systemd, backup=not no_backup, compile=not no_compile, force=force, reset=reset)
 
 
@@ -37,12 +37,12 @@ def retry_upgrade(version):
 @click.argument('apps', nargs=-1)
 @click.option('--upgrade',is_flag=True)
 def switch_to_branch(branch, apps, upgrade=False):
-	from bench.app import switch_to_branch
+	from bench.utils.app import switch_to_branch
 	switch_to_branch(branch=branch, apps=list(apps), upgrade=upgrade)
 
 
 @click.command('switch-to-develop')
 def switch_to_develop(upgrade=False):
 	"Switch frappe and erpnext to develop branch"
-	from bench.app import switch_to_develop
+	from bench.utils.app import switch_to_develop
 	switch_to_develop(apps=['frappe', 'erpnext'])

bench/commands/utils.py

@@ -11,9 +11,10 @@ import click
 @click.option('--no-prefix', is_flag=True, default=False, help="Hide process name from bench start log")
 @click.option('--concurrency', '-c', type=str)
 @click.option('--procfile', '-p', type=str)
-def start(no_dev, concurrency, procfile, no_prefix):
-	from bench.utils import start
-	start(no_dev=no_dev, concurrency=concurrency, procfile=procfile, no_prefix=no_prefix)
+@click.option('--man', '-m', help="Process Manager of your choice ;)")
+def start(no_dev, concurrency, procfile, no_prefix, man):
+	from bench.utils.system import start
+	start(no_dev=no_dev, concurrency=concurrency, procfile=procfile, no_prefix=no_prefix, procman=man)
 
 
 @click.command('restart', help="Restart supervisor processes or systemd units")
@@ -21,11 +22,14 @@ def start(no_dev, concurrency, procfile, no_prefix):
 @click.option('--supervisor', is_flag=True, default=False)
 @click.option('--systemd', is_flag=True, default=False)
 def restart(web, supervisor, systemd):
-	from bench.utils import restart_supervisor_processes, restart_systemd_processes
-	from bench.config.common_site_config import get_config
-	if get_config('.').get('restart_supervisor_on_update') or supervisor:
+	from bench.bench import Bench
+	from bench.utils.bench import restart_supervisor_processes, restart_systemd_processes
+
+	bench = Bench(".")
+
+	if bench.conf.get('restart_supervisor_on_update') or supervisor:
 		restart_supervisor_processes(bench_path='.', web_workers=web)
-	if get_config('.').get('restart_systemd_on_update') or systemd:
+	if bench.conf.get('restart_systemd_on_update') or systemd:
 		restart_systemd_processes(bench_path='.', web_workers=web)
 
 
@@ -64,7 +68,7 @@ def set_url_root(site, url_root):
 @click.command('set-mariadb-host', help="Set MariaDB host for bench")
 @click.argument('host')
 def set_mariadb_host(host):
-	from bench.utils import set_mariadb_host
+	from bench.utils.bench import set_mariadb_host
 	set_mariadb_host(host)
 
 
@@ -74,7 +78,7 @@ def set_redis_cache_host(host):
 	"""
 	Usage: bench set-redis-cache-host localhost:6379/1
 	"""
-	from bench.utils import set_redis_cache_host
+	from bench.utils.bench import set_redis_cache_host
 	set_redis_cache_host(host)
 
 
@@ -84,7 +88,7 @@ def set_redis_queue_host(host):
 	"""
 	Usage: bench set-redis-queue-host localhost:6379/2
 	"""
-	from bench.utils import set_redis_queue_host
+	from bench.utils.bench import set_redis_queue_host
 	set_redis_queue_host(host)
 
 
@@ -94,14 +98,14 @@ def set_redis_socketio_host(host):
 	"""
 	Usage: bench set-redis-socketio-host localhost:6379/3
 	"""
-	from bench.utils import set_redis_socketio_host
+	from bench.utils.bench import set_redis_socketio_host
 	set_redis_socketio_host(host)
 
 
 
 @click.command('download-translations', help="Download latest translations")
 def download_translations():
-	from bench.utils import download_translations_p
+	from bench.utils.translation import download_translations_p
 	download_translations_p()
 
 
@@ -114,8 +118,9 @@ def renew_lets_encrypt():
 @click.command('backup', help="Backup single site")
 @click.argument('site')
 def backup_site(site):
-	from bench.utils import get_sites, backup_site
-	if site not in get_sites(bench_path='.'):
+	from bench.bench import Bench
+	from bench.utils.system import backup_site
+	if site not in Bench(".").sites:
 		print(f'Site `{site}` not found')
 		sys.exit(1)
 	backup_site(site, bench_path='.')
@@ -123,7 +128,7 @@ def backup_site(site):
 
 @click.command('backup-all-sites', help="Backup all sites in current bench")
 def backup_all_sites():
-	from bench.utils import backup_all_sites
+	from bench.utils.system import backup_all_sites
 	backup_all_sites(bench_path='.')
 
 
@@ -173,7 +178,7 @@ def find_benches(location):
 @click.argument('python', type=str)
 @click.option('--no-backup', 'backup', is_flag=True, default=True)
 def migrate_env(python, backup=True):
-	from bench.utils import migrate_env
+	from bench.utils.bench import migrate_env
 	migrate_env(python=python, backup=backup)
 
 

bench/config/common_site_config.py

@@ -17,7 +17,7 @@ default_config = {
 	'live_reload': True
 }
 
-def make_config(bench_path):
+def setup_config(bench_path):
 	make_pid_folder(bench_path)
 	bench_config = get_config(bench_path)
 	bench_config.update(default_config)

bench/config/lets_encrypt.py

@@ -6,11 +6,13 @@ import click
 
 # imports - module imports
 import bench
-from bench.config.common_site_config import get_config
 from bench.config.nginx import make_nginx_conf
 from bench.config.production_setup import service
 from bench.config.site_config import get_domains, remove_domain, update_site_config
-from bench.utils import CommandFailedError, exec_cmd, update_common_site_config
+from bench.bench import Bench
+from bench.utils import exec_cmd
+from bench.utils.bench import update_common_site_config
+from bench.exceptions import CommandFailedError
 
 
 def setup_letsencrypt(site, custom_domain, bench_path, interactive):
@@ -36,7 +38,7 @@ def setup_letsencrypt(site, custom_domain, bench_path, interactive):
 			'Do you want to continue?',
 			abort=True)
 
-	if not get_config(bench_path).get("dns_multitenant"):
+	if not Bench(bench_path).conf.get("dns_multitenant"):
 		print("You cannot setup SSL without DNS Multitenancy")
 		return
 
@@ -150,7 +152,7 @@ def setup_wildcard_ssl(domain, email, bench_path, exclude_base_domain):
 
 		return domain_list
 
-	if not get_config(bench_path).get("dns_multitenant"):
+	if not Bench(bench_path).conf.get("dns_multitenant"):
 		print("You cannot setup SSL without DNS Multitenancy")
 		return
 

bench/config/nginx.py

@@ -9,7 +9,8 @@ import click
 
 # imports - module imports
 import bench
-from bench.utils import get_bench_name, get_sites
+from bench.bench import Bench
+from bench.utils import get_bench_name
 
 
 def make_nginx_conf(bench_path, yes=False):
@@ -23,7 +24,7 @@ def make_nginx_conf(bench_path, yes=False):
 	bench_path = os.path.abspath(bench_path)
 	sites_path = os.path.join(bench_path, "sites")
 
-	config = bench.config.common_site_config.get_config(bench_path)
+	config = Bench(bench_path).conf
 	sites = prepare_sites(config, bench_path)
 	bench_name = get_bench_name(bench_path)
 
@@ -56,13 +57,12 @@ def make_nginx_conf(bench_path, yes=False):
 
 def make_bench_manager_nginx_conf(bench_path, yes=False, port=23624, domain=None):
 	from bench.config.site_config import get_site_config
-	from bench.config.common_site_config import get_config
 
 	template = bench.config.env().get_template('bench_manager_nginx.conf')
 	bench_path = os.path.abspath(bench_path)
 	sites_path = os.path.join(bench_path, "sites")
 
-	config = get_config(bench_path)
+	config = Bench(bench_path).conf
 	site_config = get_site_config(domain, bench_path=bench_path)
 	bench_name = get_bench_name(bench_path)
 
@@ -182,24 +182,26 @@ def prepare_sites(config, bench_path):
 	return sites
 
 def get_sites_with_config(bench_path):
-	from bench.config.common_site_config import get_config
+	from bench.bench import Bench
 	from bench.config.site_config import get_site_config
 
-	sites = get_sites(bench_path=bench_path)
-	dns_multitenant = get_config(bench_path).get('dns_multitenant')
+	bench = Bench(bench_path)
+	sites = bench.sites
+	conf = bench.conf
+	dns_multitenant = conf.get('dns_multitenant')
 
 	ret = []
 	for site in sites:
 		try:
 			site_config = get_site_config(site, bench_path=bench_path)
 		except Exception as e:
-			strict_nginx = get_config(bench_path).get('strict_nginx')
+			strict_nginx = conf.get('strict_nginx')
 			if strict_nginx:
 				print(f"\n\nERROR: The site config for the site {site} is broken.",
 					"If you want this command to pass, instead of just throwing an error,",
 					"You may remove the 'strict_nginx' flag from common_site_config.json or set it to 0",
 					"\n\n")
-				raise (e)
+				raise e
 			else:
 				print(f"\n\nWARNING: The site config for the site {site} is broken.",
 					"If you want this command to fail, instead of just showing a warning,",
@@ -236,8 +238,8 @@ def use_wildcard_certificate(bench_path, ret):
 			"ssl_certificate_key": "/path/to/erpnext.com.key"
 		}
 	'''
-	from bench.config.common_site_config import get_config
-	config = get_config(bench_path=bench_path)
+	from bench.bench import Bench
+	config = Bench(bench_path).conf
 	wildcard = config.get('wildcard')
 
 	if not wildcard:

bench/config/procfile.py

@@ -7,12 +7,12 @@ import click
 # imports - module imports
 import bench
 from bench.app import use_rq
-from bench.config.common_site_config import get_config
 from bench.utils import which
+from bench.bench import Bench
 
 
 def setup_procfile(bench_path, yes=False, skip_redis=False):
-	config = get_config(bench_path=bench_path)
+	config = Bench(bench_path).conf
 	procfile_path = os.path.join(bench_path, 'Procfile')
 	if not yes and os.path.exists(procfile_path):
 		click.confirm('A Procfile already exists and this will overwrite it. Do you want to continue?',

bench/config/production_setup.py

@@ -5,12 +5,13 @@ import sys
 
 # imports - module imports
 import bench
-from bench.config.common_site_config import get_config
 from bench.config.nginx import make_nginx_conf
 from bench.config.supervisor import generate_supervisor_config, update_supervisord_config
 from bench.config.systemd import generate_systemd_config
-from bench.utils import CommandFailedError, exec_cmd, which, fix_prod_setup_perms, get_bench_name, get_cmd_output, log
-
+from bench.bench import Bench
+from bench.utils import exec_cmd, which, get_bench_name, get_cmd_output, log
+from bench.utils.system import fix_prod_setup_perms
+from bench.exceptions import CommandFailedError
 
 logger = logging.getLogger(bench.PROJECT_NAME)
 
@@ -30,10 +31,13 @@ def setup_production_prerequisites():
 def setup_production(user, bench_path='.', yes=False):
 	print("Setting Up prerequisites...")
 	setup_production_prerequisites()
-	if get_config(bench_path).get('restart_supervisor_on_update') and get_config(bench_path).get('restart_systemd_on_update'):
+
+	conf = Bench(bench_path).conf
+
+	if conf.get('restart_supervisor_on_update') and conf.get('restart_systemd_on_update'):
 		raise Exception("You cannot use supervisor and systemd at the same time. Modify your common_site_config accordingly." )
 
-	if get_config(bench_path).get('restart_systemd_on_update'):
+	if conf.get('restart_systemd_on_update'):
 		print("Setting Up systemd...")
 		generate_systemd_config(bench_path=bench_path, user=user, yes=yes)
 	else:
@@ -50,7 +54,7 @@ def setup_production(user, bench_path='.', yes=False):
 	nginx_conf = f'/etc/nginx/conf.d/{bench_name}.conf'
 
 	print("Setting Up symlinks and reloading services...")
-	if get_config(bench_path).get('restart_supervisor_on_update'):
+	if conf.get('restart_supervisor_on_update'):
 		supervisor_conf_extn = "ini" if is_centos7() else "conf"
 		supervisor_conf = os.path.join(get_supervisor_confdir(), f'{bench_name}.{supervisor_conf_extn}')
 
@@ -61,7 +65,7 @@ def setup_production(user, bench_path='.', yes=False):
 	if not os.path.islink(nginx_conf):
 		os.symlink(os.path.abspath(os.path.join(bench_path, 'config', 'nginx.conf')), nginx_conf)
 
-	if get_config(bench_path).get('restart_supervisor_on_update'):
+	if conf.get('restart_supervisor_on_update'):
 		reload_supervisor()
 
 	if os.environ.get('NO_SERVICE_RESTART'):
@@ -72,6 +76,7 @@ def setup_production(user, bench_path='.', yes=False):
 
 def disable_production(bench_path='.'):
 	bench_name = get_bench_name(bench_path)
+	conf = Bench(bench_path).conf
 
 	# supervisorctl
 	supervisor_conf_extn = "ini" if is_centos7() else "conf"
@@ -80,7 +85,7 @@ def disable_production(bench_path='.'):
 	if os.path.islink(supervisor_conf):
 		os.unlink(supervisor_conf)
 
-	if get_config(bench_path).get('restart_supervisor_on_update'):
+	if conf.get('restart_supervisor_on_update'):
 		reload_supervisor()
 
 	# nginx
@@ -177,7 +182,7 @@ def reload_supervisor():
 def reload_nginx():
 	try:
 		exec_cmd(f"sudo {which('nginx')} -t")
-	except:
+	except Exception:
 		raise
 
 	service('nginx', 'reload')

bench/config/redis.py

@@ -5,13 +5,13 @@ import subprocess
 
 # imports - module imports
 import bench
-from bench.config.common_site_config import get_config
 
 
 def generate_config(bench_path):
 	from urllib.parse import urlparse
+	from bench.bench import Bench
 
-	config = get_config(bench_path)
+	config = Bench(bench_path).conf
 	redis_version = get_redis_version()
 
 	ports = {}

bench/config/site_config.py

@@ -3,9 +3,6 @@ import json
 import os
 from collections import defaultdict
 
-# imports - module imports
-from bench.utils import get_sites
-
 
 def get_site_config(site, bench_path='.'):
 	config_path = os.path.join(bench_path, 'sites', site, 'site_config.json')
@@ -35,8 +32,9 @@ def set_ssl_certificate_key(site, ssl_certificate_key, bench_path='.', gen_confi
 
 def set_site_config_nginx_property(site, config, bench_path='.', gen_config=True):
 	from bench.config.nginx import make_nginx_conf
+	from bench.bench import Bench
 
-	if site not in get_sites(bench_path=bench_path):
+	if site not in Bench(bench_path).sites:
 		raise Exception("No such site")
 	update_site_config(site, config, bench_path=bench_path)
 	if gen_config:

bench/config/supervisor.py

@@ -7,7 +7,8 @@ import os
 import bench
 from bench.app import use_rq
 from bench.utils import get_bench_name, which
-from bench.config.common_site_config import get_config, update_config, get_gunicorn_workers
+from bench.bench import Bench
+from bench.config.common_site_config import update_config, get_gunicorn_workers
 
 # imports - third party imports
 import click
@@ -21,8 +22,8 @@ def generate_supervisor_config(bench_path, user=None, yes=False, skip_redis=Fals
 	if not user:
 		user = getpass.getuser()
 
+	config = Bench(bench_path).conf
 	template = bench.config.env().get_template('supervisor.conf')
-	config = get_config(bench_path=bench_path)
 	bench_dir = os.path.abspath(bench_path)
 
 	config = template.render(**{

bench/config/systemd.py

@@ -8,7 +8,8 @@ import click
 # imports - module imports
 import bench
 from bench.app import use_rq
-from bench.config.common_site_config import get_config, get_gunicorn_workers, update_config
+from bench.bench import Bench
+from bench.config.common_site_config import get_gunicorn_workers, update_config
 from bench.utils import exec_cmd, which, get_bench_name
 
 
@@ -19,7 +20,7 @@ def generate_systemd_config(bench_path, user=None, yes=False,
 	if not user:
 		user = getpass.getuser()
 
-	config = get_config(bench_path=bench_path)
+	config = Bench(bench_path).conf
 
 	bench_dir = os.path.abspath(bench_path)
 	bench_name = get_bench_name(bench_path)

bench/exceptions.py

@@ -0,0 +1,32 @@
+class InvalidBranchException(Exception):
+	pass
+
+
+class InvalidRemoteException(Exception):
+	pass
+
+
+class PatchError(Exception):
+	pass
+
+
+class CommandFailedError(Exception):
+	pass
+
+
+class BenchNotFoundError(Exception):
+	pass
+
+
+class ValidationError(Exception):
+	pass
+
+class CannotUpdateReleaseBench(ValidationError):
+	pass
+
+class FeatureDoesNotExistError(CommandFailedError):
+	pass
+
+
+class NotInBenchDirectoryError(Exception):
+	pass

bench/patches/__init__.py

@@ -29,11 +29,3 @@ def run(bench_path):
 
 			# end with an empty line
 			f.write('\n')
-
-def set_all_patches_executed(bench_path):
-	source_patch_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'patches.txt')
-	target_patch_file = os.path.join(os.path.abspath(bench_path), 'patches.txt')
-
-	with open(target_patch_file, 'w') as tf:
-		with open(source_patch_file, 'r') as sf:
-			tf.write(sf.read())

bench/patches/patches.txt

@@ -6,4 +6,5 @@ bench.patches.v4.update_socketio
 bench.patches.v4.install_yarn #2
 bench.patches.v5.fix_user_permissions
 bench.patches.v5.fix_backup_cronjob
-bench.patches.v5.set_live_reload_config
+bench.patches.v5.set_live_reload_config
+bench.patches.v5.update_archived_sites

bench/patches/v3/celery_to_rq.py

@@ -1,7 +1,7 @@
 import click, os
 from bench.config.procfile import setup_procfile
 from bench.config.supervisor import generate_supervisor_config
-from bench.app import get_current_frappe_version, get_current_branch
+from bench.utils.app import get_current_frappe_version, get_current_branch
 
 def execute(bench_path):
 	frappe_branch = get_current_branch('frappe', bench_path)

bench/patches/v5/update_archived_sites.py

@@ -0,0 +1,51 @@
+"""
+Deprecate archived_sites folder for consistency. This change is
+only for Frappe v14 benches. If not a v14 bench yet, skip this
+patch and try again later.
+
+1. Rename folder `./archived_sites` to `./archived/sites`
+2. Create a symlink `./archived_sites` => `./archived/sites`
+
+Corresponding changes in frappe/frappe via https://github.com/frappe/frappe/pull/15060
+"""
+import os
+import shutil
+from pathlib import Path
+
+import click
+from bench.utils.app import get_current_version
+from semantic_version import Version
+
+
+def execute(bench_path):
+	frappe_version = Version(get_current_version('frappe'))
+
+	if frappe_version.major < 14 or os.name != "posix":
+		# Returning False means patch has been skipped
+		return False
+
+	pre_patch_dir = os.getcwd()
+	old_directory = Path(bench_path, "archived_sites")
+	new_directory = Path(bench_path, "archived", "sites")
+
+	if old_directory.is_symlink():
+		return True
+
+	os.chdir(bench_path)
+
+	if not os.path.exists(new_directory):
+		os.makedirs(new_directory)
+
+	for archived_site_path in old_directory.glob("*"):
+		shutil.move(archived_site_path, new_directory)
+
+	click.secho(f"Archived sites are now stored under {new_directory}")
+
+	if not os.listdir(old_directory):
+		os.rmdir(old_directory)
+
+	os.symlink(new_directory, old_directory)
+
+	click.secho(f"Symlink {old_directory} that points to {new_directory}")
+
+	os.chdir(pre_patch_dir)

bench/prepare_staging.py

@@ -1,71 +0,0 @@
-#! env python
-import os
-import git
-import click
-from .config.common_site_config import get_config
-
-github_username = None
-github_password = None
-
-def prepare_staging(bench_path, app, remote='upstream'):
-	from .release import get_release_message
-	validate(bench_path)
-
-	repo_path = os.path.join(bench_path, 'apps', app)
-	update_branches(repo_path, remote)
-	message = get_release_message(repo_path, from_branch='develop', to_branch='staging', remote=remote)
-
-	if not message:
-		print('No commits to release')
-		return
-
-	print(message)
-
-	click.confirm('Do you want to continue?', abort=True)
-
-	create_staging(repo_path)
-	push_commits(repo_path)
-
-def validate(bench_path):
-	from .release import validate
-
-	config = get_config(bench_path)
-	validate(bench_path, config)
-
-def update_branches(repo_path, remote):
-	from .release import update_branch
-	update_branch(repo_path, 'staging', remote)
-	update_branch(repo_path, 'develop', remote)
-
-	git.Repo(repo_path).git.checkout('develop')
-
-def create_staging(repo_path, from_branch='develop'):
-	from .release import handle_merge_error
-
-	print('creating staging from', from_branch)
-	repo = git.Repo(repo_path)
-	g = repo.git
-	g.checkout('staging')
-	try:
-		g.merge(from_branch, '--no-ff')
-	except git.exc.GitCommandError as e:
-		handle_merge_error(e, source=from_branch, target='staging')
-
-	g.checkout(from_branch)
-	try:
-		g.merge('staging')
-	except git.exc.GitCommandError as e:
-		handle_merge_error(e, source='staging', target=from_branch)
-
-def push_commits(repo_path, remote='upstream'):
-	print('pushing staging branch of', repo_path)
-
-	repo = git.Repo(repo_path)
-	g = repo.git
-
-	args = [
-		'develop:develop',
-		'staging:staging'
-	]
-
-	print(g.push(remote, *args))

bench/release.py

@@ -7,6 +7,8 @@ import git
 import getpass
 import re
 from time import sleep
+
+from bench.exceptions import ValidationError
 from .config.common_site_config import get_config
 import click
 
@@ -189,10 +191,10 @@ def get_bumped_version(version, bump_type):
 			v.prerelease = ('beta', str(int(v.prerelease[1]) + 1))
 
 		else:
-			raise ("Something wen't wrong while doing a prerelease")
+			raise ValidationError("Something wen't wrong while doing a prerelease")
 
 	else:
-		raise ("bump_type not amongst [major, minor, patch, prerelease]")
+		raise ValidationError("bump_type not amongst [major, minor, patch, prerelease]")
 
 	return str(v)
 

bench/tests/test_base.py

@@ -10,7 +10,9 @@ import unittest
 
 # imports - module imports
 import bench
-import bench.utils
+from bench.utils import paths_in_bench, exec_cmd
+from bench.utils.system import init
+from bench.bench import Bench
 
 if sys.version_info.major == 2:
 	FRAPPE_BRANCH = "version-12"
@@ -25,15 +27,16 @@ class TestBenchBase(unittest.TestCase):
 	def tearDown(self):
 		for bench_name in self.benches:
 			bench_path = os.path.join(self.benches_path, bench_name)
+			bench = Bench(bench_path)
 			mariadb_password = "travis" if os.environ.get("CI") else getpass.getpass(prompt="Enter MariaDB root Password: ")
-			if os.path.exists(bench_path):
-				sites = bench.utils.get_sites(bench_path=bench_path)
-				for site in sites:
+
+			if bench.exists:
+				for site in bench.sites:
 					subprocess.call(["bench", "drop-site", site, "--force", "--no-backup", "--root-password", mariadb_password], cwd=bench_path)
 				shutil.rmtree(bench_path, ignore_errors=True)
 
 	def assert_folders(self, bench_name):
-		for folder in bench.utils.folders_in_bench:
+		for folder in paths_in_bench:
 			self.assert_exists(bench_name, folder)
 		self.assert_exists(bench_name, "apps", "frappe")
 
@@ -81,7 +84,7 @@ class TestBenchBase(unittest.TestCase):
 		frappe_tmp_path = "/tmp/frappe"
 
 		if not os.path.exists(frappe_tmp_path):
-			bench.utils.exec_cmd(f"git clone https://github.com/frappe/frappe -b {FRAPPE_BRANCH} --depth 1 --origin upstream {frappe_tmp_path}")
+			exec_cmd(f"git clone https://github.com/frappe/frappe -b {FRAPPE_BRANCH} --depth 1 --origin upstream {frappe_tmp_path}")
 
 		kwargs.update(dict(
 			python=sys.executable,
@@ -91,8 +94,8 @@ class TestBenchBase(unittest.TestCase):
 		))
 
 		if not os.path.exists(os.path.join(self.benches_path, bench_name)):
-			bench.utils.init(bench_name, **kwargs)
-			bench.utils.exec_cmd("git remote set-url upstream https://github.com/frappe/frappe", cwd=os.path.join(self.benches_path, bench_name, "apps", "frappe"))
+			init(bench_name, **kwargs)
+			exec_cmd("git remote set-url upstream https://github.com/frappe/frappe", cwd=os.path.join(self.benches_path, bench_name, "apps", "frappe"))
 
 	def file_exists(self, path):
 		if os.environ.get("CI"):

bench/tests/test_init.py

@@ -8,13 +8,16 @@ import unittest
 import git
 
 # imports - module imports
-import bench
-import bench.cli
-import bench.utils
+from bench.utils import exec_cmd
 from bench.release import get_bumped_version
 from bench.tests.test_base import FRAPPE_BRANCH, TestBenchBase
 
 
+# changed from frappe_theme because it wasn't maintained and incompatible,
+# chat app & wiki was breaking too. hopefully frappe_docs will be maintained
+# for longer since docs.erpnext.com is powered by it ;)
+TEST_FRAPPE_APP = "frappe_docs"
+
 class TestBenchInit(TestBenchBase):
 	def test_semantic_version(self):
 		self.assertEqual( get_bumped_version('11.0.4', 'major'), '12.0.0' )
@@ -78,7 +81,7 @@ class TestBenchInit(TestBenchBase):
 		site_config_path = os.path.join(site_path, "site_config.json")
 
 		self.init_bench(bench_name)
-		bench.utils.exec_cmd("bench setup requirements --node", cwd=bench_path)
+		exec_cmd("bench setup requirements --node", cwd=bench_path)
 		self.new_site(site_name, bench_name)
 
 		self.assertTrue(os.path.exists(site_path))
@@ -97,9 +100,9 @@ class TestBenchInit(TestBenchBase):
 	def test_get_app(self):
 		self.init_bench("test-bench")
 		bench_path = os.path.join(self.benches_path, "test-bench")
-		bench.utils.exec_cmd("bench get-app frappe_theme", cwd=bench_path)
-		self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "frappe_theme")))
-		app_installed_in_env = "frappe_theme" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8')
+		exec_cmd(f"bench get-app {TEST_FRAPPE_APP}", cwd=bench_path)
+		self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", TEST_FRAPPE_APP)))
+		app_installed_in_env = TEST_FRAPPE_APP in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8')
 		self.assertTrue(app_installed_in_env)
 
 
@@ -109,38 +112,38 @@ class TestBenchInit(TestBenchBase):
 		bench_path = os.path.join(self.benches_path, "test-bench")
 
 		self.init_bench(bench_name)
-		bench.utils.exec_cmd("bench setup requirements --node", cwd=bench_path)
-		bench.utils.exec_cmd("bench build", cwd=bench_path)
-		bench.utils.exec_cmd("bench get-app frappe_theme --branch master", cwd=bench_path)
+		exec_cmd("bench setup requirements --node", cwd=bench_path)
+		exec_cmd("bench build", cwd=bench_path)
+		exec_cmd(f"bench get-app {TEST_FRAPPE_APP} --branch master", cwd=bench_path)
 
-		self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "frappe_theme")))
+		self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", TEST_FRAPPE_APP)))
 
 		# check if app is installed
-		app_installed_in_env = "frappe_theme" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8')
+		app_installed_in_env = TEST_FRAPPE_APP in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8')
 		self.assertTrue(app_installed_in_env)
 
 		# create and install app on site
 		self.new_site(site_name, bench_name)
-		installed_app = not bench.utils.exec_cmd(f"bench --site {site_name} install-app frappe_theme", cwd=bench_path)
+		installed_app = not exec_cmd(f"bench --site {site_name} install-app {TEST_FRAPPE_APP}", cwd=bench_path)
 
 		app_installed_on_site = subprocess.check_output(["bench", "--site", site_name, "list-apps"], cwd=bench_path).decode('utf8')
 
 		if installed_app:
-			self.assertTrue("frappe_theme" in app_installed_on_site)
+			self.assertTrue(TEST_FRAPPE_APP in app_installed_on_site)
 
 
 	def test_remove_app(self):
 		self.init_bench("test-bench")
 		bench_path = os.path.join(self.benches_path, "test-bench")
 
-		bench.utils.exec_cmd("bench setup requirements --node", cwd=bench_path)
-		bench.utils.exec_cmd("bench get-app frappe_theme --branch master --overwrite", cwd=bench_path)
-		bench.utils.exec_cmd("bench remove-app frappe_theme", cwd=bench_path)
+		exec_cmd("bench setup requirements --node", cwd=bench_path)
+		exec_cmd(f"bench get-app {TEST_FRAPPE_APP} --branch master --overwrite", cwd=bench_path)
+		exec_cmd(f"bench remove-app {TEST_FRAPPE_APP}", cwd=bench_path)
 
 		with open(os.path.join(bench_path, "sites", "apps.txt")) as f:
-			self.assertFalse("frappe_theme" in f.read())
-		self.assertFalse("frappe_theme" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8'))
-		self.assertFalse(os.path.exists(os.path.join(bench_path, "apps", "frappe_theme")))
+			self.assertFalse(TEST_FRAPPE_APP in f.read())
+		self.assertFalse(TEST_FRAPPE_APP in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8'))
+		self.assertFalse(os.path.exists(os.path.join(bench_path, "apps", TEST_FRAPPE_APP)))
 
 
 	def test_switch_to_branch(self):
@@ -148,12 +151,12 @@ class TestBenchInit(TestBenchBase):
 		bench_path = os.path.join(self.benches_path, "test-bench")
 		app_path = os.path.join(bench_path, "apps", "frappe")
 
-		successful_switch = not bench.utils.exec_cmd("bench switch-to-branch version-13 frappe --upgrade", cwd=bench_path)
+		successful_switch = not exec_cmd("bench switch-to-branch version-13 frappe --upgrade", cwd=bench_path)
 		app_branch_after_switch = str(git.Repo(path=app_path).active_branch)
 		if successful_switch:
 			self.assertEqual("version-13", app_branch_after_switch)
 
-		successful_switch = not bench.utils.exec_cmd("bench switch-to-branch develop frappe --upgrade", cwd=bench_path)
+		successful_switch = not exec_cmd("bench switch-to-branch develop frappe --upgrade", cwd=bench_path)
 		app_branch_after_second_switch = str(git.Repo(path=app_path).active_branch)
 		if successful_switch:
 			self.assertEqual("develop", app_branch_after_second_switch)

bench/tests/test_setup_production.py

@@ -7,7 +7,7 @@ import time
 import unittest
 
 # imports - module imports
-import bench.utils
+from bench.utils import exec_cmd, get_cmd_output, which
 from bench.config.production_setup import get_supervisor_confdir
 from bench.tests.test_base import TestBenchBase
 
@@ -19,18 +19,18 @@ class TestSetupProduction(TestBenchBase):
 		for bench_name in ("test-bench-1", "test-bench-2"):
 			bench_path = os.path.join(os.path.abspath(self.benches_path), bench_name)
 			self.init_bench(bench_name)
-			bench.utils.exec_cmd(f"sudo bench setup production {user} --yes", cwd=bench_path)
+			exec_cmd(f"sudo bench setup production {user} --yes", cwd=bench_path)
 			self.assert_nginx_config(bench_name)
 			self.assert_supervisor_config(bench_name)
 			self.assert_supervisor_process(bench_name)
 
 		self.assert_nginx_process()
-		bench.utils.exec_cmd(f"sudo bench setup sudoers {user}")
+		exec_cmd(f"sudo bench setup sudoers {user}")
 		self.assert_sudoers(user)
 
 		for bench_name in self.benches:
 			bench_path = os.path.join(os.path.abspath(self.benches_path), bench_name)
-			bench.utils.exec_cmd("sudo bench disable-production", cwd=bench_path)
+			exec_cmd("sudo bench disable-production", cwd=bench_path)
 
 
 	def production(self):
@@ -62,14 +62,14 @@ class TestSetupProduction(TestBenchBase):
 
 
 	def assert_nginx_process(self):
-		out = bench.utils.get_cmd_output("sudo nginx -t 2>&1")
+		out = get_cmd_output("sudo nginx -t 2>&1")
 		self.assertTrue("nginx: configuration file /etc/nginx/nginx.conf test is successful" in out)
 
 
 	def assert_sudoers(self, user):
 		sudoers_file = '/etc/sudoers.d/frappe'
-		service = bench.utils.which("service")
-		nginx = bench.utils.which("nginx")
+		service = which("service")
+		nginx = which("nginx")
 
 		self.assertTrue(self.file_exists(sudoers_file))
 
@@ -133,12 +133,12 @@ class TestSetupProduction(TestBenchBase):
 
 
 	def assert_supervisor_process(self, bench_name, use_rq=True, disable_production=False):
-		out = bench.utils.get_cmd_output("supervisorctl status")
+		out = get_cmd_output("supervisorctl status")
 
 		while "STARTING" in out:
 			print ("Waiting for all processes to start...")
 			time.sleep(10)
-			out = bench.utils.get_cmd_output("supervisorctl status")
+			out = get_cmd_output("supervisorctl status")
 
 		tests = [
 			"{bench_name}-web:{bench_name}-frappe-web[\s]+RUNNING",

bench/utils/__init__.py

@@ -0,0 +1,473 @@
+# imports - standard imports
+import json
+import logging
+import os
+import subprocess
+import re
+import sys
+from glob import glob
+from shlex import split
+from typing import List, Tuple
+
+# imports - third party imports
+import click
+
+# imports - module imports
+from bench import PROJECT_NAME, VERSION
+
+from bench.exceptions import CommandFailedError, InvalidRemoteException, ValidationError
+
+
+logger = logging.getLogger(PROJECT_NAME)
+bench_cache_file = ".bench.cmd"
+paths_in_app = ("hooks.py", "modules.txt", "patches.txt")
+paths_in_bench = ("apps", "sites", "config", "logs", "config/pids")
+sudoers_file = "/etc/sudoers.d/frappe"
+
+
+def is_bench_directory(directory=os.path.curdir):
+	is_bench = True
+
+	for folder in paths_in_bench:
+		path = os.path.abspath(os.path.join(directory, folder))
+		is_bench = is_bench and os.path.exists(path)
+
+	return is_bench
+
+
+def is_frappe_app(directory: str) -> bool:
+	is_frappe_app = True
+
+	for folder in paths_in_app:
+		if not is_frappe_app:
+			break
+
+		path = glob(os.path.join(directory, "**", folder))
+		is_frappe_app = is_frappe_app and path
+
+	return bool(is_frappe_app)
+
+
+def log(message, level=0, no_log=False):
+	import bench
+	import bench.cli
+
+	levels = {
+		0: ("blue", "INFO"),  # normal
+		1: ("green", "SUCCESS"),  # success
+		2: ("red", "ERROR"),  # fail
+		3: ("yellow", "WARN"),  # warn/suggest
+	}
+
+	color, prefix = levels.get(level, levels[0])
+
+	if bench.cli.from_command_line and bench.cli.dynamic_feed:
+		bench.LOG_BUFFER.append(
+			{"prefix": prefix, "message": message, "color": color}
+		)
+
+	if no_log:
+		click.secho(message, fg=color)
+	else:
+		loggers = {2: logger.error, 3: logger.warning}
+		level_logger = loggers.get(level, logger.info)
+
+		level_logger(message)
+		click.secho(f"{prefix}: {message}", fg=color)
+
+
+def check_latest_version():
+	if VERSION.endswith("dev"):
+		return
+
+	import requests
+	from semantic_version import Version
+
+	try:
+		pypi_request = requests.get("https://pypi.org/pypi/frappe-bench/json")
+	except Exception:
+		# Exceptions thrown are defined in requests.exceptions
+		# ignore checking on all Exceptions
+		return
+
+	if pypi_request.status_code == 200:
+		pypi_version_str = pypi_request.json().get("info").get("version")
+		pypi_version = Version(pypi_version_str)
+		local_version = Version(VERSION)
+
+		if pypi_version > local_version:
+			log(f"A newer version of bench is available: {local_version} → {pypi_version}")
+
+
+def pause_exec(seconds=10):
+	from time import sleep
+
+	for i in range(seconds, 0, -1):
+		print(f"Will continue execution in {i} seconds...", end="\r")
+		sleep(1)
+
+	print(" " * 40, end="\r")
+
+
+def exec_cmd(cmd, cwd=".", env=None, _raise=True):
+	if env:
+		env.update(os.environ.copy())
+
+	click.secho(f"$ {cmd}", fg="bright_black")
+
+	cwd_info = f"cd {cwd} && " if cwd != "." else ""
+	cmd_log = f"{cwd_info}{cmd}"
+	logger.debug(cmd_log)
+	cmd = split(cmd)
+	return_code = subprocess.call(cmd, cwd=cwd, universal_newlines=True, env=env)
+	if return_code:
+		logger.warning(f"{cmd_log} executed with exit code {return_code}")
+		if _raise:
+			raise CommandFailedError
+	return return_code
+
+
+def which(executable: str, raise_err: bool = False) -> str:
+	from shutil import which
+
+	exec_ = which(executable)
+
+	if not exec_ and raise_err:
+		raise ValueError(f"{executable} not found.")
+
+	return exec_
+
+
+def setup_logging(bench_path=".") -> "logger":
+	LOG_LEVEL = 15
+	logging.addLevelName(LOG_LEVEL, "LOG")
+
+	def logv(self, message, *args, **kws):
+		if self.isEnabledFor(LOG_LEVEL):
+			self._log(LOG_LEVEL, message, args, **kws)
+
+	logging.Logger.log = logv
+
+	if os.path.exists(os.path.join(bench_path, "logs")):
+		log_file = os.path.join(bench_path, "logs", "bench.log")
+		hdlr = logging.FileHandler(log_file)
+	else:
+		hdlr = logging.NullHandler()
+
+	logger = logging.getLogger(PROJECT_NAME)
+	formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s")
+	hdlr.setFormatter(formatter)
+	logger.addHandler(hdlr)
+	logger.setLevel(logging.DEBUG)
+
+	return logger
+
+
+def get_process_manager() -> str:
+	for proc_man in ["honcho", "foreman", "forego"]:
+		proc_man_path = which(proc_man)
+		if proc_man_path:
+			return proc_man_path
+
+
+def get_git_version() -> float:
+	"""returns git version from `git --version`
+	extracts version number from string `get version 1.9.1` etc"""
+	version = get_cmd_output("git --version")
+	version = version.strip().split()[2]
+	version = ".".join(version.split(".")[0:2])
+	return float(version)
+
+
+def get_cmd_output(cmd, cwd=".", _raise=True):
+	output = ""
+	try:
+		output = subprocess.check_output(
+			cmd, cwd=cwd, shell=True, stderr=subprocess.PIPE, encoding="utf-8"
+		).strip()
+	except subprocess.CalledProcessError as e:
+		if e.output:
+			output = e.output
+		elif _raise:
+			raise
+	return output
+
+
+def is_root():
+	return os.getuid() == 0
+
+
+def run_frappe_cmd(*args, **kwargs):
+	from bench.cli import from_command_line
+	from bench.utils.bench import get_env_cmd
+
+	bench_path = kwargs.get("bench_path", ".")
+	f = get_env_cmd("python", bench_path=bench_path)
+	sites_dir = os.path.join(bench_path, "sites")
+
+	is_async = False if from_command_line else True
+	if is_async:
+		stderr = stdout = subprocess.PIPE
+	else:
+		stderr = stdout = None
+
+	p = subprocess.Popen(
+		(f, "-m", "frappe.utils.bench_helper", "frappe") + args,
+		cwd=sites_dir,
+		stdout=stdout,
+		stderr=stderr,
+	)
+
+	if is_async:
+		return_code = print_output(p)
+	else:
+		return_code = p.wait()
+
+	if return_code > 0:
+		sys.exit(return_code)
+
+
+def print_output(p):
+	from select import select
+
+	while p.poll() is None:
+		readx = select([p.stdout.fileno(), p.stderr.fileno()], [], [])[0]
+		send_buffer = []
+		for fd in readx:
+			if fd == p.stdout.fileno():
+				while 1:
+					buf = p.stdout.read(1)
+					if not len(buf):
+						break
+					if buf == "\r" or buf == "\n":
+						send_buffer.append(buf)
+						log_line("".join(send_buffer), "stdout")
+						send_buffer = []
+					else:
+						send_buffer.append(buf)
+
+			if fd == p.stderr.fileno():
+				log_line(p.stderr.readline(), "stderr")
+	return p.poll()
+
+
+def log_line(data, stream):
+	if stream == "stderr":
+		return sys.stderr.write(data)
+	return sys.stdout.write(data)
+
+
+def get_bench_name(bench_path):
+	return os.path.basename(os.path.abspath(bench_path))
+
+
+def set_git_remote_url(git_url, bench_path="."):
+	"Set app remote git url"
+	from bench.app import get_repo_dir
+	from bench.bench import Bench
+
+	app = git_url.rsplit("/", 1)[1].rsplit(".", 1)[0]
+
+	if app not in Bench(bench_path).apps:
+		raise ValidationError(f"No app named {app}")
+
+	app_dir = get_repo_dir(app, bench_path=bench_path)
+
+	if os.path.exists(os.path.join(app_dir, ".git")):
+		exec_cmd(f"git remote set-url upstream {git_url}", cwd=app_dir)
+
+
+def run_playbook(playbook_name, extra_vars=None, tag=None):
+	import bench
+
+	if not which("ansible"):
+		print(
+			"Ansible is needed to run this command, please install it using 'pip"
+			" install ansible'"
+		)
+		sys.exit(1)
+	args = ["ansible-playbook", "-c", "local", playbook_name, "-vvvv"]
+
+	if extra_vars:
+		args.extend(["-e", json.dumps(extra_vars)])
+
+	if tag:
+		args.extend(["-t", tag])
+
+	subprocess.check_call(args, cwd=os.path.join(bench.__path__[0], "playbooks"))
+
+
+def find_benches(directory: str = None) -> List:
+	if not directory:
+		directory = os.path.expanduser("~")
+	elif os.path.exists(directory):
+		directory = os.path.abspath(directory)
+	else:
+		log("Directory doesn't exist", level=2)
+		sys.exit(1)
+
+	if is_bench_directory(directory):
+		if os.path.curdir == directory:
+			print("You are in a bench directory!")
+		else:
+			print(f"{directory} is a bench directory!")
+		return
+
+	benches = []
+	for sub in os.listdir(directory):
+		sub = os.path.join(directory, sub)
+		if os.path.isdir(sub) and not os.path.islink(sub):
+			if is_bench_directory(sub):
+				print(f"{sub} found!")
+				benches.append(sub)
+			else:
+				benches.extend(find_benches(sub))
+
+	return benches
+
+
+def is_dist_editable(dist: str) -> bool:
+	"""Is distribution an editable install?"""
+	for path_item in sys.path:
+		egg_link = os.path.join(path_item, f"{dist}.egg-link")
+		if os.path.isfile(egg_link):
+			return True
+	return False
+
+
+def find_parent_bench(path: str) -> str:
+	"""Checks if parent directories are benches"""
+	if is_bench_directory(directory=path):
+		return path
+
+	home_path = os.path.expanduser("~")
+	root_path = os.path.abspath(os.sep)
+
+	if path not in {home_path, root_path}:
+		# NOTE: the os.path.split assumes that given path is absolute
+		parent_dir = os.path.split(path)[0]
+		return find_parent_bench(parent_dir)
+
+
+def generate_command_cache(bench_path=".") -> List:
+	"""Caches all available commands (even custom apps) via Frappe
+	Default caching behaviour: generated the first time any command (for a specific bench directory)
+	"""
+	from bench.utils.bench import get_env_cmd
+
+	python = get_env_cmd("python", bench_path=bench_path)
+	sites_path = os.path.join(bench_path, "sites")
+
+	if os.path.exists(bench_cache_file):
+		os.remove(bench_cache_file)
+
+	try:
+		output = get_cmd_output(
+			f"{python} -m frappe.utils.bench_helper get-frappe-commands", cwd=sites_path
+		)
+		with open(bench_cache_file, "w") as f:
+			json.dump(eval(output), f)
+		return json.loads(output)
+
+	except subprocess.CalledProcessError as e:
+		if hasattr(e, "stderr"):
+			print(e.stderr)
+
+	return []
+
+
+def clear_command_cache(bench_path="."):
+	"""Clears commands cached
+	Default invalidation behaviour: destroyed on each run of `bench update`
+	"""
+
+	if os.path.exists(bench_cache_file):
+		os.remove(bench_cache_file)
+	else:
+		print("Bench command cache doesn't exist in this folder!")
+
+
+def find_org(org_repo):
+	import requests
+
+	org_repo = org_repo[0]
+
+	for org in ["frappe", "erpnext"]:
+		res = requests.head(f"https://api.github.com/repos/{org}/{org_repo}")
+		if res.ok:
+			return org, org_repo
+
+	raise InvalidRemoteException
+
+
+def fetch_details_from_tag(_tag: str) -> Tuple[str, str, str]:
+	if not _tag:
+		raise Exception("Tag is not provided")
+
+	app_tag = _tag.split("@")
+	org_repo = app_tag[0].split("/")
+
+	try:
+		repo, tag = app_tag
+	except ValueError:
+		repo, tag = app_tag + [None]
+
+	try:
+		org, repo = org_repo
+	except Exception:
+		org, repo = find_org(org_repo)
+
+	return org, repo, tag
+
+
+def is_git_url(url: str) -> bool:
+	# modified to allow without the tailing .git from https://github.com/jonschlinkert/is-git-url.git
+	pattern = r"(?:git|ssh|https?|\w*@[-\w.]+):(\/\/)?(.*?)(\.git)?(\/?|\#[-\d\w._]+?)$"
+	return bool(re.match(pattern, url))
+
+
+def drop_privileges(uid_name="nobody", gid_name="nogroup"):
+	import grp
+	import pwd
+
+	# from http://stackoverflow.com/a/2699996
+	if os.getuid() != 0:
+		# We're not root so, like, whatever dude
+		return
+
+	# Get the uid/gid from the name
+	running_uid = pwd.getpwnam(uid_name).pw_uid
+	running_gid = grp.getgrnam(gid_name).gr_gid
+
+	# Remove group privileges
+	os.setgroups([])
+
+	# Try setting the new uid/gid
+	os.setgid(running_gid)
+	os.setuid(running_uid)
+
+	# Ensure a very conservative umask
+	os.umask(0o22)
+
+
+def get_available_folder_name(name: str, path: str) -> str:
+	"""Subfixes the passed name with -1 uptil -100 whatever's available"""
+	if os.path.exists(os.path.join(path, name)):
+		for num in range(1, 100):
+			_dt = f"{name}_{num}"
+			if not os.path.exists(os.path.join(path, _dt)):
+				return _dt
+	return name
+
+
+def get_traceback() -> str:
+	"""Returns the traceback of the Exception"""
+	from traceback import format_exception
+
+	exc_type, exc_value, exc_tb = sys.exc_info()
+
+	if not any([exc_type, exc_value, exc_tb]):
+		return ""
+
+	trace_list = format_exception(exc_type, exc_value, exc_tb)
+	return "".join(trace_list)

bench/utils/app.py

@@ -0,0 +1,225 @@
+import os
+import re
+from setuptools.config import read_configuration
+import sys
+import subprocess
+from bench.exceptions import (
+	InvalidRemoteException,
+	InvalidBranchException,
+	CommandFailedError,
+)
+from bench.app import get_repo_dir
+
+
+def is_version_upgrade(app="frappe", bench_path=".", branch=None):
+	upstream_version = get_upstream_version(app=app, branch=branch, bench_path=bench_path)
+
+	if not upstream_version:
+		raise InvalidBranchException(
+			f"Specified branch of app {app} is not in upstream remote"
+		)
+
+	local_version = get_major_version(get_current_version(app, bench_path=bench_path))
+	upstream_version = get_major_version(upstream_version)
+
+	if upstream_version > local_version:
+		return (True, local_version, upstream_version)
+
+	return (False, local_version, upstream_version)
+
+
+def switch_branch(branch, apps=None, bench_path=".", upgrade=False, check_upgrade=True):
+	import git
+	from bench.bench import Bench
+	from bench.utils import log, exec_cmd
+	from bench.utils.bench import (
+		build_assets,
+		patch_sites,
+		post_upgrade,
+	)
+	from bench.utils.system import backup_all_sites
+
+	apps_dir = os.path.join(bench_path, "apps")
+	version_upgrade = (False,)
+	switched_apps = []
+
+	if not apps:
+		apps = [
+			name for name in os.listdir(apps_dir) if os.path.isdir(os.path.join(apps_dir, name))
+		]
+
+	for app in apps:
+		app_dir = os.path.join(apps_dir, app)
+
+		if not os.path.exists(app_dir):
+			log(f"{app} does not exist!", level=2)
+			continue
+
+		repo = git.Repo(app_dir)
+		unshallow_flag = os.path.exists(os.path.join(app_dir, ".git", "shallow"))
+		log(f"Fetching upstream {'unshallow ' if unshallow_flag else ''}for {app}")
+
+		exec_cmd("git remote set-branches upstream  '*'", cwd=app_dir)
+		exec_cmd(
+			f"git fetch --all{' --unshallow' if unshallow_flag else ''} --quiet", cwd=app_dir
+		)
+
+		if check_upgrade:
+			version_upgrade = is_version_upgrade(app=app, bench_path=bench_path, branch=branch)
+			if version_upgrade[0] and not upgrade:
+				log(
+					f"Switching to {branch} will cause upgrade from"
+					f" {version_upgrade[1]} to {version_upgrade[2]}. Pass --upgrade to"
+					" confirm",
+					level=2,
+				)
+				sys.exit(1)
+
+		print("Switching for " + app)
+		exec_cmd(f"git checkout -f {branch}", cwd=app_dir)
+
+		if str(repo.active_branch) == branch:
+			switched_apps.append(app)
+		else:
+			log(f"Switching branches failed for: {app}", level=2)
+
+	if switched_apps:
+		log(f"Successfully switched branches for: {', '.join(switched_apps)}", level=1)
+		print(
+			"Please run `bench update --patch` to be safe from any differences in"
+			" database schema"
+		)
+
+	if version_upgrade[0] and upgrade:
+		Bench(bench_path).setup.requirements()
+		backup_all_sites()
+		patch_sites()
+		build_assets()
+		post_upgrade(version_upgrade[1], version_upgrade[2])
+
+
+def switch_to_branch(branch=None, apps=None, bench_path=".", upgrade=False):
+	switch_branch(branch, apps=apps, bench_path=bench_path, upgrade=upgrade)
+
+
+def switch_to_develop(apps=None, bench_path=".", upgrade=True):
+	switch_branch("develop", apps=apps, bench_path=bench_path, upgrade=upgrade)
+
+
+def get_version_from_string(contents, field="__version__"):
+	match = re.search(r"^(\s*%s\s*=\s*['\\\"])(.+?)(['\"])(?sm)" % field, contents)
+	return match.group(2)
+
+
+def get_major_version(version):
+	import semantic_version
+
+	return semantic_version.Version(version).major
+
+
+def get_develop_version(app, bench_path="."):
+	repo_dir = get_repo_dir(app, bench_path=bench_path)
+	with open(os.path.join(repo_dir, os.path.basename(repo_dir), "hooks.py")) as f:
+		return get_version_from_string(f.read(), field="develop_version")
+
+
+def get_upstream_version(app, branch=None, bench_path="."):
+	repo_dir = get_repo_dir(app, bench_path=bench_path)
+	if not branch:
+		branch = get_current_branch(app, bench_path=bench_path)
+
+	try:
+		subprocess.call(
+			f"git fetch --depth=1 --no-tags upstream {branch}", shell=True, cwd=repo_dir
+		)
+	except CommandFailedError:
+		raise InvalidRemoteException(f"Failed to fetch from remote named upstream for {app}")
+
+	try:
+		contents = subprocess.check_output(
+			f"git show upstream/{branch}:{app}/__init__.py",
+			shell=True,
+			cwd=repo_dir,
+			stderr=subprocess.STDOUT,
+		)
+		contents = contents.decode("utf-8")
+	except subprocess.CalledProcessError as e:
+		if b"Invalid object" in e.output:
+			return None
+		else:
+			raise
+	return get_version_from_string(contents)
+
+
+def get_current_frappe_version(bench_path="."):
+	try:
+		return get_major_version(get_current_version("frappe", bench_path=bench_path))
+	except IOError:
+		return 0
+
+
+def get_current_branch(app, bench_path="."):
+	from bench.utils import get_cmd_output
+
+	repo_dir = get_repo_dir(app, bench_path=bench_path)
+	return get_cmd_output("basename $(git symbolic-ref -q HEAD)", cwd=repo_dir)
+
+
+def get_remote(app, bench_path="."):
+	repo_dir = get_repo_dir(app, bench_path=bench_path)
+	contents = subprocess.check_output(
+		["git", "remote", "-v"], cwd=repo_dir, stderr=subprocess.STDOUT
+	)
+	contents = contents.decode("utf-8")
+	if re.findall(r"upstream[\s]+", contents):
+		return "upstream"
+	elif not contents:
+		# if contents is an empty string => remote doesn't exist
+		return False
+	else:
+		# get the first remote
+		return contents.splitlines()[0].split()[0]
+
+
+def get_app_name(bench_path, repo_name):
+	app_name = None
+	apps_path = os.path.join(os.path.abspath(bench_path), "apps")
+	config_path = os.path.join(apps_path, repo_name, "setup.cfg")
+	if os.path.exists(config_path):
+		config = read_configuration(config_path)
+		app_name = config.get("metadata", {}).get("name")
+
+	if not app_name:
+		# retrieve app name from setup.py as fallback
+		app_path = os.path.join(apps_path, repo_name, "setup.py")
+		with open(app_path, "rb") as f:
+			app_name = re.search(r'name\s*=\s*[\'"](.*)[\'"]', f.read().decode("utf-8")).group(1)
+
+	if app_name and repo_name != app_name:
+		os.rename(os.path.join(apps_path, repo_name), os.path.join(apps_path, app_name))
+		return app_name
+
+	return repo_name
+
+
+def get_current_version(app, bench_path="."):
+	current_version = None
+	repo_dir = get_repo_dir(app, bench_path=bench_path)
+	config_path = os.path.join(repo_dir, "setup.cfg")
+	init_path = os.path.join(repo_dir, os.path.basename(repo_dir), "__init__.py")
+	setup_path = os.path.join(repo_dir, "setup.py")
+
+	try:
+		if os.path.exists(config_path):
+			config = read_configuration(config_path)
+			current_version = config.get("metadata", {}).get("version")
+		if not current_version:
+			with open(init_path) as f:
+				current_version = get_version_from_string(f.read())
+
+	except AttributeError:
+		# backward compatibility
+		with open(setup_path) as f:
+			current_version = get_version_from_string(f.read(), field="version")
+
+	return current_version

bench/utils/bench.py

@@ -0,0 +1,593 @@
+# imports - standard imports
+import json
+import logging
+import os
+import re
+import subprocess
+import sys
+from json.decoder import JSONDecodeError
+import typing
+
+# imports - third party imports
+import click
+import bench
+
+# imports - module imports
+from bench.utils import (
+	which,
+	log,
+	exec_cmd,
+	get_bench_name,
+	get_cmd_output,
+)
+from bench.exceptions import PatchError, ValidationError
+
+
+if typing.TYPE_CHECKING:
+	from bench.bench import Bench
+
+logger = logging.getLogger(bench.PROJECT_NAME)
+
+
+def get_env_cmd(cmd, bench_path="."):
+	return os.path.abspath(os.path.join(bench_path, "env", "bin", cmd))
+
+
+def get_venv_path():
+	venv = which("virtualenv")
+
+	if not venv:
+		current_python = sys.executable
+		with open(os.devnull, "wb") as devnull:
+			is_venv_installed = not subprocess.call(
+				[current_python, "-m", "venv", "--help"], stdout=devnull
+			)
+		if is_venv_installed:
+			venv = f"{current_python} -m venv"
+
+	return venv or log("virtualenv cannot be found", level=2)
+
+
+def update_node_packages(bench_path="."):
+	print("Updating node packages...")
+	from bench.utils.app import get_develop_version
+	from distutils.version import LooseVersion
+
+	v = LooseVersion(get_develop_version("frappe", bench_path=bench_path))
+
+	# After rollup was merged, frappe_version = 10.1
+	# if develop_verion is 11 and up, only then install yarn
+	if v < LooseVersion("11.x.x-develop"):
+		update_npm_packages(bench_path)
+	else:
+		update_yarn_packages(bench_path)
+
+
+def install_python_dev_dependencies(bench_path=".", apps=None, verbose=False):
+	import bench.cli
+	from bench.bench import Bench
+
+	verbose = bench.cli.verbose or verbose
+	quiet_flag = "" if verbose else "--quiet"
+
+	bench = Bench(bench_path)
+
+	if isinstance(apps, str):
+		apps = [apps]
+	elif apps is None:
+		apps = [app for app in bench.apps if app not in bench.excluded_apps]
+
+	for app in apps:
+		app_path = os.path.join(bench_path, "apps", app)
+
+		dev_requirements_path = os.path.join(app_path, "dev-requirements.txt")
+
+		if os.path.exists(dev_requirements_path):
+			bench.run(f"{bench.python} -m pip install {quiet_flag} --upgrade -r {dev_requirements_path}")
+
+
+def update_yarn_packages(bench_path="."):
+	from bench.bench import Bench
+
+	bench = Bench(bench_path)
+	apps = [app for app in bench.apps if app not in bench.excluded_apps]
+	apps_dir = os.path.join(bench.name, "apps")
+
+	# TODO: Check for stuff like this early on only??
+	if not which("yarn"):
+		print("Please install yarn using below command and try again.")
+		print("`npm install -g yarn`")
+		return
+
+	for app in apps:
+		app_path = os.path.join(apps_dir, app)
+		if os.path.exists(os.path.join(app_path, "package.json")):
+			click.secho(f"\nInstalling node dependencies for {app}", fg="yellow")
+			bench.run("yarn install", cwd=app_path)
+
+
+def update_npm_packages(bench_path="."):
+	apps_dir = os.path.join(bench_path, "apps")
+	package_json = {}
+
+	for app in os.listdir(apps_dir):
+		package_json_path = os.path.join(apps_dir, app, "package.json")
+
+		if os.path.exists(package_json_path):
+			with open(package_json_path, "r") as f:
+				app_package_json = json.loads(f.read())
+				# package.json is usually a dict in a dict
+				for key, value in app_package_json.items():
+					if key not in package_json:
+						package_json[key] = value
+					else:
+						if isinstance(value, dict):
+							package_json[key].update(value)
+						elif isinstance(value, list):
+							package_json[key].extend(value)
+						else:
+							package_json[key] = value
+
+	if package_json is {}:
+		with open(os.path.join(os.path.dirname(__file__), "package.json"), "r") as f:
+			package_json = json.loads(f.read())
+
+	with open(os.path.join(bench_path, "package.json"), "w") as f:
+		f.write(json.dumps(package_json, indent=1, sort_keys=True))
+
+	exec_cmd("npm install", cwd=bench_path)
+
+
+def migrate_env(python, backup=False):
+	import shutil
+	from urllib.parse import urlparse
+	from bench.bench import Bench
+
+	bench = Bench(".")
+	nvenv = "env"
+	path = os.getcwd()
+	python = which(python)
+	virtualenv = which("virtualenv")
+	pvenv = os.path.join(path, nvenv)
+
+	# Clear Cache before Bench Dies.
+	try:
+		config = bench.conf
+		rredis = urlparse(config["redis_cache"])
+		redis = f"{which('redis-cli')} -p {rredis.port}"
+
+		logger.log("Clearing Redis Cache...")
+		exec_cmd(f"{redis} FLUSHALL")
+		logger.log("Clearing Redis DataBase...")
+		exec_cmd(f"{redis} FLUSHDB")
+	except Exception:
+		logger.warning("Please ensure Redis Connections are running or Daemonized.")
+
+	# Backup venv: restore using `virtualenv --relocatable` if needed
+	if backup:
+		from datetime import datetime
+
+		parch = os.path.join(path, "archived", "envs")
+		if not os.path.exists(parch):
+			os.mkdir(parch)
+
+		source = os.path.join(path, "env")
+		target = parch
+
+		logger.log("Backing up Virtual Environment")
+		stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+		dest = os.path.join(path, str(stamp))
+
+		os.rename(source, dest)
+		shutil.move(dest, target)
+
+	# Create virtualenv using specified python
+	venv_creation, packages_setup = 1, 1
+	try:
+		logger.log(f"Setting up a New Virtual {python} Environment")
+		venv_creation = exec_cmd(f"{virtualenv} --python {python} {pvenv}")
+
+		apps = " ".join([f"-e {os.path.join('apps', app)}" for app in bench.apps])
+		packages_setup = exec_cmd(f"{pvenv} -m pip install --upgrade {apps}")
+
+		logger.log(f"Migration Successful to {python}")
+	except Exception:
+		if venv_creation or packages_setup:
+			logger.warning("Migration Error")
+
+
+def validate_upgrade(from_ver, to_ver, bench_path="."):
+	if to_ver >= 6:
+		if not which("npm") and not (which("node") or which("nodejs")):
+			raise Exception("Please install nodejs and npm")
+
+
+def post_upgrade(from_ver, to_ver, bench_path="."):
+	from bench.config import redis
+	from bench.config.supervisor import generate_supervisor_config
+	from bench.config.nginx import make_nginx_conf
+	from bench.bench import Bench
+
+	conf = Bench(bench_path).conf
+	print("-" * 80 + f"Your bench was upgraded to version {to_ver}")
+
+	if conf.get("restart_supervisor_on_update"):
+		redis.generate_config(bench_path=bench_path)
+		generate_supervisor_config(bench_path=bench_path)
+		make_nginx_conf(bench_path=bench_path)
+		print(
+			"As you have setup your bench for production, you will have to reload"
+			" configuration for nginx and supervisor. To complete the migration, please"
+			" run the following commands:\nsudo service nginx restart\nsudo"
+			" supervisorctl reload"
+		)
+
+
+def patch_sites(bench_path="."):
+	from bench.bench import Bench
+	from bench.utils.system import migrate_site
+
+	bench = Bench(bench_path)
+
+	for site in bench.sites:
+		try:
+			migrate_site(site, bench_path=bench_path)
+		except subprocess.CalledProcessError:
+			raise PatchError
+
+
+def restart_supervisor_processes(bench_path=".", web_workers=False):
+	from bench.bench import Bench
+
+	bench = Bench(bench_path)
+	conf = bench.conf
+	cmd = conf.get("supervisor_restart_cmd")
+	bench_name = get_bench_name(bench_path)
+
+	if cmd:
+		bench.run(cmd)
+
+	else:
+		supervisor_status = get_cmd_output("supervisorctl status", cwd=bench_path)
+
+		if web_workers and f"{bench_name}-web:" in supervisor_status:
+			group = f"{bench_name}-web:\t"
+
+		elif f"{bench_name}-workers:" in supervisor_status:
+			group = f"{bench_name}-workers: {bench_name}-web:"
+
+		# backward compatibility
+		elif f"{bench_name}-processes:" in supervisor_status:
+			group = f"{bench_name}-processes:"
+
+		# backward compatibility
+		else:
+			group = "frappe:"
+
+		bench.run(f"supervisorctl restart {group}")
+
+
+def restart_systemd_processes(bench_path=".", web_workers=False):
+	bench_name = get_bench_name(bench_path)
+	exec_cmd(
+		f"sudo systemctl stop -- $(systemctl show -p Requires {bench_name}.target | cut"
+		" -d= -f2)"
+	)
+	exec_cmd(
+		f"sudo systemctl start -- $(systemctl show -p Requires {bench_name}.target |"
+		" cut -d= -f2)"
+	)
+
+
+def restart_process_manager(bench_path="."):
+	# only overmind has the restart feature, not sure other supported procmans do
+	if which("overmind") and os.path.exists(
+		os.path.join(bench_path, ".overmind.sock")
+	):
+		exec_cmd("overmind restart", cwd=bench_path)
+
+
+def build_assets(bench_path=".", app=None):
+	command = "bench build"
+	if app:
+		command += f" --app {app}"
+	exec_cmd(command, cwd=bench_path, env={"BENCH_DEVELOPER": "1"})
+
+
+def handle_version_upgrade(version_upgrade, bench_path, force, reset, conf):
+	from bench.utils import pause_exec, log
+
+	if version_upgrade[0]:
+		if force:
+			log(
+				"""Force flag has been used for a major version change in Frappe and it's apps.
+This will take significant time to migrate and might break custom apps.""",
+				level=3,
+			)
+		else:
+			print(
+				f"""This update will cause a major version change in Frappe/ERPNext from {version_upgrade[1]} to {version_upgrade[2]}.
+This would take significant time to migrate and might break custom apps."""
+			)
+			click.confirm("Do you want to continue?", abort=True)
+
+	if not reset and conf.get("shallow_clone"):
+		log(
+			"""shallow_clone is set in your bench config.
+However without passing the --reset flag, your repositories will be unshallowed.
+To avoid this, cancel this operation and run `bench update --reset`.
+
+Consider the consequences of `git reset --hard` on your apps before you run that.
+To avoid seeing this warning, set shallow_clone to false in your common_site_config.json
+		""",
+			level=3,
+		)
+		pause_exec(seconds=10)
+
+	if version_upgrade[0] or (not version_upgrade[0] and force):
+		validate_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path)
+
+
+def update(
+	pull: bool = False,
+	apps: str = None,
+	patch: bool = False,
+	build: bool = False,
+	requirements: bool = False,
+	backup: bool = True,
+	compile: bool = True,
+	force: bool = False,
+	reset: bool = False,
+	restart_supervisor: bool = False,
+	restart_systemd: bool = False,
+):
+	"""command: bench update"""
+	import re
+	from bench import patches
+
+	from bench.app import pull_apps
+	from bench.bench import Bench
+	from bench.config.common_site_config import update_config
+	from bench.exceptions import CannotUpdateReleaseBench
+
+	from bench.utils import clear_command_cache
+	from bench.utils.app import is_version_upgrade
+	from bench.utils.bench import (
+		restart_supervisor_processes,
+		restart_systemd_processes,
+	)
+	from bench.utils.system import backup_all_sites
+
+	bench_path = os.path.abspath(".")
+	bench = Bench(bench_path)
+	patches.run(bench_path=bench_path)
+	conf = bench.conf
+
+	clear_command_cache(bench_path=".")
+
+	if conf.get("release_bench"):
+		raise CannotUpdateReleaseBench("Release bench detected, cannot update!")
+
+	if not (pull or patch or build or requirements):
+		pull, patch, build, requirements = True, True, True, True
+
+	if apps and pull:
+		apps = [app.strip() for app in re.split(",| ", apps) if app]
+	else:
+		apps = []
+
+	validate_branch()
+
+	version_upgrade = is_version_upgrade()
+	handle_version_upgrade(version_upgrade, bench_path, force, reset, conf)
+
+	conf.update({"maintenance_mode": 1, "pause_scheduler": 1})
+	update_config(conf, bench_path=bench_path)
+
+	if backup:
+		print("Backing up sites...")
+		backup_all_sites(bench_path=bench_path)
+
+	if pull:
+		print("Updating apps source...")
+		pull_apps(apps=apps, bench_path=bench_path, reset=reset)
+
+	if requirements:
+		print("Setting up requirements...")
+		bench.setup.requirements()
+
+	if patch:
+		print("Patching sites...")
+		patch_sites(bench_path=bench_path)
+
+	if build:
+		print("Building assets...")
+		bench.build()
+
+	if version_upgrade[0] or (not version_upgrade[0] and force):
+		post_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path)
+
+	if pull and compile:
+		from compileall import compile_dir
+
+		print("Compiling Python files...")
+		apps_dir = os.path.join(bench_path, "apps")
+		compile_dir(apps_dir, quiet=1, rx=re.compile(".*node_modules.*"))
+
+	if restart_supervisor or conf.get("restart_supervisor_on_update"):
+		restart_supervisor_processes(bench_path=bench_path)
+
+	if restart_systemd or conf.get("restart_systemd_on_update"):
+		restart_systemd_processes(bench_path=bench_path)
+
+	conf.update({"maintenance_mode": 0, "pause_scheduler": 0})
+	update_config(conf, bench_path=bench_path)
+
+	print(
+		"_" * 80
+		+ "\nBench: Deployment tool for Frappe and Frappe Applications"
+		" (https://frappe.io/bench).\nOpen source depends on your contributions, so do"
+		" give back by submitting bug reports, patches and fixes and be a part of the"
+		" community :)"
+	)
+
+
+def clone_apps_from(bench_path, clone_from, update_app=True):
+	from bench.app import install_app
+
+	print(f"Copying apps from {clone_from}...")
+	subprocess.check_output(["cp", "-R", os.path.join(clone_from, "apps"), bench_path])
+
+	node_modules_path = os.path.join(clone_from, "node_modules")
+	if os.path.exists(node_modules_path):
+		print(f"Copying node_modules from {clone_from}...")
+		subprocess.check_output(["cp", "-R", node_modules_path, bench_path])
+
+	def setup_app(app):
+		# run git reset --hard in each branch, pull latest updates and install_app
+		app_path = os.path.join(bench_path, "apps", app)
+
+		# remove .egg-ino
+		subprocess.check_output(["rm", "-rf", app + ".egg-info"], cwd=app_path)
+
+		if update_app and os.path.exists(os.path.join(app_path, ".git")):
+			remotes = subprocess.check_output(["git", "remote"], cwd=app_path).strip().split()
+			if "upstream" in remotes:
+				remote = "upstream"
+			else:
+				remote = remotes[0]
+			print(f"Cleaning up {app}")
+			branch = subprocess.check_output(
+				["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=app_path
+			).strip()
+			subprocess.check_output(["git", "reset", "--hard"], cwd=app_path)
+			subprocess.check_output(["git", "pull", "--rebase", remote, branch], cwd=app_path)
+
+		install_app(app, bench_path, restart_bench=False)
+
+	with open(os.path.join(clone_from, "sites", "apps.txt"), "r") as f:
+		apps = f.read().splitlines()
+
+	for app in apps:
+		setup_app(app)
+
+
+def remove_backups_crontab(bench_path="."):
+	from crontab import CronTab
+	from bench.bench import Bench
+
+	logger.log("removing backup cronjob")
+
+	bench_dir = os.path.abspath(bench_path)
+	user = Bench(bench_dir).conf.get("frappe_user")
+	logfile = os.path.join(bench_dir, "logs", "backup.log")
+	system_crontab = CronTab(user=user)
+	backup_command = f"cd {bench_dir} && {sys.argv[0]} --verbose --site all backup"
+	job_command = f"{backup_command} >> {logfile} 2>&1"
+
+	system_crontab.remove_all(command=job_command)
+
+
+def set_mariadb_host(host, bench_path="."):
+	update_common_site_config({"db_host": host}, bench_path=bench_path)
+
+
+def set_redis_cache_host(host, bench_path="."):
+	update_common_site_config({"redis_cache": f"redis://{host}"}, bench_path=bench_path)
+
+
+def set_redis_queue_host(host, bench_path="."):
+	update_common_site_config({"redis_queue": f"redis://{host}"}, bench_path=bench_path)
+
+
+def set_redis_socketio_host(host, bench_path="."):
+	update_common_site_config({"redis_socketio": f"redis://{host}"}, bench_path=bench_path)
+
+
+def update_common_site_config(ddict, bench_path="."):
+	filename = os.path.join(bench_path, "sites", "common_site_config.json")
+
+	if os.path.exists(filename):
+		with open(filename, "r") as f:
+			content = json.load(f)
+
+	else:
+		content = {}
+
+	content.update(ddict)
+	with open(filename, "w") as f:
+		json.dump(content, f, indent=1, sort_keys=True)
+
+
+def validate_app_installed_on_sites(app, bench_path="."):
+	print("Checking if app installed on active sites...")
+	ret = check_app_installed(app, bench_path=bench_path)
+
+	if ret is None:
+		check_app_installed_legacy(app, bench_path=bench_path)
+	else:
+		return ret
+
+
+def check_app_installed(app, bench_path="."):
+	try:
+		out = subprocess.check_output(
+			["bench", "--site", "all", "list-apps", "--format", "json"],
+			stderr=open(os.devnull, "wb"),
+			cwd=bench_path,
+		).decode("utf-8")
+	except subprocess.CalledProcessError:
+		return None
+
+	try:
+		apps_sites_dict = json.loads(out)
+	except JSONDecodeError:
+		return None
+
+	for site, apps in apps_sites_dict.items():
+		if app in apps:
+			raise ValidationError(f"Cannot remove, app is installed on site: {site}")
+
+
+def check_app_installed_legacy(app, bench_path="."):
+	site_path = os.path.join(bench_path, "sites")
+
+	for site in os.listdir(site_path):
+		req_file = os.path.join(site_path, site, "site_config.json")
+		if os.path.exists(req_file):
+			out = subprocess.check_output(
+				["bench", "--site", site, "list-apps"], cwd=bench_path
+			).decode("utf-8")
+			if re.search(r"\b" + app + r"\b", out):
+				print(f"Cannot remove, app is installed on site: {site}")
+				sys.exit(1)
+
+
+def validate_branch():
+	from bench.bench import Bench
+	from bench.utils.app import get_current_branch
+
+	apps = Bench(".").apps
+
+	installed_apps = set(apps)
+	check_apps = set(["frappe", "erpnext"])
+	intersection_apps = installed_apps.intersection(check_apps)
+
+	for app in intersection_apps:
+		branch = get_current_branch(app)
+
+		if branch == "master":
+			print(
+				"""'master' branch is renamed to 'version-11' since 'version-12' release.
+As of January 2020, the following branches are
+version		Frappe			ERPNext
+11		version-11		version-11
+12		version-12		version-12
+13		version-13		version-13
+14		develop			develop
+
+Please switch to new branches to get future updates.
+To switch to your required branch, run the following commands: bench switch-to-branch [branch-name]"""
+			)
+
+			sys.exit(1)

bench/utils/cli.py

@@ -0,0 +1,70 @@
+import click
+from click.core import _check_multicommand
+
+
+def print_bench_version(ctx, param, value):
+	"""Prints current bench version"""
+	if not value or ctx.resilient_parsing:
+		return
+
+	import bench
+
+	click.echo(bench.VERSION)
+	ctx.exit()
+
+
+class MultiCommandGroup(click.Group):
+	def add_command(self, cmd, name=None):
+		"""Registers another :class:`Command` with this group.  If the name
+		is not provided, the name of the command is used.
+
+		Note: This is a custom Group that allows passing a list of names for
+		the command name.
+		"""
+		name = name or cmd.name
+		if name is None:
+			raise TypeError("Command has no name.")
+		_check_multicommand(self, name, cmd, register=True)
+
+		try:
+			self.commands[name] = cmd
+		except TypeError:
+			if isinstance(name, list):
+				for _name in name:
+					self.commands[_name] = cmd
+
+
+def use_experimental_feature(ctx, param, value):
+	if not value:
+		return
+
+	if value == "dynamic-feed":
+		import bench.cli
+
+		bench.cli.dynamic_feed = True
+		bench.cli.verbose = True
+	else:
+		from bench.exceptions import FeatureDoesNotExistError
+
+		raise FeatureDoesNotExistError(f"Feature {value} does not exist")
+
+	from bench.cli import is_envvar_warn_set
+
+	if is_envvar_warn_set:
+		return
+
+	click.secho(
+		"WARNING: bench is using it's new CLI rendering engine. This behaviour has"
+		f" been enabled by passing --{value} in the command. This feature is"
+		" experimental and may not be implemented for all commands yet.",
+		fg="yellow",
+	)
+
+
+def setup_verbosity(ctx, param, value):
+	if not value:
+		return
+
+	import bench.cli
+
+	bench.cli.verbose = True

bench/utils/render.py

@@ -0,0 +1,113 @@
+# imports - standard imports
+import sys
+from io import StringIO
+
+# imports - third party imports
+import click
+
+# imports - module imports
+import bench
+
+
+class Capturing(list):
+	"""
+	Util to consume the stdout encompassed in it and push it to a list
+
+	with Capturing() as output:
+		subprocess.check_output("ls", shell=True)
+
+	print(output)
+	# ["b'Applications\\nDesktop\\nDocuments\\nDownloads\\n'"]
+	"""
+
+	def __enter__(self):
+		self._stdout = sys.stdout
+		sys.stdout = self._stringio = StringIO()
+		return self
+
+	def __exit__(self, *args):
+		self.extend(self._stringio.getvalue().splitlines())
+		del self._stringio  # free up some memory
+		sys.stdout = self._stdout
+
+
+class Rendering:
+	def __init__(self, success, title, is_parent, args, kwargs):
+		import bench.cli
+
+		self.dynamic_feed = bench.cli.from_command_line and bench.cli.dynamic_feed
+
+		if not self.dynamic_feed:
+			return
+
+		try:
+			self.kw = args[0].__dict__
+		except Exception:
+			self.kw = kwargs
+
+		self.is_parent = is_parent
+		self.title = title
+		self.success = success
+
+	def __enter__(self, *args, **kwargs):
+		if not self.dynamic_feed:
+			return
+
+		_prefix = click.style('⏼', fg='bright_yellow')
+		_hierarchy = "  " if not self.is_parent else ""
+		self._title = self.title.format(**self.kw)
+		click.secho(f"{_hierarchy}{_prefix} {self._title}")
+
+		bench.LOG_BUFFER.append(
+			{"message": self._title, "prefix": _prefix, "color": None, "is_parent": self.is_parent}
+		)
+
+	def __exit__(self, *args, **kwargs):
+		if not self.dynamic_feed:
+			return
+
+		self._prefix = click.style('✔', fg='green')
+		self._success = self.success.format(**self.kw)
+
+		self.render_screen()
+
+	def render_screen(self):
+		click.clear()
+
+		for l in bench.LOG_BUFFER:
+			if l["message"] == self._title:
+				l["prefix"] = self._prefix
+				l["message"] = self._success
+			_hierarchy = "  " if not l["is_parent"] else ""
+			click.secho(f'{_hierarchy}{l["prefix"]} {l["message"]}', fg=l["color"])
+
+
+def job(title: str = None, success: str = None):
+	"""Supposed to be wrapped around an atomic job in a given process.
+	For instance, the `get-app` command consists of two jobs: `initializing bench`
+	and `fetching and installing app`.
+	"""
+	def innfn(fn):
+		def wrapper_fn(*args, **kwargs):
+			with Rendering(
+				success=success, title=title, is_parent=True, args=args, kwargs=kwargs,
+			):
+				return fn(*args, **kwargs)
+
+		return wrapper_fn
+	return innfn
+
+
+def step(title: str = None, success: str = None):
+	"""Supposed to be wrapped around the smallest possible atomic step in a given operation.
+	For instance, `building assets` is a step in the update operation.
+	"""
+	def innfn(fn):
+		def wrapper_fn(*args, **kwargs):
+			with Rendering(
+				success=success, title=title, is_parent=False, args=args, kwargs=kwargs,
+			):
+				return fn(*args, **kwargs)
+
+		return wrapper_fn
+	return innfn

bench/utils/system.py

@@ -0,0 +1,201 @@
+# imports - standard imports
+import grp
+import os
+import pwd
+import shutil
+import sys
+
+# imports - module imports
+import bench
+from bench.utils import (
+	exec_cmd,
+	get_process_manager,
+	log,
+	run_frappe_cmd,
+	sudoers_file,
+	which,
+)
+from bench.utils.bench import build_assets, clone_apps_from
+from bench.utils.render import job
+
+
+@job(title="Initializing Bench {path}", success="Bench {path} initialized")
+def init(
+	path,
+	apps_path=None,
+	no_procfile=False,
+	no_backups=False,
+	frappe_path=None,
+	frappe_branch=None,
+	verbose=False,
+	clone_from=None,
+	skip_redis_config_generation=False,
+	clone_without_update=False,
+	skip_assets=False,
+	python="python3",
+	install_app=None,
+):
+	"""Initialize a new bench directory
+
+	* create a bench directory in the given path
+	* setup logging for the bench
+	* setup env for the bench
+	* setup config (dir/pids/redis/procfile) for the bench
+	* setup patches.txt for bench
+	* clone & install frappe
+		* install python & node dependencies
+		* build assets
+	* setup backups crontab
+	"""
+
+	# Use print("\033c", end="") to clear entire screen after each step and re-render each list
+	# another way => https://stackoverflow.com/a/44591228/10309266
+
+	import bench.cli
+	from bench.app import get_app, install_apps_from_path
+	from bench.bench import Bench
+
+	verbose = bench.cli.verbose or verbose
+
+	bench = Bench(path)
+
+	bench.setup.dirs()
+	bench.setup.logging()
+	bench.setup.env(python=python)
+	bench.setup.config(redis=not skip_redis_config_generation, procfile=not no_procfile)
+	bench.setup.patches()
+
+	# local apps
+	if clone_from:
+		clone_apps_from(
+			bench_path=path, clone_from=clone_from, update_app=not clone_without_update
+		)
+
+	# remote apps
+	else:
+		frappe_path = frappe_path or "https://github.com/frappe/frappe.git"
+
+		get_app(
+			frappe_path, branch=frappe_branch, bench_path=path, skip_assets=True, verbose=verbose
+		)
+
+		# fetch remote apps using config file - deprecate this!
+		if apps_path:
+			install_apps_from_path(apps_path, bench_path=path)
+
+	# getting app on bench init using --install-app
+	if install_app:
+		get_app(
+			install_app, branch=frappe_branch, bench_path=path, skip_assets=True, verbose=verbose
+		)
+
+	if not skip_assets:
+		build_assets(bench_path=path)
+
+	if not no_backups:
+		bench.setup.backups()
+
+
+def setup_sudoers(user):
+	if not os.path.exists("/etc/sudoers.d"):
+		os.makedirs("/etc/sudoers.d")
+
+		set_permissions = False
+		if not os.path.exists("/etc/sudoers"):
+			set_permissions = True
+
+		with open("/etc/sudoers", "a") as f:
+			f.write("\n#includedir /etc/sudoers.d\n")
+
+		if set_permissions:
+			os.chmod("/etc/sudoers", 0o440)
+
+	template = bench.config.env().get_template("frappe_sudoers")
+	frappe_sudoers = template.render(
+		**{
+			"user": user,
+			"service": which("service"),
+			"systemctl": which("systemctl"),
+			"nginx": which("nginx"),
+		}
+	)
+
+	with open(sudoers_file, "w") as f:
+		f.write(frappe_sudoers)
+
+	os.chmod(sudoers_file, 0o440)
+	log(f"Sudoers was set up for user {user}", level=1)
+
+
+def start(no_dev=False, concurrency=None, procfile=None, no_prefix=False, procman=None):
+	if procman:
+		program = which(procman)
+	else:
+		program = get_process_manager()
+
+	if not program:
+		raise Exception("No process manager found")
+
+	os.environ["PYTHONUNBUFFERED"] = "true"
+	if not no_dev:
+		os.environ["DEV_SERVER"] = "true"
+
+	command = [program, "start"]
+	if concurrency:
+		command.extend(["-c", concurrency])
+
+	if procfile:
+		command.extend(["-f", procfile])
+
+	if no_prefix:
+		command.extend(["--no-prefix"])
+
+	os.execv(program, command)
+
+
+def migrate_site(site, bench_path="."):
+	run_frappe_cmd("--site", site, "migrate", bench_path=bench_path)
+
+
+def backup_site(site, bench_path="."):
+	run_frappe_cmd("--site", site, "backup", bench_path=bench_path)
+
+
+def backup_all_sites(bench_path="."):
+	from bench.bench import Bench
+
+	for site in Bench(bench_path).sites:
+		backup_site(site, bench_path=bench_path)
+
+
+def fix_prod_setup_perms(bench_path=".", frappe_user=None):
+	from glob import glob
+	from bench.bench import Bench
+
+	frappe_user = frappe_user or Bench(bench_path).conf.get("frappe_user")
+
+	if not frappe_user:
+		print("frappe user not set")
+		sys.exit(1)
+
+	globs = ["logs/*", "config/*"]
+	for glob_name in globs:
+		for path in glob(glob_name):
+			uid = pwd.getpwnam(frappe_user).pw_uid
+			gid = grp.getgrnam(frappe_user).gr_gid
+			os.chown(path, uid, gid)
+
+
+def setup_fonts():
+	fonts_path = os.path.join("/tmp", "fonts")
+
+	if os.path.exists("/etc/fonts_backup"):
+		return
+
+	exec_cmd("git clone https://github.com/frappe/fonts.git", cwd="/tmp")
+	os.rename("/etc/fonts", "/etc/fonts_backup")
+	os.rename("/usr/share/fonts", "/usr/share/fonts_backup")
+	os.rename(os.path.join(fonts_path, "etc_fonts"), "/etc/fonts")
+	os.rename(os.path.join(fonts_path, "usr_share_fonts"), "/usr/share/fonts")
+	shutil.rmtree(fonts_path)
+	exec_cmd("fc-cache -fv")

bench/utils/translation.py

@@ -0,0 +1,58 @@
+# imports - standard imports
+import itertools
+import json
+import os
+
+
+def update_translations_p(args):
+	import requests
+
+	try:
+		update_translations(*args)
+	except requests.exceptions.HTTPError:
+		print("Download failed for", args[0], args[1])
+
+
+def download_translations_p():
+	import multiprocessing
+
+	pool = multiprocessing.Pool(multiprocessing.cpu_count())
+
+	langs = get_langs()
+	apps = ("frappe", "erpnext")
+	args = list(itertools.product(apps, langs))
+
+	pool.map(update_translations_p, args)
+
+
+def download_translations():
+	langs = get_langs()
+	apps = ("frappe", "erpnext")
+	for app, lang in itertools.product(apps, langs):
+		update_translations(app, lang)
+
+
+def get_langs():
+	lang_file = "apps/frappe/frappe/geo/languages.json"
+	with open(lang_file) as f:
+		langs = json.loads(f.read())
+	return [d["code"] for d in langs]
+
+
+def update_translations(app, lang):
+	import requests
+
+	translations_dir = os.path.join("apps", app, app, "translations")
+	csv_file = os.path.join(translations_dir, lang + ".csv")
+	url = f"https://translate.erpnext.com/files/{app}-{lang}.csv"
+	r = requests.get(url, stream=True)
+	r.raise_for_status()
+
+	with open(csv_file, "wb") as f:
+		for chunk in r.iter_content(chunk_size=1024):
+			# filter out keep-alive new chunks
+			if chunk:
+				f.write(chunk)
+				f.flush()
+
+	print("downloaded for", app, lang)

setup.py

@@ -1,22 +1,43 @@
 from setuptools import find_packages, setup
 from bench import PROJECT_NAME, VERSION
 
-with open('requirements.txt') as f:
-	install_requires = f.read().strip().split('\n')
+with open("requirements.txt") as f:
+	install_requires = f.read().strip().split("\n")
+
+with open("README.md") as f:
+	long_description = f.read()
 
 setup(
 	name=PROJECT_NAME,
-	description='CLI to manage Multi-tenant deployments for Frappe apps',
-	author='Frappe Technologies',
-	author_email='info@frappe.io',
+	description="CLI to manage Multi-tenant deployments for Frappe apps",
+	long_description=long_description,
+	long_description_content_type="text/markdown",
 	version=VERSION,
+	license="GPLv3",
+	author="Frappe Technologies Pvt Ltd",
+	author_email="developers@frappe.io",
+	url="https://frappe.io/bench",
+	project_urls={
+		"Documentation": "https://frappeframework.com/docs/user/en/bench",
+		"Source": "https://github.com/frappe/bench",
+		"Changelog": "https://github.com/frappe/bench/releases",
+	},
+	classifiers=[
+		"Development Status :: 5 - Production/Stable",
+		"Environment :: Console",
+		"Framework :: Frappe Framework",
+		"License :: OSI Approved :: GNU Affero General Public License v3",
+		"Natural Language :: English",
+		"Operating System :: MacOS",
+		"Operating System :: OS Independent",
+		"Topic :: Software Development :: Build Tools",
+		"Topic :: Software Development :: User Interfaces",
+		"Topic :: System :: Installation/Setup",
+	],
 	packages=find_packages(),
-	python_requires='~=3.6',
+	python_requires="~=3.6",
 	zip_safe=False,
 	include_package_data=True,
 	install_requires=install_requires,
-	entry_points='''
-[console_scripts]
-bench=bench.cli:cli
-''',
+	entry_points={"console_scripts": ["bench=bench.cli:cli"]},
 )