Merge branch 'develop' into staging

bench/app.py

@@ -9,6 +9,7 @@ import subprocess
 import sys
 import typing
 from datetime import date
+from urllib.parse import urlparse
 
 # imports - third party imports
 import click
@@ -62,6 +63,9 @@ class AppMeta:
 		self.from_apps = False
 		self.is_url = False
 		self.branch = branch
+		self.mount_path = os.path.abspath(
+			os.path.join(urlparse(self.name).netloc, urlparse(self.name).path)
+		)
 		self.setup_details()
 
 	def setup_details(self):
@@ -75,7 +79,7 @@ class AppMeta:
 			self._setup_details_from_installed_apps()
 
 		# fetch meta for repo on mounted disk
-		elif os.path.exists(self.name):
+		elif os.path.exists(self.mount_path):
 			self.on_disk = True
 			self._setup_details_from_mounted_disk()
 
@@ -91,7 +95,9 @@ class AppMeta:
 			self._setup_details_from_name_tag()
 
 	def _setup_details_from_mounted_disk(self):
-		self.org, self.repo, self.tag = os.path.split(self.name)[-2:] + (self.branch,)
+		self.org, self.repo, self.tag = os.path.split(self.mount_path)[-2:] + (
+			self.branch,
+		)
 
 	def _setup_details_from_name_tag(self):
 		self.org, self.repo, self.tag = fetch_details_from_tag(self.name)
@@ -122,7 +128,7 @@ class AppMeta:
 			return os.path.abspath(os.path.join("apps", self.name))
 
 		if self.on_disk:
-			return os.path.abspath(self.name)
+			return self.mount_path
 
 		if self.is_url:
 			return self.name
@@ -173,7 +179,7 @@ class App(AppMeta):
 		shutil.move(active_app_path, archived_app_path)
 
 	@step(title="Installing App {repo}", success="App {repo} Installed")
-	def install(self, skip_assets=False, verbose=False):
+	def install(self, skip_assets=False, verbose=False, restart_bench=True):
 		import bench.cli
 		from bench.utils.app import get_app_name
 
@@ -190,7 +196,11 @@ class App(AppMeta):
 		)
 
 		install_app(
-			app=app_name, bench_path=self.bench.name, verbose=verbose, skip_assets=skip_assets,
+			app=app_name,
+			bench_path=self.bench.name,
+			verbose=verbose,
+			skip_assets=skip_assets,
+			restart_bench=restart_bench
 		)
 
 	@step(title="Uninstalling App {repo}", success="App {repo} Uninstalled")
@@ -310,6 +320,7 @@ def get_app(
 	repo_name = app.repo
 	branch = app.tag
 	bench_setup = False
+	restart_bench = not init_bench
 
 	if not is_bench_directory(bench_path):
 		if not init_bench:
@@ -333,7 +344,6 @@ def get_app(
 			"color": None,
 		})
 
-
 	cloned_path = os.path.join(bench_path, "apps", repo_name)
 	dir_already_exists = os.path.isdir(cloned_path)
 	to_clone = not dir_already_exists
@@ -358,7 +368,7 @@ def get_app(
 		or overwrite
 		or click.confirm("Do you want to reinstall the existing application?")
 	):
-		app.install(verbose=verbose, skip_assets=skip_assets)
+		app.install(verbose=verbose, skip_assets=skip_assets, restart_bench=restart_bench)
 
 
 def new_app(app, no_git=None, bench_path="."):

bench/bench.py

@@ -143,6 +143,14 @@ class Bench(Base, Validator):
 		if systemd and conf.get("restart_systemd_on_update"):
 			restart_systemd_processes(bench_path=self.name, web_workers=web)
 
+	def get_installed_apps(self) -> List:
+		"""Returns list of installed apps on bench, not in excluded_apps.txt
+		"""
+		apps = [app for app in self.apps if app not in self.excluded_apps]
+		apps.remove("frappe")
+		apps.insert(0, "frappe")
+		return apps
+
 
 class BenchApps(MutableSequence):
 	def __init__(self, bench: Bench):
@@ -317,35 +325,29 @@ class BenchSetup(Base):
 
 		logger.log("backups were set up")
 
-	def __get_installed_apps(self) -> List:
-		"""Returns list of installed apps on bench, not in excluded_apps.txt
-		"""
-		apps = [app for app in self.bench.apps if app not in self.bench.excluded_apps]
-		apps.remove("frappe")
-		apps.insert(0, "frappe")
-		return apps
-
 	@job(title="Setting Up Bench Dependencies", success="Bench Dependencies Set Up")
-	def requirements(self):
+	def requirements(self, apps=None):
-		"""Install and upgrade all installed apps on given Bench
+		"""Install and upgrade specified / all installed apps on given Bench
 		"""
 		from bench.app import App
 
-		apps = self.__get_installed_apps()
+		if not apps:
+			apps = self.bench.get_installed_apps()
 
 		self.pip()
 
 		print(f"Installing {len(apps)} applications...")
 
 		for app in apps:
-			App(app, bench=self.bench, to_clone=False).install()
+			App(app, bench=self.bench, to_clone=False).install( skip_assets=True, restart_bench=False)
 
-	def python(self):
+	def python(self, apps=None):
-		"""Install and upgrade Python dependencies for installed apps on given Bench
+		"""Install and upgrade Python dependencies for specified / all installed apps on given Bench
 		"""
 		import bench.cli
 
-		apps = self.__get_installed_apps()
+		if not apps:
+			apps = self.bench.get_installed_apps()
 
 		quiet_flag = "" if bench.cli.verbose else "--quiet"
 
@@ -356,12 +358,12 @@ class BenchSetup(Base):
 			log(f"\nInstalling python dependencies for {app}", level=3, no_log=True)
 			self.run(f"{self.bench.python} -m pip install {quiet_flag} --upgrade -e {app_path}")
 
-	def node(self):
+	def node(self, apps=None):
-		"""Install and upgrade Node dependencies for all apps on given Bench
+		"""Install and upgrade Node dependencies for specified / all apps on given Bench
 		"""
 		from bench.utils.bench import update_node_packages
 
-		return update_node_packages(bench_path=self.bench.name)
+		return update_node_packages(bench_path=self.bench.name, apps=apps)
 
 
 class BenchTearDown:

bench/commands/setup.py

@@ -6,7 +6,7 @@ import sys
 import click
 
 # imports - module imports
-from bench.utils import exec_cmd, run_playbook
+from bench.utils import exec_cmd, run_playbook, which
 
 
 @click.group(help="Setup command group for enabling setting up a Frappe environment")
@@ -41,9 +41,14 @@ def reload_nginx():
 @click.option("--yes", help="Yes to regeneration of supervisor config", is_flag=True, default=False)
 @click.option("--skip-redis", help="Skip redis configuration", is_flag=True, default=False)
 def setup_supervisor(user=None, yes=False, skip_redis=False):
+	from bench.utils import get_cmd_output
 	from bench.config.supervisor import update_supervisord_config, generate_supervisor_config
 
-	update_supervisord_config(user=user, yes=yes)
+	which("supervisorctl", raise_err=True)
+
+	if "Permission denied" in get_cmd_output("supervisorctl status"):
+		update_supervisord_config(user=user, yes=yes)
+
 	generate_supervisor_config(bench_path=".", user=user, yes=yes, skip_redis=skip_redis)
 
 
@@ -131,27 +136,33 @@ def setup_procfile():
 def setup_socketio():
 	return
 
-@click.command("requirements", help="Setup Python and Node dependencies")
+@click.command("requirements")
 @click.option("--node", help="Update only Node packages", default=False, is_flag=True)
 @click.option("--python", help="Update only Python packages", default=False, is_flag=True)
 @click.option("--dev", help="Install optional python development dependencies", default=False, is_flag=True)
-def setup_requirements(node=False, python=False, dev=False):
+@click.argument("apps", nargs=-1)
+def setup_requirements(node=False, python=False, dev=False, apps=None):
+	"""
+	Setup Python and Node dependencies.
+
+	You can optionally specify one or more apps to setup dependencies for.
+	"""
 	from bench.bench import Bench
 
 	bench = Bench(".")
 
 	if not (node or python or dev):
-		bench.setup.requirements()
+		bench.setup.requirements(apps=apps)
 
 	elif not node and not dev:
-		bench.setup.python()
+		bench.setup.python(apps=apps)
 
 	elif not python and not dev:
-		bench.setup.node()
+		bench.setup.node(apps=apps)
 
 	else:
 		from bench.utils.bench import install_python_dev_dependencies
-		install_python_dev_dependencies()
+		install_python_dev_dependencies(apps=apps)
 
 		if node:
 			click.secho("--dev flag only supports python dependencies. All node development dependencies are installed by default.", fg="yellow")

bench/commands/utils.py

@@ -23,6 +23,9 @@ def start(no_dev, concurrency, procfile, no_prefix, man):
 @click.option('--systemd', is_flag=True, default=False)
 def restart(web, supervisor, systemd):
 	from bench.bench import Bench
+	if not systemd and not web:
+		supervisor = True
+
 	Bench(".").reload(web, supervisor, systemd)
 
 

bench/config/supervisor.py

@@ -109,7 +109,9 @@ def update_supervisord_config(user=None, yes=False):
 			supervisord_conf_changes += '\n' + action
 
 	if not supervisord_conf_changes:
-		logger.log("supervisord.conf not updated")
+		logger.error("supervisord.conf not updated")
+		contents = "\n".join(f"{x}={y}" for x, y in updated_values.items())
+		print(f"Update your {supervisord_conf} with the following values:\n[{section}]\n{contents}")
 		return
 
 	if not yes:

bench/tests/test_base.py

@@ -14,10 +14,14 @@ from bench.utils import paths_in_bench, exec_cmd
 from bench.utils.system import init
 from bench.bench import Bench
 
-if sys.version_info.major == 2:
+PYTHON_VER = sys.version_info
-	FRAPPE_BRANCH = "version-12"
+
-else:
+FRAPPE_BRANCH = "version-12"
-	FRAPPE_BRANCH = "develop"
+if PYTHON_VER.major == 3:
+	if PYTHON_VER.minor in [6, 7]:
+		FRAPPE_BRANCH = "version-13"
+	else:
+		FRAPPE_BRANCH = "develop"
 
 class TestBenchBase(unittest.TestCase):
 	def setUp(self):

bench/tests/test_init.py

@@ -10,6 +10,7 @@ import git
 # imports - module imports
 from bench.utils import exec_cmd
 from bench.release import get_bumped_version
+from bench.app import App
 from bench.tests.test_base import FRAPPE_BRANCH, TestBenchBase
 
 
@@ -34,9 +35,10 @@ class TestBenchInit(TestBenchBase):
 	def test_utils(self):
 		self.assertEqual(subprocess.call("bench"), 0)
 
-
 	def test_init(self, bench_name="test-bench", **kwargs):
 		self.init_bench(bench_name, **kwargs)
+		app = App("file:///tmp/frappe")
+		self.assertEqual(app.url, "/tmp/frappe")
 		self.assert_folders(bench_name)
 		self.assert_virtual_env(bench_name)
 		self.assert_config(bench_name)
@@ -151,15 +153,21 @@ class TestBenchInit(TestBenchBase):
 		bench_path = os.path.join(self.benches_path, "test-bench")
 		app_path = os.path.join(bench_path, "apps", "frappe")
 
-		successful_switch = not exec_cmd("bench switch-to-branch version-13 frappe --upgrade", cwd=bench_path)
+		# * chore: change to 14 when avalible
+		prevoius_branch = "version-13"
+		if FRAPPE_BRANCH != "develop":
+			# assuming we follow `version-#`
+			prevoius_branch = f"version-{int(FRAPPE_BRANCH.split('-')[1]) - 1}"
+
+		successful_switch = not exec_cmd(f"bench switch-to-branch {prevoius_branch} frappe --upgrade", cwd=bench_path)
 		app_branch_after_switch = str(git.Repo(path=app_path).active_branch)
 		if successful_switch:
-			self.assertEqual("version-13", app_branch_after_switch)
+			self.assertEqual(prevoius_branch, app_branch_after_switch)
 
-		successful_switch = not exec_cmd("bench switch-to-branch develop frappe --upgrade", cwd=bench_path)
+		successful_switch = not exec_cmd(f"bench switch-to-branch {FRAPPE_BRANCH} frappe --upgrade", cwd=bench_path)
 		app_branch_after_second_switch = str(git.Repo(path=app_path).active_branch)
 		if successful_switch:
-			self.assertEqual("develop", app_branch_after_second_switch)
+			self.assertEqual(FRAPPE_BRANCH, app_branch_after_second_switch)
 
 
 if __name__ == '__main__':

bench/utils/__init__.py

@@ -133,7 +133,7 @@ def which(executable: str, raise_err: bool = False) -> str:
 	exec_ = which(executable)
 
 	if not exec_ and raise_err:
-		raise ValueError(f"{executable} not found.")
+		raise FileNotFoundError(f"{executable} not found in PATH")
 
 	return exec_
 
@@ -314,7 +314,13 @@ def find_benches(directory: str = None) -> List:
 		return
 
 	benches = []
-	for sub in os.listdir(directory):
+
+	try:
+		sub_directories = os.listdir(directory)
+	except PermissionError:
+		return benches
+
+	for sub in sub_directories:
 		sub = os.path.join(directory, sub)
 		if os.path.isdir(sub) and not os.path.islink(sub):
 			if is_bench_directory(sub):

bench/utils/bench.py

@@ -48,7 +48,7 @@ def get_venv_path():
 	return venv or log("virtualenv cannot be found", level=2)
 
 
-def update_node_packages(bench_path="."):
+def update_node_packages(bench_path=".", apps=None):
 	print("Updating node packages...")
 	from bench.utils.app import get_develop_version
 	from distutils.version import LooseVersion
@@ -58,9 +58,9 @@ def update_node_packages(bench_path="."):
 	# After rollup was merged, frappe_version = 10.1
 	# if develop_verion is 11 and up, only then install yarn
 	if v < LooseVersion("11.x.x-develop"):
-		update_npm_packages(bench_path)
+		update_npm_packages(bench_path, apps=apps)
 	else:
-		update_yarn_packages(bench_path)
+		update_yarn_packages(bench_path, apps=apps)
 
 
 def install_python_dev_dependencies(bench_path=".", apps=None, verbose=False):
@@ -75,7 +75,7 @@ def install_python_dev_dependencies(bench_path=".", apps=None, verbose=False):
 	if isinstance(apps, str):
 		apps = [apps]
 	elif apps is None:
-		apps = [app for app in bench.apps if app not in bench.excluded_apps]
+		apps = bench.get_installed_apps()
 
 	for app in apps:
 		app_path = os.path.join(bench_path, "apps", app)
@@ -86,11 +86,14 @@ def install_python_dev_dependencies(bench_path=".", apps=None, verbose=False):
 			bench.run(f"{bench.python} -m pip install {quiet_flag} --upgrade -r {dev_requirements_path}")
 
 
-def update_yarn_packages(bench_path="."):
+def update_yarn_packages(bench_path=".", apps=None):
 	from bench.bench import Bench
 
 	bench = Bench(bench_path)
-	apps = [app for app in bench.apps if app not in bench.excluded_apps]
+
+	if not apps:
+		apps = bench.get_installed_apps()
+
 	apps_dir = os.path.join(bench.name, "apps")
 
 	# TODO: Check for stuff like this early on only??
@@ -106,11 +109,14 @@ def update_yarn_packages(bench_path="."):
 			bench.run("yarn install", cwd=app_path)
 
 
-def update_npm_packages(bench_path="."):
+def update_npm_packages(bench_path=".", apps=None):
 	apps_dir = os.path.join(bench_path, "apps")
 	package_json = {}
 
-	for app in os.listdir(apps_dir):
+	if not apps:
+		apps = os.listdir(apps_dir)
+
+	for app in apps:
 		package_json_path = os.path.join(apps_dir, app, "package.json")
 
 		if os.path.exists(package_json_path):
@@ -168,8 +174,7 @@ def migrate_env(python, backup=False):
 		from datetime import datetime
 
 		parch = os.path.join(path, "archived", "envs")
-		if not os.path.exists(parch):
+		os.makedirs(parch, exist_ok=True)
-			os.mkdir(parch)
 
 		source = os.path.join(path, "env")
 		target = parch
@@ -248,7 +253,15 @@ def restart_supervisor_processes(bench_path=".", web_workers=False):
 		bench.run(cmd)
 
 	else:
-		supervisor_status = get_cmd_output("supervisorctl status", cwd=bench_path)
+		sudo = ""
+		try:
+			supervisor_status = get_cmd_output("supervisorctl status", cwd=bench_path)
+		except Exception as e:
+			if e.returncode == 127:
+				log("restart failed: Couldn't find supervisorctl in PATH", level=3)
+				return
+			sudo = "sudo "
+			supervisor_status = get_cmd_output("sudo supervisorctl status", cwd=bench_path)
 
 		if web_workers and f"{bench_name}-web:" in supervisor_status:
 			group = f"{bench_name}-web:\t"
@@ -264,7 +277,7 @@ def restart_supervisor_processes(bench_path=".", web_workers=False):
 		else:
 			group = "frappe:"
 
-		bench.run(f"supervisorctl restart {group}")
+		bench.run(f"{sudo}supervisorctl restart {group}")
 
 
 def restart_systemd_processes(bench_path=".", web_workers=False):

requirements.txt

@@ -1,7 +1,7 @@
-Click
+Click>=7.0
 GitPython~=2.1.15
 honcho
-Jinja2~=2.11.3
+Jinja2~=3.0.3
 python-crontab~=2.4.0
 requests
 semantic-version~=2.8.2