Merge pull request #1257 from Aradhya-Tripathi/dependency-resolution

feat: Resolve and install

.travis.yml

@@ -48,17 +48,17 @@ matrix:
   - name: "Python 3.7 Tests"
     python: 3.7
     env: TEST=bench
-    script: python -m unittest -v bench.tests.test_init
+    script: python -m unittest -v bench.tests.test_utils && python -m unittest -v bench.tests.test_init
 
   - name: "Python 3.8 Tests"
     python: 3.8
     env: TEST=bench
-    script: python -m unittest -v bench.tests.test_init
+    script: python -m unittest -v bench.tests.test_utils && python -m unittest -v bench.tests.test_init
 
   - name: "Python 3.9 Tests"
     python: 3.9
     env: TEST=bench
-    script: python -m unittest -v bench.tests.test_init
+    script: python -m unittest -v bench.tests.test_utils && python -m unittest -v bench.tests.test_init
 
   - name: "Python 3.7 Easy Install"
     python: 3.7

bench/app.py

@@ -8,6 +8,7 @@ import shutil
 import subprocess
 import sys
 import typing
+from collections import OrderedDict
 from datetime import date
 from urllib.parse import urlparse
 
@@ -22,6 +23,7 @@ from bench.utils import (
 	get_available_folder_name,
 	is_bench_directory,
 	is_git_url,
+	is_valid_frappe_branch,
 	log,
 	run_frappe_cmd,
 )
@@ -55,7 +57,7 @@ class AppMeta:
 		class Healthcare(AppConfig):
 			dependencies = [{"frappe/erpnext": "~13.17.0"}]
 		"""
-		self.name = name.rstrip('/')
+		self.name = name.rstrip("/")
 		self.remote_server = "github.com"
 		self.to_clone = to_clone
 		self.on_disk = False
@@ -95,9 +97,7 @@ class AppMeta:
 			self._setup_details_from_name_tag()
 
 	def _setup_details_from_mounted_disk(self):
-		self.org, self.repo, self.tag = os.path.split(self.mount_path)[-2:] + (
+		self.org, self.repo, self.tag = os.path.split(self.mount_path)[-2:] + (self.branch,)
-			self.branch,
-		)
 
 	def _setup_details_from_name_tag(self):
 		self.org, self.repo, self.tag = fetch_details_from_tag(self.name)
@@ -147,10 +147,10 @@ class AppMeta:
 
 @functools.lru_cache(maxsize=None)
 class App(AppMeta):
-	def __init__(
+	def __init__(self, name: str, branch: str = None, bench: "Bench" = None, *args, **kwargs):
-		self, name: str, branch: str = None, bench: "Bench" = None, *args, **kwargs
-	):
 		self.bench = bench
+		self.required_by = None
+		self.local_resolution = []
 		super().__init__(name, branch, *args, **kwargs)
 
 	@step(title="Fetching App {repo}", success="App {repo} Fetched")
@@ -171,42 +171,90 @@ class App(AppMeta):
 	def remove(self):
 		active_app_path = os.path.join("apps", self.repo)
 		archived_path = os.path.join("archived", "apps")
-		archived_name = get_available_folder_name(
+		archived_name = get_available_folder_name(f"{self.repo}-{date.today()}", archived_path)
-			f"{self.repo}-{date.today()}", archived_path
-		)
 		archived_app_path = os.path.join(archived_path, archived_name)
 		log(f"App moved from {active_app_path} to {archived_app_path}")
 		shutil.move(active_app_path, archived_app_path)
 
 	@step(title="Installing App {repo}", success="App {repo} Installed")
-	def install(self, skip_assets=False, verbose=False, restart_bench=True):
+	def install(
+		self,
+		skip_assets=False,
+		verbose=False,
+		resolved=False,
+		restart_bench=True,
+		ignore_resolution=False,
+	):
 		import bench.cli
 		from bench.utils.app import get_app_name
 
 		verbose = bench.cli.verbose or verbose
 		app_name = get_app_name(self.bench.name, self.repo)
-
+		if not resolved and self.repo != "frappe" and not ignore_resolution:
-		# TODO: this should go inside install_app only tho - issue: default/resolved branch
+			click.secho(
-		setup_app_dependencies(
+				f"Ignoring dependencies of {self.name}. To install dependencies use --resolve-deps",
-			repo_name=self.repo,
+				fg="yellow",
-			bench_path=self.bench.name,
+			)
-			branch=self.tag,
-			verbose=verbose,
-			skip_assets=skip_assets,
-		)
 
 		install_app(
 			app=app_name,
+			tag=self.tag,
 			bench_path=self.bench.name,
 			verbose=verbose,
 			skip_assets=skip_assets,
-			restart_bench=restart_bench
+			restart_bench=restart_bench,
+			resolution=self.local_resolution
 		)
 
+	@step(title="Cloning and installing {repo}", success="App {repo} Installed")
+	def install_resolved_apps(self, *args, **kwargs):
+		self.get()
+		self.install(*args, **kwargs, resolved=True)
+
 	@step(title="Uninstalling App {repo}", success="App {repo} Uninstalled")
 	def uninstall(self):
 		self.bench.run(f"{self.bench.python} -m pip uninstall -y {self.repo}")
 
+	def _get_dependencies(self):
+		from bench.utils.app import get_required_deps, required_apps_from_hooks
+
+		if self.on_disk:
+			required_deps = os.path.join(self.mount_path, self.repo,'hooks.py')
+			try:
+				return required_apps_from_hooks(required_deps, local=True)
+			except IndexError:
+				return []
+		try:
+			required_deps = get_required_deps(self.org, self.repo, self.tag or self.branch)
+			return required_apps_from_hooks(required_deps)
+		except Exception:
+			return []
+
+	def update_app_state(self):
+		from bench.bench import Bench
+		bench = Bench(self.bench.name)
+		bench.apps.sync(self.name, self.tag, self.local_resolution)
+
+
+def make_resolution_plan(app: App, bench: "Bench"):
+	"""
+	decide what apps and versions to install and in what order
+	"""
+	resolution = OrderedDict()
+	resolution[app.repo] = app
+
+	for app_name in app._get_dependencies():
+		dep_app = App(app_name, bench=bench)
+		is_valid_frappe_branch(dep_app.url, dep_app.branch)
+		dep_app.required_by = app.name
+		if dep_app.repo in resolution:
+			click.secho(f"{dep_app.repo} is already resolved skipping", fg="yellow")
+			continue
+		resolution[dep_app.repo] = dep_app
+		resolution.update(make_resolution_plan(dep_app, bench))
+		app.local_resolution = [repo_name for repo_name, _ in reversed(resolution.items())]
+	return resolution
+
 
 def add_to_appstxt(app, bench_path="."):
 	from bench.bench import Bench
@@ -264,40 +312,6 @@ def remove_from_excluded_apps_txt(app, bench_path="."):
 		return write_excluded_apps_txt(apps, bench_path=bench_path)
 
 
-def setup_app_dependencies(
-	repo_name, bench_path=".", branch=None, skip_assets=False, verbose=False
-):
-	# branch kwarg is somewhat of a hack here; since we're assuming the same branches for all apps
-	# for eg: if you're installing erpnext@develop, you'll want frappe@develop and healthcare@develop too
-	import glob
-	import bench.cli
-	from bench.bench import Bench
-
-	verbose = bench.cli.verbose or verbose
-	apps_path = os.path.join(os.path.abspath(bench_path), "apps")
-	files = glob.glob(os.path.join(apps_path, repo_name, "**", "hooks.py"))
-
-	if files:
-		with open(files[0]) as f:
-			lines = [x for x in f.read().split("\n") if x.strip().startswith("required_apps")]
-		if lines:
-			required_apps = eval(lines[0].strip("required_apps").strip().lstrip("=").strip())
-
-			if isinstance(required_apps, str):
-				required_apps = [required_apps]
-
-			# TODO: when the time comes, add version check here
-			for app in required_apps:
-				if app not in Bench(bench_path).apps:
-					get_app(
-						app,
-						bench_path=bench_path,
-						branch=branch,
-						skip_assets=skip_assets,
-						verbose=verbose,
-					)
-
-
 def get_app(
 	git_url,
 	branch=None,
@@ -306,6 +320,7 @@ def get_app(
 	verbose=False,
 	overwrite=False,
 	init_bench=False,
+	resolve_deps=False,
 ):
 	"""bench get-app clones a Frappe App from remote (GitHub or any other git server),
 	and installs it on the current bench. This also resolves dependencies based on the
@@ -314,9 +329,10 @@ def get_app(
 	If the bench_path is not a bench directory, a new bench is created named using the
 	git_url parameter.
 	"""
-	from bench.bench import Bench
 	import bench as _bench
 	import bench.cli as bench_cli
+	from bench.bench import Bench
+	from bench.utils.app import check_existing_dir
 
 	bench = Bench(bench_path)
 	app = App(git_url, branch=branch, bench=bench)
@@ -325,6 +341,17 @@ def get_app(
 	branch = app.tag
 	bench_setup = False
 	restart_bench = not init_bench
+	frappe_path, frappe_branch = None, None
+
+	if resolve_deps:
+		resolution = make_resolution_plan(app, bench)
+		click.secho("Following apps will be installed", fg="bright_blue")
+		for idx, app in enumerate(reversed(resolution.values()), start=1):
+			print(f"{idx}. {app.name} {f'(required by {app.required_by})' if app.required_by else ''}")
+
+		if "frappe" in resolution:
+			# Todo: Make frappe a terminal dependency for all frappe apps.
+			frappe_path, frappe_branch = resolution["frappe"].url, resolution["frappe"].tag
 
 	if not is_bench_directory(bench_path):
 		if not init_bench:
@@ -336,20 +363,35 @@ def get_app(
 		from bench.utils.system import init
 
 		bench_path = get_available_folder_name(f"{app.repo}-bench", bench_path)
-		init(path=bench_path, frappe_branch=branch)
+		init(
+			path=bench_path,
+			frappe_path=frappe_path,
+			frappe_branch=frappe_branch if frappe_branch else branch,
+		)
 		os.chdir(bench_path)
 		bench_setup = True
 
 	if bench_setup and bench_cli.from_command_line and bench_cli.dynamic_feed:
-		_bench.LOG_BUFFER.append({
+		_bench.LOG_BUFFER.append(
-			"message": f"Fetching App {repo_name}",
+			{
-			"prefix": click.style('⏼', fg='bright_yellow'),
+				"message": f"Fetching App {repo_name}",
-			"is_parent": True,
+				"prefix": click.style("⏼", fg="bright_yellow"),
-			"color": None,
+				"is_parent": True,
-		})
+				"color": None,
+			}
+		)
 
-	cloned_path = os.path.join(bench_path, "apps", repo_name)
+	if resolve_deps:
-	dir_already_exists = os.path.isdir(cloned_path)
+		install_resolved_deps(
+			bench,
+			resolution,
+			bench_path=bench_path,
+			skip_assets=skip_assets,
+			verbose=verbose,
+		)
+		return
+
+	dir_already_exists, cloned_path = check_existing_dir(bench_path, repo_name)
 	to_clone = not dir_already_exists
 
 	# application directory already exists
@@ -375,11 +417,74 @@ def get_app(
 		app.install(verbose=verbose, skip_assets=skip_assets, restart_bench=restart_bench)
 
 
+def install_resolved_deps(
+	bench,
+	resolution,
+	bench_path=".",
+	skip_assets=False,
+	verbose=False,
+):
+	from bench.utils.app import check_existing_dir
+
+	if "frappe" in resolution:
+		# Terminal dependency
+		del resolution["frappe"]
+
+	for repo_name, app in reversed(resolution.items()):
+		existing_dir, path_to_app = check_existing_dir(bench_path, repo_name)
+		if existing_dir:
+			is_compatible = False
+
+			try:
+				installed_branch = bench.apps.states[repo_name]["resolution"]["branch"].strip()
+			except Exception:
+				installed_branch = (
+					subprocess.
+					check_output("git rev-parse --abbrev-ref HEAD", shell=True, cwd=path_to_app)
+					.decode("utf-8")
+					.rstrip()
+					)
+			try:
+				if app.tag is None:
+					current_remote = (
+						subprocess.check_output(f"git config branch.{installed_branch}.remote", shell=True, cwd=path_to_app)
+						.decode("utf-8")
+						.rstrip()
+					)
+
+					default_branch = (
+						subprocess.check_output(
+							f"git symbolic-ref refs/remotes/{current_remote}/HEAD", shell=True, cwd=path_to_app
+						)
+						.decode("utf-8")
+						.rsplit("/")[-1]
+						.strip()
+					)
+					is_compatible = default_branch == installed_branch
+				else:
+					is_compatible = installed_branch == app.tag
+			except Exception:
+				is_compatible = False
+
+			prefix = 'C' if is_compatible else 'Inc'
+			click.secho(
+				f"{prefix}ompatible version of {repo_name} is already installed",
+				fg="green" if is_compatible else "red",
+			)
+			app.update_app_state()
+			if click.confirm(
+				f"Do you wish to clone and install the already installed {prefix}ompatible app"
+			):
+				click.secho(f"Removing installed app {app.name}", fg="yellow")
+				shutil.rmtree(path_to_app)
+			else:
+				continue
+		app.install_resolved_apps(skip_assets=skip_assets, verbose=verbose)
+
+
 def new_app(app, no_git=None, bench_path="."):
 	if bench.FRAPPE_VERSION in (0, None):
-		raise NotInBenchDirectoryError(
+		raise NotInBenchDirectoryError(f"{os.path.realpath(bench_path)} is not a valid bench directory.")
-			f"{os.path.realpath(bench_path)} is not a valid bench directory."
-		)
 
 	# For backwards compatibility
 	app = app.lower().replace(" ", "_").replace("-", "_")
@@ -387,10 +492,7 @@ def new_app(app, no_git=None, bench_path="."):
 	args = ["make-app", apps, app]
 	if no_git:
 		if bench.FRAPPE_VERSION < 14:
-			click.secho(
+			click.secho("Frappe v14 or greater is needed for '--no-git' flag", fg="red")
-				"Frappe v14 or greater is needed for '--no-git' flag",
-				fg="red"
-			)
 			return
 		args.append(no_git)
 
@@ -401,11 +503,13 @@ def new_app(app, no_git=None, bench_path="."):
 
 def install_app(
 	app,
+	tag=None,
 	bench_path=".",
 	verbose=False,
 	no_cache=False,
 	restart_bench=True,
 	skip_assets=False,
+	resolution=[]
 ):
 	import bench.cli as bench_cli
 	from bench.bench import Bench
@@ -431,7 +535,7 @@ def install_app(
 	if os.path.exists(os.path.join(app_path, "package.json")):
 		bench.run("yarn install", cwd=app_path)
 
-	bench.apps.sync()
+	bench.apps.sync(app, required=resolution, branch=tag)
 
 	if not skip_assets:
 		build_assets(bench_path=bench_path, app=app)
@@ -530,7 +634,10 @@ def install_apps_from_path(path, bench_path="."):
 	apps = get_apps_json(path)
 	for app in apps:
 		get_app(
-			app["url"], branch=app.get("branch"), bench_path=bench_path, skip_assets=True,
+			app["url"],
+			branch=app.get("branch"),
+			bench_path=bench_path,
+			skip_assets=True,
 		)
 
 

bench/bench.py

@@ -1,10 +1,12 @@
 # imports - standard imports
+import subprocess
 import functools
 import os
 import shutil
+import json
 import sys
 import logging
-from typing import List, MutableSequence, TYPE_CHECKING
+from typing import List, MutableSequence, TYPE_CHECKING, Union
 
 # imports - module imports
 import bench
@@ -30,6 +32,7 @@ from bench.utils.bench import (
 	get_env_cmd,
 )
 from bench.utils.render import job, step
+from bench.utils.app import get_current_version
 
 
 if TYPE_CHECKING:
@@ -155,12 +158,80 @@ class Bench(Base, Validator):
 class BenchApps(MutableSequence):
 	def __init__(self, bench: Bench):
 		self.bench = bench
+		self.states_path = os.path.join(self.bench.name, "sites", "apps.json")
+		self.apps_path = os.path.join(self.bench.name, "apps")
 		self.initialize_apps()
+		self.set_states()
 
-	def sync(self):
+	def set_states(self):
+		try:
+			with open(self.states_path, "r") as f:
+				self.states = json.loads(f.read() or "{}")
+		except FileNotFoundError:
+			self.states = {}
+
+	def update_apps_states(self, app_name: Union[str, None] = None, branch: Union[str, None] = None, required:List = []):
+		if self.apps and not os.path.exists(self.states_path):
+			# idx according to apps listed in apps.txt (backwards compatibility)
+			# Keeping frappe as the first app.
+			if "frappe" in self.apps:
+				self.apps.remove("frappe")
+				self.apps.insert(0, "frappe")
+				with open(self.bench.apps_txt, "w") as f:
+					f.write("\n".join(self.apps))
+
+			print("Found existing apps updating states...")
+			for idx, app in enumerate(self.apps, start=1):
+				self.states[app] = {
+					"resolution": {
+					"commit_hash": None,
+					"branch": None
+				},
+				"required": required,
+				"idx": idx,
+				"version": get_current_version(app, self.bench.name),
+				}
+
+		apps_to_remove = []
+		for app in self.states:
+			if app not in self.apps:
+				apps_to_remove.append(app)
+
+		for app in apps_to_remove:
+			del self.states[app]
+
+		if app_name and app_name not in self.states:
+			version = get_current_version(app_name, self.bench.name)
+
+			app_dir = os.path.join(self.apps_path, app_name)
+			if not branch:
+				branch = (
+						subprocess
+						.check_output("git rev-parse --abbrev-ref HEAD", shell=True, cwd=app_dir)
+						.decode("utf-8")
+						.rstrip()
+						)
+
+			commit_hash = subprocess.check_output(f"git rev-parse {branch}", shell=True, cwd=app_dir).decode("utf-8").rstrip()
+
+			self.states[app_name] = {
+				"resolution": {
+					"commit_hash":commit_hash,
+					"branch": branch
+				},
+				"required":required,
+				"idx":len(self.states) + 1,
+				"version": version,
+			}
+
+		with open(self.states_path, "w") as f:
+			f.write(json.dumps(self.states, indent=4))
+
+	def sync(self,app_name: Union[str, None] = None, branch: Union[str, None] = None, required:List = []):
 		self.initialize_apps()
 		with open(self.bench.apps_txt, "w") as f:
-			return f.write("\n".join(self.apps))
+			f.write("\n".join(self.apps))
+		self.update_apps_states(app_name, branch, required)
 
 	def initialize_apps(self):
 		is_installed = lambda app: app in installed_packages
@@ -180,7 +251,6 @@ class BenchApps(MutableSequence):
 					and is_installed(x)
 				)
 			]
-			self.apps.sort()
 		except FileNotFoundError:
 			self.apps = []
 
@@ -248,6 +318,9 @@ class BenchSetup(Base):
 		- install frappe python dependencies
 		"""
 		import bench.cli
+		import click
+
+		click.secho("Setting Up Environment", fg="yellow")
 
 		frappe = os.path.join(self.bench.name, "apps", "frappe")
 		virtualenv = get_venv_path()
@@ -339,7 +412,9 @@ class BenchSetup(Base):
 		print(f"Installing {len(apps)} applications...")
 
 		for app in apps:
-			App(app, bench=self.bench, to_clone=False).install( skip_assets=True, restart_bench=False)
+			App(app, bench=self.bench, to_clone=False).install(
+				skip_assets=True, restart_bench=False, ignore_resolution=True
+			)
 
 	def python(self, apps=None):
 		"""Install and upgrade Python dependencies for specified / all installed apps on given Bench

bench/commands/make.py

@@ -133,8 +133,20 @@ def drop(path):
 @click.option(
 	"--init-bench", is_flag=True, default=False, help="Initialize Bench if not in one"
 )
+@click.option(
+	"--resolve-deps",
+	is_flag=True,
+	default=False,
+	help="Resolve dependencies before installing app",
+)
 def get_app(
-	git_url, branch, name=None, overwrite=False, skip_assets=False, init_bench=False
+	git_url,
+	branch,
+	name=None,
+	overwrite=False,
+	skip_assets=False,
+	init_bench=False,
+	resolve_deps=False,
 ):
 	"clone an app from the internet and set it up in your bench"
 	from bench.app import get_app
@@ -145,9 +157,9 @@ def get_app(
 		skip_assets=skip_assets,
 		overwrite=overwrite,
 		init_bench=init_bench,
+		resolve_deps=resolve_deps,
 	)
 
-
 @click.command("new-app", help="Create a new Frappe application under apps folder")
 @click.option(
 	"--no-git",

bench/exceptions.py

@@ -30,3 +30,7 @@ class FeatureDoesNotExistError(CommandFailedError):
 
 class NotInBenchDirectoryError(Exception):
 	pass
+
+
+class VersionNotFound(Exception):
+	pass

bench/tests/test_init.py

@@ -107,6 +107,20 @@ class TestBenchInit(TestBenchBase):
 		app_installed_in_env = TEST_FRAPPE_APP in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8')
 		self.assertTrue(app_installed_in_env)
 
+	def test_get_app_resolve_deps(self):
+		FRAPPE_APP = "healthcare"
+		self.init_bench("test-bench")
+		bench_path = os.path.join(self.benches_path, "test-bench")
+		exec_cmd(f"bench get-app {FRAPPE_APP} --resolve-deps", cwd=bench_path)
+		self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", FRAPPE_APP)))
+
+		states_path = os.path.join(bench_path, "sites", "apps.json")
+		self.assert_(os.path.exists(states_path))
+
+		with open(states_path, "r") as f:
+			states = json.load(f)
+
+		self.assert_(FRAPPE_APP in states)
 
 	def test_install_app(self):
 		bench_name = "test-bench"

bench/tests/test_utils.py

@@ -0,0 +1,84 @@
+import os
+import shutil
+import subprocess
+import unittest
+from tabnanny import check
+
+from bench.app import App
+from bench.bench import Bench
+from bench.exceptions import InvalidRemoteException
+from bench.utils import is_valid_frappe_branch
+
+
+class TestUtils(unittest.TestCase):
+	def test_app_utils(self):
+		git_url = "https://github.com/frappe/frappe"
+		branch = "develop"
+		app = App(name=git_url, branch=branch, bench=Bench("."))
+		self.assertTrue(
+			all(
+				[
+					app.name == git_url,
+					app.branch == branch,
+					app.tag == branch,
+					app.is_url == True,
+					app.on_disk == False,
+					app.org == "frappe",
+					app.url == git_url,
+				]
+			)
+		)
+
+	def test_is_valid_frappe_branch(self):
+		with self.assertRaises(InvalidRemoteException):
+			is_valid_frappe_branch("https://github.com/frappe/frappe.git", frappe_branch="random-branch")
+			is_valid_frappe_branch("https://github.com/random/random.git", frappe_branch="random-branch")
+
+		is_valid_frappe_branch("https://github.com/frappe/frappe.git", frappe_branch="develop")
+
+	def test_app_states(self):
+		bench_dir = "./sandbox"
+		sites_dir = os.path.join(bench_dir, "sites")
+
+		if not os.path.exists(sites_dir):
+			os.makedirs(sites_dir)
+
+		fake_bench = Bench(bench_dir)
+
+		self.assertTrue(hasattr(fake_bench.apps, "states"))
+
+		fake_bench.apps.states = {
+			"frappe": {"resolution": {"branch": "develop", "commit_hash": "234rwefd"}, "version": "14.0.0-dev"}
+		}
+		fake_bench.apps.update_apps_states()
+
+		self.assertEqual(fake_bench.apps.states, {})
+
+		frappe_path = os.path.join(bench_dir, "apps", "frappe")
+
+		os.makedirs(os.path.join(frappe_path, "frappe"))
+
+		subprocess.run(["git", "init"], cwd=frappe_path, capture_output=True, check=True)
+
+		with open(os.path.join(frappe_path, "frappe", "__init__.py"), "w+") as f:
+			f.write("__version__ = '11.0'")
+
+		subprocess.run(["git", "add", "."], cwd=frappe_path, capture_output=True, check=True)
+		subprocess.run(["git", "commit", "-m", "temp"], cwd=frappe_path, capture_output=True, check=True)
+
+		fake_bench.apps.update_apps_states("frappe")
+
+		self.assertIn("frappe", fake_bench.apps.states)
+		self.assertIn("version", fake_bench.apps.states["frappe"])
+		self.assertEqual("11.0", fake_bench.apps.states["frappe"]["version"])
+
+		shutil.rmtree(bench_dir)
+
+	def test_get_dependencies(self):
+		git_url = "https://github.com/frappe/healthcare"
+		branch = "develop"
+		fake_app = App(git_url, branch=branch)
+		self.assertIn("erpnext", fake_app._get_dependencies())
+		git_url = git_url.replace("healthcare", "erpnext")
+		fake_app = App(git_url)
+		self.assertTrue(len(fake_app._get_dependencies()) == 0)

bench/utils/__init__.py

@@ -8,9 +8,11 @@ import sys
 from glob import glob
 from shlex import split
 from typing import List, Tuple
+from functools import lru_cache
 
 # imports - third party imports
 import click
+import requests
 
 # imports - module imports
 from bench import PROJECT_NAME, VERSION
@@ -48,6 +50,37 @@ def is_frappe_app(directory: str) -> bool:
 	return bool(is_frappe_app)
 
 
+def is_valid_frappe_branch(frappe_path:str, frappe_branch:str):
+	""" Check if a branch exists in a repo. Throws InvalidRemoteException if branch is not found
+
+	Uses github's api without auth to query branch.
+	If rate limited by gitapi, requests are sent to github.com
+
+	:param frappe_path: git url
+	:type frappe_path: str
+	:param frappe_branch: branch to check
+	:type frappe_branch: str
+	:raises InvalidRemoteException: branch for this repo doesn't exist
+	"""
+	if "http" in frappe_path and frappe_branch:
+		frappe_path = frappe_path.replace(".git", "")
+		try:
+			owner, repo = frappe_path.split("/")[3], frappe_path.split("/")[4]
+		except IndexError:
+			raise InvalidRemoteException("Invalid git url")
+		git_api_req = f"https://api.github.com/repos/{owner}/{repo}/branches"
+		res = requests.get(git_api_req).json()
+
+		if "message" in res:
+			# slower alternative with no rate limit
+			github_req = f'https://github.com/{owner}/{repo}/tree/{frappe_branch}'
+			if requests.get(github_req).status_code != 200:
+				raise InvalidRemoteException("Invalid git url")
+
+		elif frappe_branch not in [x["name"] for x in res]:
+			raise InvalidRemoteException("Frappe branch does not exist")
+
+
 def log(message, level=0, no_log=False):
 	import bench
 	import bench.cli
@@ -62,9 +95,7 @@ def log(message, level=0, no_log=False):
 	color, prefix = levels.get(level, levels[0])
 
 	if bench.cli.from_command_line and bench.cli.dynamic_feed:
-		bench.LOG_BUFFER.append(
+		bench.LOG_BUFFER.append({"prefix": prefix, "message": message, "color": color})
-			{"prefix": prefix, "message": message, "color": color}
-		)
 
 	if no_log:
 		click.secho(message, fg=color)
@@ -182,9 +213,7 @@ def get_git_version() -> float:
 def get_cmd_output(cmd, cwd=".", _raise=True):
 	output = ""
 	try:
-		output = subprocess.check_output(
+		output = subprocess.check_output(cmd, cwd=cwd, shell=True, stderr=subprocess.PIPE, encoding="utf-8").strip()
-			cmd, cwd=cwd, shell=True, stderr=subprocess.PIPE, encoding="utf-8"
-		).strip()
 	except subprocess.CalledProcessError as e:
 		if e.output:
 			output = e.output
@@ -400,10 +429,12 @@ def find_org(org_repo):
 
 	for org in ["frappe", "erpnext"]:
 		res = requests.head(f"https://api.github.com/repos/{org}/{org_repo}")
+		if res.status_code == 400:
+			res = requests.head(f"https://github.com/{org}/{org_repo}")
 		if res.ok:
 			return org, org_repo
 
-	raise InvalidRemoteException
+	raise InvalidRemoteException(f"{org_repo} Not foung in frappe or erpnext")
 
 
 def fetch_details_from_tag(_tag: str) -> Tuple[str, str, str]:

bench/utils/app.py

@@ -7,8 +7,10 @@ from bench.exceptions import (
 	InvalidRemoteException,
 	InvalidBranchException,
 	CommandFailedError,
+	VersionNotFound,
 )
 from bench.app import get_repo_dir
+from functools import lru_cache
 
 
 def is_version_upgrade(app="frappe", bench_path=".", branch=None):
@@ -107,7 +109,9 @@ def switch_to_develop(apps=None, bench_path=".", upgrade=True):
 
 
 def get_version_from_string(contents, field="__version__"):
-	match = re.search(r"^(\s*%s\s*=\s*['\\\"])(.+?)(['\"])(?sm)" % field, contents)
+	match = re.search(r"^(\s*%s\s*=\s*['\\\"])(.+?)(['\"])" % field, contents, flags=(re.S | re.M))
+	if not match:
+		raise VersionNotFound(f"{contents} is not a valid version")
 	return match.group(2)
 
 
@@ -165,6 +169,31 @@ def get_current_branch(app, bench_path="."):
 	return get_cmd_output("basename $(git symbolic-ref -q HEAD)", cwd=repo_dir)
 
 
+@lru_cache(maxsize=5)
+def get_required_deps(org, name, branch, deps="hooks.py"):
+	import requests
+	import base64
+
+	git_api_url = f"https://api.github.com/repos/{org}/{name}/contents/{name}/{deps}"
+	params = {"branch": branch or "develop"}
+	res = requests.get(url=git_api_url, params=params).json()
+
+	if "message" in res:
+		git_url = f"https://raw.githubusercontent.com/{org}/{name}/{params['branch']}/{deps}"
+		return requests.get(git_url).text
+
+	return base64.decodebytes(res["content"].encode()).decode()
+
+
+def required_apps_from_hooks(required_deps, local=False):
+	if local:
+		with open(required_deps) as f:
+			required_deps = f.read()
+	lines = [x for x in required_deps.split("\n") if x.strip().startswith("required_apps")]
+	required_apps = eval(lines[0].strip("required_apps").strip().lstrip("=").strip())
+	return required_apps
+
+
 def get_remote(app, bench_path="."):
 	repo_dir = get_repo_dir(app, bench_path=bench_path)
 	contents = subprocess.check_output(
@@ -201,6 +230,11 @@ def get_app_name(bench_path, repo_name):
 
 	return repo_name
 
+def check_existing_dir(bench_path, repo_name):
+	cloned_path = os.path.join(bench_path, "apps", repo_name)
+	dir_already_exists = os.path.isdir(cloned_path)
+	return dir_already_exists, cloned_path
+
 
 def get_current_version(app, bench_path="."):
 	current_version = None

bench/utils/system.py

@@ -14,6 +14,7 @@ from bench.utils import (
 	run_frappe_cmd,
 	sudoers_file,
 	which,
+	is_valid_frappe_branch,
 )
 from bench.utils.bench import build_assets, clone_apps_from
 from bench.utils.render import job
@@ -74,9 +75,14 @@ def init(
 	# remote apps
 	else:
 		frappe_path = frappe_path or "https://github.com/frappe/frappe.git"
-
+		is_valid_frappe_branch(frappe_path=frappe_path, frappe_branch=frappe_branch)
 		get_app(
-			frappe_path, branch=frappe_branch, bench_path=path, skip_assets=True, verbose=verbose
+			frappe_path,
+			branch=frappe_branch,
+			bench_path=path,
+			skip_assets=True,
+			verbose=verbose,
+			resolve_deps=False,
 		)
 
 		# fetch remote apps using config file - deprecate this!
@@ -86,7 +92,12 @@ def init(
 	# getting app on bench init using --install-app
 	if install_app:
 		get_app(
-			install_app, branch=frappe_branch, bench_path=path, skip_assets=True, verbose=verbose
+			install_app,
+			branch=frappe_branch,
+			bench_path=path,
+			skip_assets=True,
+			verbose=verbose,
+			resolve_deps=False,
 		)
 
 	if not skip_assets: