diff --git a/bench/app.py b/bench/app.py index 0dc2fba7..f8c1ba22 100755 --- a/bench/app.py +++ b/bench/app.py @@ -9,6 +9,7 @@ import subprocess import sys import typing from datetime import date +from urllib.parse import urlparse # imports - third party imports import click @@ -62,6 +63,9 @@ class AppMeta: self.from_apps = False self.is_url = False self.branch = branch + self.mount_path = os.path.abspath( + os.path.join(urlparse(self.name).netloc, urlparse(self.name).path) + ) self.setup_details() def setup_details(self): @@ -75,7 +79,7 @@ class AppMeta: self._setup_details_from_installed_apps() # fetch meta for repo on mounted disk - elif os.path.exists(self.name): + elif os.path.exists(self.mount_path): self.on_disk = True self._setup_details_from_mounted_disk() @@ -91,7 +95,9 @@ class AppMeta: self._setup_details_from_name_tag() def _setup_details_from_mounted_disk(self): - self.org, self.repo, self.tag = os.path.split(self.name)[-2:] + (self.branch,) + self.org, self.repo, self.tag = os.path.split(self.mount_path)[-2:] + ( + self.branch, + ) def _setup_details_from_name_tag(self): self.org, self.repo, self.tag = fetch_details_from_tag(self.name) @@ -122,7 +128,7 @@ class AppMeta: return os.path.abspath(os.path.join("apps", self.name)) if self.on_disk: - return os.path.abspath(self.name) + return self.mount_path if self.is_url: return self.name @@ -173,7 +179,7 @@ class App(AppMeta): shutil.move(active_app_path, archived_app_path) @step(title="Installing App {repo}", success="App {repo} Installed") - def install(self, skip_assets=False, verbose=False): + def install(self, skip_assets=False, verbose=False, restart_bench=True): import bench.cli from bench.utils.app import get_app_name @@ -190,7 +196,11 @@ class App(AppMeta): ) install_app( - app=app_name, bench_path=self.bench.name, verbose=verbose, skip_assets=skip_assets, + app=app_name, + bench_path=self.bench.name, + verbose=verbose, + skip_assets=skip_assets, + restart_bench=restart_bench ) @step(title="Uninstalling App {repo}", success="App {repo} Uninstalled") @@ -310,6 +320,7 @@ def get_app( repo_name = app.repo branch = app.tag bench_setup = False + restart_bench = not init_bench if not is_bench_directory(bench_path): if not init_bench: @@ -333,7 +344,6 @@ def get_app( "color": None, }) - cloned_path = os.path.join(bench_path, "apps", repo_name) dir_already_exists = os.path.isdir(cloned_path) to_clone = not dir_already_exists @@ -358,7 +368,7 @@ def get_app( or overwrite or click.confirm("Do you want to reinstall the existing application?") ): - app.install(verbose=verbose, skip_assets=skip_assets) + app.install(verbose=verbose, skip_assets=skip_assets, restart_bench=restart_bench) def new_app(app, no_git=None, bench_path="."): diff --git a/bench/bench.py b/bench/bench.py index db04fd34..f359be7c 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -143,6 +143,14 @@ class Bench(Base, Validator): if systemd and conf.get("restart_systemd_on_update"): restart_systemd_processes(bench_path=self.name, web_workers=web) + def get_installed_apps(self) -> List: + """Returns list of installed apps on bench, not in excluded_apps.txt + """ + apps = [app for app in self.apps if app not in self.excluded_apps] + apps.remove("frappe") + apps.insert(0, "frappe") + return apps + class BenchApps(MutableSequence): def __init__(self, bench: Bench): @@ -317,35 +325,29 @@ class BenchSetup(Base): logger.log("backups were set up") - def __get_installed_apps(self) -> List: - """Returns list of installed apps on bench, not in excluded_apps.txt - """ - apps = [app for app in self.bench.apps if app not in self.bench.excluded_apps] - apps.remove("frappe") - apps.insert(0, "frappe") - return apps - @job(title="Setting Up Bench Dependencies", success="Bench Dependencies Set Up") - def requirements(self): - """Install and upgrade all installed apps on given Bench + def requirements(self, apps=None): + """Install and upgrade specified / all installed apps on given Bench """ from bench.app import App - apps = self.__get_installed_apps() + if not apps: + apps = self.bench.get_installed_apps() self.pip() print(f"Installing {len(apps)} applications...") for app in apps: - App(app, bench=self.bench, to_clone=False).install() + App(app, bench=self.bench, to_clone=False).install( skip_assets=True, restart_bench=False) - def python(self): - """Install and upgrade Python dependencies for installed apps on given Bench + def python(self, apps=None): + """Install and upgrade Python dependencies for specified / all installed apps on given Bench """ import bench.cli - apps = self.__get_installed_apps() + if not apps: + apps = self.bench.get_installed_apps() quiet_flag = "" if bench.cli.verbose else "--quiet" @@ -356,12 +358,12 @@ class BenchSetup(Base): log(f"\nInstalling python dependencies for {app}", level=3, no_log=True) self.run(f"{self.bench.python} -m pip install {quiet_flag} --upgrade -e {app_path}") - def node(self): - """Install and upgrade Node dependencies for all apps on given Bench + def node(self, apps=None): + """Install and upgrade Node dependencies for specified / all apps on given Bench """ from bench.utils.bench import update_node_packages - return update_node_packages(bench_path=self.bench.name) + return update_node_packages(bench_path=self.bench.name, apps=apps) class BenchTearDown: diff --git a/bench/commands/setup.py b/bench/commands/setup.py index 006eba74..42672aa1 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -6,7 +6,7 @@ import sys import click # imports - module imports -from bench.utils import exec_cmd, run_playbook +from bench.utils import exec_cmd, run_playbook, which @click.group(help="Setup command group for enabling setting up a Frappe environment") @@ -41,9 +41,14 @@ def reload_nginx(): @click.option("--yes", help="Yes to regeneration of supervisor config", is_flag=True, default=False) @click.option("--skip-redis", help="Skip redis configuration", is_flag=True, default=False) def setup_supervisor(user=None, yes=False, skip_redis=False): + from bench.utils import get_cmd_output from bench.config.supervisor import update_supervisord_config, generate_supervisor_config - update_supervisord_config(user=user, yes=yes) + which("supervisorctl", raise_err=True) + + if "Permission denied" in get_cmd_output("supervisorctl status"): + update_supervisord_config(user=user, yes=yes) + generate_supervisor_config(bench_path=".", user=user, yes=yes, skip_redis=skip_redis) @@ -131,27 +136,33 @@ def setup_procfile(): def setup_socketio(): return -@click.command("requirements", help="Setup Python and Node dependencies") +@click.command("requirements") @click.option("--node", help="Update only Node packages", default=False, is_flag=True) @click.option("--python", help="Update only Python packages", default=False, is_flag=True) @click.option("--dev", help="Install optional python development dependencies", default=False, is_flag=True) -def setup_requirements(node=False, python=False, dev=False): +@click.argument("apps", nargs=-1) +def setup_requirements(node=False, python=False, dev=False, apps=None): + """ + Setup Python and Node dependencies. + + You can optionally specify one or more apps to setup dependencies for. + """ from bench.bench import Bench bench = Bench(".") if not (node or python or dev): - bench.setup.requirements() + bench.setup.requirements(apps=apps) elif not node and not dev: - bench.setup.python() + bench.setup.python(apps=apps) elif not python and not dev: - bench.setup.node() + bench.setup.node(apps=apps) else: from bench.utils.bench import install_python_dev_dependencies - install_python_dev_dependencies() + install_python_dev_dependencies(apps=apps) if node: click.secho("--dev flag only supports python dependencies. All node development dependencies are installed by default.", fg="yellow") diff --git a/bench/commands/utils.py b/bench/commands/utils.py index 20b5a796..b2d8a301 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -23,6 +23,9 @@ def start(no_dev, concurrency, procfile, no_prefix, man): @click.option('--systemd', is_flag=True, default=False) def restart(web, supervisor, systemd): from bench.bench import Bench + if not systemd and not web: + supervisor = True + Bench(".").reload(web, supervisor, systemd) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 8db76c3a..765e9989 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -109,7 +109,9 @@ def update_supervisord_config(user=None, yes=False): supervisord_conf_changes += '\n' + action if not supervisord_conf_changes: - logger.log("supervisord.conf not updated") + logger.error("supervisord.conf not updated") + contents = "\n".join(f"{x}={y}" for x, y in updated_values.items()) + print(f"Update your {supervisord_conf} with the following values:\n[{section}]\n{contents}") return if not yes: diff --git a/bench/tests/test_init.py b/bench/tests/test_init.py index 52c23282..80018b59 100755 --- a/bench/tests/test_init.py +++ b/bench/tests/test_init.py @@ -10,6 +10,7 @@ import git # imports - module imports from bench.utils import exec_cmd from bench.release import get_bumped_version +from bench.app import App from bench.tests.test_base import FRAPPE_BRANCH, TestBenchBase @@ -34,9 +35,10 @@ class TestBenchInit(TestBenchBase): def test_utils(self): self.assertEqual(subprocess.call("bench"), 0) - def test_init(self, bench_name="test-bench", **kwargs): self.init_bench(bench_name, **kwargs) + app = App("file:///tmp/frappe") + self.assertEqual(app.url, "/tmp/frappe") self.assert_folders(bench_name) self.assert_virtual_env(bench_name) self.assert_config(bench_name) diff --git a/bench/utils/__init__.py b/bench/utils/__init__.py index f75fe263..9c8d98a4 100644 --- a/bench/utils/__init__.py +++ b/bench/utils/__init__.py @@ -133,7 +133,7 @@ def which(executable: str, raise_err: bool = False) -> str: exec_ = which(executable) if not exec_ and raise_err: - raise ValueError(f"{executable} not found.") + raise FileNotFoundError(f"{executable} not found in PATH") return exec_ @@ -314,7 +314,13 @@ def find_benches(directory: str = None) -> List: return benches = [] - for sub in os.listdir(directory): + + try: + sub_directories = os.listdir(directory) + except PermissionError: + return benches + + for sub in sub_directories: sub = os.path.join(directory, sub) if os.path.isdir(sub) and not os.path.islink(sub): if is_bench_directory(sub): diff --git a/bench/utils/bench.py b/bench/utils/bench.py index aba4b943..e0be48b9 100644 --- a/bench/utils/bench.py +++ b/bench/utils/bench.py @@ -48,7 +48,7 @@ def get_venv_path(): return venv or log("virtualenv cannot be found", level=2) -def update_node_packages(bench_path="."): +def update_node_packages(bench_path=".", apps=None): print("Updating node packages...") from bench.utils.app import get_develop_version from distutils.version import LooseVersion @@ -58,9 +58,9 @@ def update_node_packages(bench_path="."): # After rollup was merged, frappe_version = 10.1 # if develop_verion is 11 and up, only then install yarn if v < LooseVersion("11.x.x-develop"): - update_npm_packages(bench_path) + update_npm_packages(bench_path, apps=apps) else: - update_yarn_packages(bench_path) + update_yarn_packages(bench_path, apps=apps) def install_python_dev_dependencies(bench_path=".", apps=None, verbose=False): @@ -75,7 +75,7 @@ def install_python_dev_dependencies(bench_path=".", apps=None, verbose=False): if isinstance(apps, str): apps = [apps] elif apps is None: - apps = [app for app in bench.apps if app not in bench.excluded_apps] + apps = bench.get_installed_apps() for app in apps: app_path = os.path.join(bench_path, "apps", app) @@ -86,11 +86,14 @@ def install_python_dev_dependencies(bench_path=".", apps=None, verbose=False): bench.run(f"{bench.python} -m pip install {quiet_flag} --upgrade -r {dev_requirements_path}") -def update_yarn_packages(bench_path="."): +def update_yarn_packages(bench_path=".", apps=None): from bench.bench import Bench bench = Bench(bench_path) - apps = [app for app in bench.apps if app not in bench.excluded_apps] + + if not apps: + apps = bench.get_installed_apps() + apps_dir = os.path.join(bench.name, "apps") # TODO: Check for stuff like this early on only?? @@ -106,11 +109,14 @@ def update_yarn_packages(bench_path="."): bench.run("yarn install", cwd=app_path) -def update_npm_packages(bench_path="."): +def update_npm_packages(bench_path=".", apps=None): apps_dir = os.path.join(bench_path, "apps") package_json = {} - for app in os.listdir(apps_dir): + if not apps: + apps = os.listdir(apps_dir) + + for app in apps: package_json_path = os.path.join(apps_dir, app, "package.json") if os.path.exists(package_json_path): @@ -168,8 +174,7 @@ def migrate_env(python, backup=False): from datetime import datetime parch = os.path.join(path, "archived", "envs") - if not os.path.exists(parch): - os.mkdir(parch) + os.makedirs(parch, exist_ok=True) source = os.path.join(path, "env") target = parch @@ -248,7 +253,15 @@ def restart_supervisor_processes(bench_path=".", web_workers=False): bench.run(cmd) else: - supervisor_status = get_cmd_output("supervisorctl status", cwd=bench_path) + sudo = "" + try: + supervisor_status = get_cmd_output("supervisorctl status", cwd=bench_path) + except Exception as e: + if e.returncode == 127: + log("restart failed: Couldn't find supervisorctl in PATH", level=3) + return + sudo = "sudo " + supervisor_status = get_cmd_output("sudo supervisorctl status", cwd=bench_path) if web_workers and f"{bench_name}-web:" in supervisor_status: group = f"{bench_name}-web:\t" @@ -264,7 +277,7 @@ def restart_supervisor_processes(bench_path=".", web_workers=False): else: group = "frappe:" - bench.run(f"supervisorctl restart {group}") + bench.run(f"{sudo}supervisorctl restart {group}") def restart_systemd_processes(bench_path=".", web_workers=False):