diff --git a/.travis.yml b/.travis.yml index e2b718ae..f1d8b83f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,32 +48,17 @@ matrix: - name: "Python 3.7 Tests" python: 3.7 env: TEST=bench - script: python -m unittest -v bench.tests.test_init + script: python -m unittest -v bench.tests.test_utils && python -m unittest -v bench.tests.test_init - name: "Python 3.8 Tests" python: 3.8 env: TEST=bench - script: python -m unittest -v bench.tests.test_init + script: python -m unittest -v bench.tests.test_utils && python -m unittest -v bench.tests.test_init - name: "Python 3.9 Tests" python: 3.9 env: TEST=bench - script: python -m unittest -v bench.tests.test_init - - - name: "Python 3.7 Easy Install" - python: 3.7 - env: TEST=easy_install - script: sudo python $TRAVIS_BUILD_DIR/install.py --user travis --run-travis --production --verbose - - - name: "Python 3.8 Easy Install" - python: 3.8 - env: TEST=easy_install - script: sudo python $TRAVIS_BUILD_DIR/install.py --user travis --run-travis --production --verbose - - - name: "Python 3.9 Easy Install" - python: 3.9 - env: TEST=easy_install - script: sudo python $TRAVIS_BUILD_DIR/install.py --user travis --run-travis --production --verbose + script: python -m unittest -v bench.tests.test_utils && python -m unittest -v bench.tests.test_init install: - pip3 install urllib3 pyOpenSSL ndg-httpsclient pyasn1 diff --git a/README.md b/README.md index de2f4365..4d5c0f40 100755 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ The setup for each of these installations can be achieved in multiple ways: We recommend using either the Docker Installation or the Easy Install Script to setup a Production Environment. For Development, you may choose either of the three methods to setup an instance. -Otherwise, if you are looking to evaluate ERPNext, you can also download the [Virtual Machine Image](https://erpnext.com/download) or register for [a free trial on erpnext.com](https://erpnext.com/pricing). +Otherwise, if you are looking to evaluate ERPNext, you can register for [a free trial on erpnext.com](https://erpnext.com/pricing). ### Containerized Installation diff --git a/bench/app.py b/bench/app.py index f8c1ba22..4b527366 100755 --- a/bench/app.py +++ b/bench/app.py @@ -8,11 +8,14 @@ import shutil import subprocess import sys import typing +from collections import OrderedDict from datetime import date from urllib.parse import urlparse +import os # imports - third party imports import click +from git import Repo # imports - module imports import bench @@ -22,6 +25,7 @@ from bench.utils import ( get_available_folder_name, is_bench_directory, is_git_url, + is_valid_frappe_branch, log, run_frappe_cmd, ) @@ -55,7 +59,7 @@ class AppMeta: class Healthcare(AppConfig): dependencies = [{"frappe/erpnext": "~13.17.0"}] """ - self.name = name.rstrip('/') + self.name = name.rstrip("/") self.remote_server = "github.com" self.to_clone = to_clone self.on_disk = False @@ -63,6 +67,8 @@ class AppMeta: self.from_apps = False self.is_url = False self.branch = branch + self.app_name = None + self.git_repo = None self.mount_path = os.path.abspath( os.path.join(urlparse(self.name).netloc, urlparse(self.name).path) ) @@ -70,13 +76,12 @@ class AppMeta: def setup_details(self): # fetch meta from installed apps - if ( - not self.to_clone - and hasattr(self, "bench") - and os.path.exists(os.path.join(self.bench.name, "apps", self.name)) + if self.bench and os.path.exists( + os.path.join(self.bench.name, "apps", self.name) ): + self.mount_path = os.path.join(self.bench.name, "apps", self.name) self.from_apps = True - self._setup_details_from_installed_apps() + self._setup_details_from_mounted_disk() # fetch meta for repo on mounted disk elif os.path.exists(self.mount_path): @@ -86,50 +91,56 @@ class AppMeta: # fetch meta for repo from remote git server - traditional get-app url elif is_git_url(self.name): self.is_url = True - if self.name.startswith("git@") or self.name.startswith("ssh://"): - self.use_ssh = True self._setup_details_from_git_url() # fetch meta from new styled name tags & first party apps on github else: self._setup_details_from_name_tag() + if self.git_repo: + self.app_name = os.path.basename( + os.path.normpath(self.git_repo.working_tree_dir) + ) + else: + self.app_name = self.repo + def _setup_details_from_mounted_disk(self): - self.org, self.repo, self.tag = os.path.split(self.mount_path)[-2:] + ( - self.branch, - ) + # If app is a git repo + self.git_repo = Repo(self.mount_path) + try: + self._setup_details_from_git_url(self.git_repo.remotes[0].url) + if not (self.branch or self.tag): + self.tag = self.branch = self.git_repo.active_branch.name + except IndexError: + self.org, self.repo, self.tag = os.path.split(self.mount_path)[-2:] + (self.branch,) + except TypeError: + # faced a "a detached symbolic reference as it points" in case you're in the middle of + # some git shenanigans + self.tag = self.branch = None def _setup_details_from_name_tag(self): self.org, self.repo, self.tag = fetch_details_from_tag(self.name) self.tag = self.tag or self.branch - def _setup_details_from_installed_apps(self): - self.org, self.repo, self.tag = os.path.split( - os.path.join(self.bench.name, "apps", self.name) - )[-2:] + (self.branch,) + def _setup_details_from_git_url(self, url=None): + return self.__setup_details_from_git(url) - def _setup_details_from_git_url(self): - return self.__setup_details_from_git() - - def __setup_details_from_git(self): - if self.use_ssh: - _first_part, _second_part = self.name.split(":") + def __setup_details_from_git(self, url=None): + name = url if url else self.name + if name.startswith("git@") or name.startswith("ssh://"): + self.use_ssh = True + _first_part, _second_part = name.rsplit(":", 1) self.remote_server = _first_part.split("@")[-1] self.org, _repo = _second_part.rsplit("/", 1) else: - self.remote_server, self.org, _repo = self.name.rsplit("/", 2) + protocal = "https://" if "https://" in name else "http://" + self.remote_server, self.org, _repo = name.replace(protocal, "").rsplit("/", 2) self.tag = self.branch self.repo = _repo.split(".")[0] @property def url(self): - if self.from_apps: - return os.path.abspath(os.path.join("apps", self.name)) - - if self.on_disk: - return self.mount_path - if self.is_url: return self.name @@ -147,10 +158,10 @@ class AppMeta: @functools.lru_cache(maxsize=None) class App(AppMeta): - def __init__( - self, name: str, branch: str = None, bench: "Bench" = None, *args, **kwargs - ): + def __init__(self, name: str, branch: str = None, bench: "Bench" = None, *args, **kwargs): self.bench = bench + self.required_by = None + self.local_resolution = [] super().__init__(name, branch, *args, **kwargs) @step(title="Fetching App {repo}", success="App {repo} Fetched") @@ -168,44 +179,100 @@ class App(AppMeta): ) @step(title="Archiving App {repo}", success="App {repo} Archived") - def remove(self): - active_app_path = os.path.join("apps", self.repo) - archived_path = os.path.join("archived", "apps") - archived_name = get_available_folder_name( - f"{self.repo}-{date.today()}", archived_path - ) - archived_app_path = os.path.join(archived_path, archived_name) - log(f"App moved from {active_app_path} to {archived_app_path}") - shutil.move(active_app_path, archived_app_path) + def remove(self, no_backup: bool = False): + active_app_path = os.path.join("apps", self.name) + + if no_backup: + shutil.rmtree(active_app_path) + log(f"App deleted from {active_app_path}") + else: + archived_path = os.path.join("archived", "apps") + archived_name = get_available_folder_name(f"{self.repo}-{date.today()}", archived_path) + archived_app_path = os.path.join(archived_path, archived_name) + + shutil.move(active_app_path, archived_app_path) + log(f"App moved from {active_app_path} to {archived_app_path}") @step(title="Installing App {repo}", success="App {repo} Installed") - def install(self, skip_assets=False, verbose=False, restart_bench=True): + def install( + self, + skip_assets=False, + verbose=False, + resolved=False, + restart_bench=True, + ignore_resolution=False, + ): import bench.cli from bench.utils.app import get_app_name verbose = bench.cli.verbose or verbose - app_name = get_app_name(self.bench.name, self.repo) - - # TODO: this should go inside install_app only tho - issue: default/resolved branch - setup_app_dependencies( - repo_name=self.repo, - bench_path=self.bench.name, - branch=self.tag, - verbose=verbose, - skip_assets=skip_assets, - ) + app_name = get_app_name(self.bench.name, self.app_name) + if not resolved and self.repo != "frappe" and not ignore_resolution: + click.secho( + f"Ignoring dependencies of {self.name}. To install dependencies use --resolve-deps", + fg="yellow", + ) install_app( app=app_name, + tag=self.tag, bench_path=self.bench.name, verbose=verbose, skip_assets=skip_assets, - restart_bench=restart_bench + restart_bench=restart_bench, + resolution=self.local_resolution ) + @step(title="Cloning and installing {repo}", success="App {repo} Installed") + def install_resolved_apps(self, *args, **kwargs): + self.get() + self.install(*args, **kwargs, resolved=True) + @step(title="Uninstalling App {repo}", success="App {repo} Uninstalled") def uninstall(self): - self.bench.run(f"{self.bench.python} -m pip uninstall -y {self.repo}") + self.bench.run(f"{self.bench.python} -m pip uninstall -y {self.name}") + + def _get_dependencies(self): + from bench.utils.app import get_required_deps, required_apps_from_hooks + + if self.on_disk: + required_deps = os.path.join(self.mount_path, self.repo,'hooks.py') + try: + return required_apps_from_hooks(required_deps, local=True) + except IndexError: + return [] + try: + required_deps = get_required_deps(self.org, self.repo, self.tag or self.branch) + return required_apps_from_hooks(required_deps) + except Exception: + return [] + + def update_app_state(self): + from bench.bench import Bench + bench = Bench(self.bench.name) + bench.apps.sync(app_dir=self.app_name, app_name=self.name, + branch=self.tag, required_list=self.local_resolution) + + + +def make_resolution_plan(app: App, bench: "Bench"): + """ + decide what apps and versions to install and in what order + """ + resolution = OrderedDict() + resolution[app.repo] = app + + for app_name in app._get_dependencies(): + dep_app = App(app_name, bench=bench) + is_valid_frappe_branch(dep_app.url, dep_app.branch) + dep_app.required_by = app.name + if dep_app.repo in resolution: + click.secho(f"{dep_app.repo} is already resolved skipping", fg="yellow") + continue + resolution[dep_app.repo] = dep_app + resolution.update(make_resolution_plan(dep_app, bench)) + app.local_resolution = [repo_name for repo_name, _ in reversed(resolution.items())] + return resolution def add_to_appstxt(app, bench_path="."): @@ -264,36 +331,6 @@ def remove_from_excluded_apps_txt(app, bench_path="."): return write_excluded_apps_txt(apps, bench_path=bench_path) -def setup_app_dependencies( - repo_name, bench_path=".", branch=None, skip_assets=False, verbose=False -): - # branch kwarg is somewhat of a hack here; since we're assuming the same branches for all apps - # for eg: if you're installing erpnext@develop, you'll want frappe@develop and healthcare@develop too - import glob - import bench.cli - from bench.bench import Bench - - verbose = bench.cli.verbose or verbose - apps_path = os.path.join(os.path.abspath(bench_path), "apps") - files = glob.glob(os.path.join(apps_path, repo_name, "**", "hooks.py")) - - if files: - with open(files[0]) as f: - lines = [x for x in f.read().split("\n") if x.strip().startswith("required_apps")] - if lines: - required_apps = eval(lines[0].strip("required_apps").strip().lstrip("=").strip()) - # TODO: when the time comes, add version check here - for app in required_apps: - if app not in Bench(bench_path).apps: - get_app( - app, - bench_path=bench_path, - branch=branch, - skip_assets=skip_assets, - verbose=verbose, - ) - - def get_app( git_url, branch=None, @@ -302,6 +339,7 @@ def get_app( verbose=False, overwrite=False, init_bench=False, + resolve_deps=False, ): """bench get-app clones a Frappe App from remote (GitHub or any other git server), and installs it on the current bench. This also resolves dependencies based on the @@ -310,9 +348,10 @@ def get_app( If the bench_path is not a bench directory, a new bench is created named using the git_url parameter. """ - from bench.bench import Bench import bench as _bench import bench.cli as bench_cli + from bench.bench import Bench + from bench.utils.app import check_existing_dir bench = Bench(bench_path) app = App(git_url, branch=branch, bench=bench) @@ -321,6 +360,17 @@ def get_app( branch = app.tag bench_setup = False restart_bench = not init_bench + frappe_path, frappe_branch = None, None + + if resolve_deps: + resolution = make_resolution_plan(app, bench) + click.secho("Following apps will be installed", fg="bright_blue") + for idx, app in enumerate(reversed(resolution.values()), start=1): + print(f"{idx}. {app.name} {f'(required by {app.required_by})' if app.required_by else ''}") + + if "frappe" in resolution: + # Todo: Make frappe a terminal dependency for all frappe apps. + frappe_path, frappe_branch = resolution["frappe"].url, resolution["frappe"].tag if not is_bench_directory(bench_path): if not init_bench: @@ -332,20 +382,35 @@ def get_app( from bench.utils.system import init bench_path = get_available_folder_name(f"{app.repo}-bench", bench_path) - init(path=bench_path, frappe_branch=branch) + init( + path=bench_path, + frappe_path=frappe_path, + frappe_branch=frappe_branch if frappe_branch else branch, + ) os.chdir(bench_path) bench_setup = True if bench_setup and bench_cli.from_command_line and bench_cli.dynamic_feed: - _bench.LOG_BUFFER.append({ - "message": f"Fetching App {repo_name}", - "prefix": click.style('⏼', fg='bright_yellow'), - "is_parent": True, - "color": None, - }) + _bench.LOG_BUFFER.append( + { + "message": f"Fetching App {repo_name}", + "prefix": click.style("⏼", fg="bright_yellow"), + "is_parent": True, + "color": None, + } + ) - cloned_path = os.path.join(bench_path, "apps", repo_name) - dir_already_exists = os.path.isdir(cloned_path) + if resolve_deps: + install_resolved_deps( + bench, + resolution, + bench_path=bench_path, + skip_assets=skip_assets, + verbose=verbose, + ) + return + + dir_already_exists, cloned_path = check_existing_dir(bench_path, repo_name) to_clone = not dir_already_exists # application directory already exists @@ -371,22 +436,89 @@ def get_app( app.install(verbose=verbose, skip_assets=skip_assets, restart_bench=restart_bench) +def install_resolved_deps( + bench, + resolution, + bench_path=".", + skip_assets=False, + verbose=False, +): + from bench.utils.app import check_existing_dir + + if "frappe" in resolution: + # Terminal dependency + del resolution["frappe"] + + for repo_name, app in reversed(resolution.items()): + existing_dir, path_to_app = check_existing_dir(bench_path, repo_name) + if existing_dir: + is_compatible = False + + try: + installed_branch = bench.apps.states[repo_name]["resolution"]["branch"].strip() + except Exception: + installed_branch = ( + subprocess. + check_output("git rev-parse --abbrev-ref HEAD", shell=True, cwd=path_to_app) + .decode("utf-8") + .rstrip() + ) + try: + if app.tag is None: + current_remote = ( + subprocess.check_output(f"git config branch.{installed_branch}.remote", shell=True, cwd=path_to_app) + .decode("utf-8") + .rstrip() + ) + + default_branch = ( + subprocess.check_output( + f"git symbolic-ref refs/remotes/{current_remote}/HEAD", shell=True, cwd=path_to_app + ) + .decode("utf-8") + .rsplit("/")[-1] + .strip() + ) + is_compatible = default_branch == installed_branch + else: + is_compatible = installed_branch == app.tag + except Exception: + is_compatible = False + + prefix = 'C' if is_compatible else 'Inc' + click.secho( + f"{prefix}ompatible version of {repo_name} is already installed", + fg="green" if is_compatible else "red", + ) + app.update_app_state() + if click.confirm( + f"Do you wish to clone and install the already installed {prefix}ompatible app" + ): + click.secho(f"Removing installed app {app.name}", fg="yellow") + shutil.rmtree(path_to_app) + else: + continue + app.install_resolved_apps(skip_assets=skip_assets, verbose=verbose) + + def new_app(app, no_git=None, bench_path="."): if bench.FRAPPE_VERSION in (0, None): - raise NotInBenchDirectoryError( - f"{os.path.realpath(bench_path)} is not a valid bench directory." - ) + raise NotInBenchDirectoryError(f"{os.path.realpath(bench_path)} is not a valid bench directory.") # For backwards compatibility app = app.lower().replace(" ", "_").replace("-", "_") + if app[0].isdigit() or "." in app: + click.secho( + "App names cannot start with numbers(digits) or have dot(.) in them", + fg="red" + ) + return + apps = os.path.abspath(os.path.join(bench_path, "apps")) args = ["make-app", apps, app] if no_git: if bench.FRAPPE_VERSION < 14: - click.secho( - "Frappe v14 or greater is needed for '--no-git' flag", - fg="red" - ) + click.secho("Frappe v14 or greater is needed for '--no-git' flag", fg="red") return args.append(no_git) @@ -397,11 +529,13 @@ def new_app(app, no_git=None, bench_path="."): def install_app( app, + tag=None, bench_path=".", verbose=False, no_cache=False, restart_bench=True, skip_assets=False, + resolution=[] ): import bench.cli as bench_cli from bench.bench import Bench @@ -427,7 +561,7 @@ def install_app( if os.path.exists(os.path.join(app_path, "package.json")): bench.run("yarn install", cwd=app_path) - bench.apps.sync() + bench.apps.sync(app_name=app, required=resolution, branch=tag, app_dir=app_path) if not skip_assets: build_assets(bench_path=bench_path, app=app) @@ -526,7 +660,10 @@ def install_apps_from_path(path, bench_path="."): apps = get_apps_json(path) for app in apps: get_app( - app["url"], branch=app.get("branch"), bench_path=bench_path, skip_assets=True, + app["url"], + branch=app.get("branch"), + bench_path=bench_path, + skip_assets=True, ) diff --git a/bench/bench.py b/bench/bench.py index f359be7c..3585769c 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -1,14 +1,16 @@ # imports - standard imports +import subprocess import functools import os import shutil +import json import sys import logging -from typing import List, MutableSequence, TYPE_CHECKING +from typing import List, MutableSequence, TYPE_CHECKING, Union # imports - module imports import bench -from bench.exceptions import ValidationError +from bench.exceptions import AppNotInstalledError, InvalidRemoteException from bench.config.common_site_config import setup_config from bench.utils import ( paths_in_bench, @@ -27,9 +29,11 @@ from bench.utils.bench import ( restart_process_manager, remove_backups_crontab, get_venv_path, + get_virtualenv_path, get_env_cmd, ) from bench.utils.render import job, step +from bench.utils.app import get_current_version if TYPE_CHECKING: @@ -46,7 +50,7 @@ class Base: class Validator: def validate_app_uninstall(self, app): if app not in self.apps: - raise ValidationError(f"No app named {app}") + raise AppNotInstalledError(f"No app named {app}") validate_app_installed_on_sites(app, bench_path=self.name) @@ -116,11 +120,16 @@ class Bench(Base, Validator): self.apps.append(app) self.apps.sync() - def uninstall(self, app): + def uninstall(self, app, no_backup=False, force=False): from bench.app import App - self.validate_app_uninstall(app) - self.apps.remove(App(app, bench=self, to_clone=False)) + if not force: + self.validate_app_uninstall(app) + try: + self.apps.remove(App(app, bench=self, to_clone=False), no_backup=no_backup) + except InvalidRemoteException: + if not force: + raise self.apps.sync() # self.build() - removed because it seems unnecessary self.reload() @@ -155,12 +164,102 @@ class Bench(Base, Validator): class BenchApps(MutableSequence): def __init__(self, bench: Bench): self.bench = bench + self.states_path = os.path.join(self.bench.name, "sites", "apps.json") + self.apps_path = os.path.join(self.bench.name, "apps") + self.initialize_apps() + self.set_states() + + def set_states(self): + try: + with open(self.states_path, "r") as f: + self.states = json.loads(f.read() or "{}") + except FileNotFoundError: + self.states = {} + + def update_apps_states( + self, + app_dir: str = None, + app_name: Union[str, None] = None, + branch: Union[str, None] = None, + required: List = [], + ): + if self.apps and not os.path.exists(self.states_path): + # idx according to apps listed in apps.txt (backwards compatibility) + # Keeping frappe as the first app. + if "frappe" in self.apps: + self.apps.remove("frappe") + self.apps.insert(0, "frappe") + with open(self.bench.apps_txt, "w") as f: + f.write("\n".join(self.apps)) + + print("Found existing apps updating states...") + for idx, app in enumerate(self.apps, start=1): + self.states[app] = { + "resolution": { + "commit_hash": None, + "branch": None + }, + "required": required, + "idx": idx, + "version": get_current_version(app, self.bench.name), + } + + apps_to_remove = [] + for app in self.states: + if app not in self.apps: + apps_to_remove.append(app) + + for app in apps_to_remove: + del self.states[app] + + if app_name and not app_dir: + app_dir = app_name + + if app_name and app_name not in self.states: + version = get_current_version(app_name, self.bench.name) + + app_dir = os.path.join(self.apps_path, app_dir) + if not branch: + branch = ( + subprocess + .check_output("git rev-parse --abbrev-ref HEAD", shell=True, cwd=app_dir) + .decode("utf-8") + .rstrip() + ) + + commit_hash = subprocess.check_output(f"git rev-parse {branch}", shell=True, cwd=app_dir).decode("utf-8").rstrip() + + self.states[app_name] = { + "resolution": { + "commit_hash":commit_hash, + "branch": branch + }, + "required":required, + "idx":len(self.states) + 1, + "version": version, + } + + with open(self.states_path, "w") as f: + f.write(json.dumps(self.states, indent=4)) + + def sync( + self, + app_name: Union[str, None] = None, + app_dir: Union[str, None] = None, + branch: Union[str, None] = None, + required: List = [] + ): self.initialize_apps() - def sync(self): - self.initialize_apps() with open(self.bench.apps_txt, "w") as f: - return f.write("\n".join(self.apps)) + f.write("\n".join(self.apps)) + + self.update_apps_states( + app_name=app_name, + app_dir=app_dir, + branch=branch, + required=required + ) def initialize_apps(self): is_installed = lambda app: app in installed_packages @@ -180,7 +279,6 @@ class BenchApps(MutableSequence): and is_installed(x) ) ] - self.apps.sort() except FileNotFoundError: self.apps = [] @@ -213,9 +311,9 @@ class BenchApps(MutableSequence): super().append(app.repo) self.apps.sort() - def remove(self, app: "App"): + def remove(self, app: "App", no_backup: bool = False): app.uninstall() - app.remove() + app.remove(no_backup=no_backup) super().remove(app.repo) def append(self, app: "App"): @@ -248,13 +346,22 @@ class BenchSetup(Base): - install frappe python dependencies """ import bench.cli + import click + + verbose = bench.cli.verbose + + click.secho("Setting Up Environment", fg="yellow") frappe = os.path.join(self.bench.name, "apps", "frappe") - virtualenv = get_venv_path() - quiet_flag = "" if bench.cli.verbose else "--quiet" + virtualenv = get_virtualenv_path(verbose=verbose) + quiet_flag = "" if verbose else "--quiet" if not os.path.exists(self.bench.python): - self.run(f"{virtualenv} {quiet_flag} env -p {python}") + if virtualenv: + self.run(f"{virtualenv} {quiet_flag} env -p {python}") + else: + venv = get_venv_path(verbose=verbose) + self.run(f"{venv} env") self.pip() @@ -339,7 +446,10 @@ class BenchSetup(Base): print(f"Installing {len(apps)} applications...") for app in apps: - App(app, bench=self.bench, to_clone=False).install( skip_assets=True, restart_bench=False) + path_to_app = os.path.join(self.bench.name, "apps", app) + app = App(path_to_app, bench=self.bench, to_clone=False).install( + skip_assets=True, restart_bench=False, ignore_resolution=True + ) def python(self, apps=None): """Install and upgrade Python dependencies for specified / all installed apps on given Bench diff --git a/bench/commands/make.py b/bench/commands/make.py index 88b37c61..8fe6382b 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -133,8 +133,20 @@ def drop(path): @click.option( "--init-bench", is_flag=True, default=False, help="Initialize Bench if not in one" ) +@click.option( + "--resolve-deps", + is_flag=True, + default=False, + help="Resolve dependencies before installing app", +) def get_app( - git_url, branch, name=None, overwrite=False, skip_assets=False, init_bench=False + git_url, + branch, + name=None, + overwrite=False, + skip_assets=False, + init_bench=False, + resolve_deps=False, ): "clone an app from the internet and set it up in your bench" from bench.app import get_app @@ -145,9 +157,9 @@ def get_app( skip_assets=skip_assets, overwrite=overwrite, init_bench=init_bench, + resolve_deps=resolve_deps, ) - @click.command("new-app", help="Create a new Frappe application under apps folder") @click.option( "--no-git", @@ -168,12 +180,14 @@ def new_app(app_name, no_git=None): "Completely remove app from bench and re-build assets if not installed on any site" ), ) +@click.option("--no-backup", is_flag=True, help="Do not backup app before removing") +@click.option("--force", is_flag=True, help="Force remove app") @click.argument("app-name") -def remove_app(app_name): +def remove_app(app_name, no_backup=False, force=False): from bench.bench import Bench bench = Bench(".") - bench.uninstall(app_name) + bench.uninstall(app_name, no_backup=no_backup, force=force) @click.command("exclude-app", help="Exclude app from updating") diff --git a/bench/config/lets_encrypt.py b/bench/config/lets_encrypt.py index 45a61308..4919ef73 100755 --- a/bench/config/lets_encrypt.py +++ b/bench/config/lets_encrypt.py @@ -10,11 +10,10 @@ from bench.config.nginx import make_nginx_conf from bench.config.production_setup import service from bench.config.site_config import get_domains, remove_domain, update_site_config from bench.bench import Bench -from bench.utils import exec_cmd +from bench.utils import exec_cmd, which from bench.utils.bench import update_common_site_config from bench.exceptions import CommandFailedError - def setup_letsencrypt(site, custom_domain, bench_path, interactive): site_path = os.path.join(bench_path, "sites", site, "site_config.json") @@ -58,7 +57,6 @@ def create_config(site, custom_domain): def run_certbot_and_setup_ssl(site, custom_domain, bench_path, interactive=True): service('nginx', 'stop') - get_certbot() try: interactive = '' if interactive else '-n' @@ -88,7 +86,7 @@ def run_certbot_and_setup_ssl(site, custom_domain, bench_path, interactive=True) def setup_crontab(): from crontab import CronTab - job_command = '/opt/certbot-auto renew -a nginx --post-hook "systemctl reload nginx"' + job_command = f'{get_certbot_path()} renew -a nginx --post-hook "systemctl reload nginx"' job_comment = 'Renew lets-encrypt every month' print(f"Setting Up cron job to {job_comment}") @@ -107,20 +105,11 @@ def create_dir_if_missing(path): os.makedirs(os.path.dirname(path)) -def get_certbot(): - from urllib.request import urlretrieve - - certbot_path = get_certbot_path() - create_dir_if_missing(certbot_path) - - if not os.path.isfile(certbot_path): - urlretrieve("https://dl.eff.org/certbot-auto", certbot_path) - os.chmod(certbot_path, 0o744) - - def get_certbot_path(): - return "/opt/certbot-auto" - + try: + return which("certbot", raise_err=True) + except FileNotFoundError: + raise CommandFailedError("Certbot is not installed on your system. Please visit https://certbot.eff.org/instructions for installation instructions, then try again.") def renew_certs(): # Needs to be run with sudo @@ -156,7 +145,6 @@ def setup_wildcard_ssl(domain, email, bench_path, exclude_base_domain): print("You cannot setup SSL without DNS Multitenancy") return - get_certbot() domain_list = _get_domains(domain.strip()) email_param = '' diff --git a/bench/config/templates/frappe_sudoers b/bench/config/templates/frappe_sudoers index 567ccf06..c0857408 100644 --- a/bench/config/templates/frappe_sudoers +++ b/bench/config/templates/frappe_sudoers @@ -15,6 +15,5 @@ {{ user }} ALL = (root) NOPASSWD: {{ nginx }} {% endif %} -{{ user }} ALL = (root) NOPASSWD: /opt/certbot-auto +{{ user }} ALL = (root) NOPASSWD: {{ certbot }} Defaults:{{ user }} !requiretty - diff --git a/bench/config/templates/nginx.conf b/bench/config/templates/nginx.conf index 77d865a2..4af72ab4 100644 --- a/bench/config/templates/nginx.conf +++ b/bench/config/templates/nginx.conf @@ -14,8 +14,10 @@ map {{ from_variable }} {{ to_variable }} { server { {% if ssl_certificate and ssl_certificate_key %} listen {{ port }} ssl; + listen [::]:{{ port }} ssl; {% else %} listen {{ port }}; + listen [::]:{{ port }}; {% endif %} server_name @@ -80,7 +82,7 @@ server { rewrite ^(.+)/index\.html$ $1 permanent; rewrite ^(.+)\.html$ $1 permanent; - location ~ ^/files/.*.(htm|html|svg|xml) { + location ~* ^/files/.*.(htm|html|svg|xml) { add_header Content-disposition "attachment"; try_files /{{ site_name }}/public/$uri @webserver; } diff --git a/bench/exceptions.py b/bench/exceptions.py index 0d517fe4..48cb6668 100644 --- a/bench/exceptions.py +++ b/bench/exceptions.py @@ -21,12 +21,22 @@ class BenchNotFoundError(Exception): class ValidationError(Exception): pass + +class AppNotInstalledError(ValidationError): + pass + + class CannotUpdateReleaseBench(ValidationError): pass + class FeatureDoesNotExistError(CommandFailedError): pass class NotInBenchDirectoryError(Exception): pass + + +class VersionNotFound(Exception): + pass \ No newline at end of file diff --git a/bench/tests/test_init.py b/bench/tests/test_init.py index 80018b59..0e2f1964 100755 --- a/bench/tests/test_init.py +++ b/bench/tests/test_init.py @@ -12,6 +12,7 @@ from bench.utils import exec_cmd from bench.release import get_bumped_version from bench.app import App from bench.tests.test_base import FRAPPE_BRANCH, TestBenchBase +from bench.bench import Bench # changed from frappe_theme because it wasn't maintained and incompatible, @@ -38,11 +39,14 @@ class TestBenchInit(TestBenchBase): def test_init(self, bench_name="test-bench", **kwargs): self.init_bench(bench_name, **kwargs) app = App("file:///tmp/frappe") - self.assertEqual(app.url, "/tmp/frappe") + self.assertEqual(app.mount_path, "/tmp/frappe") + self.assertEqual(app.url, "https://github.com/frappe/frappe.git") self.assert_folders(bench_name) self.assert_virtual_env(bench_name) self.assert_config(bench_name) - + test_bench = Bench(bench_name) + app = App("frappe", bench=test_bench) + self.assertEqual(app.from_apps, True) def basic(self): try: @@ -107,6 +111,20 @@ class TestBenchInit(TestBenchBase): app_installed_in_env = TEST_FRAPPE_APP in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8') self.assertTrue(app_installed_in_env) + def test_get_app_resolve_deps(self): + FRAPPE_APP = "healthcare" + self.init_bench("test-bench") + bench_path = os.path.join(self.benches_path, "test-bench") + exec_cmd(f"bench get-app {FRAPPE_APP} --resolve-deps", cwd=bench_path) + self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", FRAPPE_APP))) + + states_path = os.path.join(bench_path, "sites", "apps.json") + self.assert_(os.path.exists(states_path)) + + with open(states_path, "r") as f: + states = json.load(f) + + self.assert_(FRAPPE_APP in states) def test_install_app(self): bench_name = "test-bench" diff --git a/bench/tests/test_utils.py b/bench/tests/test_utils.py new file mode 100644 index 00000000..0575fa9e --- /dev/null +++ b/bench/tests/test_utils.py @@ -0,0 +1,80 @@ +import os +import shutil +import subprocess +import unittest +from tabnanny import check + +from bench.app import App +from bench.bench import Bench +from bench.exceptions import InvalidRemoteException +from bench.utils import is_valid_frappe_branch + + +class TestUtils(unittest.TestCase): + def test_app_utils(self): + git_url = "https://github.com/frappe/frappe" + branch = "develop" + app = App(name=git_url, branch=branch, bench=Bench(".")) + self.assertTrue( + all( + [ + app.name == git_url, + app.branch == branch, + app.tag == branch, + app.is_url == True, + app.on_disk == False, + app.org == "frappe", + app.url == git_url, + ] + ) + ) + + def test_is_valid_frappe_branch(self): + with self.assertRaises(InvalidRemoteException): + is_valid_frappe_branch("https://github.com/frappe/frappe.git", frappe_branch="random-branch") + is_valid_frappe_branch("https://github.com/random/random.git", frappe_branch="random-branch") + + is_valid_frappe_branch("https://github.com/frappe/frappe.git", frappe_branch="develop") + is_valid_frappe_branch("https://github.com/frappe/frappe.git", frappe_branch="v13.29.0") + + def test_app_states(self): + bench_dir = "./sandbox" + sites_dir = os.path.join(bench_dir, "sites") + + if not os.path.exists(sites_dir): + os.makedirs(sites_dir) + + fake_bench = Bench(bench_dir) + + self.assertTrue(hasattr(fake_bench.apps, "states")) + + fake_bench.apps.states = { + "frappe": {"resolution": {"branch": "develop", "commit_hash": "234rwefd"}, "version": "14.0.0-dev"} + } + fake_bench.apps.update_apps_states() + + self.assertEqual(fake_bench.apps.states, {}) + + frappe_path = os.path.join(bench_dir, "apps", "frappe") + + os.makedirs(os.path.join(frappe_path, "frappe")) + + subprocess.run(["git", "init"], cwd=frappe_path, capture_output=True, check=True) + + with open(os.path.join(frappe_path, "frappe", "__init__.py"), "w+") as f: + f.write("__version__ = '11.0'") + + subprocess.run(["git", "add", "."], cwd=frappe_path, capture_output=True, check=True) + subprocess.run(["git", "commit", "-m", "temp"], cwd=frappe_path, capture_output=True, check=True) + + fake_bench.apps.update_apps_states(app_name="frappe") + + self.assertIn("frappe", fake_bench.apps.states) + self.assertIn("version", fake_bench.apps.states["frappe"]) + self.assertEqual("11.0", fake_bench.apps.states["frappe"]["version"]) + + shutil.rmtree(bench_dir) + + def test_ssh_ports(self): + app = App("git@github.com:22:frappe/frappe") + self.assertEqual((app.use_ssh, app.org, app.repo), (True, "frappe", "frappe")) \ No newline at end of file diff --git a/bench/utils/__init__.py b/bench/utils/__init__.py index 9c8d98a4..85746877 100644 --- a/bench/utils/__init__.py +++ b/bench/utils/__init__.py @@ -8,14 +8,16 @@ import sys from glob import glob from shlex import split from typing import List, Tuple +from functools import lru_cache # imports - third party imports import click +import requests # imports - module imports from bench import PROJECT_NAME, VERSION -from bench.exceptions import CommandFailedError, InvalidRemoteException, ValidationError +from bench.exceptions import CommandFailedError, InvalidRemoteException, AppNotInstalledError logger = logging.getLogger(PROJECT_NAME) @@ -48,6 +50,33 @@ def is_frappe_app(directory: str) -> bool: return bool(is_frappe_app) +@lru_cache(maxsize=None) +def is_valid_frappe_branch(frappe_path:str, frappe_branch:str): + """Check if a branch exists in a repo. Throws InvalidRemoteException if branch is not found + + Uses native git command to check for branches on a remote. + + :param frappe_path: git url + :type frappe_path: str + :param frappe_branch: branch to check + :type frappe_branch: str + :raises InvalidRemoteException: branch for this repo doesn't exist + """ + import git + + g = git.cmd.Git() + + if frappe_branch: + try: + res = g.ls_remote("--heads", "--tags", frappe_path, frappe_branch) + if not res: + raise InvalidRemoteException( + f"Invalid branch or tag: {frappe_branch} for the remote {frappe_path}" + ) + except git.exc.GitCommandError: + raise InvalidRemoteException(f"Invalid frappe path: {frappe_path}") + + def log(message, level=0, no_log=False): import bench import bench.cli @@ -62,9 +91,7 @@ def log(message, level=0, no_log=False): color, prefix = levels.get(level, levels[0]) if bench.cli.from_command_line and bench.cli.dynamic_feed: - bench.LOG_BUFFER.append( - {"prefix": prefix, "message": message, "color": color} - ) + bench.LOG_BUFFER.append({"prefix": prefix, "message": message, "color": color}) if no_log: click.secho(message, fg=color) @@ -182,9 +209,7 @@ def get_git_version() -> float: def get_cmd_output(cmd, cwd=".", _raise=True): output = "" try: - output = subprocess.check_output( - cmd, cwd=cwd, shell=True, stderr=subprocess.PIPE, encoding="utf-8" - ).strip() + output = subprocess.check_output(cmd, cwd=cwd, shell=True, stderr=subprocess.PIPE, encoding="utf-8").strip() except subprocess.CalledProcessError as e: if e.output: output = e.output @@ -269,7 +294,7 @@ def set_git_remote_url(git_url, bench_path="."): app = git_url.rsplit("/", 1)[1].rsplit(".", 1)[0] if app not in Bench(bench_path).apps: - raise ValidationError(f"No app named {app}") + raise AppNotInstalledError(f"No app named {app}") app_dir = get_repo_dir(app, bench_path=bench_path) @@ -400,10 +425,12 @@ def find_org(org_repo): for org in ["frappe", "erpnext"]: res = requests.head(f"https://api.github.com/repos/{org}/{org_repo}") + if res.status_code in (400, 403): + res = requests.head(f"https://github.com/{org}/{org_repo}") if res.ok: return org, org_repo - raise InvalidRemoteException + raise InvalidRemoteException(f"{org_repo} not found in frappe or erpnext") def fetch_details_from_tag(_tag: str) -> Tuple[str, str, str]: diff --git a/bench/utils/app.py b/bench/utils/app.py index ae980743..158fdf96 100644 --- a/bench/utils/app.py +++ b/bench/utils/app.py @@ -7,8 +7,10 @@ from bench.exceptions import ( InvalidRemoteException, InvalidBranchException, CommandFailedError, + VersionNotFound, ) from bench.app import get_repo_dir +from functools import lru_cache def is_version_upgrade(app="frappe", bench_path=".", branch=None): @@ -107,7 +109,9 @@ def switch_to_develop(apps=None, bench_path=".", upgrade=True): def get_version_from_string(contents, field="__version__"): - match = re.search(r"^(\s*%s\s*=\s*['\\\"])(.+?)(['\"])(?sm)" % field, contents) + match = re.search(r"^(\s*%s\s*=\s*['\\\"])(.+?)(['\"])" % field, contents, flags=(re.S | re.M)) + if not match: + raise VersionNotFound(f"{contents} is not a valid version") return match.group(2) @@ -165,6 +169,31 @@ def get_current_branch(app, bench_path="."): return get_cmd_output("basename $(git symbolic-ref -q HEAD)", cwd=repo_dir) +@lru_cache(maxsize=5) +def get_required_deps(org, name, branch, deps="hooks.py"): + import requests + import base64 + + git_api_url = f"https://api.github.com/repos/{org}/{name}/contents/{name}/{deps}" + params = {"branch": branch or "develop"} + res = requests.get(url=git_api_url, params=params).json() + + if "message" in res: + git_url = f"https://raw.githubusercontent.com/{org}/{name}/{params['branch']}/{deps}" + return requests.get(git_url).text + + return base64.decodebytes(res["content"].encode()).decode() + + +def required_apps_from_hooks(required_deps, local=False): + if local: + with open(required_deps) as f: + required_deps = f.read() + lines = [x for x in required_deps.split("\n") if x.strip().startswith("required_apps")] + required_apps = eval(lines[0].strip("required_apps").strip().lstrip("=").strip()) + return required_apps + + def get_remote(app, bench_path="."): repo_dir = get_repo_dir(app, bench_path=bench_path) contents = subprocess.check_output( @@ -181,25 +210,30 @@ def get_remote(app, bench_path="."): return contents.splitlines()[0].split()[0] -def get_app_name(bench_path, repo_name): +def get_app_name(bench_path, folder_name): app_name = None apps_path = os.path.join(os.path.abspath(bench_path), "apps") - config_path = os.path.join(apps_path, repo_name, "setup.cfg") + config_path = os.path.join(apps_path, folder_name, "setup.cfg") if os.path.exists(config_path): config = read_configuration(config_path) app_name = config.get("metadata", {}).get("name") if not app_name: # retrieve app name from setup.py as fallback - app_path = os.path.join(apps_path, repo_name, "setup.py") + app_path = os.path.join(apps_path, folder_name, "setup.py") with open(app_path, "rb") as f: app_name = re.search(r'name\s*=\s*[\'"](.*)[\'"]', f.read().decode("utf-8")).group(1) - if app_name and repo_name != app_name: - os.rename(os.path.join(apps_path, repo_name), os.path.join(apps_path, app_name)) + if app_name and folder_name != app_name: + os.rename(os.path.join(apps_path, folder_name), os.path.join(apps_path, app_name)) return app_name - return repo_name + return folder_name + +def check_existing_dir(bench_path, repo_name): + cloned_path = os.path.join(bench_path, "apps", repo_name) + dir_already_exists = os.path.isdir(cloned_path) + return dir_already_exists, cloned_path def get_current_version(app, bench_path="."): diff --git a/bench/utils/bench.py b/bench/utils/bench.py index e0be48b9..8a695b7b 100644 --- a/bench/utils/bench.py +++ b/bench/utils/bench.py @@ -33,19 +33,25 @@ def get_env_cmd(cmd, bench_path="."): return os.path.abspath(os.path.join(bench_path, "env", "bin", cmd)) -def get_venv_path(): - venv = which("virtualenv") +def get_virtualenv_path(verbose=False): + virtualenv_path = which("virtualenv") - if not venv: - current_python = sys.executable - with open(os.devnull, "wb") as devnull: - is_venv_installed = not subprocess.call( - [current_python, "-m", "venv", "--help"], stdout=devnull - ) - if is_venv_installed: - venv = f"{current_python} -m venv" + if not virtualenv_path and verbose: + log("virtualenv cannot be found", level=2) - return venv or log("virtualenv cannot be found", level=2) + return virtualenv_path + + +def get_venv_path(verbose=False): + current_python = sys.executable + with open(os.devnull, "wb") as devnull: + is_venv_installed = not subprocess.call( + [current_python, "-m", "venv", "--help"], stdout=devnull + ) + if is_venv_installed: + return f"{current_python} -m venv" + else: + log("virtualenv cannot be found", level=2) def update_node_packages(bench_path=".", apps=None): @@ -74,7 +80,7 @@ def install_python_dev_dependencies(bench_path=".", apps=None, verbose=False): if isinstance(apps, str): apps = [apps] - elif apps is None: + elif not apps: apps = bench.get_installed_apps() for app in apps: diff --git a/bench/utils/system.py b/bench/utils/system.py index 50c8b1d8..131db2a9 100644 --- a/bench/utils/system.py +++ b/bench/utils/system.py @@ -14,6 +14,7 @@ from bench.utils import ( run_frappe_cmd, sudoers_file, which, + is_valid_frappe_branch, ) from bench.utils.bench import build_assets, clone_apps_from from bench.utils.render import job @@ -74,9 +75,14 @@ def init( # remote apps else: frappe_path = frappe_path or "https://github.com/frappe/frappe.git" - + is_valid_frappe_branch(frappe_path=frappe_path, frappe_branch=frappe_branch) get_app( - frappe_path, branch=frappe_branch, bench_path=path, skip_assets=True, verbose=verbose + frappe_path, + branch=frappe_branch, + bench_path=path, + skip_assets=True, + verbose=verbose, + resolve_deps=False, ) # fetch remote apps using config file - deprecate this! @@ -86,7 +92,12 @@ def init( # getting app on bench init using --install-app if install_app: get_app( - install_app, branch=frappe_branch, bench_path=path, skip_assets=True, verbose=verbose + install_app, + branch=frappe_branch, + bench_path=path, + skip_assets=True, + verbose=verbose, + resolve_deps=False, ) if not skip_assets: @@ -97,6 +108,8 @@ def init( def setup_sudoers(user): + from bench.config.lets_encrypt import get_certbot_path + if not os.path.exists("/etc/sudoers.d"): os.makedirs("/etc/sudoers.d") @@ -117,6 +130,7 @@ def setup_sudoers(user): "service": which("service"), "systemctl": which("systemctl"), "nginx": which("nginx"), + "certbot": get_certbot_path(), } ) diff --git a/requirements.txt b/requirements.txt index 68e7c7d3..c29eb08e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,3 @@ python-crontab~=2.4.0 requests semantic-version~=2.8.2 setuptools -virtualenv