accounting/bench
2
0
Fork 0
mirror of https://github.com/frappe/bench.git synced 2025-01-09 08:30:39 +00:00
Code Issues Releases Activity

Merge branch 'develop' into staging

Browse Source
This commit is contained in:
Gavin D'souza 2022-06-08 13:28:22 +05:30
commit 2005fde2cd
16 changed files with 619 additions and 196 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
.travis.yml
View File

@ -48,32 +48,17 @@ matrix:
- name: "Python 3.7 Tests"
python: 3.7
env: TEST=bench
script: python -m unittest -v bench.tests.test_init
script: python -m unittest -v bench.tests.test_utils && python -m unittest -v bench.tests.test_init
- name: "Python 3.8 Tests"
python: 3.8
env: TEST=bench
script: python -m unittest -v bench.tests.test_init
script: python -m unittest -v bench.tests.test_utils && python -m unittest -v bench.tests.test_init
- name: "Python 3.9 Tests"
python: 3.9
env: TEST=bench
script: python -m unittest -v bench.tests.test_init
- name: "Python 3.7 Easy Install"
python: 3.7
env: TEST=easy_install
script: sudo python $TRAVIS_BUILD_DIR/install.py --user travis --run-travis --production --verbose
- name: "Python 3.8 Easy Install"
python: 3.8
env: TEST=easy_install
script: sudo python $TRAVIS_BUILD_DIR/install.py --user travis --run-travis --production --verbose
- name: "Python 3.9 Easy Install"
python: 3.9
env: TEST=easy_install
script: sudo python $TRAVIS_BUILD_DIR/install.py --user travis --run-travis --production --verbose
script: python -m unittest -v bench.tests.test_utils && python -m unittest -v bench.tests.test_init
install:
- pip3 install urllib3 pyOpenSSL ndg-httpsclient pyasn1

2
README.md
View File

@ -54,7 +54,7 @@ The setup for each of these installations can be achieved in multiple ways:
We recommend using either the Docker Installation or the Easy Install Script to setup a Production Environment. For Development, you may choose either of the three methods to setup an instance.
Otherwise, if you are looking to evaluate ERPNext, you can also download the [Virtual Machine Image](https://erpnext.com/download) or register for [a free trial on erpnext.com](https://erpnext.com/pricing).
Otherwise, if you are looking to evaluate ERPNext, you can register for [a free trial on erpnext.com](https://erpnext.com/pricing).
### Containerized Installation

341
bench/app.py
View File

@ -8,11 +8,14 @@ import shutil
import subprocess
import sys
import typing
from collections import OrderedDict
from datetime import date
from urllib.parse import urlparse
import os
# imports - third party imports
import click
from git import Repo
# imports - module imports
import bench
@ -22,6 +25,7 @@ from bench.utils import (
get_available_folder_name,
is_bench_directory,
is_git_url,
is_valid_frappe_branch,
log,
run_frappe_cmd,
)
@ -55,7 +59,7 @@ class AppMeta:
class Healthcare(AppConfig):
dependencies = [{"frappe/erpnext": "~13.17.0"}]
"""
self.name = name.rstrip('/')
self.name = name.rstrip("/")
self.remote_server = "github.com"
self.to_clone = to_clone
self.on_disk = False
@ -63,6 +67,8 @@ class AppMeta:
self.from_apps = False
self.is_url = False
self.branch = branch
self.app_name = None
self.git_repo = None
self.mount_path = os.path.abspath(
os.path.join(urlparse(self.name).netloc, urlparse(self.name).path)
)
@ -70,13 +76,12 @@ class AppMeta:
def setup_details(self):
# fetch meta from installed apps
if (
not self.to_clone
and hasattr(self, "bench")
and os.path.exists(os.path.join(self.bench.name, "apps", self.name))
if self.bench and os.path.exists(
os.path.join(self.bench.name, "apps", self.name)
):
self.mount_path = os.path.join(self.bench.name, "apps", self.name)
self.from_apps = True
self._setup_details_from_installed_apps()
self._setup_details_from_mounted_disk()
# fetch meta for repo on mounted disk
elif os.path.exists(self.mount_path):
@ -86,50 +91,56 @@ class AppMeta:
# fetch meta for repo from remote git server - traditional get-app url
elif is_git_url(self.name):
self.is_url = True
if self.name.startswith("git@") or self.name.startswith("ssh://"):
self.use_ssh = True
self._setup_details_from_git_url()
# fetch meta from new styled name tags & first party apps on github
else:
self._setup_details_from_name_tag()
if self.git_repo:
self.app_name = os.path.basename(
os.path.normpath(self.git_repo.working_tree_dir)
)
else:
self.app_name = self.repo
def _setup_details_from_mounted_disk(self):
self.org, self.repo, self.tag = os.path.split(self.mount_path)[-2:] + (
self.branch,
)
# If app is a git repo
self.git_repo = Repo(self.mount_path)
try:
self._setup_details_from_git_url(self.git_repo.remotes[0].url)
if not (self.branch or self.tag):
self.tag = self.branch = self.git_repo.active_branch.name
except IndexError:
self.org, self.repo, self.tag = os.path.split(self.mount_path)[-2:] + (self.branch,)
except TypeError:
# faced a "a detached symbolic reference as it points" in case you're in the middle of
# some git shenanigans
self.tag = self.branch = None
def _setup_details_from_name_tag(self):
self.org, self.repo, self.tag = fetch_details_from_tag(self.name)
self.tag = self.tag or self.branch
def _setup_details_from_installed_apps(self):
self.org, self.repo, self.tag = os.path.split(
os.path.join(self.bench.name, "apps", self.name)
)[-2:] + (self.branch,)
def _setup_details_from_git_url(self, url=None):
return self.__setup_details_from_git(url)
def _setup_details_from_git_url(self):
return self.__setup_details_from_git()
def __setup_details_from_git(self):
if self.use_ssh:
_first_part, _second_part = self.name.split(":")
def __setup_details_from_git(self, url=None):
name = url if url else self.name
if name.startswith("git@") or name.startswith("ssh://"):
self.use_ssh = True
_first_part, _second_part = name.rsplit(":", 1)
self.remote_server = _first_part.split("@")[-1]
self.org, _repo = _second_part.rsplit("/", 1)
else:
self.remote_server, self.org, _repo = self.name.rsplit("/", 2)
protocal = "https://" if "https://" in name else "http://"
self.remote_server, self.org, _repo = name.replace(protocal, "").rsplit("/", 2)
self.tag = self.branch
self.repo = _repo.split(".")[0]
@property
def url(self):
if self.from_apps:
return os.path.abspath(os.path.join("apps", self.name))
if self.on_disk:
return self.mount_path
if self.is_url:
return self.name
@ -147,10 +158,10 @@ class AppMeta:
@functools.lru_cache(maxsize=None)
class App(AppMeta):
def __init__(
self, name: str, branch: str = None, bench: "Bench" = None, *args, **kwargs
):
def __init__(self, name: str, branch: str = None, bench: "Bench" = None, *args, **kwargs):
self.bench = bench
self.required_by = None
self.local_resolution = []
super().__init__(name, branch, *args, **kwargs)
@step(title="Fetching App {repo}", success="App {repo} Fetched")
@ -168,44 +179,100 @@ class App(AppMeta):
)
@step(title="Archiving App {repo}", success="App {repo} Archived")
def remove(self):
active_app_path = os.path.join("apps", self.repo)
archived_path = os.path.join("archived", "apps")
archived_name = get_available_folder_name(
f"{self.repo}-{date.today()}", archived_path
)
archived_app_path = os.path.join(archived_path, archived_name)
log(f"App moved from {active_app_path} to {archived_app_path}")
shutil.move(active_app_path, archived_app_path)
def remove(self, no_backup: bool = False):
active_app_path = os.path.join("apps", self.name)
if no_backup:
shutil.rmtree(active_app_path)
log(f"App deleted from {active_app_path}")
else:
archived_path = os.path.join("archived", "apps")
archived_name = get_available_folder_name(f"{self.repo}-{date.today()}", archived_path)
archived_app_path = os.path.join(archived_path, archived_name)
shutil.move(active_app_path, archived_app_path)
log(f"App moved from {active_app_path} to {archived_app_path}")
@step(title="Installing App {repo}", success="App {repo} Installed")
def install(self, skip_assets=False, verbose=False, restart_bench=True):
def install(
self,
skip_assets=False,
verbose=False,
resolved=False,
restart_bench=True,
ignore_resolution=False,
):
import bench.cli
from bench.utils.app import get_app_name
verbose = bench.cli.verbose or verbose
app_name = get_app_name(self.bench.name, self.repo)
# TODO: this should go inside install_app only tho - issue: default/resolved branch
setup_app_dependencies(
repo_name=self.repo,
bench_path=self.bench.name,
branch=self.tag,
verbose=verbose,
skip_assets=skip_assets,
)
app_name = get_app_name(self.bench.name, self.app_name)
if not resolved and self.repo != "frappe" and not ignore_resolution:
click.secho(
f"Ignoring dependencies of {self.name}. To install dependencies use --resolve-deps",
fg="yellow",
)
install_app(
app=app_name,
tag=self.tag,
bench_path=self.bench.name,
verbose=verbose,
skip_assets=skip_assets,
restart_bench=restart_bench
restart_bench=restart_bench,
resolution=self.local_resolution
)
@step(title="Cloning and installing {repo}", success="App {repo} Installed")
def install_resolved_apps(self, *args, **kwargs):
self.get()
self.install(*args, **kwargs, resolved=True)
@step(title="Uninstalling App {repo}", success="App {repo} Uninstalled")
def uninstall(self):
self.bench.run(f"{self.bench.python} -m pip uninstall -y {self.repo}")
self.bench.run(f"{self.bench.python} -m pip uninstall -y {self.name}")
def _get_dependencies(self):
from bench.utils.app import get_required_deps, required_apps_from_hooks
if self.on_disk:
required_deps = os.path.join(self.mount_path, self.repo,'hooks.py')
try:
return required_apps_from_hooks(required_deps, local=True)
except IndexError:
return []
try:
required_deps = get_required_deps(self.org, self.repo, self.tag or self.branch)
return required_apps_from_hooks(required_deps)
except Exception:
return []
def update_app_state(self):
from bench.bench import Bench
bench = Bench(self.bench.name)
bench.apps.sync(app_dir=self.app_name, app_name=self.name,
branch=self.tag, required_list=self.local_resolution)
def make_resolution_plan(app: App, bench: "Bench"):
"""
decide what apps and versions to install and in what order
"""
resolution = OrderedDict()
resolution[app.repo] = app
for app_name in app._get_dependencies():
dep_app = App(app_name, bench=bench)
is_valid_frappe_branch(dep_app.url, dep_app.branch)
dep_app.required_by = app.name
if dep_app.repo in resolution:
click.secho(f"{dep_app.repo} is already resolved skipping", fg="yellow")
continue
resolution[dep_app.repo] = dep_app
resolution.update(make_resolution_plan(dep_app, bench))
app.local_resolution = [repo_name for repo_name, _ in reversed(resolution.items())]
return resolution
def add_to_appstxt(app, bench_path="."):
@ -264,36 +331,6 @@ def remove_from_excluded_apps_txt(app, bench_path="."):
return write_excluded_apps_txt(apps, bench_path=bench_path)
def setup_app_dependencies(
repo_name, bench_path=".", branch=None, skip_assets=False, verbose=False
):
# branch kwarg is somewhat of a hack here; since we're assuming the same branches for all apps
# for eg: if you're installing erpnext@develop, you'll want frappe@develop and healthcare@develop too
import glob
import bench.cli
from bench.bench import Bench
verbose = bench.cli.verbose or verbose
apps_path = os.path.join(os.path.abspath(bench_path), "apps")
files = glob.glob(os.path.join(apps_path, repo_name, "**", "hooks.py"))
if files:
with open(files[0]) as f:
lines = [x for x in f.read().split("\n") if x.strip().startswith("required_apps")]
if lines:
required_apps = eval(lines[0].strip("required_apps").strip().lstrip("=").strip())
# TODO: when the time comes, add version check here
for app in required_apps:
if app not in Bench(bench_path).apps:
get_app(
app,
bench_path=bench_path,
branch=branch,
skip_assets=skip_assets,
verbose=verbose,
)
def get_app(
git_url,
branch=None,
@ -302,6 +339,7 @@ def get_app(
verbose=False,
overwrite=False,
init_bench=False,
resolve_deps=False,
):
"""bench get-app clones a Frappe App from remote (GitHub or any other git server),
and installs it on the current bench. This also resolves dependencies based on the
@ -310,9 +348,10 @@ def get_app(
If the bench_path is not a bench directory, a new bench is created named using the
git_url parameter.
"""
from bench.bench import Bench
import bench as _bench
import bench.cli as bench_cli
from bench.bench import Bench
from bench.utils.app import check_existing_dir
bench = Bench(bench_path)
app = App(git_url, branch=branch, bench=bench)
@ -321,6 +360,17 @@ def get_app(
branch = app.tag
bench_setup = False
restart_bench = not init_bench
frappe_path, frappe_branch = None, None
if resolve_deps:
resolution = make_resolution_plan(app, bench)
click.secho("Following apps will be installed", fg="bright_blue")
for idx, app in enumerate(reversed(resolution.values()), start=1):
print(f"{idx}. {app.name} {f'(required by {app.required_by})' if app.required_by else ''}")
if "frappe" in resolution:
# Todo: Make frappe a terminal dependency for all frappe apps.
frappe_path, frappe_branch = resolution["frappe"].url, resolution["frappe"].tag
if not is_bench_directory(bench_path):
if not init_bench:
@ -332,20 +382,35 @@ def get_app(
from bench.utils.system import init
bench_path = get_available_folder_name(f"{app.repo}-bench", bench_path)
init(path=bench_path, frappe_branch=branch)
init(
path=bench_path,
frappe_path=frappe_path,
frappe_branch=frappe_branch if frappe_branch else branch,
)
os.chdir(bench_path)
bench_setup = True
if bench_setup and bench_cli.from_command_line and bench_cli.dynamic_feed:
_bench.LOG_BUFFER.append({
"message": f"Fetching App {repo_name}",
"prefix": click.style('⏼', fg='bright_yellow'),
"is_parent": True,
"color": None,
})
_bench.LOG_BUFFER.append(
{
"message": f"Fetching App {repo_name}",
"prefix": click.style("⏼", fg="bright_yellow"),
"is_parent": True,
"color": None,
}
)
cloned_path = os.path.join(bench_path, "apps", repo_name)
dir_already_exists = os.path.isdir(cloned_path)
if resolve_deps:
install_resolved_deps(
bench,
resolution,
bench_path=bench_path,
skip_assets=skip_assets,
verbose=verbose,
)
return
dir_already_exists, cloned_path = check_existing_dir(bench_path, repo_name)
to_clone = not dir_already_exists
# application directory already exists
@ -371,22 +436,89 @@ def get_app(
app.install(verbose=verbose, skip_assets=skip_assets, restart_bench=restart_bench)
def install_resolved_deps(
bench,
resolution,
bench_path=".",
skip_assets=False,
verbose=False,
):
from bench.utils.app import check_existing_dir
if "frappe" in resolution:
# Terminal dependency
del resolution["frappe"]
for repo_name, app in reversed(resolution.items()):
existing_dir, path_to_app = check_existing_dir(bench_path, repo_name)
if existing_dir:
is_compatible = False
try:
installed_branch = bench.apps.states[repo_name]["resolution"]["branch"].strip()
except Exception:
installed_branch = (
subprocess.
check_output("git rev-parse --abbrev-ref HEAD", shell=True, cwd=path_to_app)
.decode("utf-8")
.rstrip()
)
try:
if app.tag is None:
current_remote = (
subprocess.check_output(f"git config branch.{installed_branch}.remote", shell=True, cwd=path_to_app)
.decode("utf-8")
.rstrip()
)
default_branch = (
subprocess.check_output(
f"git symbolic-ref refs/remotes/{current_remote}/HEAD", shell=True, cwd=path_to_app
)
.decode("utf-8")
.rsplit("/")[-1]
.strip()
)
is_compatible = default_branch == installed_branch
else:
is_compatible = installed_branch == app.tag
except Exception:
is_compatible = False
prefix = 'C' if is_compatible else 'Inc'
click.secho(
f"{prefix}ompatible version of {repo_name} is already installed",
fg="green" if is_compatible else "red",
)
app.update_app_state()
if click.confirm(
f"Do you wish to clone and install the already installed {prefix}ompatible app"
):
click.secho(f"Removing installed app {app.name}", fg="yellow")
shutil.rmtree(path_to_app)
else:
continue
app.install_resolved_apps(skip_assets=skip_assets, verbose=verbose)
def new_app(app, no_git=None, bench_path="."):
if bench.FRAPPE_VERSION in (0, None):
raise NotInBenchDirectoryError(
f"{os.path.realpath(bench_path)} is not a valid bench directory."
)
raise NotInBenchDirectoryError(f"{os.path.realpath(bench_path)} is not a valid bench directory.")
# For backwards compatibility
app = app.lower().replace(" ", "_").replace("-", "_")
if app[0].isdigit() or "." in app:
click.secho(
"App names cannot start with numbers(digits) or have dot(.) in them",
fg="red"
)
return
apps = os.path.abspath(os.path.join(bench_path, "apps"))
args = ["make-app", apps, app]
if no_git:
if bench.FRAPPE_VERSION < 14:
click.secho(
"Frappe v14 or greater is needed for '--no-git' flag",
fg="red"
)
click.secho("Frappe v14 or greater is needed for '--no-git' flag", fg="red")
return
args.append(no_git)
@ -397,11 +529,13 @@ def new_app(app, no_git=None, bench_path="."):
def install_app(
app,
tag=None,
bench_path=".",
verbose=False,
no_cache=False,
restart_bench=True,
skip_assets=False,
resolution=[]
):
import bench.cli as bench_cli
from bench.bench import Bench
@ -427,7 +561,7 @@ def install_app(
if os.path.exists(os.path.join(app_path, "package.json")):
bench.run("yarn install", cwd=app_path)
bench.apps.sync()
bench.apps.sync(app_name=app, required=resolution, branch=tag, app_dir=app_path)
if not skip_assets:
build_assets(bench_path=bench_path, app=app)
@ -526,7 +660,10 @@ def install_apps_from_path(path, bench_path="."):
apps = get_apps_json(path)
for app in apps:
get_app(
app["url"], branch=app.get("branch"), bench_path=bench_path, skip_assets=True,
app["url"],
branch=app.get("branch"),
bench_path=bench_path,
skip_assets=True,
)

142
bench/bench.py
View File

@ -1,14 +1,16 @@
# imports - standard imports
import subprocess
import functools
import os
import shutil
import json
import sys
import logging
from typing import List, MutableSequence, TYPE_CHECKING
from typing import List, MutableSequence, TYPE_CHECKING, Union
# imports - module imports
import bench
from bench.exceptions import ValidationError
from bench.exceptions import AppNotInstalledError, InvalidRemoteException
from bench.config.common_site_config import setup_config
from bench.utils import (
paths_in_bench,
@ -27,9 +29,11 @@ from bench.utils.bench import (
restart_process_manager,
remove_backups_crontab,
get_venv_path,
get_virtualenv_path,
get_env_cmd,
)
from bench.utils.render import job, step
from bench.utils.app import get_current_version
if TYPE_CHECKING:
@ -46,7 +50,7 @@ class Base:
class Validator:
def validate_app_uninstall(self, app):
if app not in self.apps:
raise ValidationError(f"No app named {app}")
raise AppNotInstalledError(f"No app named {app}")
validate_app_installed_on_sites(app, bench_path=self.name)
@ -116,11 +120,16 @@ class Bench(Base, Validator):
self.apps.append(app)
self.apps.sync()
def uninstall(self, app):
def uninstall(self, app, no_backup=False, force=False):
from bench.app import App
self.validate_app_uninstall(app)
self.apps.remove(App(app, bench=self, to_clone=False))
if not force:
self.validate_app_uninstall(app)
try:
self.apps.remove(App(app, bench=self, to_clone=False), no_backup=no_backup)
except InvalidRemoteException:
if not force:
raise
self.apps.sync()
# self.build() - removed because it seems unnecessary
self.reload()
@ -155,12 +164,102 @@ class Bench(Base, Validator):
class BenchApps(MutableSequence):
def __init__(self, bench: Bench):
self.bench = bench
self.states_path = os.path.join(self.bench.name, "sites", "apps.json")
self.apps_path = os.path.join(self.bench.name, "apps")
self.initialize_apps()
self.set_states()
def set_states(self):
try:
with open(self.states_path, "r") as f:
self.states = json.loads(f.read() or "{}")
except FileNotFoundError:
self.states = {}
def update_apps_states(
self,
app_dir: str = None,
app_name: Union[str, None] = None,
branch: Union[str, None] = None,
required: List = [],
):
if self.apps and not os.path.exists(self.states_path):
# idx according to apps listed in apps.txt (backwards compatibility)
# Keeping frappe as the first app.
if "frappe" in self.apps:
self.apps.remove("frappe")
self.apps.insert(0, "frappe")
with open(self.bench.apps_txt, "w") as f:
f.write("\n".join(self.apps))
print("Found existing apps updating states...")
for idx, app in enumerate(self.apps, start=1):
self.states[app] = {
"resolution": {
"commit_hash": None,
"branch": None
},
"required": required,
"idx": idx,
"version": get_current_version(app, self.bench.name),
}
apps_to_remove = []
for app in self.states:
if app not in self.apps:
apps_to_remove.append(app)
for app in apps_to_remove:
del self.states[app]
if app_name and not app_dir:
app_dir = app_name
if app_name and app_name not in self.states:
version = get_current_version(app_name, self.bench.name)
app_dir = os.path.join(self.apps_path, app_dir)
if not branch:
branch = (
subprocess
.check_output("git rev-parse --abbrev-ref HEAD", shell=True, cwd=app_dir)
.decode("utf-8")
.rstrip()
)
commit_hash = subprocess.check_output(f"git rev-parse {branch}", shell=True, cwd=app_dir).decode("utf-8").rstrip()
self.states[app_name] = {
"resolution": {
"commit_hash":commit_hash,
"branch": branch
},
"required":required,
"idx":len(self.states) + 1,
"version": version,
}
with open(self.states_path, "w") as f:
f.write(json.dumps(self.states, indent=4))
def sync(
self,
app_name: Union[str, None] = None,
app_dir: Union[str, None] = None,
branch: Union[str, None] = None,
required: List = []
):
self.initialize_apps()
def sync(self):
self.initialize_apps()
with open(self.bench.apps_txt, "w") as f:
return f.write("\n".join(self.apps))
f.write("\n".join(self.apps))
self.update_apps_states(
app_name=app_name,
app_dir=app_dir,
branch=branch,
required=required
)
def initialize_apps(self):
is_installed = lambda app: app in installed_packages
@ -180,7 +279,6 @@ class BenchApps(MutableSequence):
and is_installed(x)
)
]
self.apps.sort()
except FileNotFoundError:
self.apps = []
@ -213,9 +311,9 @@ class BenchApps(MutableSequence):
super().append(app.repo)
self.apps.sort()
def remove(self, app: "App"):
def remove(self, app: "App", no_backup: bool = False):
app.uninstall()
app.remove()
app.remove(no_backup=no_backup)
super().remove(app.repo)
def append(self, app: "App"):
@ -248,13 +346,22 @@ class BenchSetup(Base):
- install frappe python dependencies
"""
import bench.cli
import click
verbose = bench.cli.verbose
click.secho("Setting Up Environment", fg="yellow")
frappe = os.path.join(self.bench.name, "apps", "frappe")
virtualenv = get_venv_path()
quiet_flag = "" if bench.cli.verbose else "--quiet"
virtualenv = get_virtualenv_path(verbose=verbose)
quiet_flag = "" if verbose else "--quiet"
if not os.path.exists(self.bench.python):
self.run(f"{virtualenv} {quiet_flag} env -p {python}")
if virtualenv:
self.run(f"{virtualenv} {quiet_flag} env -p {python}")
else:
venv = get_venv_path(verbose=verbose)
self.run(f"{venv} env")
self.pip()
@ -339,7 +446,10 @@ class BenchSetup(Base):
print(f"Installing {len(apps)} applications...")
for app in apps:
App(app, bench=self.bench, to_clone=False).install( skip_assets=True, restart_bench=False)
path_to_app = os.path.join(self.bench.name, "apps", app)
app = App(path_to_app, bench=self.bench, to_clone=False).install(
skip_assets=True, restart_bench=False, ignore_resolution=True
)
def python(self, apps=None):
"""Install and upgrade Python dependencies for specified / all installed apps on given Bench

22
bench/commands/make.py
View File

@ -133,8 +133,20 @@ def drop(path):
@click.option(
"--init-bench", is_flag=True, default=False, help="Initialize Bench if not in one"
)
@click.option(
"--resolve-deps",
is_flag=True,
default=False,
help="Resolve dependencies before installing app",
)
def get_app(
git_url, branch, name=None, overwrite=False, skip_assets=False, init_bench=False
git_url,
branch,
name=None,
overwrite=False,
skip_assets=False,
init_bench=False,
resolve_deps=False,
):
"clone an app from the internet and set it up in your bench"
from bench.app import get_app
@ -145,9 +157,9 @@ def get_app(
skip_assets=skip_assets,
overwrite=overwrite,
init_bench=init_bench,
resolve_deps=resolve_deps,
)
@click.command("new-app", help="Create a new Frappe application under apps folder")
@click.option(
"--no-git",
@ -168,12 +180,14 @@ def new_app(app_name, no_git=None):
"Completely remove app from bench and re-build assets if not installed on any site"
),
)
@click.option("--no-backup", is_flag=True, help="Do not backup app before removing")
@click.option("--force", is_flag=True, help="Force remove app")
@click.argument("app-name")
def remove_app(app_name):
def remove_app(app_name, no_backup=False, force=False):
from bench.bench import Bench
bench = Bench(".")
bench.uninstall(app_name)
bench.uninstall(app_name, no_backup=no_backup, force=force)
@click.command("exclude-app", help="Exclude app from updating")

24
bench/config/lets_encrypt.py
View File

@ -10,11 +10,10 @@ from bench.config.nginx import make_nginx_conf
from bench.config.production_setup import service
from bench.config.site_config import get_domains, remove_domain, update_site_config
from bench.bench import Bench
from bench.utils import exec_cmd
from bench.utils import exec_cmd, which
from bench.utils.bench import update_common_site_config
from bench.exceptions import CommandFailedError
def setup_letsencrypt(site, custom_domain, bench_path, interactive):
site_path = os.path.join(bench_path, "sites", site, "site_config.json")
@ -58,7 +57,6 @@ def create_config(site, custom_domain):
def run_certbot_and_setup_ssl(site, custom_domain, bench_path, interactive=True):
service('nginx', 'stop')
get_certbot()
try:
interactive = '' if interactive else '-n'
@ -88,7 +86,7 @@ def run_certbot_and_setup_ssl(site, custom_domain, bench_path, interactive=True)
def setup_crontab():
from crontab import CronTab
job_command = '/opt/certbot-auto renew -a nginx --post-hook "systemctl reload nginx"'
job_command = f'{get_certbot_path()} renew -a nginx --post-hook "systemctl reload nginx"'
job_comment = 'Renew lets-encrypt every month'
print(f"Setting Up cron job to {job_comment}")
@ -107,20 +105,11 @@ def create_dir_if_missing(path):
os.makedirs(os.path.dirname(path))
def get_certbot():
from urllib.request import urlretrieve
certbot_path = get_certbot_path()
create_dir_if_missing(certbot_path)
if not os.path.isfile(certbot_path):
urlretrieve("https://dl.eff.org/certbot-auto", certbot_path)
os.chmod(certbot_path, 0o744)
def get_certbot_path():
return "/opt/certbot-auto"
try:
return which("certbot", raise_err=True)
except FileNotFoundError:
raise CommandFailedError("Certbot is not installed on your system. Please visit https://certbot.eff.org/instructions for installation instructions, then try again.")
def renew_certs():
# Needs to be run with sudo
@ -156,7 +145,6 @@ def setup_wildcard_ssl(domain, email, bench_path, exclude_base_domain):
print("You cannot setup SSL without DNS Multitenancy")
return
get_certbot()
domain_list = _get_domains(domain.strip())
email_param = ''

3
bench/config/templates/frappe_sudoers
View File

@ -15,6 +15,5 @@
{{ user }} ALL = (root) NOPASSWD: {{ nginx }}
{% endif %}
{{ user }} ALL = (root) NOPASSWD: /opt/certbot-auto
{{ user }} ALL = (root) NOPASSWD: {{ certbot }}
Defaults:{{ user }} !requiretty

4
bench/config/templates/nginx.conf
View File

@ -14,8 +14,10 @@ map {{ from_variable }} {{ to_variable }} {
server {
{% if ssl_certificate and ssl_certificate_key %}
listen {{ port }} ssl;
listen [::]:{{ port }} ssl;
{% else %}
listen {{ port }};
listen [::]:{{ port }};
{% endif %}
server_name
@ -80,7 +82,7 @@ server {
rewrite ^(.+)/index\.html$ $1 permanent;
rewrite ^(.+)\.html$ $1 permanent;
location ~ ^/files/.*.(htm|html|svg|xml) {
location ~* ^/files/.*.(htm|html|svg|xml) {
add_header Content-disposition "attachment";
try_files /{{ site_name }}/public/$uri @webserver;
}

10
bench/exceptions.py
View File

@ -21,12 +21,22 @@ class BenchNotFoundError(Exception):
class ValidationError(Exception):
pass
class AppNotInstalledError(ValidationError):
pass
class CannotUpdateReleaseBench(ValidationError):
pass
class FeatureDoesNotExistError(CommandFailedError):
pass
class NotInBenchDirectoryError(Exception):
pass
class VersionNotFound(Exception):
pass

22
bench/tests/test_init.py
View File

@ -12,6 +12,7 @@ from bench.utils import exec_cmd
from bench.release import get_bumped_version
from bench.app import App
from bench.tests.test_base import FRAPPE_BRANCH, TestBenchBase
from bench.bench import Bench
# changed from frappe_theme because it wasn't maintained and incompatible,
@ -38,11 +39,14 @@ class TestBenchInit(TestBenchBase):
def test_init(self, bench_name="test-bench", **kwargs):
self.init_bench(bench_name, **kwargs)
app = App("file:///tmp/frappe")
self.assertEqual(app.url, "/tmp/frappe")
self.assertEqual(app.mount_path, "/tmp/frappe")
self.assertEqual(app.url, "https://github.com/frappe/frappe.git")
self.assert_folders(bench_name)
self.assert_virtual_env(bench_name)
self.assert_config(bench_name)
test_bench = Bench(bench_name)
app = App("frappe", bench=test_bench)
self.assertEqual(app.from_apps, True)
def basic(self):
try:
@ -107,6 +111,20 @@ class TestBenchInit(TestBenchBase):
app_installed_in_env = TEST_FRAPPE_APP in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8')
self.assertTrue(app_installed_in_env)
def test_get_app_resolve_deps(self):
FRAPPE_APP = "healthcare"
self.init_bench("test-bench")
bench_path = os.path.join(self.benches_path, "test-bench")
exec_cmd(f"bench get-app {FRAPPE_APP} --resolve-deps", cwd=bench_path)
self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", FRAPPE_APP)))
states_path = os.path.join(bench_path, "sites", "apps.json")
self.assert_(os.path.exists(states_path))
with open(states_path, "r") as f:
states = json.load(f)
self.assert_(FRAPPE_APP in states)
def test_install_app(self):
bench_name = "test-bench"

80
bench/tests/test_utils.py Normal file
View File

@ -0,0 +1,80 @@
import os
import shutil
import subprocess
import unittest
from tabnanny import check
from bench.app import App
from bench.bench import Bench
from bench.exceptions import InvalidRemoteException
from bench.utils import is_valid_frappe_branch
class TestUtils(unittest.TestCase):
def test_app_utils(self):
git_url = "https://github.com/frappe/frappe"
branch = "develop"
app = App(name=git_url, branch=branch, bench=Bench("."))
self.assertTrue(
all(
[
app.name == git_url,
app.branch == branch,
app.tag == branch,
app.is_url == True,
app.on_disk == False,
app.org == "frappe",
app.url == git_url,
]
)
)
def test_is_valid_frappe_branch(self):
with self.assertRaises(InvalidRemoteException):
is_valid_frappe_branch("https://github.com/frappe/frappe.git", frappe_branch="random-branch")
is_valid_frappe_branch("https://github.com/random/random.git", frappe_branch="random-branch")
is_valid_frappe_branch("https://github.com/frappe/frappe.git", frappe_branch="develop")
is_valid_frappe_branch("https://github.com/frappe/frappe.git", frappe_branch="v13.29.0")
def test_app_states(self):
bench_dir = "./sandbox"
sites_dir = os.path.join(bench_dir, "sites")
if not os.path.exists(sites_dir):
os.makedirs(sites_dir)
fake_bench = Bench(bench_dir)
self.assertTrue(hasattr(fake_bench.apps, "states"))
fake_bench.apps.states = {
"frappe": {"resolution": {"branch": "develop", "commit_hash": "234rwefd"}, "version": "14.0.0-dev"}
}
fake_bench.apps.update_apps_states()
self.assertEqual(fake_bench.apps.states, {})
frappe_path = os.path.join(bench_dir, "apps", "frappe")
os.makedirs(os.path.join(frappe_path, "frappe"))
subprocess.run(["git", "init"], cwd=frappe_path, capture_output=True, check=True)
with open(os.path.join(frappe_path, "frappe", "__init__.py"), "w+") as f:
f.write("__version__ = '11.0'")
subprocess.run(["git", "add", "."], cwd=frappe_path, capture_output=True, check=True)
subprocess.run(["git", "commit", "-m", "temp"], cwd=frappe_path, capture_output=True, check=True)
fake_bench.apps.update_apps_states(app_name="frappe")
self.assertIn("frappe", fake_bench.apps.states)
self.assertIn("version", fake_bench.apps.states["frappe"])
self.assertEqual("11.0", fake_bench.apps.states["frappe"]["version"])
shutil.rmtree(bench_dir)
def test_ssh_ports(self):
app = App("git@github.com:22:frappe/frappe")
self.assertEqual((app.use_ssh, app.org, app.repo), (True, "frappe", "frappe"))

45
bench/utils/__init__.py
View File

@ -8,14 +8,16 @@ import sys
from glob import glob
from shlex import split
from typing import List, Tuple
from functools import lru_cache
# imports - third party imports
import click
import requests
# imports - module imports
from bench import PROJECT_NAME, VERSION
from bench.exceptions import CommandFailedError, InvalidRemoteException, ValidationError
from bench.exceptions import CommandFailedError, InvalidRemoteException, AppNotInstalledError
logger = logging.getLogger(PROJECT_NAME)
@ -48,6 +50,33 @@ def is_frappe_app(directory: str) -> bool:
return bool(is_frappe_app)
@lru_cache(maxsize=None)
def is_valid_frappe_branch(frappe_path:str, frappe_branch:str):
"""Check if a branch exists in a repo. Throws InvalidRemoteException if branch is not found
Uses native git command to check for branches on a remote.
:param frappe_path: git url
:type frappe_path: str
:param frappe_branch: branch to check
:type frappe_branch: str
:raises InvalidRemoteException: branch for this repo doesn't exist
"""
import git
g = git.cmd.Git()
if frappe_branch:
try:
res = g.ls_remote("--heads", "--tags", frappe_path, frappe_branch)
if not res:
raise InvalidRemoteException(
f"Invalid branch or tag: {frappe_branch} for the remote {frappe_path}"
)
except git.exc.GitCommandError:
raise InvalidRemoteException(f"Invalid frappe path: {frappe_path}")
def log(message, level=0, no_log=False):
import bench
import bench.cli
@ -62,9 +91,7 @@ def log(message, level=0, no_log=False):
color, prefix = levels.get(level, levels[0])
if bench.cli.from_command_line and bench.cli.dynamic_feed:
bench.LOG_BUFFER.append(
{"prefix": prefix, "message": message, "color": color}
)
bench.LOG_BUFFER.append({"prefix": prefix, "message": message, "color": color})
if no_log:
click.secho(message, fg=color)
@ -182,9 +209,7 @@ def get_git_version() -> float:
def get_cmd_output(cmd, cwd=".", _raise=True):
output = ""
try:
output = subprocess.check_output(
cmd, cwd=cwd, shell=True, stderr=subprocess.PIPE, encoding="utf-8"
).strip()
output = subprocess.check_output(cmd, cwd=cwd, shell=True, stderr=subprocess.PIPE, encoding="utf-8").strip()
except subprocess.CalledProcessError as e:
if e.output:
output = e.output
@ -269,7 +294,7 @@ def set_git_remote_url(git_url, bench_path="."):
app = git_url.rsplit("/", 1)[1].rsplit(".", 1)[0]
if app not in Bench(bench_path).apps:
raise ValidationError(f"No app named {app}")
raise AppNotInstalledError(f"No app named {app}")
app_dir = get_repo_dir(app, bench_path=bench_path)
@ -400,10 +425,12 @@ def find_org(org_repo):
for org in ["frappe", "erpnext"]:
res = requests.head(f"https://api.github.com/repos/{org}/{org_repo}")
if res.status_code in (400, 403):
res = requests.head(f"https://github.com/{org}/{org_repo}")
if res.ok:
return org, org_repo
raise InvalidRemoteException
raise InvalidRemoteException(f"{org_repo} not found in frappe or erpnext")
def fetch_details_from_tag(_tag: str) -> Tuple[str, str, str]:

48
bench/utils/app.py
View File

@ -7,8 +7,10 @@ from bench.exceptions import (
InvalidRemoteException,
InvalidBranchException,
CommandFailedError,
VersionNotFound,
)
from bench.app import get_repo_dir
from functools import lru_cache
def is_version_upgrade(app="frappe", bench_path=".", branch=None):
@ -107,7 +109,9 @@ def switch_to_develop(apps=None, bench_path=".", upgrade=True):
def get_version_from_string(contents, field="__version__"):
match = re.search(r"^(\s*%s\s*=\s*['\\\"])(.+?)(['\"])(?sm)" % field, contents)
match = re.search(r"^(\s*%s\s*=\s*['\\\"])(.+?)(['\"])" % field, contents, flags=(re.S | re.M))
if not match:
raise VersionNotFound(f"{contents} is not a valid version")
return match.group(2)
@ -165,6 +169,31 @@ def get_current_branch(app, bench_path="."):
return get_cmd_output("basename $(git symbolic-ref -q HEAD)", cwd=repo_dir)
@lru_cache(maxsize=5)
def get_required_deps(org, name, branch, deps="hooks.py"):
import requests
import base64
git_api_url = f"https://api.github.com/repos/{org}/{name}/contents/{name}/{deps}"
params = {"branch": branch or "develop"}
res = requests.get(url=git_api_url, params=params).json()
if "message" in res:
git_url = f"https://raw.githubusercontent.com/{org}/{name}/{params['branch']}/{deps}"
return requests.get(git_url).text
return base64.decodebytes(res["content"].encode()).decode()
def required_apps_from_hooks(required_deps, local=False):
if local:
with open(required_deps) as f:
required_deps = f.read()
lines = [x for x in required_deps.split("\n") if x.strip().startswith("required_apps")]
required_apps = eval(lines[0].strip("required_apps").strip().lstrip("=").strip())
return required_apps
def get_remote(app, bench_path="."):
repo_dir = get_repo_dir(app, bench_path=bench_path)
contents = subprocess.check_output(
@ -181,25 +210,30 @@ def get_remote(app, bench_path="."):
return contents.splitlines()[0].split()[0]
def get_app_name(bench_path, repo_name):
def get_app_name(bench_path, folder_name):
app_name = None
apps_path = os.path.join(os.path.abspath(bench_path), "apps")
config_path = os.path.join(apps_path, repo_name, "setup.cfg")
config_path = os.path.join(apps_path, folder_name, "setup.cfg")
if os.path.exists(config_path):
config = read_configuration(config_path)
app_name = config.get("metadata", {}).get("name")
if not app_name:
# retrieve app name from setup.py as fallback
app_path = os.path.join(apps_path, repo_name, "setup.py")
app_path = os.path.join(apps_path, folder_name, "setup.py")
with open(app_path, "rb") as f:
app_name = re.search(r'name\s*=\s*[\'"](.*)[\'"]', f.read().decode("utf-8")).group(1)
if app_name and repo_name != app_name:
os.rename(os.path.join(apps_path, repo_name), os.path.join(apps_path, app_name))
if app_name and folder_name != app_name:
os.rename(os.path.join(apps_path, folder_name), os.path.join(apps_path, app_name))
return app_name
return repo_name
return folder_name
def check_existing_dir(bench_path, repo_name):
cloned_path = os.path.join(bench_path, "apps", repo_name)
dir_already_exists = os.path.isdir(cloned_path)
return dir_already_exists, cloned_path
def get_current_version(app, bench_path="."):

30
bench/utils/bench.py
View File

@ -33,19 +33,25 @@ def get_env_cmd(cmd, bench_path="."):
return os.path.abspath(os.path.join(bench_path, "env", "bin", cmd))
def get_venv_path():
venv = which("virtualenv")
def get_virtualenv_path(verbose=False):
virtualenv_path = which("virtualenv")
if not venv:
current_python = sys.executable
with open(os.devnull, "wb") as devnull:
is_venv_installed = not subprocess.call(
[current_python, "-m", "venv", "--help"], stdout=devnull
)
if is_venv_installed:
venv = f"{current_python} -m venv"
if not virtualenv_path and verbose:
log("virtualenv cannot be found", level=2)
return venv or log("virtualenv cannot be found", level=2)
return virtualenv_path
def get_venv_path(verbose=False):
current_python = sys.executable
with open(os.devnull, "wb") as devnull:
is_venv_installed = not subprocess.call(
[current_python, "-m", "venv", "--help"], stdout=devnull
)
if is_venv_installed:
return f"{current_python} -m venv"
else:
log("virtualenv cannot be found", level=2)
def update_node_packages(bench_path=".", apps=None):
@ -74,7 +80,7 @@ def install_python_dev_dependencies(bench_path=".", apps=None, verbose=False):
if isinstance(apps, str):
apps = [apps]
elif apps is None:
elif not apps:
apps = bench.get_installed_apps()
for app in apps:

20
bench/utils/system.py
View File

@ -14,6 +14,7 @@ from bench.utils import (
run_frappe_cmd,
sudoers_file,
which,
is_valid_frappe_branch,
)
from bench.utils.bench import build_assets, clone_apps_from
from bench.utils.render import job
@ -74,9 +75,14 @@ def init(
# remote apps
else:
frappe_path = frappe_path or "https://github.com/frappe/frappe.git"
is_valid_frappe_branch(frappe_path=frappe_path, frappe_branch=frappe_branch)
get_app(
frappe_path, branch=frappe_branch, bench_path=path, skip_assets=True, verbose=verbose
frappe_path,
branch=frappe_branch,
bench_path=path,
skip_assets=True,
verbose=verbose,
resolve_deps=False,
)
# fetch remote apps using config file - deprecate this!
@ -86,7 +92,12 @@ def init(
# getting app on bench init using --install-app
if install_app:
get_app(
install_app, branch=frappe_branch, bench_path=path, skip_assets=True, verbose=verbose
install_app,
branch=frappe_branch,
bench_path=path,
skip_assets=True,
verbose=verbose,
resolve_deps=False,
)
if not skip_assets:
@ -97,6 +108,8 @@ def init(
def setup_sudoers(user):
from bench.config.lets_encrypt import get_certbot_path
if not os.path.exists("/etc/sudoers.d"):
os.makedirs("/etc/sudoers.d")
@ -117,6 +130,7 @@ def setup_sudoers(user):
"service": which("service"),
"systemctl": which("systemctl"),
"nginx": which("nginx"),
"certbot": get_certbot_path(),
}
)

1
requirements.txt
View File

@ -6,4 +6,3 @@ python-crontab~=2.4.0
requests
semantic-version~=2.8.2
setuptools
virtualenv