2
0
mirror of https://github.com/frappe/bench.git synced 2024-11-16 10:05:21 +00:00

style: Flake8 + Black-ish

Tried styling and following standards over teh following modules:
* bench.app
* bench.bench
* bench.exceptions
* bench.utils
This commit is contained in:
Gavin D'souza 2021-11-13 02:59:01 +05:30
parent d2fba5fe52
commit 1330e66d07
9 changed files with 577 additions and 386 deletions

View File

@ -5,8 +5,9 @@ current_path = None
updated_path = None updated_path = None
def set_frappe_version(bench_path='.'): def set_frappe_version(bench_path="."):
from .utils.app import get_current_frappe_version from .utils.app import get_current_frappe_version
global FRAPPE_VERSION global FRAPPE_VERSION
if not FRAPPE_VERSION: if not FRAPPE_VERSION:
FRAPPE_VERSION = get_current_frappe_version(bench_path=bench_path) FRAPPE_VERSION = get_current_frappe_version(bench_path=bench_path)

View File

@ -13,8 +13,19 @@ import click
# imports - module imports # imports - module imports
import bench import bench
from bench.utils import exec_cmd, is_bench_directory, run_frappe_cmd, is_git_url, fetch_details_from_tag from bench.utils import (
from bench.utils.bench import get_env_cmd, build_assets, restart_supervisor_processes, restart_systemd_processes exec_cmd,
is_bench_directory,
run_frappe_cmd,
is_git_url,
fetch_details_from_tag,
)
from bench.utils.bench import (
get_env_cmd,
build_assets,
restart_supervisor_processes,
restart_systemd_processes,
)
logger = logging.getLogger(bench.PROJECT_NAME) logger = logging.getLogger(bench.PROJECT_NAME)
@ -22,8 +33,9 @@ logger = logging.getLogger(bench.PROJECT_NAME)
if typing.TYPE_CHECKING: if typing.TYPE_CHECKING:
from bench.bench import Bench from bench.bench import Bench
class AppMeta: class AppMeta:
def __init__(self, name: str, branch : str = None): def __init__(self, name: str, branch: str = None):
""" """
name (str): This could look something like name (str): This could look something like
1. https://github.com/frappe/healthcare.git 1. https://github.com/frappe/healthcare.git
@ -96,23 +108,22 @@ class AppMeta:
class App(AppMeta): class App(AppMeta):
def __init__(self, name: str, branch : str = None, bench : "Bench" = None): def __init__(self, name: str, branch: str = None, bench: "Bench" = None):
super().__init__(name, branch) super().__init__(name, branch)
self.bench = bench self.bench = bench
def get(self): def get(self):
branch = f'--branch {self.tag}' if self.tag else '' branch = f"--branch {self.tag}" if self.tag else ""
shallow = '--depth 1' if self.bench.shallow_clone else '' shallow = "--depth 1" if self.bench.shallow_clone else ""
self.bench.run( self.bench.run(
f"git clone {self.url} {branch} {shallow} --origin upstream", f"git clone {self.url} {branch} {shallow} --origin upstream",
cwd=os.path.join(self.bench.name, 'apps') cwd=os.path.join(self.bench.name, "apps"),
) )
def remove(self): def remove(self):
shutil.move( shutil.move(
os.path.join("apps", self.repo), os.path.join("apps", self.repo), os.path.join("archived", "apps", self.repo),
os.path.join("archived", "apps", self.repo),
) )
def install(self, skip_assets=False, verbose=False): def install(self, skip_assets=False, verbose=False):
@ -131,12 +142,10 @@ class App(AppMeta):
def uninstall(self): def uninstall(self):
env_python = get_env_cmd("python", bench_path=self.bench.name) env_python = get_env_cmd("python", bench_path=self.bench.name)
self.bench.run( self.bench.run(f"{env_python} -m pip uninstall -y {self.repo}")
f"{env_python} -m pip uninstall -y {self.repo}"
)
def add_to_appstxt(app, bench_path='.'): def add_to_appstxt(app, bench_path="."):
from bench.bench import Bench from bench.bench import Bench
apps = Bench(bench_path).apps apps = Bench(bench_path).apps
@ -145,7 +154,8 @@ def add_to_appstxt(app, bench_path='.'):
apps.append(app) apps.append(app)
return write_appstxt(apps, bench_path=bench_path) return write_appstxt(apps, bench_path=bench_path)
def remove_from_appstxt(app, bench_path='.'):
def remove_from_appstxt(app, bench_path="."):
from bench.bench import Bench from bench.bench import Bench
apps = Bench(bench_path).apps apps = Bench(bench_path).apps
@ -154,37 +164,43 @@ def remove_from_appstxt(app, bench_path='.'):
apps.remove(app) apps.remove(app)
return write_appstxt(apps, bench_path=bench_path) return write_appstxt(apps, bench_path=bench_path)
def write_appstxt(apps, bench_path='.'):
with open(os.path.join(bench_path, 'sites', 'apps.txt'), 'w') as f:
return f.write('\n'.join(apps))
def get_excluded_apps(bench_path='.'): def write_appstxt(apps, bench_path="."):
with open(os.path.join(bench_path, "sites", "apps.txt"), "w") as f:
return f.write("\n".join(apps))
def get_excluded_apps(bench_path="."):
try: try:
with open(os.path.join(bench_path, 'sites', 'excluded_apps.txt')) as f: with open(os.path.join(bench_path, "sites", "excluded_apps.txt")) as f:
return f.read().strip().split('\n') return f.read().strip().split("\n")
except IOError: except IOError:
return [] return []
def add_to_excluded_apps_txt(app, bench_path='.'):
if app == 'frappe': def add_to_excluded_apps_txt(app, bench_path="."):
raise ValueError('Frappe app cannot be excludeed from update') if app == "frappe":
if app not in os.listdir('apps'): raise ValueError("Frappe app cannot be excludeed from update")
raise ValueError(f'The app {app} does not exist') if app not in os.listdir("apps"):
raise ValueError(f"The app {app} does not exist")
apps = get_excluded_apps(bench_path=bench_path) apps = get_excluded_apps(bench_path=bench_path)
if app not in apps: if app not in apps:
apps.append(app) apps.append(app)
return write_excluded_apps_txt(apps, bench_path=bench_path) return write_excluded_apps_txt(apps, bench_path=bench_path)
def write_excluded_apps_txt(apps, bench_path='.'):
with open(os.path.join(bench_path, 'sites', 'excluded_apps.txt'), 'w') as f:
return f.write('\n'.join(apps))
def remove_from_excluded_apps_txt(app, bench_path='.'): def write_excluded_apps_txt(apps, bench_path="."):
with open(os.path.join(bench_path, "sites", "excluded_apps.txt"), "w") as f:
return f.write("\n".join(apps))
def remove_from_excluded_apps_txt(app, bench_path="."):
apps = get_excluded_apps(bench_path=bench_path) apps = get_excluded_apps(bench_path=bench_path)
if app in apps: if app in apps:
apps.remove(app) apps.remove(app)
return write_excluded_apps_txt(apps, bench_path=bench_path) return write_excluded_apps_txt(apps, bench_path=bench_path)
def generate_bench_name(git_url, bench_path): def generate_bench_name(git_url, bench_path):
if os.path.exists(git_url): if os.path.exists(git_url):
guessed_app_name = os.path.basename(git_url) guessed_app_name = os.path.basename(git_url)
@ -194,25 +210,29 @@ def generate_bench_name(git_url, bench_path):
return os.path.join(bench_path, f"{guessed_app_name}-bench") return os.path.join(bench_path, f"{guessed_app_name}-bench")
def setup_app_dependencies(repo_name, bench_path='.', branch=None):
def setup_app_dependencies(repo_name, bench_path=".", branch=None):
# branch kwarg is somewhat of a hack here; since we're assuming the same branches for all apps # branch kwarg is somewhat of a hack here; since we're assuming the same branches for all apps
# for eg: if you're installing erpnext@develop, you'll want frappe@develop and healthcare@develop too # for eg: if you're installing erpnext@develop, you'll want frappe@develop and healthcare@develop too
import glob import glob
apps_path = os.path.join(os.path.abspath(bench_path), 'apps') apps_path = os.path.join(os.path.abspath(bench_path), "apps")
files = glob.glob(os.path.join(apps_path, repo_name, '**', 'hooks.py')) files = glob.glob(os.path.join(apps_path, repo_name, "**", "hooks.py"))
if files: if files:
with open(files[0]) as f: with open(files[0]) as f:
lines = [x for x in f.read().split('\n') if x.strip().startswith('required_apps')] lines = [x for x in f.read().split("\n") if x.strip().startswith("required_apps")]
if lines: if lines:
required_apps = eval(lines[0].strip('required_apps').strip().lstrip('=').strip()) required_apps = eval(lines[0].strip("required_apps").strip().lstrip("=").strip())
# TODO: when the time comes, add version check here # TODO: when the time comes, add version check here
for app in required_apps: for app in required_apps:
if app not in Bench(bench_path).apps: if app not in Bench(bench_path).apps:
get_app(app, bench_path=bench_path, branch=branch) get_app(app, bench_path=bench_path, branch=branch)
def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=False, overwrite=False):
def get_app(
git_url, branch=None, bench_path=".", skip_assets=False, verbose=False, overwrite=False
):
"""bench get-app clones a Frappe App from remote (GitHub or any other git server), """bench get-app clones a Frappe App from remote (GitHub or any other git server),
and installs it on the current bench. This also resolves dependencies based on the and installs it on the current bench. This also resolves dependencies based on the
apps' required_apps defined in the hooks.py file. apps' required_apps defined in the hooks.py file.
@ -231,9 +251,10 @@ def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=Fal
if not is_bench_directory(bench_path): if not is_bench_directory(bench_path):
bench_path = generate_bench_name(git_url, bench_path) bench_path = generate_bench_name(git_url, bench_path)
from bench.commands.make import init from bench.commands.make import init
click.get_current_context().invoke(init, path=bench_path, frappe_branch=branch) click.get_current_context().invoke(init, path=bench_path, frappe_branch=branch)
cloned_path = os.path.join(bench_path, 'apps', repo_name) cloned_path = os.path.join(bench_path, "apps", repo_name)
dir_already_exists = os.path.isdir(cloned_path) dir_already_exists = os.path.isdir(cloned_path)
if dir_already_exists: if dir_already_exists:
@ -244,6 +265,7 @@ def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=Fal
"Do you want to continue and overwrite it?" "Do you want to continue and overwrite it?"
): ):
import shutil import shutil
shutil.rmtree(cloned_path) shutil.rmtree(cloned_path)
elif click.confirm("Do you want to reinstall the existing application?", abort=True): elif click.confirm("Do you want to reinstall the existing application?", abort=True):
pass pass
@ -256,20 +278,27 @@ def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=Fal
app.install(verbose=verbose, skip_assets=skip_assets) app.install(verbose=verbose, skip_assets=skip_assets)
def new_app(app, bench_path='.'): def new_app(app, bench_path="."):
# For backwards compatibility # For backwards compatibility
app = app.lower().replace(" ", "_").replace("-", "_") app = app.lower().replace(" ", "_").replace("-", "_")
logger.log(f'creating new app {app}') logger.log(f"creating new app {app}")
apps = os.path.abspath(os.path.join(bench_path, 'apps')) apps = os.path.abspath(os.path.join(bench_path, "apps"))
run_frappe_cmd('make-app', apps, app, bench_path=bench_path) run_frappe_cmd("make-app", apps, app, bench_path=bench_path)
install_app(app, bench_path=bench_path) install_app(app, bench_path=bench_path)
def install_app(app, bench_path=".", verbose=False, no_cache=False, restart_bench=True, skip_assets=False): def install_app(
app,
bench_path=".",
verbose=False,
no_cache=False,
restart_bench=True,
skip_assets=False,
):
from bench.bench import Bench from bench.bench import Bench
from bench.utils.bench import get_env_cmd from bench.utils.bench import get_env_cmd
install_text = f'Installing {app}' install_text = f"Installing {app}"
click.secho(install_text, fg="yellow") click.secho(install_text, fg="yellow")
logger.log(install_text) logger.log(install_text)
@ -280,7 +309,7 @@ def install_app(app, bench_path=".", verbose=False, no_cache=False, restart_benc
exec_cmd(f"{python_path} -m pip install {quiet_flag} -U -e {app_path} {cache_flag}") exec_cmd(f"{python_path} -m pip install {quiet_flag} -U -e {app_path} {cache_flag}")
if os.path.exists(os.path.join(app_path, 'package.json')): if os.path.exists(os.path.join(app_path, "package.json")):
exec_cmd("yarn install", cwd=app_path) exec_cmd("yarn install", cwd=app_path)
add_to_appstxt(app, bench_path=bench_path) add_to_appstxt(app, bench_path=bench_path)
@ -289,25 +318,26 @@ def install_app(app, bench_path=".", verbose=False, no_cache=False, restart_benc
if conf.get("developer_mode"): if conf.get("developer_mode"):
from bench.utils.bench import install_python_dev_dependencies from bench.utils.bench import install_python_dev_dependencies
install_python_dev_dependencies(apps=app) install_python_dev_dependencies(apps=app)
if not skip_assets: if not skip_assets:
build_assets(bench_path=bench_path, app=app) build_assets(bench_path=bench_path, app=app)
if restart_bench: if restart_bench:
if conf.get('restart_supervisor_on_update'): if conf.get("restart_supervisor_on_update"):
restart_supervisor_processes(bench_path=bench_path) restart_supervisor_processes(bench_path=bench_path)
if conf.get('restart_systemd_on_update'): if conf.get("restart_systemd_on_update"):
restart_systemd_processes(bench_path=bench_path) restart_systemd_processes(bench_path=bench_path)
def pull_apps(apps=None, bench_path='.', reset=False): def pull_apps(apps=None, bench_path=".", reset=False):
'''Check all apps if there no local changes, pull''' """Check all apps if there no local changes, pull"""
from bench.bench import Bench from bench.bench import Bench
from bench.utils.app import get_remote, get_current_branch from bench.utils.app import get_remote, get_current_branch
bench = Bench(bench_path) bench = Bench(bench_path)
rebase = '--rebase' if bench.conf.get('rebase_on_pull') else '' rebase = "--rebase" if bench.conf.get("rebase_on_pull") else ""
apps = apps or bench.apps apps = apps or bench.apps
excluded_apps = bench.excluded_apps excluded_apps = bench.excluded_apps
@ -318,11 +348,12 @@ def pull_apps(apps=None, bench_path='.', reset=False):
print(f"Skipping reset for app {app}") print(f"Skipping reset for app {app}")
continue continue
app_dir = get_repo_dir(app, bench_path=bench_path) app_dir = get_repo_dir(app, bench_path=bench_path)
if os.path.exists(os.path.join(app_dir, '.git')): if os.path.exists(os.path.join(app_dir, ".git")):
out = subprocess.check_output('git status', shell=True, cwd=app_dir) out = subprocess.check_output("git status", shell=True, cwd=app_dir)
out = out.decode('utf-8') out = out.decode("utf-8")
if not re.search(r'nothing to commit, working (directory|tree) clean', out): if not re.search(r"nothing to commit, working (directory|tree) clean", out):
print(f''' print(
f"""
Cannot proceed with update: You have local changes in app "{app}" that are not committed. Cannot proceed with update: You have local changes in app "{app}" that are not committed.
@ -332,7 +363,8 @@ Here are your choices:
1. Temporarily remove your changes with "git stash" or discard them completely 1. Temporarily remove your changes with "git stash" or discard them completely
with "bench update --reset" or for individual repositries "git reset --hard" with "bench update --reset" or for individual repositries "git reset --hard"
2. If your changes are helpful for others, send in a pull request via GitHub and 2. If your changes are helpful for others, send in a pull request via GitHub and
wait for them to be merged in the core.''') wait for them to be merged in the core."""
)
sys.exit(1) sys.exit(1)
for app in apps: for app in apps:
@ -340,15 +372,18 @@ Here are your choices:
print(f"Skipping pull for app {app}") print(f"Skipping pull for app {app}")
continue continue
app_dir = get_repo_dir(app, bench_path=bench_path) app_dir = get_repo_dir(app, bench_path=bench_path)
if os.path.exists(os.path.join(app_dir, '.git')): if os.path.exists(os.path.join(app_dir, ".git")):
remote = get_remote(app) remote = get_remote(app)
if not remote: if not remote:
# remote is False, i.e. remote doesn't exist, add the app to excluded_apps.txt # remote is False, i.e. remote doesn't exist, add the app to excluded_apps.txt
add_to_excluded_apps_txt(app, bench_path=bench_path) add_to_excluded_apps_txt(app, bench_path=bench_path)
print(f"Skipping pull for app {app}, since remote doesn't exist, and adding it to excluded apps") print(
f"Skipping pull for app {app}, since remote doesn't exist, and"
" adding it to excluded apps"
)
continue continue
if not bench.conf.get('shallow_clone') or not reset: if not bench.conf.get("shallow_clone") or not reset:
is_shallow = os.path.exists(os.path.join(app_dir, ".git", "shallow")) is_shallow = os.path.exists(os.path.join(app_dir, ".git", "shallow"))
if is_shallow: if is_shallow:
s = " to safely pull remote changes." if not reset else "" s = " to safely pull remote changes." if not reset else ""
@ -356,10 +391,10 @@ Here are your choices:
bench.run(f"git fetch {remote} --unshallow", cwd=app_dir) bench.run(f"git fetch {remote} --unshallow", cwd=app_dir)
branch = get_current_branch(app, bench_path=bench_path) branch = get_current_branch(app, bench_path=bench_path)
logger.log(f'pulling {app}') logger.log(f"pulling {app}")
if reset: if reset:
reset_cmd = f"git reset --hard {remote}/{branch}" reset_cmd = f"git reset --hard {remote}/{branch}"
if bench.conf.get('shallow_clone'): if bench.conf.get("shallow_clone"):
bench.run(f"git fetch --depth=1 --no-tags {remote} {branch}", cwd=app_dir) bench.run(f"git fetch --depth=1 --no-tags {remote} {branch}", cwd=app_dir)
bench.run(reset_cmd, cwd=app_dir) bench.run(reset_cmd, cwd=app_dir)
bench.run("git reflog expire --all", cwd=app_dir) bench.run("git reflog expire --all", cwd=app_dir)
@ -374,21 +409,24 @@ Here are your choices:
def use_rq(bench_path): def use_rq(bench_path):
bench_path = os.path.abspath(bench_path) bench_path = os.path.abspath(bench_path)
celery_app = os.path.join(bench_path, 'apps', 'frappe', 'frappe', 'celery_app.py') celery_app = os.path.join(bench_path, "apps", "frappe", "frappe", "celery_app.py")
return not os.path.exists(celery_app) return not os.path.exists(celery_app)
def get_repo_dir(app, bench_path='.'):
return os.path.join(bench_path, 'apps', app)
def install_apps_from_path(path, bench_path='.'): def get_repo_dir(app, bench_path="."):
return os.path.join(bench_path, "apps", app)
def install_apps_from_path(path, bench_path="."):
apps = get_apps_json(path) apps = get_apps_json(path)
for app in apps: for app in apps:
get_app(app['url'], branch=app.get('branch'), bench_path=bench_path, skip_assets=True) get_app(app["url"], branch=app.get("branch"), bench_path=bench_path, skip_assets=True)
def get_apps_json(path): def get_apps_json(path):
import requests import requests
if path.startswith('http'): if path.startswith("http"):
r = requests.get(path) r = requests.get(path)
return r.json() return r.json()

View File

@ -7,8 +7,21 @@ from typing import MutableSequence, TYPE_CHECKING
import bench import bench
from bench.exceptions import ValidationError from bench.exceptions import ValidationError
from bench.config.common_site_config import setup_config from bench.config.common_site_config import setup_config
from bench.utils import paths_in_bench, exec_cmd, is_frappe_app, get_git_version, run_frappe_cmd from bench.utils import (
from bench.utils.bench import validate_app_installed_on_sites, restart_supervisor_processes, restart_systemd_processes, remove_backups_crontab, get_venv_path, get_env_cmd paths_in_bench,
exec_cmd,
is_frappe_app,
get_git_version,
run_frappe_cmd,
)
from bench.utils.bench import (
validate_app_installed_on_sites,
restart_supervisor_processes,
restart_systemd_processes,
remove_backups_crontab,
get_venv_path,
get_env_cmd,
)
if TYPE_CHECKING: if TYPE_CHECKING:
@ -39,15 +52,15 @@ class Bench(Base, Validator):
self.teardown = BenchTearDown(self) self.teardown = BenchTearDown(self)
self.apps = BenchApps(self) self.apps = BenchApps(self)
self.apps_txt = os.path.join(self.name, 'sites', 'apps.txt') self.apps_txt = os.path.join(self.name, "sites", "apps.txt")
self.excluded_apps_txt = os.path.join(self.name, 'sites', 'excluded_apps.txt') self.excluded_apps_txt = os.path.join(self.name, "sites", "excluded_apps.txt")
@property @property
def shallow_clone(self): def shallow_clone(self):
config = self.conf config = self.conf
if config: if config:
if config.get('release_bench') or not config.get('shallow_clone'): if config.get("release_bench") or not config.get("shallow_clone"):
return False return False
if get_git_version() > 1.9: if get_git_version() > 1.9:
@ -57,22 +70,22 @@ class Bench(Base, Validator):
def excluded_apps(self): def excluded_apps(self):
try: try:
with open(self.excluded_apps_txt) as f: with open(self.excluded_apps_txt) as f:
return f.read().strip().split('\n') return f.read().strip().split("\n")
except Exception: except Exception:
return [] return []
@property @property
def sites(self): def sites(self):
return [ return [
path for path in os.listdir(os.path.join(self.name, 'sites')) path
if os.path.exists( for path in os.listdir(os.path.join(self.name, "sites"))
os.path.join("sites", path, "site_config.json") if os.path.exists(os.path.join("sites", path, "site_config.json"))
)
] ]
@property @property
def conf(self): def conf(self):
from bench.config.common_site_config import get_config from bench.config.common_site_config import get_config
return get_config(self.name) return get_config(self.name)
def init(self): def init(self):
@ -112,14 +125,14 @@ class Bench(Base, Validator):
def reload(self): def reload(self):
conf = self.conf conf = self.conf
if conf.get('restart_supervisor_on_update'): if conf.get("restart_supervisor_on_update"):
restart_supervisor_processes(bench_path=self.name) restart_supervisor_processes(bench_path=self.name)
if conf.get('restart_systemd_on_update'): if conf.get("restart_systemd_on_update"):
restart_systemd_processes(bench_path=self.name) restart_systemd_processes(bench_path=self.name)
class BenchApps(MutableSequence): class BenchApps(MutableSequence):
def __init__(self, bench : Bench): def __init__(self, bench: Bench):
self.bench = bench self.bench = bench
self.initialize_apps() self.initialize_apps()
@ -130,25 +143,27 @@ class BenchApps(MutableSequence):
def initialize_apps(self): def initialize_apps(self):
try: try:
self.apps = [x for x in os.listdir( self.apps = [
os.path.join(self.bench.name, "apps") x
) if is_frappe_app(os.path.join(self.bench.name, "apps", x))] for x in os.listdir(os.path.join(self.bench.name, "apps"))
if is_frappe_app(os.path.join(self.bench.name, "apps", x))
]
self.apps.sort() self.apps.sort()
except FileNotFoundError: except FileNotFoundError:
self.apps = [] self.apps = []
def __getitem__(self, key): def __getitem__(self, key):
''' retrieves an item by its index, key''' """ retrieves an item by its index, key"""
return self.apps[key] return self.apps[key]
def __setitem__(self, key, value): def __setitem__(self, key, value):
''' set the item at index, key, to value ''' """ set the item at index, key, to value """
# should probably not be allowed # should probably not be allowed
# self.apps[key] = value # self.apps[key] = value
raise NotImplementedError raise NotImplementedError
def __delitem__(self, key): def __delitem__(self, key):
''' removes the item at index, key ''' """ removes the item at index, key """
# TODO: uninstall and delete app from bench # TODO: uninstall and delete app from bench
del self.apps[key] del self.apps[key]
@ -156,7 +171,7 @@ class BenchApps(MutableSequence):
return len(self.apps) return len(self.apps)
def insert(self, key, value): def insert(self, key, value):
''' add an item, value, at index, key. ''' """ add an item, value, at index, key. """
# TODO: fetch and install app to bench # TODO: fetch and install app to bench
self.apps.insert(key, value) self.apps.insert(key, value)
@ -171,7 +186,7 @@ class BenchApps(MutableSequence):
app.remove() app.remove()
super().remove(app.repo) super().remove(app.repo)
def append(self, app : "App"): def append(self, app: "App"):
return self.add(app) return self.add(app)
def __repr__(self): def __repr__(self):
@ -182,7 +197,7 @@ class BenchApps(MutableSequence):
class BenchSetup(Base): class BenchSetup(Base):
def __init__(self, bench : Bench): def __init__(self, bench: Bench):
self.bench = bench self.bench = bench
self.cwd = self.bench.cwd self.cwd = self.bench.cwd
@ -219,41 +234,46 @@ class BenchSetup(Base):
if redis: if redis:
from bench.config.redis import generate_config from bench.config.redis import generate_config
generate_config(self.bench.name) generate_config(self.bench.name)
if procfile: if procfile:
from bench.config.procfile import setup_procfile from bench.config.procfile import setup_procfile
setup_procfile(self.bench.name, skip_redis=not redis) setup_procfile(self.bench.name, skip_redis=not redis)
def logging(self): def logging(self):
from bench.utils import setup_logging from bench.utils import setup_logging
return setup_logging(bench_path=self.bench.name) return setup_logging(bench_path=self.bench.name)
def patches(self): def patches(self):
shutil.copy( shutil.copy(
os.path.join(os.path.dirname(os.path.abspath(__file__)), 'patches', 'patches.txt'), os.path.join(os.path.dirname(os.path.abspath(__file__)), "patches", "patches.txt"),
os.path.join(self.bench.name, 'patches.txt') os.path.join(self.bench.name, "patches.txt"),
) )
def backups(self): def backups(self):
# TODO: to something better for logging data? - maybe a wrapper that auto-logs with more context # TODO: to something better for logging data? - maybe a wrapper that auto-logs with more context
logger.log('setting up backups') logger.log("setting up backups")
from crontab import CronTab from crontab import CronTab
bench_dir = os.path.abspath(self.bench.name) bench_dir = os.path.abspath(self.bench.name)
user = self.bench.conf.get('frappe_user') user = self.bench.conf.get("frappe_user")
logfile = os.path.join(bench_dir, 'logs', 'backup.log') logfile = os.path.join(bench_dir, "logs", "backup.log")
system_crontab = CronTab(user=user) system_crontab = CronTab(user=user)
backup_command = f"cd {bench_dir} && {sys.argv[0]} --verbose --site all backup" backup_command = f"cd {bench_dir} && {sys.argv[0]} --verbose --site all backup"
job_command = f"{backup_command} >> {logfile} 2>&1" job_command = f"{backup_command} >> {logfile} 2>&1"
if job_command not in str(system_crontab): if job_command not in str(system_crontab):
job = system_crontab.new(command=job_command, comment="bench auto backups set for every 6 hours") job = system_crontab.new(
command=job_command, comment="bench auto backups set for every 6 hours"
)
job.every(6).hours() job.every(6).hours()
system_crontab.write() system_crontab.write()
logger.log('backups were set up') logger.log("backups were set up")
class BenchTearDown: class BenchTearDown:

View File

@ -1,17 +1,22 @@
class InvalidBranchException(Exception): class InvalidBranchException(Exception):
pass pass
class InvalidRemoteException(Exception): class InvalidRemoteException(Exception):
pass pass
class PatchError(Exception): class PatchError(Exception):
pass pass
class CommandFailedError(Exception): class CommandFailedError(Exception):
pass pass
class BenchNotFoundError(Exception): class BenchNotFoundError(Exception):
pass pass
class ValidationError(Exception): class ValidationError(Exception):
pass pass

View File

@ -18,10 +18,10 @@ from bench.exceptions import InvalidRemoteException, ValidationError
logger = logging.getLogger(PROJECT_NAME) logger = logging.getLogger(PROJECT_NAME)
bench_cache_file = '.bench.cmd' bench_cache_file = ".bench.cmd"
paths_in_app = ('hooks.py', 'modules.txt', 'patches.txt', 'public') paths_in_app = ("hooks.py", "modules.txt", "patches.txt", "public")
paths_in_bench = ('apps', 'sites', 'config', 'logs', 'config/pids') paths_in_bench = ("apps", "sites", "config", "logs", "config/pids")
sudoers_file = '/etc/sudoers.d/frappe' sudoers_file = "/etc/sudoers.d/frappe"
def is_bench_directory(directory=os.path.curdir): def is_bench_directory(directory=os.path.curdir):
@ -49,15 +49,12 @@ def is_frappe_app(directory):
def log(message, level=0): def log(message, level=0):
levels = { levels = {
0: ("blue", "INFO"), # normal 0: ("blue", "INFO"), # normal
1: ("green", "SUCCESS"), # success 1: ("green", "SUCCESS"), # success
2: ("red", "ERROR"), # fail 2: ("red", "ERROR"), # fail
3: ("yellow", "WARN") # warn/suggest 3: ("yellow", "WARN"), # warn/suggest
}
loggers = {
2: logger.error,
3: logger.warning
} }
loggers = {2: logger.error, 3: logger.warning}
color, prefix = levels.get(level, levels[0]) color, prefix = levels.get(level, levels[0])
level_logger = loggers.get(level, logger.info) level_logger = loggers.get(level, logger.info)
@ -80,7 +77,7 @@ def check_latest_version():
return return
if pypi_request.status_code == 200: if pypi_request.status_code == 200:
pypi_version_str = pypi_request.json().get('info').get('version') pypi_version_str = pypi_request.json().get("info").get("version")
pypi_version = Version(pypi_version_str) pypi_version = Version(pypi_version_str)
local_version = Version(VERSION) local_version = Version(VERSION)
@ -98,11 +95,11 @@ def pause_exec(seconds=10):
print(" " * 40, end="\r") print(" " * 40, end="\r")
def exec_cmd(cmd, cwd='.', env=None): def exec_cmd(cmd, cwd=".", env=None):
if env: if env:
env.update(os.environ.copy()) env.update(os.environ.copy())
click.secho(f"$ {cmd}", fg='bright_black') click.secho(f"$ {cmd}", fg="bright_black")
cwd_info = f"cd {cwd} && " if cwd != "." else "" cwd_info = f"cd {cwd} && " if cwd != "." else ""
cmd_log = f"{cwd_info}{cmd}" cmd_log = f"{cwd_info}{cmd}"
@ -119,27 +116,29 @@ def which(executable, raise_err=False):
exec_ = which(executable) exec_ = which(executable)
if not exec_ and raise_err: if not exec_ and raise_err:
raise ValueError(f'{executable} not found.') raise ValueError(f"{executable} not found.")
return exec_ return exec_
def setup_logging(bench_path='.'): def setup_logging(bench_path="."):
LOG_LEVEL = 15 LOG_LEVEL = 15
logging.addLevelName(LOG_LEVEL, "LOG") logging.addLevelName(LOG_LEVEL, "LOG")
def logv(self, message, *args, **kws): def logv(self, message, *args, **kws):
if self.isEnabledFor(LOG_LEVEL): if self.isEnabledFor(LOG_LEVEL):
self._log(LOG_LEVEL, message, args, **kws) self._log(LOG_LEVEL, message, args, **kws)
logging.Logger.log = logv logging.Logger.log = logv
if os.path.exists(os.path.join(bench_path, 'logs')): if os.path.exists(os.path.join(bench_path, "logs")):
log_file = os.path.join(bench_path, 'logs', 'bench.log') log_file = os.path.join(bench_path, "logs", "bench.log")
hdlr = logging.FileHandler(log_file) hdlr = logging.FileHandler(log_file)
else: else:
hdlr = logging.NullHandler() hdlr = logging.NullHandler()
logger = logging.getLogger(PROJECT_NAME) logger = logging.getLogger(PROJECT_NAME)
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s")
hdlr.setFormatter(formatter) hdlr.setFormatter(formatter)
logger.addHandler(hdlr) logger.addHandler(hdlr)
logger.setLevel(logging.DEBUG) logger.setLevel(logging.DEBUG)
@ -148,25 +147,27 @@ def setup_logging(bench_path='.'):
def get_process_manager(): def get_process_manager():
for proc_man in ['honcho', 'foreman', 'forego']: for proc_man in ["honcho", "foreman", "forego"]:
proc_man_path = which(proc_man) proc_man_path = which(proc_man)
if proc_man_path: if proc_man_path:
return proc_man_path return proc_man_path
def get_git_version(): def get_git_version():
'''returns git version from `git --version` """returns git version from `git --version`
extracts version number from string `get version 1.9.1` etc''' extracts version number from string `get version 1.9.1` etc"""
version = get_cmd_output("git --version") version = get_cmd_output("git --version")
version = version.strip().split()[2] version = version.strip().split()[2]
version = '.'.join(version.split('.')[0:2]) version = ".".join(version.split(".")[0:2])
return float(version) return float(version)
def get_cmd_output(cmd, cwd='.', _raise=True): def get_cmd_output(cmd, cwd=".", _raise=True):
output = "" output = ""
try: try:
output = subprocess.check_output(cmd, cwd=cwd, shell=True, stderr=subprocess.PIPE, encoding="utf-8").strip() output = subprocess.check_output(
cmd, cwd=cwd, shell=True, stderr=subprocess.PIPE, encoding="utf-8"
).strip()
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
if e.output: if e.output:
output = e.output output = e.output
@ -183,9 +184,9 @@ def run_frappe_cmd(*args, **kwargs):
from bench.cli import from_command_line from bench.cli import from_command_line
from bench.utils.bench import get_env_cmd from bench.utils.bench import get_env_cmd
bench_path = kwargs.get('bench_path', '.') bench_path = kwargs.get("bench_path", ".")
f = get_env_cmd('python', bench_path=bench_path) f = get_env_cmd("python", bench_path=bench_path)
sites_dir = os.path.join(bench_path, 'sites') sites_dir = os.path.join(bench_path, "sites")
is_async = False if from_command_line else True is_async = False if from_command_line else True
if is_async: if is_async:
@ -193,8 +194,12 @@ def run_frappe_cmd(*args, **kwargs):
else: else:
stderr = stdout = None stderr = stdout = None
p = subprocess.Popen((f, '-m', 'frappe.utils.bench_helper', 'frappe') + args, p = subprocess.Popen(
cwd=sites_dir, stdout=stdout, stderr=stderr) (f, "-m", "frappe.utils.bench_helper", "frappe") + args,
cwd=sites_dir,
stdout=stdout,
stderr=stderr,
)
if is_async: if is_async:
return_code = print_output(p) return_code = print_output(p)
@ -217,20 +222,20 @@ def print_output(p):
buf = p.stdout.read(1) buf = p.stdout.read(1)
if not len(buf): if not len(buf):
break break
if buf == '\r' or buf == '\n': if buf == "\r" or buf == "\n":
send_buffer.append(buf) send_buffer.append(buf)
log_line(''.join(send_buffer), 'stdout') log_line("".join(send_buffer), "stdout")
send_buffer = [] send_buffer = []
else: else:
send_buffer.append(buf) send_buffer.append(buf)
if fd == p.stderr.fileno(): if fd == p.stderr.fileno():
log_line(p.stderr.readline(), 'stderr') log_line(p.stderr.readline(), "stderr")
return p.poll() return p.poll()
def log_line(data, stream): def log_line(data, stream):
if stream == 'stderr': if stream == "stderr":
return sys.stderr.write(data) return sys.stderr.write(data)
return sys.stdout.write(data) return sys.stdout.write(data)
@ -239,37 +244,40 @@ def get_bench_name(bench_path):
return os.path.basename(os.path.abspath(bench_path)) return os.path.basename(os.path.abspath(bench_path))
def set_git_remote_url(git_url, bench_path='.'): def set_git_remote_url(git_url, bench_path="."):
"Set app remote git url" "Set app remote git url"
from bench.app import get_repo_dir from bench.app import get_repo_dir
from bench.bench import Bench from bench.bench import Bench
app = git_url.rsplit('/', 1)[1].rsplit('.', 1)[0] app = git_url.rsplit("/", 1)[1].rsplit(".", 1)[0]
if app not in Bench(bench_path).apps: if app not in Bench(bench_path).apps:
raise ValidationError(f"No app named {app}") raise ValidationError(f"No app named {app}")
app_dir = get_repo_dir(app, bench_path=bench_path) app_dir = get_repo_dir(app, bench_path=bench_path)
if os.path.exists(os.path.join(app_dir, '.git')): if os.path.exists(os.path.join(app_dir, ".git")):
exec_cmd(f"git remote set-url upstream {git_url}", cwd=app_dir) exec_cmd(f"git remote set-url upstream {git_url}", cwd=app_dir)
def run_playbook(playbook_name, extra_vars=None, tag=None): def run_playbook(playbook_name, extra_vars=None, tag=None):
import bench import bench
if not which('ansible'): if not which("ansible"):
print("Ansible is needed to run this command, please install it using 'pip install ansible'") print(
"Ansible is needed to run this command, please install it using 'pip"
" install ansible'"
)
sys.exit(1) sys.exit(1)
args = ['ansible-playbook', '-c', 'local', playbook_name, '-vvvv'] args = ["ansible-playbook", "-c", "local", playbook_name, "-vvvv"]
if extra_vars: if extra_vars:
args.extend(['-e', json.dumps(extra_vars)]) args.extend(["-e", json.dumps(extra_vars)])
if tag: if tag:
args.extend(['-t', tag]) args.extend(["-t", tag])
subprocess.check_call(args, cwd=os.path.join(bench.__path__[0], 'playbooks')) subprocess.check_call(args, cwd=os.path.join(bench.__path__[0], "playbooks"))
def find_benches(directory=None): def find_benches(directory=None):
@ -304,7 +312,7 @@ def find_benches(directory=None):
def is_dist_editable(dist): def is_dist_editable(dist):
"""Is distribution an editable install?""" """Is distribution an editable install?"""
for path_item in sys.path: for path_item in sys.path:
egg_link = os.path.join(path_item, dist + '.egg-link') egg_link = os.path.join(path_item, dist + ".egg-link")
if os.path.isfile(egg_link): if os.path.isfile(egg_link):
return True return True
return False return False
@ -324,21 +332,23 @@ def find_parent_bench(path):
return find_parent_bench(parent_dir) return find_parent_bench(parent_dir)
def generate_command_cache(bench_path='.'): def generate_command_cache(bench_path="."):
"""Caches all available commands (even custom apps) via Frappe """Caches all available commands (even custom apps) via Frappe
Default caching behaviour: generated the first time any command (for a specific bench directory) Default caching behaviour: generated the first time any command (for a specific bench directory)
""" """
from bench.utils.bench import get_env_cmd from bench.utils.bench import get_env_cmd
python = get_env_cmd('python', bench_path=bench_path) python = get_env_cmd("python", bench_path=bench_path)
sites_path = os.path.join(bench_path, 'sites') sites_path = os.path.join(bench_path, "sites")
if os.path.exists(bench_cache_file): if os.path.exists(bench_cache_file):
os.remove(bench_cache_file) os.remove(bench_cache_file)
try: try:
output = get_cmd_output(f"{python} -m frappe.utils.bench_helper get-frappe-commands", cwd=sites_path) output = get_cmd_output(
with open(bench_cache_file, 'w') as f: f"{python} -m frappe.utils.bench_helper get-frappe-commands", cwd=sites_path
)
with open(bench_cache_file, "w") as f:
json.dump(eval(output), f) json.dump(eval(output), f)
return json.loads(output) return json.loads(output)
@ -349,7 +359,7 @@ def generate_command_cache(bench_path='.'):
return [] return []
def clear_command_cache(bench_path='.'): def clear_command_cache(bench_path="."):
"""Clears commands cached """Clears commands cached
Default invalidation behaviour: destroyed on each run of `bench update` Default invalidation behaviour: destroyed on each run of `bench update`
""" """
@ -366,7 +376,7 @@ def find_org(org_repo):
org_repo = org_repo[0] org_repo = org_repo[0]
for org in ["frappe", "erpnext"]: for org in ["frappe", "erpnext"]:
res = requests.head(f'https://api.github.com/repos/{org}/{org_repo}') res = requests.head(f"https://api.github.com/repos/{org}/{org_repo}")
if res.ok: if res.ok:
return org, org_repo return org, org_repo
@ -399,7 +409,7 @@ def is_git_url(url):
return bool(re.match(pattern, url)) return bool(re.match(pattern, url))
def drop_privileges(uid_name='nobody', gid_name='nogroup'): def drop_privileges(uid_name="nobody", gid_name="nogroup"):
import grp import grp
import pwd import pwd

View File

@ -4,15 +4,21 @@ from setuptools.config import read_configuration
import bench import bench
import sys import sys
import subprocess import subprocess
from bench.exceptions import InvalidRemoteException, InvalidBranchException, CommandFailedError from bench.exceptions import (
InvalidRemoteException,
InvalidBranchException,
CommandFailedError,
)
from bench.app import get_repo_dir from bench.app import get_repo_dir
def is_version_upgrade(app='frappe', bench_path='.', branch=None): def is_version_upgrade(app="frappe", bench_path=".", branch=None):
upstream_version = get_upstream_version(app=app, branch=branch, bench_path=bench_path) upstream_version = get_upstream_version(app=app, branch=branch, bench_path=bench_path)
if not upstream_version: if not upstream_version:
raise InvalidBranchException(f'Specified branch of app {app} is not in upstream remote') raise InvalidBranchException(
f"Specified branch of app {app} is not in upstream remote"
)
local_version = get_major_version(get_current_version(app, bench_path=bench_path)) local_version = get_major_version(get_current_version(app, bench_path=bench_path))
upstream_version = get_major_version(upstream_version) upstream_version = get_major_version(upstream_version)
@ -23,21 +29,28 @@ def is_version_upgrade(app='frappe', bench_path='.', branch=None):
return (False, local_version, upstream_version) return (False, local_version, upstream_version)
def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrade=True): def switch_branch(branch, apps=None, bench_path=".", upgrade=False, check_upgrade=True):
import git import git
import importlib import importlib
from bench.utils import update_requirements, update_node_packages, backup_all_sites, patch_sites, post_upgrade from bench.utils import (
update_requirements,
update_node_packages,
backup_all_sites,
patch_sites,
post_upgrade,
)
from bench.utils.bench import build_assets from bench.utils.bench import build_assets
apps_dir = os.path.join(bench_path, 'apps') apps_dir = os.path.join(bench_path, "apps")
version_upgrade = (False,) version_upgrade = (False,)
switched_apps = [] switched_apps = []
if not apps: if not apps:
apps = [name for name in os.listdir(apps_dir) apps = [
if os.path.isdir(os.path.join(apps_dir, name))] name for name in os.listdir(apps_dir) if os.path.isdir(os.path.join(apps_dir, name))
if branch=="v4.x.x": ]
apps.append('shopping_cart') if branch == "v4.x.x":
apps.append("shopping_cart")
for app in apps: for app in apps:
app_dir = os.path.join(apps_dir, app) app_dir = os.path.join(apps_dir, app)
@ -48,18 +61,27 @@ def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrad
repo = git.Repo(app_dir) repo = git.Repo(app_dir)
unshallow_flag = os.path.exists(os.path.join(app_dir, ".git", "shallow")) unshallow_flag = os.path.exists(os.path.join(app_dir, ".git", "shallow"))
bench.utils.log(f"Fetching upstream {'unshallow ' if unshallow_flag else ''}for {app}") bench.utils.log(
f"Fetching upstream {'unshallow ' if unshallow_flag else ''}for {app}"
)
bench.utils.exec_cmd("git remote set-branches upstream '*'", cwd=app_dir) bench.utils.exec_cmd("git remote set-branches upstream '*'", cwd=app_dir)
bench.utils.exec_cmd(f"git fetch --all{' --unshallow' if unshallow_flag else ''} --quiet", cwd=app_dir) bench.utils.exec_cmd(
f"git fetch --all{' --unshallow' if unshallow_flag else ''} --quiet", cwd=app_dir
)
if check_upgrade: if check_upgrade:
version_upgrade = is_version_upgrade(app=app, bench_path=bench_path, branch=branch) version_upgrade = is_version_upgrade(app=app, bench_path=bench_path, branch=branch)
if version_upgrade[0] and not upgrade: if version_upgrade[0] and not upgrade:
bench.utils.log(f"Switching to {branch} will cause upgrade from {version_upgrade[1]} to {version_upgrade[2]}. Pass --upgrade to confirm", level=2) bench.utils.log(
f"Switching to {branch} will cause upgrade from"
f" {version_upgrade[1]} to {version_upgrade[2]}. Pass --upgrade to"
" confirm",
level=2,
)
sys.exit(1) sys.exit(1)
print("Switching for "+app) print("Switching for " + app)
bench.utils.exec_cmd(f"git checkout -f {branch}", cwd=app_dir) bench.utils.exec_cmd(f"git checkout -f {branch}", cwd=app_dir)
if str(repo.active_branch) == branch: if str(repo.active_branch) == branch:
@ -68,8 +90,13 @@ def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrad
bench.utils.log(f"Switching branches failed for: {app}", level=2) bench.utils.log(f"Switching branches failed for: {app}", level=2)
if switched_apps: if switched_apps:
bench.utils.log("Successfully switched branches for: " + ", ".join(switched_apps), level=1) bench.utils.log(
print('Please run `bench update --patch` to be safe from any differences in database schema') "Successfully switched branches for: " + ", ".join(switched_apps), level=1
)
print(
"Please run `bench update --patch` to be safe from any differences in"
" database schema"
)
if version_upgrade[0] and upgrade: if version_upgrade[0] and upgrade:
update_requirements() update_requirements()
@ -81,40 +108,51 @@ def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrad
post_upgrade(version_upgrade[1], version_upgrade[2]) post_upgrade(version_upgrade[1], version_upgrade[2])
def switch_to_branch(branch=None, apps=None, bench_path='.', upgrade=False): def switch_to_branch(branch=None, apps=None, bench_path=".", upgrade=False):
switch_branch(branch, apps=apps, bench_path=bench_path, upgrade=upgrade) switch_branch(branch, apps=apps, bench_path=bench_path, upgrade=upgrade)
def switch_to_develop(apps=None, bench_path='.', upgrade=True):
switch_branch('develop', apps=apps, bench_path=bench_path, upgrade=upgrade)
def get_version_from_string(contents, field='__version__'): def switch_to_develop(apps=None, bench_path=".", upgrade=True):
switch_branch("develop", apps=apps, bench_path=bench_path, upgrade=upgrade)
def get_version_from_string(contents, field="__version__"):
match = re.search(r"^(\s*%s\s*=\s*['\\\"])(.+?)(['\"])(?sm)" % field, contents) match = re.search(r"^(\s*%s\s*=\s*['\\\"])(.+?)(['\"])(?sm)" % field, contents)
return match.group(2) return match.group(2)
def get_major_version(version): def get_major_version(version):
import semantic_version import semantic_version
return semantic_version.Version(version).major return semantic_version.Version(version).major
def get_develop_version(app, bench_path='.'):
repo_dir = get_repo_dir(app, bench_path=bench_path)
with open(os.path.join(repo_dir, os.path.basename(repo_dir), 'hooks.py')) as f:
return get_version_from_string(f.read(), field='develop_version')
def get_upstream_version(app, branch=None, bench_path='.'): def get_develop_version(app, bench_path="."):
repo_dir = get_repo_dir(app, bench_path=bench_path)
with open(os.path.join(repo_dir, os.path.basename(repo_dir), "hooks.py")) as f:
return get_version_from_string(f.read(), field="develop_version")
def get_upstream_version(app, branch=None, bench_path="."):
repo_dir = get_repo_dir(app, bench_path=bench_path) repo_dir = get_repo_dir(app, bench_path=bench_path)
if not branch: if not branch:
branch = get_current_branch(app, bench_path=bench_path) branch = get_current_branch(app, bench_path=bench_path)
try: try:
subprocess.call(f'git fetch --depth=1 --no-tags upstream {branch}', shell=True, cwd=repo_dir) subprocess.call(
f"git fetch --depth=1 --no-tags upstream {branch}", shell=True, cwd=repo_dir
)
except CommandFailedError: except CommandFailedError:
raise InvalidRemoteException(f'Failed to fetch from remote named upstream for {app}') raise InvalidRemoteException(f"Failed to fetch from remote named upstream for {app}")
try: try:
contents = subprocess.check_output(f'git show upstream/{branch}:{app}/__init__.py', contents = subprocess.check_output(
shell=True, cwd=repo_dir, stderr=subprocess.STDOUT) f"git show upstream/{branch}:{app}/__init__.py",
contents = contents.decode('utf-8') shell=True,
cwd=repo_dir,
stderr=subprocess.STDOUT,
)
contents = contents.decode("utf-8")
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
if b"Invalid object" in e.output: if b"Invalid object" in e.output:
return None return None
@ -123,23 +161,28 @@ def get_upstream_version(app, branch=None, bench_path='.'):
return get_version_from_string(contents) return get_version_from_string(contents)
def get_current_frappe_version(bench_path='.'): def get_current_frappe_version(bench_path="."):
try: try:
return get_major_version(get_current_version('frappe', bench_path=bench_path)) return get_major_version(get_current_version("frappe", bench_path=bench_path))
except IOError: except IOError:
return 0 return 0
def get_current_branch(app, bench_path='.'):
def get_current_branch(app, bench_path="."):
from bench.utils import get_cmd_output from bench.utils import get_cmd_output
repo_dir = get_repo_dir(app, bench_path=bench_path) repo_dir = get_repo_dir(app, bench_path=bench_path)
return get_cmd_output("basename $(git symbolic-ref -q HEAD)", cwd=repo_dir) return get_cmd_output("basename $(git symbolic-ref -q HEAD)", cwd=repo_dir)
def get_remote(app, bench_path='.'):
def get_remote(app, bench_path="."):
repo_dir = get_repo_dir(app, bench_path=bench_path) repo_dir = get_repo_dir(app, bench_path=bench_path)
contents = subprocess.check_output(['git', 'remote', '-v'], cwd=repo_dir, stderr=subprocess.STDOUT) contents = subprocess.check_output(
contents = contents.decode('utf-8') ["git", "remote", "-v"], cwd=repo_dir, stderr=subprocess.STDOUT
if re.findall('upstream[\s]+', contents): )
return 'upstream' contents = contents.decode("utf-8")
if re.findall(r"upstream[\s]+", contents):
return "upstream"
elif not contents: elif not contents:
# if contents is an empty string => remote doesn't exist # if contents is an empty string => remote doesn't exist
return False return False
@ -150,17 +193,17 @@ def get_remote(app, bench_path='.'):
def get_app_name(bench_path, repo_name): def get_app_name(bench_path, repo_name):
app_name = None app_name = None
apps_path = os.path.join(os.path.abspath(bench_path), 'apps') apps_path = os.path.join(os.path.abspath(bench_path), "apps")
config_path = os.path.join(apps_path, repo_name, 'setup.cfg') config_path = os.path.join(apps_path, repo_name, "setup.cfg")
if os.path.exists(config_path): if os.path.exists(config_path):
config = read_configuration(config_path) config = read_configuration(config_path)
app_name = config.get('metadata', {}).get('name') app_name = config.get("metadata", {}).get("name")
if not app_name: if not app_name:
# retrieve app name from setup.py as fallback # retrieve app name from setup.py as fallback
app_path = os.path.join(apps_path, repo_name, 'setup.py') app_path = os.path.join(apps_path, repo_name, "setup.py")
with open(app_path, 'rb') as f: with open(app_path, "rb") as f:
app_name = re.search(r'name\s*=\s*[\'"](.*)[\'"]', f.read().decode('utf-8')).group(1) app_name = re.search(r'name\s*=\s*[\'"](.*)[\'"]', f.read().decode("utf-8")).group(1)
if app_name and repo_name != app_name: if app_name and repo_name != app_name:
os.rename(os.path.join(apps_path, repo_name), os.path.join(apps_path, app_name)) os.rename(os.path.join(apps_path, repo_name), os.path.join(apps_path, app_name))
@ -169,12 +212,12 @@ def get_app_name(bench_path, repo_name):
return repo_name return repo_name
def get_current_version(app, bench_path='.'): def get_current_version(app, bench_path="."):
current_version = None current_version = None
repo_dir = get_repo_dir(app, bench_path=bench_path) repo_dir = get_repo_dir(app, bench_path=bench_path)
config_path = os.path.join(repo_dir, "setup.cfg") config_path = os.path.join(repo_dir, "setup.cfg")
init_path = os.path.join(repo_dir, os.path.basename(repo_dir), '__init__.py') init_path = os.path.join(repo_dir, os.path.basename(repo_dir), "__init__.py")
setup_path = os.path.join(repo_dir, 'setup.py') setup_path = os.path.join(repo_dir, "setup.py")
try: try:
if os.path.exists(config_path): if os.path.exists(config_path):
@ -187,6 +230,6 @@ def get_current_version(app, bench_path='.'):
except AttributeError: except AttributeError:
# backward compatibility # backward compatibility
with open(setup_path) as f: with open(setup_path) as f:
current_version = get_version_from_string(f.read(), field='version') current_version = get_version_from_string(f.read(), field="version")
return current_version return current_version

View File

@ -18,17 +18,19 @@ from bench.exceptions import PatchError, ValidationError
logger = logging.getLogger(bench.PROJECT_NAME) logger = logging.getLogger(bench.PROJECT_NAME)
def get_env_cmd(cmd, bench_path='.'): def get_env_cmd(cmd, bench_path="."):
return os.path.abspath(os.path.join(bench_path, 'env', 'bin', cmd)) return os.path.abspath(os.path.join(bench_path, "env", "bin", cmd))
def get_venv_path(): def get_venv_path():
venv = which('virtualenv') venv = which("virtualenv")
if not venv: if not venv:
current_python = sys.executable current_python = sys.executable
with open(os.devnull, "wb") as devnull: with open(os.devnull, "wb") as devnull:
is_venv_installed = not subprocess.call([current_python, "-m", "venv", "--help"], stdout=devnull) is_venv_installed = not subprocess.call(
[current_python, "-m", "venv", "--help"], stdout=devnull
)
if is_venv_installed: if is_venv_installed:
venv = f"{current_python} -m venv" venv = f"{current_python} -m venv"
@ -40,14 +42,14 @@ def update_env_pip(bench_path):
exec_cmd(f"{env_py} -m pip install -q -U pip") exec_cmd(f"{env_py} -m pip install -q -U pip")
def update_requirements(bench_path='.'): def update_requirements(bench_path="."):
from bench.app import install_app from bench.app import install_app
from bench.bench import Bench from bench.bench import Bench
bench = Bench(bench_path) bench = Bench(bench_path)
apps = [app for app in bench.apps if app not in bench.excluded_apps] apps = [app for app in bench.apps if app not in bench.excluded_apps]
print(f"Updating env pip...") print("Updating env pip...")
update_env_pip(bench_path) update_env_pip(bench_path)
@ -57,14 +59,14 @@ def update_requirements(bench_path='.'):
install_app(app, bench_path=bench_path, skip_assets=True, restart_bench=False) install_app(app, bench_path=bench_path, skip_assets=True, restart_bench=False)
def update_python_packages(bench_path='.'): def update_python_packages(bench_path="."):
from bench.bench import Bench from bench.bench import Bench
bench = Bench(bench_path) bench = Bench(bench_path)
env_py = get_env_cmd("python") env_py = get_env_cmd("python")
apps = [app for app in bench.apps if app not in bench.excluded_apps] apps = [app for app in bench.apps if app not in bench.excluded_apps]
print('Updating Python libraries...') print("Updating Python libraries...")
update_env_pip(bench_path) update_env_pip(bench_path)
@ -74,21 +76,22 @@ def update_python_packages(bench_path='.'):
bench.run(f"{env_py} -m pip install -q -U -e {app_path}") bench.run(f"{env_py} -m pip install -q -U -e {app_path}")
def update_node_packages(bench_path='.'): def update_node_packages(bench_path="."):
print('Updating node packages...') print("Updating node packages...")
from bench.utils.app import get_develop_version from bench.utils.app import get_develop_version
from distutils.version import LooseVersion from distutils.version import LooseVersion
v = LooseVersion(get_develop_version('frappe', bench_path = bench_path))
v = LooseVersion(get_develop_version("frappe", bench_path=bench_path))
# After rollup was merged, frappe_version = 10.1 # After rollup was merged, frappe_version = 10.1
# if develop_verion is 11 and up, only then install yarn # if develop_verion is 11 and up, only then install yarn
if v < LooseVersion('11.x.x-develop'): if v < LooseVersion("11.x.x-develop"):
update_npm_packages(bench_path) update_npm_packages(bench_path)
else: else:
update_yarn_packages(bench_path) update_yarn_packages(bench_path)
def install_python_dev_dependencies(bench_path='.', apps=None): def install_python_dev_dependencies(bench_path=".", apps=None):
from bench.bench import Bench from bench.bench import Bench
bench = Bench(bench_path) bench = Bench(bench_path)
@ -104,43 +107,43 @@ def install_python_dev_dependencies(bench_path='.', apps=None):
dev_requirements_path = os.path.join(app_path, "dev-requirements.txt") dev_requirements_path = os.path.join(app_path, "dev-requirements.txt")
if os.path.exists(dev_requirements_path): if os.path.exists(dev_requirements_path):
log(f'Installing python development dependencies for {app}') log(f"Installing python development dependencies for {app}")
bench.run(f"{env_py} -m pip install -q -r {dev_requirements_path}") bench.run(f"{env_py} -m pip install -q -r {dev_requirements_path}")
def update_yarn_packages(bench_path='.'): def update_yarn_packages(bench_path="."):
from bench.bench import Bench from bench.bench import Bench
bench = Bench(bench_path) bench = Bench(bench_path)
apps = [app for app in bench.apps if app not in bench.excluded_apps] apps = [app for app in bench.apps if app not in bench.excluded_apps]
apps_dir = os.path.join(bench.name, 'apps') apps_dir = os.path.join(bench.name, "apps")
# TODO: Check for stuff like this early on only?? # TODO: Check for stuff like this early on only??
if not which('yarn'): if not which("yarn"):
print("Please install yarn using below command and try again.") print("Please install yarn using below command and try again.")
print("`npm install -g yarn`") print("`npm install -g yarn`")
return return
for app in apps: for app in apps:
app_path = os.path.join(apps_dir, app) app_path = os.path.join(apps_dir, app)
if os.path.exists(os.path.join(app_path, 'package.json')): if os.path.exists(os.path.join(app_path, "package.json")):
click.secho(f"\nInstalling node dependencies for {app}", fg="yellow") click.secho(f"\nInstalling node dependencies for {app}", fg="yellow")
bench.run('yarn install', cwd=app_path) bench.run("yarn install", cwd=app_path)
def update_npm_packages(bench_path='.'): def update_npm_packages(bench_path="."):
apps_dir = os.path.join(bench_path, 'apps') apps_dir = os.path.join(bench_path, "apps")
package_json = {} package_json = {}
for app in os.listdir(apps_dir): for app in os.listdir(apps_dir):
package_json_path = os.path.join(apps_dir, app, 'package.json') package_json_path = os.path.join(apps_dir, app, "package.json")
if os.path.exists(package_json_path): if os.path.exists(package_json_path):
with open(package_json_path, "r") as f: with open(package_json_path, "r") as f:
app_package_json = json.loads(f.read()) app_package_json = json.loads(f.read())
# package.json is usually a dict in a dict # package.json is usually a dict in a dict
for key, value in app_package_json.items(): for key, value in app_package_json.items():
if not key in package_json: if key not in package_json:
package_json[key] = value package_json[key] = value
else: else:
if isinstance(value, dict): if isinstance(value, dict):
@ -151,13 +154,13 @@ def update_npm_packages(bench_path='.'):
package_json[key] = value package_json[key] = value
if package_json is {}: if package_json is {}:
with open(os.path.join(os.path.dirname(__file__), 'package.json'), 'r') as f: with open(os.path.join(os.path.dirname(__file__), "package.json"), "r") as f:
package_json = json.loads(f.read()) package_json = json.loads(f.read())
with open(os.path.join(bench_path, 'package.json'), 'w') as f: with open(os.path.join(bench_path, "package.json"), "w") as f:
f.write(json.dumps(package_json, indent=1, sort_keys=True)) f.write(json.dumps(package_json, indent=1, sort_keys=True))
exec_cmd('npm install', cwd=bench_path) exec_cmd("npm install", cwd=bench_path)
def migrate_env(python, backup=False): def migrate_env(python, backup=False):
@ -166,38 +169,38 @@ def migrate_env(python, backup=False):
from bench.bench import Bench from bench.bench import Bench
bench = Bench(".") bench = Bench(".")
nvenv = 'env' nvenv = "env"
path = os.getcwd() path = os.getcwd()
python = which(python) python = which(python)
virtualenv = which('virtualenv') virtualenv = which("virtualenv")
pvenv = os.path.join(path, nvenv) pvenv = os.path.join(path, nvenv)
# Clear Cache before Bench Dies. # Clear Cache before Bench Dies.
try: try:
config = bench.conf config = bench.conf
rredis = urlparse(config['redis_cache']) rredis = urlparse(config["redis_cache"])
redis = f"{which('redis-cli')} -p {rredis.port}" redis = f"{which('redis-cli')} -p {rredis.port}"
logger.log('Clearing Redis Cache...') logger.log("Clearing Redis Cache...")
exec_cmd(f'{redis} FLUSHALL') exec_cmd(f"{redis} FLUSHALL")
logger.log('Clearing Redis DataBase...') logger.log("Clearing Redis DataBase...")
exec_cmd(f'{redis} FLUSHDB') exec_cmd(f"{redis} FLUSHDB")
except Exception: except Exception:
logger.warning('Please ensure Redis Connections are running or Daemonized.') logger.warning("Please ensure Redis Connections are running or Daemonized.")
# Backup venv: restore using `virtualenv --relocatable` if needed # Backup venv: restore using `virtualenv --relocatable` if needed
if backup: if backup:
from datetime import datetime from datetime import datetime
parch = os.path.join(path, 'archived', 'envs') parch = os.path.join(path, "archived", "envs")
if not os.path.exists(parch): if not os.path.exists(parch):
os.mkdir(parch) os.mkdir(parch)
source = os.path.join(path, 'env') source = os.path.join(path, "env")
target = parch target = parch
logger.log('Backing up Virtual Environment') logger.log("Backing up Virtual Environment")
stamp = datetime.now().strftime('%Y%m%d_%H%M%S') stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
dest = os.path.join(path, str(stamp)) dest = os.path.join(path, str(stamp))
os.rename(source, dest) os.rename(source, dest)
@ -206,24 +209,25 @@ def migrate_env(python, backup=False):
# Create virtualenv using specified python # Create virtualenv using specified python
venv_creation, packages_setup = 1, 1 venv_creation, packages_setup = 1, 1
try: try:
logger.log(f'Setting up a New Virtual {python} Environment') logger.log(f"Setting up a New Virtual {python} Environment")
venv_creation = exec_cmd(f'{virtualenv} --python {python} {pvenv}') venv_creation = exec_cmd(f"{virtualenv} --python {python} {pvenv}")
apps = ' '.join([f"-e {os.path.join('apps', app)}" for app in bench.apps]) apps = " ".join([f"-e {os.path.join('apps', app)}" for app in bench.apps])
packages_setup = exec_cmd(f'{pvenv} -m pip install -q -U {apps}') packages_setup = exec_cmd(f"{pvenv} -m pip install -q -U {apps}")
logger.log(f'Migration Successful to {python}') logger.log(f"Migration Successful to {python}")
except Exception: except Exception:
if venv_creation or packages_setup: if venv_creation or packages_setup:
logger.warning('Migration Error') logger.warning("Migration Error")
def validate_upgrade(from_ver, to_ver, bench_path='.'):
def validate_upgrade(from_ver, to_ver, bench_path="."):
if to_ver >= 6: if to_ver >= 6:
if not which('npm') and not (which('node') or which('nodejs')): if not which("npm") and not (which("node") or which("nodejs")):
raise Exception("Please install nodejs and npm") raise Exception("Please install nodejs and npm")
def post_upgrade(from_ver, to_ver, bench_path='.'): def post_upgrade(from_ver, to_ver, bench_path="."):
from bench.config import redis from bench.config import redis
from bench.config.supervisor import generate_supervisor_config from bench.config.supervisor import generate_supervisor_config
from bench.config.nginx import make_nginx_conf from bench.config.nginx import make_nginx_conf
@ -232,18 +236,19 @@ def post_upgrade(from_ver, to_ver, bench_path='.'):
conf = Bench(bench_path).conf conf = Bench(bench_path).conf
print("-" * 80 + f"Your bench was upgraded to version {to_ver}") print("-" * 80 + f"Your bench was upgraded to version {to_ver}")
if conf.get('restart_supervisor_on_update'): if conf.get("restart_supervisor_on_update"):
redis.generate_config(bench_path=bench_path) redis.generate_config(bench_path=bench_path)
generate_supervisor_config(bench_path=bench_path) generate_supervisor_config(bench_path=bench_path)
make_nginx_conf(bench_path=bench_path) make_nginx_conf(bench_path=bench_path)
print( print(
"As you have setup your bench for production, you will have to reload configuration for " "As you have setup your bench for production, you will have to reload"
"nginx and supervisor. To complete the migration, please run the following commands:" " configuration for nginx and supervisor. To complete the migration, please"
"\nsudo service nginx restart" " run the following commands:\nsudo service nginx restart\nsudo"
"\nsudo supervisorctl reload" " supervisorctl reload"
) )
def patch_sites(bench_path='.'):
def patch_sites(bench_path="."):
from bench.bench import Bench from bench.bench import Bench
from bench.utils.system import migrate_site from bench.utils.system import migrate_site
@ -255,56 +260,79 @@ def patch_sites(bench_path='.'):
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
raise PatchError raise PatchError
def restart_supervisor_processes(bench_path='.', web_workers=False):
def restart_supervisor_processes(bench_path=".", web_workers=False):
from bench.bench import Bench from bench.bench import Bench
bench = Bench(bench_path) bench = Bench(bench_path)
conf = bench.conf conf = bench.conf
cmd = conf.get('supervisor_restart_cmd') cmd = conf.get("supervisor_restart_cmd")
bench_name = get_bench_name(bench_path) bench_name = get_bench_name(bench_path)
if cmd: if cmd:
bench.run(cmd) bench.run(cmd)
else: else:
supervisor_status = get_cmd_output('supervisorctl status', cwd=bench_path) supervisor_status = get_cmd_output("supervisorctl status", cwd=bench_path)
if web_workers and f'{bench_name}-web:' in supervisor_status: if web_workers and f"{bench_name}-web:" in supervisor_status:
group = f'{bench_name}-web:\t' group = f"{bench_name}-web:\t"
elif f'{bench_name}-workers:' in supervisor_status: elif f"{bench_name}-workers:" in supervisor_status:
group = f'{bench_name}-workers: {bench_name}-web:' group = f"{bench_name}-workers: {bench_name}-web:"
# backward compatibility # backward compatibility
elif f'{bench_name}-processes:' in supervisor_status: elif f"{bench_name}-processes:" in supervisor_status:
group = f'{bench_name}-processes:' group = f"{bench_name}-processes:"
# backward compatibility # backward compatibility
else: else:
group = 'frappe:' group = "frappe:"
bench.run(f"supervisorctl restart {group}") bench.run(f"supervisorctl restart {group}")
def restart_systemd_processes(bench_path='.', web_workers=False): def restart_systemd_processes(bench_path=".", web_workers=False):
bench_name = get_bench_name(bench_path) bench_name = get_bench_name(bench_path)
exec_cmd(f'sudo systemctl stop -- $(systemctl show -p Requires {bench_name}.target | cut -d= -f2)') exec_cmd(
exec_cmd(f'sudo systemctl start -- $(systemctl show -p Requires {bench_name}.target | cut -d= -f2)') f"sudo systemctl stop -- $(systemctl show -p Requires {bench_name}.target | cut"
" -d= -f2)"
)
exec_cmd(
f"sudo systemctl start -- $(systemctl show -p Requires {bench_name}.target |"
" cut -d= -f2)"
)
def build_assets(bench_path='.', app=None): def build_assets(bench_path=".", app=None):
command = 'bench build' command = "bench build"
if app: if app:
command += f' --app {app}' command += f" --app {app}"
exec_cmd(command, cwd=bench_path, env={"BENCH_DEVELOPER": "1"}) exec_cmd(command, cwd=bench_path, env={"BENCH_DEVELOPER": "1"})
def update(pull=False, apps=None, patch=False, build=False, requirements=False, backup=True, compile=True,
force=False, reset=False, restart_supervisor=False, restart_systemd=False): def update(
pull=False,
apps=None,
patch=False,
build=False,
requirements=False,
backup=True,
compile=True,
force=False,
reset=False,
restart_supervisor=False,
restart_systemd=False,
):
"""command: bench update""" """command: bench update"""
import re import re
from bench import patches from bench import patches
from bench.utils import clear_command_cache, pause_exec, log from bench.utils import clear_command_cache, pause_exec, log
from bench.utils.bench import restart_supervisor_processes, restart_systemd_processes, backup_all_sites from bench.utils.bench import (
restart_supervisor_processes,
restart_systemd_processes,
backup_all_sites,
)
from bench.app import pull_apps from bench.app import pull_apps
from bench.utils.app import is_version_upgrade from bench.utils.app import is_version_upgrade
from bench.config.common_site_config import update_config from bench.config.common_site_config import update_config
@ -318,10 +346,10 @@ def update(pull=False, apps=None, patch=False, build=False, requirements=False,
if apps and not pull: if apps and not pull:
apps = [] apps = []
clear_command_cache(bench_path='.') clear_command_cache(bench_path=".")
if conf.get('release_bench'): if conf.get("release_bench"):
print('Release bench detected, cannot update!') print("Release bench detected, cannot update!")
sys.exit(1) sys.exit(1)
if not (pull or patch or build or requirements): if not (pull or patch or build or requirements):
@ -332,50 +360,58 @@ def update(pull=False, apps=None, patch=False, build=False, requirements=False,
if version_upgrade[0]: if version_upgrade[0]:
if force: if force:
log("""Force flag has been used for a major version change in Frappe and it's apps. log(
This will take significant time to migrate and might break custom apps.""", level=3) """Force flag has been used for a major version change in Frappe and it's apps.
This will take significant time to migrate and might break custom apps.""",
level=3,
)
else: else:
print(f"""This update will cause a major version change in Frappe/ERPNext from {version_upgrade[1]} to {version_upgrade[2]}. print(
This would take significant time to migrate and might break custom apps.""") f"""This update will cause a major version change in Frappe/ERPNext from {version_upgrade[1]} to {version_upgrade[2]}.
click.confirm('Do you want to continue?', abort=True) This would take significant time to migrate and might break custom apps."""
)
click.confirm("Do you want to continue?", abort=True)
if not reset and conf.get('shallow_clone'): if not reset and conf.get("shallow_clone"):
log("""shallow_clone is set in your bench config. log(
"""shallow_clone is set in your bench config.
However without passing the --reset flag, your repositories will be unshallowed. However without passing the --reset flag, your repositories will be unshallowed.
To avoid this, cancel this operation and run `bench update --reset`. To avoid this, cancel this operation and run `bench update --reset`.
Consider the consequences of `git reset --hard` on your apps before you run that. Consider the consequences of `git reset --hard` on your apps before you run that.
To avoid seeing this warning, set shallow_clone to false in your common_site_config.json To avoid seeing this warning, set shallow_clone to false in your common_site_config.json
""", level=3) """,
level=3,
)
pause_exec(seconds=10) pause_exec(seconds=10)
if version_upgrade[0] or (not version_upgrade[0] and force): if version_upgrade[0] or (not version_upgrade[0] and force):
validate_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) validate_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path)
conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 }) conf.update({"maintenance_mode": 1, "pause_scheduler": 1})
update_config(conf, bench_path=bench_path) update_config(conf, bench_path=bench_path)
if backup: if backup:
print('Backing up sites...') print("Backing up sites...")
backup_all_sites(bench_path=bench_path) backup_all_sites(bench_path=bench_path)
if apps: if apps:
apps = [app.strip() for app in re.split(",| ", apps) if app] apps = [app.strip() for app in re.split(",| ", apps) if app]
if pull: if pull:
print('Updating apps source...') print("Updating apps source...")
pull_apps(apps=apps, bench_path=bench_path, reset=reset) pull_apps(apps=apps, bench_path=bench_path, reset=reset)
if requirements: if requirements:
print('Setting up requirements...') print("Setting up requirements...")
update_requirements(bench_path=bench_path) update_requirements(bench_path=bench_path)
update_node_packages(bench_path=bench_path) update_node_packages(bench_path=bench_path)
if patch: if patch:
print('Patching sites...') print("Patching sites...")
patch_sites(bench_path=bench_path) patch_sites(bench_path=bench_path)
if build: if build:
print('Building assets...') print("Building assets...")
build_assets(bench_path=bench_path) build_assets(bench_path=bench_path)
if version_upgrade[0] or (not version_upgrade[0] and force): if version_upgrade[0] or (not version_upgrade[0] and force):
@ -384,102 +420,112 @@ To avoid seeing this warning, set shallow_clone to false in your common_site_con
if pull and compile: if pull and compile:
from compileall import compile_dir from compileall import compile_dir
print('Compiling Python files...') print("Compiling Python files...")
apps_dir = os.path.join(bench_path, 'apps') apps_dir = os.path.join(bench_path, "apps")
compile_dir(apps_dir, quiet=1, rx=re.compile('.*node_modules.*')) compile_dir(apps_dir, quiet=1, rx=re.compile(".*node_modules.*"))
if restart_supervisor or conf.get('restart_supervisor_on_update'): if restart_supervisor or conf.get("restart_supervisor_on_update"):
restart_supervisor_processes(bench_path=bench_path) restart_supervisor_processes(bench_path=bench_path)
if restart_systemd or conf.get('restart_systemd_on_update'): if restart_systemd or conf.get("restart_systemd_on_update"):
restart_systemd_processes(bench_path=bench_path) restart_systemd_processes(bench_path=bench_path)
conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 }) conf.update({"maintenance_mode": 0, "pause_scheduler": 0})
update_config(conf, bench_path=bench_path) update_config(conf, bench_path=bench_path)
print("_" * 80 + "\nBench: Deployment tool for Frappe and Frappe Applications (https://frappe.io/bench).\nOpen source depends on your contributions, so do give back by submitting bug reports, patches and fixes and be a part of the community :)") print(
"_" * 80
+ "\nBench: Deployment tool for Frappe and Frappe Applications"
" (https://frappe.io/bench).\nOpen source depends on your contributions, so do"
" give back by submitting bug reports, patches and fixes and be a part of the"
" community :)"
)
def clone_apps_from(bench_path, clone_from, update_app=True): def clone_apps_from(bench_path, clone_from, update_app=True):
from bench.app import install_app from bench.app import install_app
print(f'Copying apps from {clone_from}...')
subprocess.check_output(['cp', '-R', os.path.join(clone_from, 'apps'), bench_path])
node_modules_path = os.path.join(clone_from, 'node_modules') print(f"Copying apps from {clone_from}...")
subprocess.check_output(["cp", "-R", os.path.join(clone_from, "apps"), bench_path])
node_modules_path = os.path.join(clone_from, "node_modules")
if os.path.exists(node_modules_path): if os.path.exists(node_modules_path):
print(f'Copying node_modules from {clone_from}...') print(f"Copying node_modules from {clone_from}...")
subprocess.check_output(['cp', '-R', node_modules_path, bench_path]) subprocess.check_output(["cp", "-R", node_modules_path, bench_path])
def setup_app(app): def setup_app(app):
# run git reset --hard in each branch, pull latest updates and install_app # run git reset --hard in each branch, pull latest updates and install_app
app_path = os.path.join(bench_path, 'apps', app) app_path = os.path.join(bench_path, "apps", app)
# remove .egg-ino # remove .egg-ino
subprocess.check_output(['rm', '-rf', app + '.egg-info'], cwd=app_path) subprocess.check_output(["rm", "-rf", app + ".egg-info"], cwd=app_path)
if update_app and os.path.exists(os.path.join(app_path, '.git')): if update_app and os.path.exists(os.path.join(app_path, ".git")):
remotes = subprocess.check_output(['git', 'remote'], cwd=app_path).strip().split() remotes = subprocess.check_output(["git", "remote"], cwd=app_path).strip().split()
if 'upstream' in remotes: if "upstream" in remotes:
remote = 'upstream' remote = "upstream"
else: else:
remote = remotes[0] remote = remotes[0]
print(f'Cleaning up {app}') print(f"Cleaning up {app}")
branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], cwd=app_path).strip() branch = subprocess.check_output(
subprocess.check_output(['git', 'reset', '--hard'], cwd=app_path) ["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=app_path
subprocess.check_output(['git', 'pull', '--rebase', remote, branch], cwd=app_path) ).strip()
subprocess.check_output(["git", "reset", "--hard"], cwd=app_path)
subprocess.check_output(["git", "pull", "--rebase", remote, branch], cwd=app_path)
install_app(app, bench_path, restart_bench=False) install_app(app, bench_path, restart_bench=False)
with open(os.path.join(clone_from, 'sites', 'apps.txt'), 'r') as f: with open(os.path.join(clone_from, "sites", "apps.txt"), "r") as f:
apps = f.read().splitlines() apps = f.read().splitlines()
for app in apps: for app in apps:
setup_app(app) setup_app(app)
def remove_backups_crontab(bench_path='.'): def remove_backups_crontab(bench_path="."):
from crontab import CronTab from crontab import CronTab
from bench.bench import Bench from bench.bench import Bench
logger.log('removing backup cronjob') logger.log("removing backup cronjob")
bench_dir = os.path.abspath(bench_path) bench_dir = os.path.abspath(bench_path)
user = Bench(bench_dir).conf.get('frappe_user') user = Bench(bench_dir).conf.get("frappe_user")
logfile = os.path.join(bench_dir, 'logs', 'backup.log') logfile = os.path.join(bench_dir, "logs", "backup.log")
system_crontab = CronTab(user=user) system_crontab = CronTab(user=user)
backup_command = f"cd {bench_dir} && {sys.argv[0]} --verbose --site all backup" backup_command = f"cd {bench_dir} && {sys.argv[0]} --verbose --site all backup"
job_command = f"{backup_command} >> {logfile} 2>&1" job_command = f"{backup_command} >> {logfile} 2>&1"
system_crontab.remove_all(command=job_command) system_crontab.remove_all(command=job_command)
def set_mariadb_host(host, bench_path='.'):
update_common_site_config({'db_host': host}, bench_path=bench_path) def set_mariadb_host(host, bench_path="."):
update_common_site_config({"db_host": host}, bench_path=bench_path)
def set_redis_cache_host(host, bench_path='.'): def set_redis_cache_host(host, bench_path="."):
update_common_site_config({'redis_cache': f"redis://{host}"}, bench_path=bench_path) update_common_site_config({"redis_cache": f"redis://{host}"}, bench_path=bench_path)
def set_redis_queue_host(host, bench_path='.'): def set_redis_queue_host(host, bench_path="."):
update_common_site_config({'redis_queue': f"redis://{host}"}, bench_path=bench_path) update_common_site_config({"redis_queue": f"redis://{host}"}, bench_path=bench_path)
def set_redis_socketio_host(host, bench_path='.'): def set_redis_socketio_host(host, bench_path="."):
update_common_site_config({'redis_socketio': f"redis://{host}"}, bench_path=bench_path) update_common_site_config({"redis_socketio": f"redis://{host}"}, bench_path=bench_path)
def update_common_site_config(ddict, bench_path='.'): def update_common_site_config(ddict, bench_path="."):
filename = os.path.join(bench_path, 'sites', 'common_site_config.json') filename = os.path.join(bench_path, "sites", "common_site_config.json")
if os.path.exists(filename): if os.path.exists(filename):
with open(filename, 'r') as f: with open(filename, "r") as f:
content = json.load(f) content = json.load(f)
else: else:
content = {} content = {}
content.update(ddict) content.update(ddict)
with open(filename, 'w') as f: with open(filename, "w") as f:
json.dump(content, f, indent=1, sort_keys=True) json.dump(content, f, indent=1, sort_keys=True)
@ -499,7 +545,7 @@ def check_app_installed(app, bench_path="."):
["bench", "--site", "all", "list-apps", "--format", "json"], ["bench", "--site", "all", "list-apps", "--format", "json"],
stderr=open(os.devnull, "wb"), stderr=open(os.devnull, "wb"),
cwd=bench_path, cwd=bench_path,
).decode('utf-8') ).decode("utf-8")
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
return None return None
@ -514,16 +560,19 @@ def check_app_installed(app, bench_path="."):
def check_app_installed_legacy(app, bench_path="."): def check_app_installed_legacy(app, bench_path="."):
site_path = os.path.join(bench_path, 'sites') site_path = os.path.join(bench_path, "sites")
for site in os.listdir(site_path): for site in os.listdir(site_path):
req_file = os.path.join(site_path, site, 'site_config.json') req_file = os.path.join(site_path, site, "site_config.json")
if os.path.exists(req_file): if os.path.exists(req_file):
out = subprocess.check_output(["bench", "--site", site, "list-apps"], cwd=bench_path).decode('utf-8') out = subprocess.check_output(
if re.search(r'\b' + app + r'\b', out): ["bench", "--site", site, "list-apps"], cwd=bench_path
).decode("utf-8")
if re.search(r"\b" + app + r"\b", out):
print(f"Cannot remove, app is installed on site: {site}") print(f"Cannot remove, app is installed on site: {site}")
sys.exit(1) sys.exit(1)
def validate_branch(): def validate_branch():
from bench.bench import Bench from bench.bench import Bench
from bench.utils.app import get_current_branch from bench.utils.app import get_current_branch
@ -531,14 +580,15 @@ def validate_branch():
apps = Bench(".").apps apps = Bench(".").apps
installed_apps = set(apps) installed_apps = set(apps)
check_apps = set(['frappe', 'erpnext']) check_apps = set(["frappe", "erpnext"])
intersection_apps = installed_apps.intersection(check_apps) intersection_apps = installed_apps.intersection(check_apps)
for app in intersection_apps: for app in intersection_apps:
branch = get_current_branch(app) branch = get_current_branch(app)
if branch == "master": if branch == "master":
print("""'master' branch is renamed to 'version-11' since 'version-12' release. print(
"""'master' branch is renamed to 'version-11' since 'version-12' release.
As of January 2020, the following branches are As of January 2020, the following branches are
version Frappe ERPNext version Frappe ERPNext
11 version-11 version-11 11 version-11 version-11
@ -547,6 +597,7 @@ version Frappe ERPNext
14 develop develop 14 develop develop
Please switch to new branches to get future updates. Please switch to new branches to get future updates.
To switch to your required branch, run the following commands: bench switch-to-branch [branch-name]""") To switch to your required branch, run the following commands: bench switch-to-branch [branch-name]"""
)
sys.exit(1) sys.exit(1)

View File

@ -7,15 +7,31 @@ import sys
# imports - module imports # imports - module imports
import bench import bench
from bench.utils import exec_cmd, get_process_manager, log, run_frappe_cmd, sudoers_file, which from bench.utils import (
exec_cmd,
get_process_manager,
log,
run_frappe_cmd,
sudoers_file,
which,
)
from bench.utils.bench import build_assets, clone_apps_from from bench.utils.bench import build_assets, clone_apps_from
def init(
def init(path, apps_path=None, no_procfile=False, no_backups=False, path,
frappe_path=None, frappe_branch=None, verbose=False, clone_from=None, apps_path=None,
skip_redis_config_generation=False, clone_without_update=False, skip_assets=False, no_procfile=False,
python='python3'): no_backups=False,
frappe_path=None,
frappe_branch=None,
verbose=False,
clone_from=None,
skip_redis_config_generation=False,
clone_without_update=False,
skip_assets=False,
python="python3",
):
"""Initialize a new bench directory """Initialize a new bench directory
* create a bench directory in the given path * create a bench directory in the given path
@ -45,13 +61,17 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False,
# local apps # local apps
if clone_from: if clone_from:
clone_apps_from(bench_path=path, clone_from=clone_from, update_app=not clone_without_update) clone_apps_from(
bench_path=path, clone_from=clone_from, update_app=not clone_without_update
)
# remote apps # remote apps
else: else:
frappe_path = frappe_path or 'https://github.com/frappe/frappe.git' frappe_path = frappe_path or "https://github.com/frappe/frappe.git"
get_app(frappe_path, branch=frappe_branch, bench_path=path, skip_assets=True, verbose=verbose) get_app(
frappe_path, branch=frappe_branch, bench_path=path, skip_assets=True, verbose=verbose
)
# fetch remote apps using config file - deprecate this! # fetch remote apps using config file - deprecate this!
if apps_path: if apps_path:
@ -65,29 +85,32 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False,
if not no_backups: if not no_backups:
bench.setup.backups() bench.setup.backups()
def setup_sudoers(user): def setup_sudoers(user):
if not os.path.exists('/etc/sudoers.d'): if not os.path.exists("/etc/sudoers.d"):
os.makedirs('/etc/sudoers.d') os.makedirs("/etc/sudoers.d")
set_permissions = False set_permissions = False
if not os.path.exists('/etc/sudoers'): if not os.path.exists("/etc/sudoers"):
set_permissions = True set_permissions = True
with open('/etc/sudoers', 'a') as f: with open("/etc/sudoers", "a") as f:
f.write('\n#includedir /etc/sudoers.d\n') f.write("\n#includedir /etc/sudoers.d\n")
if set_permissions: if set_permissions:
os.chmod('/etc/sudoers', 0o440) os.chmod("/etc/sudoers", 0o440)
template = bench.config.env().get_template('frappe_sudoers') template = bench.config.env().get_template("frappe_sudoers")
frappe_sudoers = template.render(**{ frappe_sudoers = template.render(
'user': user, **{
'service': which('service'), "user": user,
'systemctl': which('systemctl'), "service": which("service"),
'nginx': which('nginx'), "systemctl": which("systemctl"),
}) "nginx": which("nginx"),
}
)
with open(sudoers_file, 'w') as f: with open(sudoers_file, "w") as f:
f.write(frappe_sudoers) f.write(frappe_sudoers)
os.chmod(sudoers_file, 0o440) os.chmod(sudoers_file, 0o440)
@ -103,43 +126,43 @@ def start(no_dev=False, concurrency=None, procfile=None, no_prefix=False, procma
if not program: if not program:
raise Exception("No process manager found") raise Exception("No process manager found")
os.environ['PYTHONUNBUFFERED'] = "true" os.environ["PYTHONUNBUFFERED"] = "true"
if not no_dev: if not no_dev:
os.environ['DEV_SERVER'] = "true" os.environ["DEV_SERVER"] = "true"
command = [program, 'start'] command = [program, "start"]
if concurrency: if concurrency:
command.extend(['-c', concurrency]) command.extend(["-c", concurrency])
if procfile: if procfile:
command.extend(['-f', procfile]) command.extend(["-f", procfile])
if no_prefix: if no_prefix:
command.extend(['--no-prefix']) command.extend(["--no-prefix"])
os.execv(program, command) os.execv(program, command)
def migrate_site(site, bench_path='.'): def migrate_site(site, bench_path="."):
run_frappe_cmd('--site', site, 'migrate', bench_path=bench_path) run_frappe_cmd("--site", site, "migrate", bench_path=bench_path)
def backup_site(site, bench_path='.'): def backup_site(site, bench_path="."):
run_frappe_cmd('--site', site, 'backup', bench_path=bench_path) run_frappe_cmd("--site", site, "backup", bench_path=bench_path)
def backup_all_sites(bench_path='.'): def backup_all_sites(bench_path="."):
from bench.bench import Bench from bench.bench import Bench
for site in Bench(bench_path).sites: for site in Bench(bench_path).sites:
backup_site(site, bench_path=bench_path) backup_site(site, bench_path=bench_path)
def fix_prod_setup_perms(bench_path='.', frappe_user=None): def fix_prod_setup_perms(bench_path=".", frappe_user=None):
from glob import glob from glob import glob
from bench.bench import Bench from bench.bench import Bench
frappe_user = frappe_user or Bench(bench_path).conf.get('frappe_user') frappe_user = frappe_user or Bench(bench_path).conf.get("frappe_user")
if not frappe_user: if not frappe_user:
print("frappe user not set") print("frappe user not set")
@ -154,15 +177,15 @@ def fix_prod_setup_perms(bench_path='.', frappe_user=None):
def setup_fonts(): def setup_fonts():
fonts_path = os.path.join('/tmp', 'fonts') fonts_path = os.path.join("/tmp", "fonts")
if os.path.exists('/etc/fonts_backup'): if os.path.exists("/etc/fonts_backup"):
return return
exec_cmd("git clone https://github.com/frappe/fonts.git", cwd='/tmp') exec_cmd("git clone https://github.com/frappe/fonts.git", cwd="/tmp")
os.rename('/etc/fonts', '/etc/fonts_backup') os.rename("/etc/fonts", "/etc/fonts_backup")
os.rename('/usr/share/fonts', '/usr/share/fonts_backup') os.rename("/usr/share/fonts", "/usr/share/fonts_backup")
os.rename(os.path.join(fonts_path, 'etc_fonts'), '/etc/fonts') os.rename(os.path.join(fonts_path, "etc_fonts"), "/etc/fonts")
os.rename(os.path.join(fonts_path, 'usr_share_fonts'), '/usr/share/fonts') os.rename(os.path.join(fonts_path, "usr_share_fonts"), "/usr/share/fonts")
shutil.rmtree(fonts_path) shutil.rmtree(fonts_path)
exec_cmd("fc-cache -fv") exec_cmd("fc-cache -fv")

View File

@ -10,7 +10,7 @@ def update_translations_p(args):
try: try:
update_translations(*args) update_translations(*args)
except requests.exceptions.HTTPError: except requests.exceptions.HTTPError:
print('Download failed for', args[0], args[1]) print("Download failed for", args[0], args[1])
def download_translations_p(): def download_translations_p():
@ -19,7 +19,7 @@ def download_translations_p():
pool = multiprocessing.Pool(multiprocessing.cpu_count()) pool = multiprocessing.Pool(multiprocessing.cpu_count())
langs = get_langs() langs = get_langs()
apps = ('frappe', 'erpnext') apps = ("frappe", "erpnext")
args = list(itertools.product(apps, langs)) args = list(itertools.product(apps, langs))
pool.map(update_translations_p, args) pool.map(update_translations_p, args)
@ -27,32 +27,32 @@ def download_translations_p():
def download_translations(): def download_translations():
langs = get_langs() langs = get_langs()
apps = ('frappe', 'erpnext') apps = ("frappe", "erpnext")
for app, lang in itertools.product(apps, langs): for app, lang in itertools.product(apps, langs):
update_translations(app, lang) update_translations(app, lang)
def get_langs(): def get_langs():
lang_file = 'apps/frappe/frappe/geo/languages.json' lang_file = "apps/frappe/frappe/geo/languages.json"
with open(lang_file) as f: with open(lang_file) as f:
langs = json.loads(f.read()) langs = json.loads(f.read())
return [d['code'] for d in langs] return [d["code"] for d in langs]
def update_translations(app, lang): def update_translations(app, lang):
import requests import requests
translations_dir = os.path.join('apps', app, app, 'translations') translations_dir = os.path.join("apps", app, app, "translations")
csv_file = os.path.join(translations_dir, lang + '.csv') csv_file = os.path.join(translations_dir, lang + ".csv")
url = f"https://translate.erpnext.com/files/{app}-{lang}.csv" url = f"https://translate.erpnext.com/files/{app}-{lang}.csv"
r = requests.get(url, stream=True) r = requests.get(url, stream=True)
r.raise_for_status() r.raise_for_status()
with open(csv_file, 'wb') as f: with open(csv_file, "wb") as f:
for chunk in r.iter_content(chunk_size=1024): for chunk in r.iter_content(chunk_size=1024):
# filter out keep-alive new chunks # filter out keep-alive new chunks
if chunk: if chunk:
f.write(chunk) f.write(chunk)
f.flush() f.flush()
print('downloaded for', app, lang) print("downloaded for", app, lang)