diff --git a/bench/__init__.py b/bench/__init__.py index e2f6fd6d..77c6e130 100644 --- a/bench/__init__.py +++ b/bench/__init__.py @@ -5,8 +5,9 @@ current_path = None updated_path = None -def set_frappe_version(bench_path='.'): +def set_frappe_version(bench_path="."): from .utils.app import get_current_frappe_version + global FRAPPE_VERSION if not FRAPPE_VERSION: - FRAPPE_VERSION = get_current_frappe_version(bench_path=bench_path) \ No newline at end of file + FRAPPE_VERSION = get_current_frappe_version(bench_path=bench_path) diff --git a/bench/app.py b/bench/app.py index d3aee384..e36cc73f 100755 --- a/bench/app.py +++ b/bench/app.py @@ -13,8 +13,19 @@ import click # imports - module imports import bench -from bench.utils import exec_cmd, is_bench_directory, run_frappe_cmd, is_git_url, fetch_details_from_tag -from bench.utils.bench import get_env_cmd, build_assets, restart_supervisor_processes, restart_systemd_processes +from bench.utils import ( + exec_cmd, + is_bench_directory, + run_frappe_cmd, + is_git_url, + fetch_details_from_tag, +) +from bench.utils.bench import ( + get_env_cmd, + build_assets, + restart_supervisor_processes, + restart_systemd_processes, +) logger = logging.getLogger(bench.PROJECT_NAME) @@ -22,8 +33,9 @@ logger = logging.getLogger(bench.PROJECT_NAME) if typing.TYPE_CHECKING: from bench.bench import Bench + class AppMeta: - def __init__(self, name: str, branch : str = None): + def __init__(self, name: str, branch: str = None): """ name (str): This could look something like 1. https://github.com/frappe/healthcare.git @@ -96,23 +108,22 @@ class AppMeta: class App(AppMeta): - def __init__(self, name: str, branch : str = None, bench : "Bench" = None): + def __init__(self, name: str, branch: str = None, bench: "Bench" = None): super().__init__(name, branch) self.bench = bench def get(self): - branch = f'--branch {self.tag}' if self.tag else '' - shallow = '--depth 1' if self.bench.shallow_clone else '' + branch = f"--branch {self.tag}" if self.tag else "" + shallow = "--depth 1" if self.bench.shallow_clone else "" self.bench.run( f"git clone {self.url} {branch} {shallow} --origin upstream", - cwd=os.path.join(self.bench.name, 'apps') + cwd=os.path.join(self.bench.name, "apps"), ) def remove(self): shutil.move( - os.path.join("apps", self.repo), - os.path.join("archived", "apps", self.repo), + os.path.join("apps", self.repo), os.path.join("archived", "apps", self.repo), ) def install(self, skip_assets=False, verbose=False): @@ -131,12 +142,10 @@ class App(AppMeta): def uninstall(self): env_python = get_env_cmd("python", bench_path=self.bench.name) - self.bench.run( - f"{env_python} -m pip uninstall -y {self.repo}" - ) + self.bench.run(f"{env_python} -m pip uninstall -y {self.repo}") -def add_to_appstxt(app, bench_path='.'): +def add_to_appstxt(app, bench_path="."): from bench.bench import Bench apps = Bench(bench_path).apps @@ -145,7 +154,8 @@ def add_to_appstxt(app, bench_path='.'): apps.append(app) return write_appstxt(apps, bench_path=bench_path) -def remove_from_appstxt(app, bench_path='.'): + +def remove_from_appstxt(app, bench_path="."): from bench.bench import Bench apps = Bench(bench_path).apps @@ -154,37 +164,43 @@ def remove_from_appstxt(app, bench_path='.'): apps.remove(app) return write_appstxt(apps, bench_path=bench_path) -def write_appstxt(apps, bench_path='.'): - with open(os.path.join(bench_path, 'sites', 'apps.txt'), 'w') as f: - return f.write('\n'.join(apps)) -def get_excluded_apps(bench_path='.'): +def write_appstxt(apps, bench_path="."): + with open(os.path.join(bench_path, "sites", "apps.txt"), "w") as f: + return f.write("\n".join(apps)) + + +def get_excluded_apps(bench_path="."): try: - with open(os.path.join(bench_path, 'sites', 'excluded_apps.txt')) as f: - return f.read().strip().split('\n') + with open(os.path.join(bench_path, "sites", "excluded_apps.txt")) as f: + return f.read().strip().split("\n") except IOError: return [] -def add_to_excluded_apps_txt(app, bench_path='.'): - if app == 'frappe': - raise ValueError('Frappe app cannot be excludeed from update') - if app not in os.listdir('apps'): - raise ValueError(f'The app {app} does not exist') + +def add_to_excluded_apps_txt(app, bench_path="."): + if app == "frappe": + raise ValueError("Frappe app cannot be excludeed from update") + if app not in os.listdir("apps"): + raise ValueError(f"The app {app} does not exist") apps = get_excluded_apps(bench_path=bench_path) if app not in apps: apps.append(app) return write_excluded_apps_txt(apps, bench_path=bench_path) -def write_excluded_apps_txt(apps, bench_path='.'): - with open(os.path.join(bench_path, 'sites', 'excluded_apps.txt'), 'w') as f: - return f.write('\n'.join(apps)) -def remove_from_excluded_apps_txt(app, bench_path='.'): +def write_excluded_apps_txt(apps, bench_path="."): + with open(os.path.join(bench_path, "sites", "excluded_apps.txt"), "w") as f: + return f.write("\n".join(apps)) + + +def remove_from_excluded_apps_txt(app, bench_path="."): apps = get_excluded_apps(bench_path=bench_path) if app in apps: apps.remove(app) return write_excluded_apps_txt(apps, bench_path=bench_path) + def generate_bench_name(git_url, bench_path): if os.path.exists(git_url): guessed_app_name = os.path.basename(git_url) @@ -194,25 +210,29 @@ def generate_bench_name(git_url, bench_path): return os.path.join(bench_path, f"{guessed_app_name}-bench") -def setup_app_dependencies(repo_name, bench_path='.', branch=None): + +def setup_app_dependencies(repo_name, bench_path=".", branch=None): # branch kwarg is somewhat of a hack here; since we're assuming the same branches for all apps # for eg: if you're installing erpnext@develop, you'll want frappe@develop and healthcare@develop too import glob - apps_path = os.path.join(os.path.abspath(bench_path), 'apps') - files = glob.glob(os.path.join(apps_path, repo_name, '**', 'hooks.py')) + apps_path = os.path.join(os.path.abspath(bench_path), "apps") + files = glob.glob(os.path.join(apps_path, repo_name, "**", "hooks.py")) if files: with open(files[0]) as f: - lines = [x for x in f.read().split('\n') if x.strip().startswith('required_apps')] + lines = [x for x in f.read().split("\n") if x.strip().startswith("required_apps")] if lines: - required_apps = eval(lines[0].strip('required_apps').strip().lstrip('=').strip()) + required_apps = eval(lines[0].strip("required_apps").strip().lstrip("=").strip()) # TODO: when the time comes, add version check here for app in required_apps: if app not in Bench(bench_path).apps: get_app(app, bench_path=bench_path, branch=branch) -def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=False, overwrite=False): + +def get_app( + git_url, branch=None, bench_path=".", skip_assets=False, verbose=False, overwrite=False +): """bench get-app clones a Frappe App from remote (GitHub or any other git server), and installs it on the current bench. This also resolves dependencies based on the apps' required_apps defined in the hooks.py file. @@ -231,9 +251,10 @@ def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=Fal if not is_bench_directory(bench_path): bench_path = generate_bench_name(git_url, bench_path) from bench.commands.make import init + click.get_current_context().invoke(init, path=bench_path, frappe_branch=branch) - cloned_path = os.path.join(bench_path, 'apps', repo_name) + cloned_path = os.path.join(bench_path, "apps", repo_name) dir_already_exists = os.path.isdir(cloned_path) if dir_already_exists: @@ -244,6 +265,7 @@ def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=Fal "Do you want to continue and overwrite it?" ): import shutil + shutil.rmtree(cloned_path) elif click.confirm("Do you want to reinstall the existing application?", abort=True): pass @@ -256,20 +278,27 @@ def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=Fal app.install(verbose=verbose, skip_assets=skip_assets) -def new_app(app, bench_path='.'): +def new_app(app, bench_path="."): # For backwards compatibility app = app.lower().replace(" ", "_").replace("-", "_") - logger.log(f'creating new app {app}') - apps = os.path.abspath(os.path.join(bench_path, 'apps')) - run_frappe_cmd('make-app', apps, app, bench_path=bench_path) + logger.log(f"creating new app {app}") + apps = os.path.abspath(os.path.join(bench_path, "apps")) + run_frappe_cmd("make-app", apps, app, bench_path=bench_path) install_app(app, bench_path=bench_path) -def install_app(app, bench_path=".", verbose=False, no_cache=False, restart_bench=True, skip_assets=False): +def install_app( + app, + bench_path=".", + verbose=False, + no_cache=False, + restart_bench=True, + skip_assets=False, +): from bench.bench import Bench from bench.utils.bench import get_env_cmd - install_text = f'Installing {app}' + install_text = f"Installing {app}" click.secho(install_text, fg="yellow") logger.log(install_text) @@ -280,7 +309,7 @@ def install_app(app, bench_path=".", verbose=False, no_cache=False, restart_benc exec_cmd(f"{python_path} -m pip install {quiet_flag} -U -e {app_path} {cache_flag}") - if os.path.exists(os.path.join(app_path, 'package.json')): + if os.path.exists(os.path.join(app_path, "package.json")): exec_cmd("yarn install", cwd=app_path) add_to_appstxt(app, bench_path=bench_path) @@ -289,25 +318,26 @@ def install_app(app, bench_path=".", verbose=False, no_cache=False, restart_benc if conf.get("developer_mode"): from bench.utils.bench import install_python_dev_dependencies + install_python_dev_dependencies(apps=app) if not skip_assets: build_assets(bench_path=bench_path, app=app) if restart_bench: - if conf.get('restart_supervisor_on_update'): + if conf.get("restart_supervisor_on_update"): restart_supervisor_processes(bench_path=bench_path) - if conf.get('restart_systemd_on_update'): + if conf.get("restart_systemd_on_update"): restart_systemd_processes(bench_path=bench_path) -def pull_apps(apps=None, bench_path='.', reset=False): - '''Check all apps if there no local changes, pull''' +def pull_apps(apps=None, bench_path=".", reset=False): + """Check all apps if there no local changes, pull""" from bench.bench import Bench from bench.utils.app import get_remote, get_current_branch bench = Bench(bench_path) - rebase = '--rebase' if bench.conf.get('rebase_on_pull') else '' + rebase = "--rebase" if bench.conf.get("rebase_on_pull") else "" apps = apps or bench.apps excluded_apps = bench.excluded_apps @@ -318,11 +348,12 @@ def pull_apps(apps=None, bench_path='.', reset=False): print(f"Skipping reset for app {app}") continue app_dir = get_repo_dir(app, bench_path=bench_path) - if os.path.exists(os.path.join(app_dir, '.git')): - out = subprocess.check_output('git status', shell=True, cwd=app_dir) - out = out.decode('utf-8') - if not re.search(r'nothing to commit, working (directory|tree) clean', out): - print(f''' + if os.path.exists(os.path.join(app_dir, ".git")): + out = subprocess.check_output("git status", shell=True, cwd=app_dir) + out = out.decode("utf-8") + if not re.search(r"nothing to commit, working (directory|tree) clean", out): + print( + f""" Cannot proceed with update: You have local changes in app "{app}" that are not committed. @@ -332,7 +363,8 @@ Here are your choices: 1. Temporarily remove your changes with "git stash" or discard them completely with "bench update --reset" or for individual repositries "git reset --hard" 2. If your changes are helpful for others, send in a pull request via GitHub and - wait for them to be merged in the core.''') + wait for them to be merged in the core.""" + ) sys.exit(1) for app in apps: @@ -340,15 +372,18 @@ Here are your choices: print(f"Skipping pull for app {app}") continue app_dir = get_repo_dir(app, bench_path=bench_path) - if os.path.exists(os.path.join(app_dir, '.git')): + if os.path.exists(os.path.join(app_dir, ".git")): remote = get_remote(app) if not remote: # remote is False, i.e. remote doesn't exist, add the app to excluded_apps.txt add_to_excluded_apps_txt(app, bench_path=bench_path) - print(f"Skipping pull for app {app}, since remote doesn't exist, and adding it to excluded apps") + print( + f"Skipping pull for app {app}, since remote doesn't exist, and" + " adding it to excluded apps" + ) continue - if not bench.conf.get('shallow_clone') or not reset: + if not bench.conf.get("shallow_clone") or not reset: is_shallow = os.path.exists(os.path.join(app_dir, ".git", "shallow")) if is_shallow: s = " to safely pull remote changes." if not reset else "" @@ -356,10 +391,10 @@ Here are your choices: bench.run(f"git fetch {remote} --unshallow", cwd=app_dir) branch = get_current_branch(app, bench_path=bench_path) - logger.log(f'pulling {app}') + logger.log(f"pulling {app}") if reset: reset_cmd = f"git reset --hard {remote}/{branch}" - if bench.conf.get('shallow_clone'): + if bench.conf.get("shallow_clone"): bench.run(f"git fetch --depth=1 --no-tags {remote} {branch}", cwd=app_dir) bench.run(reset_cmd, cwd=app_dir) bench.run("git reflog expire --all", cwd=app_dir) @@ -374,21 +409,24 @@ Here are your choices: def use_rq(bench_path): bench_path = os.path.abspath(bench_path) - celery_app = os.path.join(bench_path, 'apps', 'frappe', 'frappe', 'celery_app.py') + celery_app = os.path.join(bench_path, "apps", "frappe", "frappe", "celery_app.py") return not os.path.exists(celery_app) -def get_repo_dir(app, bench_path='.'): - return os.path.join(bench_path, 'apps', app) -def install_apps_from_path(path, bench_path='.'): +def get_repo_dir(app, bench_path="."): + return os.path.join(bench_path, "apps", app) + + +def install_apps_from_path(path, bench_path="."): apps = get_apps_json(path) for app in apps: - get_app(app['url'], branch=app.get('branch'), bench_path=bench_path, skip_assets=True) + get_app(app["url"], branch=app.get("branch"), bench_path=bench_path, skip_assets=True) + def get_apps_json(path): import requests - if path.startswith('http'): + if path.startswith("http"): r = requests.get(path) return r.json() diff --git a/bench/bench.py b/bench/bench.py index e1db12b6..8add1744 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -7,8 +7,21 @@ from typing import MutableSequence, TYPE_CHECKING import bench from bench.exceptions import ValidationError from bench.config.common_site_config import setup_config -from bench.utils import paths_in_bench, exec_cmd, is_frappe_app, get_git_version, run_frappe_cmd -from bench.utils.bench import validate_app_installed_on_sites, restart_supervisor_processes, restart_systemd_processes, remove_backups_crontab, get_venv_path, get_env_cmd +from bench.utils import ( + paths_in_bench, + exec_cmd, + is_frappe_app, + get_git_version, + run_frappe_cmd, +) +from bench.utils.bench import ( + validate_app_installed_on_sites, + restart_supervisor_processes, + restart_systemd_processes, + remove_backups_crontab, + get_venv_path, + get_env_cmd, +) if TYPE_CHECKING: @@ -39,15 +52,15 @@ class Bench(Base, Validator): self.teardown = BenchTearDown(self) self.apps = BenchApps(self) - self.apps_txt = os.path.join(self.name, 'sites', 'apps.txt') - self.excluded_apps_txt = os.path.join(self.name, 'sites', 'excluded_apps.txt') + self.apps_txt = os.path.join(self.name, "sites", "apps.txt") + self.excluded_apps_txt = os.path.join(self.name, "sites", "excluded_apps.txt") @property def shallow_clone(self): config = self.conf if config: - if config.get('release_bench') or not config.get('shallow_clone'): + if config.get("release_bench") or not config.get("shallow_clone"): return False if get_git_version() > 1.9: @@ -57,22 +70,22 @@ class Bench(Base, Validator): def excluded_apps(self): try: with open(self.excluded_apps_txt) as f: - return f.read().strip().split('\n') + return f.read().strip().split("\n") except Exception: return [] @property def sites(self): return [ - path for path in os.listdir(os.path.join(self.name, 'sites')) - if os.path.exists( - os.path.join("sites", path, "site_config.json") - ) + path + for path in os.listdir(os.path.join(self.name, "sites")) + if os.path.exists(os.path.join("sites", path, "site_config.json")) ] @property def conf(self): from bench.config.common_site_config import get_config + return get_config(self.name) def init(self): @@ -112,14 +125,14 @@ class Bench(Base, Validator): def reload(self): conf = self.conf - if conf.get('restart_supervisor_on_update'): + if conf.get("restart_supervisor_on_update"): restart_supervisor_processes(bench_path=self.name) - if conf.get('restart_systemd_on_update'): + if conf.get("restart_systemd_on_update"): restart_systemd_processes(bench_path=self.name) class BenchApps(MutableSequence): - def __init__(self, bench : Bench): + def __init__(self, bench: Bench): self.bench = bench self.initialize_apps() @@ -130,25 +143,27 @@ class BenchApps(MutableSequence): def initialize_apps(self): try: - self.apps = [x for x in os.listdir( - os.path.join(self.bench.name, "apps") - ) if is_frappe_app(os.path.join(self.bench.name, "apps", x))] + self.apps = [ + x + for x in os.listdir(os.path.join(self.bench.name, "apps")) + if is_frappe_app(os.path.join(self.bench.name, "apps", x)) + ] self.apps.sort() except FileNotFoundError: self.apps = [] def __getitem__(self, key): - ''' retrieves an item by its index, key''' + """ retrieves an item by its index, key""" return self.apps[key] def __setitem__(self, key, value): - ''' set the item at index, key, to value ''' + """ set the item at index, key, to value """ # should probably not be allowed # self.apps[key] = value raise NotImplementedError def __delitem__(self, key): - ''' removes the item at index, key ''' + """ removes the item at index, key """ # TODO: uninstall and delete app from bench del self.apps[key] @@ -156,7 +171,7 @@ class BenchApps(MutableSequence): return len(self.apps) def insert(self, key, value): - ''' add an item, value, at index, key. ''' + """ add an item, value, at index, key. """ # TODO: fetch and install app to bench self.apps.insert(key, value) @@ -171,7 +186,7 @@ class BenchApps(MutableSequence): app.remove() super().remove(app.repo) - def append(self, app : "App"): + def append(self, app: "App"): return self.add(app) def __repr__(self): @@ -182,7 +197,7 @@ class BenchApps(MutableSequence): class BenchSetup(Base): - def __init__(self, bench : Bench): + def __init__(self, bench: Bench): self.bench = bench self.cwd = self.bench.cwd @@ -219,41 +234,46 @@ class BenchSetup(Base): if redis: from bench.config.redis import generate_config + generate_config(self.bench.name) if procfile: from bench.config.procfile import setup_procfile + setup_procfile(self.bench.name, skip_redis=not redis) def logging(self): from bench.utils import setup_logging + return setup_logging(bench_path=self.bench.name) def patches(self): shutil.copy( - os.path.join(os.path.dirname(os.path.abspath(__file__)), 'patches', 'patches.txt'), - os.path.join(self.bench.name, 'patches.txt') + os.path.join(os.path.dirname(os.path.abspath(__file__)), "patches", "patches.txt"), + os.path.join(self.bench.name, "patches.txt"), ) def backups(self): # TODO: to something better for logging data? - maybe a wrapper that auto-logs with more context - logger.log('setting up backups') + logger.log("setting up backups") from crontab import CronTab bench_dir = os.path.abspath(self.bench.name) - user = self.bench.conf.get('frappe_user') - logfile = os.path.join(bench_dir, 'logs', 'backup.log') + user = self.bench.conf.get("frappe_user") + logfile = os.path.join(bench_dir, "logs", "backup.log") system_crontab = CronTab(user=user) backup_command = f"cd {bench_dir} && {sys.argv[0]} --verbose --site all backup" job_command = f"{backup_command} >> {logfile} 2>&1" if job_command not in str(system_crontab): - job = system_crontab.new(command=job_command, comment="bench auto backups set for every 6 hours") + job = system_crontab.new( + command=job_command, comment="bench auto backups set for every 6 hours" + ) job.every(6).hours() system_crontab.write() - logger.log('backups were set up') + logger.log("backups were set up") class BenchTearDown: diff --git a/bench/exceptions.py b/bench/exceptions.py index b3f206c7..3002314b 100644 --- a/bench/exceptions.py +++ b/bench/exceptions.py @@ -1,17 +1,22 @@ class InvalidBranchException(Exception): pass + class InvalidRemoteException(Exception): pass + class PatchError(Exception): pass + class CommandFailedError(Exception): pass + class BenchNotFoundError(Exception): pass + class ValidationError(Exception): pass diff --git a/bench/utils/__init__.py b/bench/utils/__init__.py index 8bcb9ee0..525c6079 100644 --- a/bench/utils/__init__.py +++ b/bench/utils/__init__.py @@ -18,10 +18,10 @@ from bench.exceptions import InvalidRemoteException, ValidationError logger = logging.getLogger(PROJECT_NAME) -bench_cache_file = '.bench.cmd' -paths_in_app = ('hooks.py', 'modules.txt', 'patches.txt', 'public') -paths_in_bench = ('apps', 'sites', 'config', 'logs', 'config/pids') -sudoers_file = '/etc/sudoers.d/frappe' +bench_cache_file = ".bench.cmd" +paths_in_app = ("hooks.py", "modules.txt", "patches.txt", "public") +paths_in_bench = ("apps", "sites", "config", "logs", "config/pids") +sudoers_file = "/etc/sudoers.d/frappe" def is_bench_directory(directory=os.path.curdir): @@ -49,15 +49,12 @@ def is_frappe_app(directory): def log(message, level=0): levels = { - 0: ("blue", "INFO"), # normal - 1: ("green", "SUCCESS"), # success - 2: ("red", "ERROR"), # fail - 3: ("yellow", "WARN") # warn/suggest - } - loggers = { - 2: logger.error, - 3: logger.warning + 0: ("blue", "INFO"), # normal + 1: ("green", "SUCCESS"), # success + 2: ("red", "ERROR"), # fail + 3: ("yellow", "WARN"), # warn/suggest } + loggers = {2: logger.error, 3: logger.warning} color, prefix = levels.get(level, levels[0]) level_logger = loggers.get(level, logger.info) @@ -80,7 +77,7 @@ def check_latest_version(): return if pypi_request.status_code == 200: - pypi_version_str = pypi_request.json().get('info').get('version') + pypi_version_str = pypi_request.json().get("info").get("version") pypi_version = Version(pypi_version_str) local_version = Version(VERSION) @@ -98,11 +95,11 @@ def pause_exec(seconds=10): print(" " * 40, end="\r") -def exec_cmd(cmd, cwd='.', env=None): +def exec_cmd(cmd, cwd=".", env=None): if env: env.update(os.environ.copy()) - click.secho(f"$ {cmd}", fg='bright_black') + click.secho(f"$ {cmd}", fg="bright_black") cwd_info = f"cd {cwd} && " if cwd != "." else "" cmd_log = f"{cwd_info}{cmd}" @@ -119,27 +116,29 @@ def which(executable, raise_err=False): exec_ = which(executable) if not exec_ and raise_err: - raise ValueError(f'{executable} not found.') + raise ValueError(f"{executable} not found.") return exec_ -def setup_logging(bench_path='.'): +def setup_logging(bench_path="."): LOG_LEVEL = 15 logging.addLevelName(LOG_LEVEL, "LOG") + def logv(self, message, *args, **kws): if self.isEnabledFor(LOG_LEVEL): self._log(LOG_LEVEL, message, args, **kws) + logging.Logger.log = logv - if os.path.exists(os.path.join(bench_path, 'logs')): - log_file = os.path.join(bench_path, 'logs', 'bench.log') + if os.path.exists(os.path.join(bench_path, "logs")): + log_file = os.path.join(bench_path, "logs", "bench.log") hdlr = logging.FileHandler(log_file) else: hdlr = logging.NullHandler() logger = logging.getLogger(PROJECT_NAME) - formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') + formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s") hdlr.setFormatter(formatter) logger.addHandler(hdlr) logger.setLevel(logging.DEBUG) @@ -148,25 +147,27 @@ def setup_logging(bench_path='.'): def get_process_manager(): - for proc_man in ['honcho', 'foreman', 'forego']: + for proc_man in ["honcho", "foreman", "forego"]: proc_man_path = which(proc_man) if proc_man_path: return proc_man_path def get_git_version(): - '''returns git version from `git --version` - extracts version number from string `get version 1.9.1` etc''' + """returns git version from `git --version` + extracts version number from string `get version 1.9.1` etc""" version = get_cmd_output("git --version") version = version.strip().split()[2] - version = '.'.join(version.split('.')[0:2]) + version = ".".join(version.split(".")[0:2]) return float(version) -def get_cmd_output(cmd, cwd='.', _raise=True): +def get_cmd_output(cmd, cwd=".", _raise=True): output = "" try: - output = subprocess.check_output(cmd, cwd=cwd, shell=True, stderr=subprocess.PIPE, encoding="utf-8").strip() + output = subprocess.check_output( + cmd, cwd=cwd, shell=True, stderr=subprocess.PIPE, encoding="utf-8" + ).strip() except subprocess.CalledProcessError as e: if e.output: output = e.output @@ -183,9 +184,9 @@ def run_frappe_cmd(*args, **kwargs): from bench.cli import from_command_line from bench.utils.bench import get_env_cmd - bench_path = kwargs.get('bench_path', '.') - f = get_env_cmd('python', bench_path=bench_path) - sites_dir = os.path.join(bench_path, 'sites') + bench_path = kwargs.get("bench_path", ".") + f = get_env_cmd("python", bench_path=bench_path) + sites_dir = os.path.join(bench_path, "sites") is_async = False if from_command_line else True if is_async: @@ -193,8 +194,12 @@ def run_frappe_cmd(*args, **kwargs): else: stderr = stdout = None - p = subprocess.Popen((f, '-m', 'frappe.utils.bench_helper', 'frappe') + args, - cwd=sites_dir, stdout=stdout, stderr=stderr) + p = subprocess.Popen( + (f, "-m", "frappe.utils.bench_helper", "frappe") + args, + cwd=sites_dir, + stdout=stdout, + stderr=stderr, + ) if is_async: return_code = print_output(p) @@ -217,20 +222,20 @@ def print_output(p): buf = p.stdout.read(1) if not len(buf): break - if buf == '\r' or buf == '\n': + if buf == "\r" or buf == "\n": send_buffer.append(buf) - log_line(''.join(send_buffer), 'stdout') + log_line("".join(send_buffer), "stdout") send_buffer = [] else: send_buffer.append(buf) if fd == p.stderr.fileno(): - log_line(p.stderr.readline(), 'stderr') + log_line(p.stderr.readline(), "stderr") return p.poll() def log_line(data, stream): - if stream == 'stderr': + if stream == "stderr": return sys.stderr.write(data) return sys.stdout.write(data) @@ -239,37 +244,40 @@ def get_bench_name(bench_path): return os.path.basename(os.path.abspath(bench_path)) -def set_git_remote_url(git_url, bench_path='.'): +def set_git_remote_url(git_url, bench_path="."): "Set app remote git url" from bench.app import get_repo_dir from bench.bench import Bench - app = git_url.rsplit('/', 1)[1].rsplit('.', 1)[0] + app = git_url.rsplit("/", 1)[1].rsplit(".", 1)[0] if app not in Bench(bench_path).apps: raise ValidationError(f"No app named {app}") app_dir = get_repo_dir(app, bench_path=bench_path) - if os.path.exists(os.path.join(app_dir, '.git')): + if os.path.exists(os.path.join(app_dir, ".git")): exec_cmd(f"git remote set-url upstream {git_url}", cwd=app_dir) def run_playbook(playbook_name, extra_vars=None, tag=None): import bench - if not which('ansible'): - print("Ansible is needed to run this command, please install it using 'pip install ansible'") + if not which("ansible"): + print( + "Ansible is needed to run this command, please install it using 'pip" + " install ansible'" + ) sys.exit(1) - args = ['ansible-playbook', '-c', 'local', playbook_name, '-vvvv'] + args = ["ansible-playbook", "-c", "local", playbook_name, "-vvvv"] if extra_vars: - args.extend(['-e', json.dumps(extra_vars)]) + args.extend(["-e", json.dumps(extra_vars)]) if tag: - args.extend(['-t', tag]) + args.extend(["-t", tag]) - subprocess.check_call(args, cwd=os.path.join(bench.__path__[0], 'playbooks')) + subprocess.check_call(args, cwd=os.path.join(bench.__path__[0], "playbooks")) def find_benches(directory=None): @@ -304,7 +312,7 @@ def find_benches(directory=None): def is_dist_editable(dist): """Is distribution an editable install?""" for path_item in sys.path: - egg_link = os.path.join(path_item, dist + '.egg-link') + egg_link = os.path.join(path_item, dist + ".egg-link") if os.path.isfile(egg_link): return True return False @@ -324,21 +332,23 @@ def find_parent_bench(path): return find_parent_bench(parent_dir) -def generate_command_cache(bench_path='.'): +def generate_command_cache(bench_path="."): """Caches all available commands (even custom apps) via Frappe Default caching behaviour: generated the first time any command (for a specific bench directory) """ from bench.utils.bench import get_env_cmd - python = get_env_cmd('python', bench_path=bench_path) - sites_path = os.path.join(bench_path, 'sites') + python = get_env_cmd("python", bench_path=bench_path) + sites_path = os.path.join(bench_path, "sites") if os.path.exists(bench_cache_file): os.remove(bench_cache_file) try: - output = get_cmd_output(f"{python} -m frappe.utils.bench_helper get-frappe-commands", cwd=sites_path) - with open(bench_cache_file, 'w') as f: + output = get_cmd_output( + f"{python} -m frappe.utils.bench_helper get-frappe-commands", cwd=sites_path + ) + with open(bench_cache_file, "w") as f: json.dump(eval(output), f) return json.loads(output) @@ -349,7 +359,7 @@ def generate_command_cache(bench_path='.'): return [] -def clear_command_cache(bench_path='.'): +def clear_command_cache(bench_path="."): """Clears commands cached Default invalidation behaviour: destroyed on each run of `bench update` """ @@ -366,7 +376,7 @@ def find_org(org_repo): org_repo = org_repo[0] for org in ["frappe", "erpnext"]: - res = requests.head(f'https://api.github.com/repos/{org}/{org_repo}') + res = requests.head(f"https://api.github.com/repos/{org}/{org_repo}") if res.ok: return org, org_repo @@ -399,7 +409,7 @@ def is_git_url(url): return bool(re.match(pattern, url)) -def drop_privileges(uid_name='nobody', gid_name='nogroup'): +def drop_privileges(uid_name="nobody", gid_name="nogroup"): import grp import pwd diff --git a/bench/utils/app.py b/bench/utils/app.py index ab2f0b50..61a22462 100644 --- a/bench/utils/app.py +++ b/bench/utils/app.py @@ -4,15 +4,21 @@ from setuptools.config import read_configuration import bench import sys import subprocess -from bench.exceptions import InvalidRemoteException, InvalidBranchException, CommandFailedError +from bench.exceptions import ( + InvalidRemoteException, + InvalidBranchException, + CommandFailedError, +) from bench.app import get_repo_dir -def is_version_upgrade(app='frappe', bench_path='.', branch=None): +def is_version_upgrade(app="frappe", bench_path=".", branch=None): upstream_version = get_upstream_version(app=app, branch=branch, bench_path=bench_path) if not upstream_version: - raise InvalidBranchException(f'Specified branch of app {app} is not in upstream remote') + raise InvalidBranchException( + f"Specified branch of app {app} is not in upstream remote" + ) local_version = get_major_version(get_current_version(app, bench_path=bench_path)) upstream_version = get_major_version(upstream_version) @@ -23,21 +29,28 @@ def is_version_upgrade(app='frappe', bench_path='.', branch=None): return (False, local_version, upstream_version) -def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrade=True): +def switch_branch(branch, apps=None, bench_path=".", upgrade=False, check_upgrade=True): import git import importlib - from bench.utils import update_requirements, update_node_packages, backup_all_sites, patch_sites, post_upgrade + from bench.utils import ( + update_requirements, + update_node_packages, + backup_all_sites, + patch_sites, + post_upgrade, + ) from bench.utils.bench import build_assets - apps_dir = os.path.join(bench_path, 'apps') + apps_dir = os.path.join(bench_path, "apps") version_upgrade = (False,) switched_apps = [] if not apps: - apps = [name for name in os.listdir(apps_dir) - if os.path.isdir(os.path.join(apps_dir, name))] - if branch=="v4.x.x": - apps.append('shopping_cart') + apps = [ + name for name in os.listdir(apps_dir) if os.path.isdir(os.path.join(apps_dir, name)) + ] + if branch == "v4.x.x": + apps.append("shopping_cart") for app in apps: app_dir = os.path.join(apps_dir, app) @@ -48,18 +61,27 @@ def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrad repo = git.Repo(app_dir) unshallow_flag = os.path.exists(os.path.join(app_dir, ".git", "shallow")) - bench.utils.log(f"Fetching upstream {'unshallow ' if unshallow_flag else ''}for {app}") + bench.utils.log( + f"Fetching upstream {'unshallow ' if unshallow_flag else ''}for {app}" + ) bench.utils.exec_cmd("git remote set-branches upstream '*'", cwd=app_dir) - bench.utils.exec_cmd(f"git fetch --all{' --unshallow' if unshallow_flag else ''} --quiet", cwd=app_dir) + bench.utils.exec_cmd( + f"git fetch --all{' --unshallow' if unshallow_flag else ''} --quiet", cwd=app_dir + ) if check_upgrade: version_upgrade = is_version_upgrade(app=app, bench_path=bench_path, branch=branch) if version_upgrade[0] and not upgrade: - bench.utils.log(f"Switching to {branch} will cause upgrade from {version_upgrade[1]} to {version_upgrade[2]}. Pass --upgrade to confirm", level=2) + bench.utils.log( + f"Switching to {branch} will cause upgrade from" + f" {version_upgrade[1]} to {version_upgrade[2]}. Pass --upgrade to" + " confirm", + level=2, + ) sys.exit(1) - print("Switching for "+app) + print("Switching for " + app) bench.utils.exec_cmd(f"git checkout -f {branch}", cwd=app_dir) if str(repo.active_branch) == branch: @@ -68,8 +90,13 @@ def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrad bench.utils.log(f"Switching branches failed for: {app}", level=2) if switched_apps: - bench.utils.log("Successfully switched branches for: " + ", ".join(switched_apps), level=1) - print('Please run `bench update --patch` to be safe from any differences in database schema') + bench.utils.log( + "Successfully switched branches for: " + ", ".join(switched_apps), level=1 + ) + print( + "Please run `bench update --patch` to be safe from any differences in" + " database schema" + ) if version_upgrade[0] and upgrade: update_requirements() @@ -81,40 +108,51 @@ def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrad post_upgrade(version_upgrade[1], version_upgrade[2]) -def switch_to_branch(branch=None, apps=None, bench_path='.', upgrade=False): +def switch_to_branch(branch=None, apps=None, bench_path=".", upgrade=False): switch_branch(branch, apps=apps, bench_path=bench_path, upgrade=upgrade) -def switch_to_develop(apps=None, bench_path='.', upgrade=True): - switch_branch('develop', apps=apps, bench_path=bench_path, upgrade=upgrade) -def get_version_from_string(contents, field='__version__'): +def switch_to_develop(apps=None, bench_path=".", upgrade=True): + switch_branch("develop", apps=apps, bench_path=bench_path, upgrade=upgrade) + + +def get_version_from_string(contents, field="__version__"): match = re.search(r"^(\s*%s\s*=\s*['\\\"])(.+?)(['\"])(?sm)" % field, contents) return match.group(2) + def get_major_version(version): import semantic_version return semantic_version.Version(version).major -def get_develop_version(app, bench_path='.'): - repo_dir = get_repo_dir(app, bench_path=bench_path) - with open(os.path.join(repo_dir, os.path.basename(repo_dir), 'hooks.py')) as f: - return get_version_from_string(f.read(), field='develop_version') -def get_upstream_version(app, branch=None, bench_path='.'): +def get_develop_version(app, bench_path="."): + repo_dir = get_repo_dir(app, bench_path=bench_path) + with open(os.path.join(repo_dir, os.path.basename(repo_dir), "hooks.py")) as f: + return get_version_from_string(f.read(), field="develop_version") + + +def get_upstream_version(app, branch=None, bench_path="."): repo_dir = get_repo_dir(app, bench_path=bench_path) if not branch: branch = get_current_branch(app, bench_path=bench_path) try: - subprocess.call(f'git fetch --depth=1 --no-tags upstream {branch}', shell=True, cwd=repo_dir) + subprocess.call( + f"git fetch --depth=1 --no-tags upstream {branch}", shell=True, cwd=repo_dir + ) except CommandFailedError: - raise InvalidRemoteException(f'Failed to fetch from remote named upstream for {app}') + raise InvalidRemoteException(f"Failed to fetch from remote named upstream for {app}") try: - contents = subprocess.check_output(f'git show upstream/{branch}:{app}/__init__.py', - shell=True, cwd=repo_dir, stderr=subprocess.STDOUT) - contents = contents.decode('utf-8') + contents = subprocess.check_output( + f"git show upstream/{branch}:{app}/__init__.py", + shell=True, + cwd=repo_dir, + stderr=subprocess.STDOUT, + ) + contents = contents.decode("utf-8") except subprocess.CalledProcessError as e: if b"Invalid object" in e.output: return None @@ -123,23 +161,28 @@ def get_upstream_version(app, branch=None, bench_path='.'): return get_version_from_string(contents) -def get_current_frappe_version(bench_path='.'): +def get_current_frappe_version(bench_path="."): try: - return get_major_version(get_current_version('frappe', bench_path=bench_path)) + return get_major_version(get_current_version("frappe", bench_path=bench_path)) except IOError: return 0 -def get_current_branch(app, bench_path='.'): + +def get_current_branch(app, bench_path="."): from bench.utils import get_cmd_output + repo_dir = get_repo_dir(app, bench_path=bench_path) return get_cmd_output("basename $(git symbolic-ref -q HEAD)", cwd=repo_dir) -def get_remote(app, bench_path='.'): + +def get_remote(app, bench_path="."): repo_dir = get_repo_dir(app, bench_path=bench_path) - contents = subprocess.check_output(['git', 'remote', '-v'], cwd=repo_dir, stderr=subprocess.STDOUT) - contents = contents.decode('utf-8') - if re.findall('upstream[\s]+', contents): - return 'upstream' + contents = subprocess.check_output( + ["git", "remote", "-v"], cwd=repo_dir, stderr=subprocess.STDOUT + ) + contents = contents.decode("utf-8") + if re.findall(r"upstream[\s]+", contents): + return "upstream" elif not contents: # if contents is an empty string => remote doesn't exist return False @@ -150,17 +193,17 @@ def get_remote(app, bench_path='.'): def get_app_name(bench_path, repo_name): app_name = None - apps_path = os.path.join(os.path.abspath(bench_path), 'apps') - config_path = os.path.join(apps_path, repo_name, 'setup.cfg') + apps_path = os.path.join(os.path.abspath(bench_path), "apps") + config_path = os.path.join(apps_path, repo_name, "setup.cfg") if os.path.exists(config_path): config = read_configuration(config_path) - app_name = config.get('metadata', {}).get('name') + app_name = config.get("metadata", {}).get("name") if not app_name: # retrieve app name from setup.py as fallback - app_path = os.path.join(apps_path, repo_name, 'setup.py') - with open(app_path, 'rb') as f: - app_name = re.search(r'name\s*=\s*[\'"](.*)[\'"]', f.read().decode('utf-8')).group(1) + app_path = os.path.join(apps_path, repo_name, "setup.py") + with open(app_path, "rb") as f: + app_name = re.search(r'name\s*=\s*[\'"](.*)[\'"]', f.read().decode("utf-8")).group(1) if app_name and repo_name != app_name: os.rename(os.path.join(apps_path, repo_name), os.path.join(apps_path, app_name)) @@ -169,12 +212,12 @@ def get_app_name(bench_path, repo_name): return repo_name -def get_current_version(app, bench_path='.'): +def get_current_version(app, bench_path="."): current_version = None repo_dir = get_repo_dir(app, bench_path=bench_path) config_path = os.path.join(repo_dir, "setup.cfg") - init_path = os.path.join(repo_dir, os.path.basename(repo_dir), '__init__.py') - setup_path = os.path.join(repo_dir, 'setup.py') + init_path = os.path.join(repo_dir, os.path.basename(repo_dir), "__init__.py") + setup_path = os.path.join(repo_dir, "setup.py") try: if os.path.exists(config_path): @@ -187,6 +230,6 @@ def get_current_version(app, bench_path='.'): except AttributeError: # backward compatibility with open(setup_path) as f: - current_version = get_version_from_string(f.read(), field='version') + current_version = get_version_from_string(f.read(), field="version") return current_version diff --git a/bench/utils/bench.py b/bench/utils/bench.py index 5bb4e649..cc8e0561 100644 --- a/bench/utils/bench.py +++ b/bench/utils/bench.py @@ -18,17 +18,19 @@ from bench.exceptions import PatchError, ValidationError logger = logging.getLogger(bench.PROJECT_NAME) -def get_env_cmd(cmd, bench_path='.'): - return os.path.abspath(os.path.join(bench_path, 'env', 'bin', cmd)) +def get_env_cmd(cmd, bench_path="."): + return os.path.abspath(os.path.join(bench_path, "env", "bin", cmd)) def get_venv_path(): - venv = which('virtualenv') + venv = which("virtualenv") if not venv: current_python = sys.executable with open(os.devnull, "wb") as devnull: - is_venv_installed = not subprocess.call([current_python, "-m", "venv", "--help"], stdout=devnull) + is_venv_installed = not subprocess.call( + [current_python, "-m", "venv", "--help"], stdout=devnull + ) if is_venv_installed: venv = f"{current_python} -m venv" @@ -40,14 +42,14 @@ def update_env_pip(bench_path): exec_cmd(f"{env_py} -m pip install -q -U pip") -def update_requirements(bench_path='.'): +def update_requirements(bench_path="."): from bench.app import install_app from bench.bench import Bench bench = Bench(bench_path) apps = [app for app in bench.apps if app not in bench.excluded_apps] - print(f"Updating env pip...") + print("Updating env pip...") update_env_pip(bench_path) @@ -57,14 +59,14 @@ def update_requirements(bench_path='.'): install_app(app, bench_path=bench_path, skip_assets=True, restart_bench=False) -def update_python_packages(bench_path='.'): +def update_python_packages(bench_path="."): from bench.bench import Bench bench = Bench(bench_path) env_py = get_env_cmd("python") apps = [app for app in bench.apps if app not in bench.excluded_apps] - print('Updating Python libraries...') + print("Updating Python libraries...") update_env_pip(bench_path) @@ -74,21 +76,22 @@ def update_python_packages(bench_path='.'): bench.run(f"{env_py} -m pip install -q -U -e {app_path}") -def update_node_packages(bench_path='.'): - print('Updating node packages...') +def update_node_packages(bench_path="."): + print("Updating node packages...") from bench.utils.app import get_develop_version from distutils.version import LooseVersion - v = LooseVersion(get_develop_version('frappe', bench_path = bench_path)) + + v = LooseVersion(get_develop_version("frappe", bench_path=bench_path)) # After rollup was merged, frappe_version = 10.1 # if develop_verion is 11 and up, only then install yarn - if v < LooseVersion('11.x.x-develop'): + if v < LooseVersion("11.x.x-develop"): update_npm_packages(bench_path) else: update_yarn_packages(bench_path) -def install_python_dev_dependencies(bench_path='.', apps=None): +def install_python_dev_dependencies(bench_path=".", apps=None): from bench.bench import Bench bench = Bench(bench_path) @@ -104,43 +107,43 @@ def install_python_dev_dependencies(bench_path='.', apps=None): dev_requirements_path = os.path.join(app_path, "dev-requirements.txt") if os.path.exists(dev_requirements_path): - log(f'Installing python development dependencies for {app}') + log(f"Installing python development dependencies for {app}") bench.run(f"{env_py} -m pip install -q -r {dev_requirements_path}") -def update_yarn_packages(bench_path='.'): +def update_yarn_packages(bench_path="."): from bench.bench import Bench bench = Bench(bench_path) apps = [app for app in bench.apps if app not in bench.excluded_apps] - apps_dir = os.path.join(bench.name, 'apps') + apps_dir = os.path.join(bench.name, "apps") # TODO: Check for stuff like this early on only?? - if not which('yarn'): + if not which("yarn"): print("Please install yarn using below command and try again.") print("`npm install -g yarn`") return for app in apps: app_path = os.path.join(apps_dir, app) - if os.path.exists(os.path.join(app_path, 'package.json')): + if os.path.exists(os.path.join(app_path, "package.json")): click.secho(f"\nInstalling node dependencies for {app}", fg="yellow") - bench.run('yarn install', cwd=app_path) + bench.run("yarn install", cwd=app_path) -def update_npm_packages(bench_path='.'): - apps_dir = os.path.join(bench_path, 'apps') +def update_npm_packages(bench_path="."): + apps_dir = os.path.join(bench_path, "apps") package_json = {} for app in os.listdir(apps_dir): - package_json_path = os.path.join(apps_dir, app, 'package.json') + package_json_path = os.path.join(apps_dir, app, "package.json") if os.path.exists(package_json_path): with open(package_json_path, "r") as f: app_package_json = json.loads(f.read()) # package.json is usually a dict in a dict for key, value in app_package_json.items(): - if not key in package_json: + if key not in package_json: package_json[key] = value else: if isinstance(value, dict): @@ -151,13 +154,13 @@ def update_npm_packages(bench_path='.'): package_json[key] = value if package_json is {}: - with open(os.path.join(os.path.dirname(__file__), 'package.json'), 'r') as f: + with open(os.path.join(os.path.dirname(__file__), "package.json"), "r") as f: package_json = json.loads(f.read()) - with open(os.path.join(bench_path, 'package.json'), 'w') as f: + with open(os.path.join(bench_path, "package.json"), "w") as f: f.write(json.dumps(package_json, indent=1, sort_keys=True)) - exec_cmd('npm install', cwd=bench_path) + exec_cmd("npm install", cwd=bench_path) def migrate_env(python, backup=False): @@ -166,38 +169,38 @@ def migrate_env(python, backup=False): from bench.bench import Bench bench = Bench(".") - nvenv = 'env' + nvenv = "env" path = os.getcwd() python = which(python) - virtualenv = which('virtualenv') + virtualenv = which("virtualenv") pvenv = os.path.join(path, nvenv) # Clear Cache before Bench Dies. try: config = bench.conf - rredis = urlparse(config['redis_cache']) - redis = f"{which('redis-cli')} -p {rredis.port}" + rredis = urlparse(config["redis_cache"]) + redis = f"{which('redis-cli')} -p {rredis.port}" - logger.log('Clearing Redis Cache...') - exec_cmd(f'{redis} FLUSHALL') - logger.log('Clearing Redis DataBase...') - exec_cmd(f'{redis} FLUSHDB') + logger.log("Clearing Redis Cache...") + exec_cmd(f"{redis} FLUSHALL") + logger.log("Clearing Redis DataBase...") + exec_cmd(f"{redis} FLUSHDB") except Exception: - logger.warning('Please ensure Redis Connections are running or Daemonized.') + logger.warning("Please ensure Redis Connections are running or Daemonized.") # Backup venv: restore using `virtualenv --relocatable` if needed if backup: from datetime import datetime - parch = os.path.join(path, 'archived', 'envs') + parch = os.path.join(path, "archived", "envs") if not os.path.exists(parch): os.mkdir(parch) - source = os.path.join(path, 'env') + source = os.path.join(path, "env") target = parch - logger.log('Backing up Virtual Environment') - stamp = datetime.now().strftime('%Y%m%d_%H%M%S') + logger.log("Backing up Virtual Environment") + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") dest = os.path.join(path, str(stamp)) os.rename(source, dest) @@ -206,24 +209,25 @@ def migrate_env(python, backup=False): # Create virtualenv using specified python venv_creation, packages_setup = 1, 1 try: - logger.log(f'Setting up a New Virtual {python} Environment') - venv_creation = exec_cmd(f'{virtualenv} --python {python} {pvenv}') + logger.log(f"Setting up a New Virtual {python} Environment") + venv_creation = exec_cmd(f"{virtualenv} --python {python} {pvenv}") - apps = ' '.join([f"-e {os.path.join('apps', app)}" for app in bench.apps]) - packages_setup = exec_cmd(f'{pvenv} -m pip install -q -U {apps}') + apps = " ".join([f"-e {os.path.join('apps', app)}" for app in bench.apps]) + packages_setup = exec_cmd(f"{pvenv} -m pip install -q -U {apps}") - logger.log(f'Migration Successful to {python}') + logger.log(f"Migration Successful to {python}") except Exception: if venv_creation or packages_setup: - logger.warning('Migration Error') + logger.warning("Migration Error") -def validate_upgrade(from_ver, to_ver, bench_path='.'): + +def validate_upgrade(from_ver, to_ver, bench_path="."): if to_ver >= 6: - if not which('npm') and not (which('node') or which('nodejs')): + if not which("npm") and not (which("node") or which("nodejs")): raise Exception("Please install nodejs and npm") -def post_upgrade(from_ver, to_ver, bench_path='.'): +def post_upgrade(from_ver, to_ver, bench_path="."): from bench.config import redis from bench.config.supervisor import generate_supervisor_config from bench.config.nginx import make_nginx_conf @@ -232,18 +236,19 @@ def post_upgrade(from_ver, to_ver, bench_path='.'): conf = Bench(bench_path).conf print("-" * 80 + f"Your bench was upgraded to version {to_ver}") - if conf.get('restart_supervisor_on_update'): + if conf.get("restart_supervisor_on_update"): redis.generate_config(bench_path=bench_path) generate_supervisor_config(bench_path=bench_path) make_nginx_conf(bench_path=bench_path) print( - "As you have setup your bench for production, you will have to reload configuration for " - "nginx and supervisor. To complete the migration, please run the following commands:" - "\nsudo service nginx restart" - "\nsudo supervisorctl reload" + "As you have setup your bench for production, you will have to reload" + " configuration for nginx and supervisor. To complete the migration, please" + " run the following commands:\nsudo service nginx restart\nsudo" + " supervisorctl reload" ) -def patch_sites(bench_path='.'): + +def patch_sites(bench_path="."): from bench.bench import Bench from bench.utils.system import migrate_site @@ -255,56 +260,79 @@ def patch_sites(bench_path='.'): except subprocess.CalledProcessError: raise PatchError -def restart_supervisor_processes(bench_path='.', web_workers=False): + +def restart_supervisor_processes(bench_path=".", web_workers=False): from bench.bench import Bench bench = Bench(bench_path) conf = bench.conf - cmd = conf.get('supervisor_restart_cmd') + cmd = conf.get("supervisor_restart_cmd") bench_name = get_bench_name(bench_path) if cmd: bench.run(cmd) else: - supervisor_status = get_cmd_output('supervisorctl status', cwd=bench_path) + supervisor_status = get_cmd_output("supervisorctl status", cwd=bench_path) - if web_workers and f'{bench_name}-web:' in supervisor_status: - group = f'{bench_name}-web:\t' + if web_workers and f"{bench_name}-web:" in supervisor_status: + group = f"{bench_name}-web:\t" - elif f'{bench_name}-workers:' in supervisor_status: - group = f'{bench_name}-workers: {bench_name}-web:' + elif f"{bench_name}-workers:" in supervisor_status: + group = f"{bench_name}-workers: {bench_name}-web:" # backward compatibility - elif f'{bench_name}-processes:' in supervisor_status: - group = f'{bench_name}-processes:' + elif f"{bench_name}-processes:" in supervisor_status: + group = f"{bench_name}-processes:" # backward compatibility else: - group = 'frappe:' + group = "frappe:" bench.run(f"supervisorctl restart {group}") -def restart_systemd_processes(bench_path='.', web_workers=False): +def restart_systemd_processes(bench_path=".", web_workers=False): bench_name = get_bench_name(bench_path) - exec_cmd(f'sudo systemctl stop -- $(systemctl show -p Requires {bench_name}.target | cut -d= -f2)') - exec_cmd(f'sudo systemctl start -- $(systemctl show -p Requires {bench_name}.target | cut -d= -f2)') + exec_cmd( + f"sudo systemctl stop -- $(systemctl show -p Requires {bench_name}.target | cut" + " -d= -f2)" + ) + exec_cmd( + f"sudo systemctl start -- $(systemctl show -p Requires {bench_name}.target |" + " cut -d= -f2)" + ) -def build_assets(bench_path='.', app=None): - command = 'bench build' +def build_assets(bench_path=".", app=None): + command = "bench build" if app: - command += f' --app {app}' + command += f" --app {app}" exec_cmd(command, cwd=bench_path, env={"BENCH_DEVELOPER": "1"}) -def update(pull=False, apps=None, patch=False, build=False, requirements=False, backup=True, compile=True, - force=False, reset=False, restart_supervisor=False, restart_systemd=False): + +def update( + pull=False, + apps=None, + patch=False, + build=False, + requirements=False, + backup=True, + compile=True, + force=False, + reset=False, + restart_supervisor=False, + restart_systemd=False, +): """command: bench update""" import re from bench import patches from bench.utils import clear_command_cache, pause_exec, log - from bench.utils.bench import restart_supervisor_processes, restart_systemd_processes, backup_all_sites + from bench.utils.bench import ( + restart_supervisor_processes, + restart_systemd_processes, + backup_all_sites, + ) from bench.app import pull_apps from bench.utils.app import is_version_upgrade from bench.config.common_site_config import update_config @@ -318,10 +346,10 @@ def update(pull=False, apps=None, patch=False, build=False, requirements=False, if apps and not pull: apps = [] - clear_command_cache(bench_path='.') + clear_command_cache(bench_path=".") - if conf.get('release_bench'): - print('Release bench detected, cannot update!') + if conf.get("release_bench"): + print("Release bench detected, cannot update!") sys.exit(1) if not (pull or patch or build or requirements): @@ -332,50 +360,58 @@ def update(pull=False, apps=None, patch=False, build=False, requirements=False, if version_upgrade[0]: if force: - log("""Force flag has been used for a major version change in Frappe and it's apps. -This will take significant time to migrate and might break custom apps.""", level=3) + log( + """Force flag has been used for a major version change in Frappe and it's apps. +This will take significant time to migrate and might break custom apps.""", + level=3, + ) else: - print(f"""This update will cause a major version change in Frappe/ERPNext from {version_upgrade[1]} to {version_upgrade[2]}. -This would take significant time to migrate and might break custom apps.""") - click.confirm('Do you want to continue?', abort=True) + print( + f"""This update will cause a major version change in Frappe/ERPNext from {version_upgrade[1]} to {version_upgrade[2]}. +This would take significant time to migrate and might break custom apps.""" + ) + click.confirm("Do you want to continue?", abort=True) - if not reset and conf.get('shallow_clone'): - log("""shallow_clone is set in your bench config. + if not reset and conf.get("shallow_clone"): + log( + """shallow_clone is set in your bench config. However without passing the --reset flag, your repositories will be unshallowed. To avoid this, cancel this operation and run `bench update --reset`. Consider the consequences of `git reset --hard` on your apps before you run that. To avoid seeing this warning, set shallow_clone to false in your common_site_config.json - """, level=3) + """, + level=3, + ) pause_exec(seconds=10) if version_upgrade[0] or (not version_upgrade[0] and force): validate_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) - conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 }) + conf.update({"maintenance_mode": 1, "pause_scheduler": 1}) update_config(conf, bench_path=bench_path) if backup: - print('Backing up sites...') + print("Backing up sites...") backup_all_sites(bench_path=bench_path) if apps: apps = [app.strip() for app in re.split(",| ", apps) if app] if pull: - print('Updating apps source...') + print("Updating apps source...") pull_apps(apps=apps, bench_path=bench_path, reset=reset) if requirements: - print('Setting up requirements...') + print("Setting up requirements...") update_requirements(bench_path=bench_path) update_node_packages(bench_path=bench_path) if patch: - print('Patching sites...') + print("Patching sites...") patch_sites(bench_path=bench_path) if build: - print('Building assets...') + print("Building assets...") build_assets(bench_path=bench_path) if version_upgrade[0] or (not version_upgrade[0] and force): @@ -384,102 +420,112 @@ To avoid seeing this warning, set shallow_clone to false in your common_site_con if pull and compile: from compileall import compile_dir - print('Compiling Python files...') - apps_dir = os.path.join(bench_path, 'apps') - compile_dir(apps_dir, quiet=1, rx=re.compile('.*node_modules.*')) + print("Compiling Python files...") + apps_dir = os.path.join(bench_path, "apps") + compile_dir(apps_dir, quiet=1, rx=re.compile(".*node_modules.*")) - if restart_supervisor or conf.get('restart_supervisor_on_update'): + if restart_supervisor or conf.get("restart_supervisor_on_update"): restart_supervisor_processes(bench_path=bench_path) - if restart_systemd or conf.get('restart_systemd_on_update'): + if restart_systemd or conf.get("restart_systemd_on_update"): restart_systemd_processes(bench_path=bench_path) - conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 }) + conf.update({"maintenance_mode": 0, "pause_scheduler": 0}) update_config(conf, bench_path=bench_path) - print("_" * 80 + "\nBench: Deployment tool for Frappe and Frappe Applications (https://frappe.io/bench).\nOpen source depends on your contributions, so do give back by submitting bug reports, patches and fixes and be a part of the community :)") + print( + "_" * 80 + + "\nBench: Deployment tool for Frappe and Frappe Applications" + " (https://frappe.io/bench).\nOpen source depends on your contributions, so do" + " give back by submitting bug reports, patches and fixes and be a part of the" + " community :)" + ) def clone_apps_from(bench_path, clone_from, update_app=True): from bench.app import install_app - print(f'Copying apps from {clone_from}...') - subprocess.check_output(['cp', '-R', os.path.join(clone_from, 'apps'), bench_path]) - node_modules_path = os.path.join(clone_from, 'node_modules') + print(f"Copying apps from {clone_from}...") + subprocess.check_output(["cp", "-R", os.path.join(clone_from, "apps"), bench_path]) + + node_modules_path = os.path.join(clone_from, "node_modules") if os.path.exists(node_modules_path): - print(f'Copying node_modules from {clone_from}...') - subprocess.check_output(['cp', '-R', node_modules_path, bench_path]) + print(f"Copying node_modules from {clone_from}...") + subprocess.check_output(["cp", "-R", node_modules_path, bench_path]) def setup_app(app): # run git reset --hard in each branch, pull latest updates and install_app - app_path = os.path.join(bench_path, 'apps', app) + app_path = os.path.join(bench_path, "apps", app) # remove .egg-ino - subprocess.check_output(['rm', '-rf', app + '.egg-info'], cwd=app_path) + subprocess.check_output(["rm", "-rf", app + ".egg-info"], cwd=app_path) - if update_app and os.path.exists(os.path.join(app_path, '.git')): - remotes = subprocess.check_output(['git', 'remote'], cwd=app_path).strip().split() - if 'upstream' in remotes: - remote = 'upstream' + if update_app and os.path.exists(os.path.join(app_path, ".git")): + remotes = subprocess.check_output(["git", "remote"], cwd=app_path).strip().split() + if "upstream" in remotes: + remote = "upstream" else: remote = remotes[0] - print(f'Cleaning up {app}') - branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], cwd=app_path).strip() - subprocess.check_output(['git', 'reset', '--hard'], cwd=app_path) - subprocess.check_output(['git', 'pull', '--rebase', remote, branch], cwd=app_path) + print(f"Cleaning up {app}") + branch = subprocess.check_output( + ["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=app_path + ).strip() + subprocess.check_output(["git", "reset", "--hard"], cwd=app_path) + subprocess.check_output(["git", "pull", "--rebase", remote, branch], cwd=app_path) install_app(app, bench_path, restart_bench=False) - with open(os.path.join(clone_from, 'sites', 'apps.txt'), 'r') as f: + with open(os.path.join(clone_from, "sites", "apps.txt"), "r") as f: apps = f.read().splitlines() for app in apps: setup_app(app) -def remove_backups_crontab(bench_path='.'): +def remove_backups_crontab(bench_path="."): from crontab import CronTab from bench.bench import Bench - logger.log('removing backup cronjob') + logger.log("removing backup cronjob") bench_dir = os.path.abspath(bench_path) - user = Bench(bench_dir).conf.get('frappe_user') - logfile = os.path.join(bench_dir, 'logs', 'backup.log') + user = Bench(bench_dir).conf.get("frappe_user") + logfile = os.path.join(bench_dir, "logs", "backup.log") system_crontab = CronTab(user=user) backup_command = f"cd {bench_dir} && {sys.argv[0]} --verbose --site all backup" job_command = f"{backup_command} >> {logfile} 2>&1" system_crontab.remove_all(command=job_command) -def set_mariadb_host(host, bench_path='.'): - update_common_site_config({'db_host': host}, bench_path=bench_path) + +def set_mariadb_host(host, bench_path="."): + update_common_site_config({"db_host": host}, bench_path=bench_path) -def set_redis_cache_host(host, bench_path='.'): - update_common_site_config({'redis_cache': f"redis://{host}"}, bench_path=bench_path) +def set_redis_cache_host(host, bench_path="."): + update_common_site_config({"redis_cache": f"redis://{host}"}, bench_path=bench_path) -def set_redis_queue_host(host, bench_path='.'): - update_common_site_config({'redis_queue': f"redis://{host}"}, bench_path=bench_path) +def set_redis_queue_host(host, bench_path="."): + update_common_site_config({"redis_queue": f"redis://{host}"}, bench_path=bench_path) -def set_redis_socketio_host(host, bench_path='.'): - update_common_site_config({'redis_socketio': f"redis://{host}"}, bench_path=bench_path) +def set_redis_socketio_host(host, bench_path="."): + update_common_site_config({"redis_socketio": f"redis://{host}"}, bench_path=bench_path) -def update_common_site_config(ddict, bench_path='.'): - filename = os.path.join(bench_path, 'sites', 'common_site_config.json') +def update_common_site_config(ddict, bench_path="."): + filename = os.path.join(bench_path, "sites", "common_site_config.json") if os.path.exists(filename): - with open(filename, 'r') as f: + with open(filename, "r") as f: content = json.load(f) else: content = {} content.update(ddict) - with open(filename, 'w') as f: + with open(filename, "w") as f: json.dump(content, f, indent=1, sort_keys=True) @@ -499,7 +545,7 @@ def check_app_installed(app, bench_path="."): ["bench", "--site", "all", "list-apps", "--format", "json"], stderr=open(os.devnull, "wb"), cwd=bench_path, - ).decode('utf-8') + ).decode("utf-8") except subprocess.CalledProcessError: return None @@ -514,16 +560,19 @@ def check_app_installed(app, bench_path="."): def check_app_installed_legacy(app, bench_path="."): - site_path = os.path.join(bench_path, 'sites') + site_path = os.path.join(bench_path, "sites") for site in os.listdir(site_path): - req_file = os.path.join(site_path, site, 'site_config.json') + req_file = os.path.join(site_path, site, "site_config.json") if os.path.exists(req_file): - out = subprocess.check_output(["bench", "--site", site, "list-apps"], cwd=bench_path).decode('utf-8') - if re.search(r'\b' + app + r'\b', out): + out = subprocess.check_output( + ["bench", "--site", site, "list-apps"], cwd=bench_path + ).decode("utf-8") + if re.search(r"\b" + app + r"\b", out): print(f"Cannot remove, app is installed on site: {site}") sys.exit(1) + def validate_branch(): from bench.bench import Bench from bench.utils.app import get_current_branch @@ -531,14 +580,15 @@ def validate_branch(): apps = Bench(".").apps installed_apps = set(apps) - check_apps = set(['frappe', 'erpnext']) + check_apps = set(["frappe", "erpnext"]) intersection_apps = installed_apps.intersection(check_apps) for app in intersection_apps: branch = get_current_branch(app) if branch == "master": - print("""'master' branch is renamed to 'version-11' since 'version-12' release. + print( + """'master' branch is renamed to 'version-11' since 'version-12' release. As of January 2020, the following branches are version Frappe ERPNext 11 version-11 version-11 @@ -547,6 +597,7 @@ version Frappe ERPNext 14 develop develop Please switch to new branches to get future updates. -To switch to your required branch, run the following commands: bench switch-to-branch [branch-name]""") +To switch to your required branch, run the following commands: bench switch-to-branch [branch-name]""" + ) sys.exit(1) diff --git a/bench/utils/system.py b/bench/utils/system.py index d0a2c428..297c3fc4 100644 --- a/bench/utils/system.py +++ b/bench/utils/system.py @@ -7,15 +7,31 @@ import sys # imports - module imports import bench -from bench.utils import exec_cmd, get_process_manager, log, run_frappe_cmd, sudoers_file, which +from bench.utils import ( + exec_cmd, + get_process_manager, + log, + run_frappe_cmd, + sudoers_file, + which, +) from bench.utils.bench import build_assets, clone_apps_from - -def init(path, apps_path=None, no_procfile=False, no_backups=False, - frappe_path=None, frappe_branch=None, verbose=False, clone_from=None, - skip_redis_config_generation=False, clone_without_update=False, skip_assets=False, - python='python3'): +def init( + path, + apps_path=None, + no_procfile=False, + no_backups=False, + frappe_path=None, + frappe_branch=None, + verbose=False, + clone_from=None, + skip_redis_config_generation=False, + clone_without_update=False, + skip_assets=False, + python="python3", +): """Initialize a new bench directory * create a bench directory in the given path @@ -45,13 +61,17 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, # local apps if clone_from: - clone_apps_from(bench_path=path, clone_from=clone_from, update_app=not clone_without_update) + clone_apps_from( + bench_path=path, clone_from=clone_from, update_app=not clone_without_update + ) # remote apps else: - frappe_path = frappe_path or 'https://github.com/frappe/frappe.git' + frappe_path = frappe_path or "https://github.com/frappe/frappe.git" - get_app(frappe_path, branch=frappe_branch, bench_path=path, skip_assets=True, verbose=verbose) + get_app( + frappe_path, branch=frappe_branch, bench_path=path, skip_assets=True, verbose=verbose + ) # fetch remote apps using config file - deprecate this! if apps_path: @@ -65,29 +85,32 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, if not no_backups: bench.setup.backups() + def setup_sudoers(user): - if not os.path.exists('/etc/sudoers.d'): - os.makedirs('/etc/sudoers.d') + if not os.path.exists("/etc/sudoers.d"): + os.makedirs("/etc/sudoers.d") set_permissions = False - if not os.path.exists('/etc/sudoers'): + if not os.path.exists("/etc/sudoers"): set_permissions = True - with open('/etc/sudoers', 'a') as f: - f.write('\n#includedir /etc/sudoers.d\n') + with open("/etc/sudoers", "a") as f: + f.write("\n#includedir /etc/sudoers.d\n") if set_permissions: - os.chmod('/etc/sudoers', 0o440) + os.chmod("/etc/sudoers", 0o440) - template = bench.config.env().get_template('frappe_sudoers') - frappe_sudoers = template.render(**{ - 'user': user, - 'service': which('service'), - 'systemctl': which('systemctl'), - 'nginx': which('nginx'), - }) + template = bench.config.env().get_template("frappe_sudoers") + frappe_sudoers = template.render( + **{ + "user": user, + "service": which("service"), + "systemctl": which("systemctl"), + "nginx": which("nginx"), + } + ) - with open(sudoers_file, 'w') as f: + with open(sudoers_file, "w") as f: f.write(frappe_sudoers) os.chmod(sudoers_file, 0o440) @@ -103,43 +126,43 @@ def start(no_dev=False, concurrency=None, procfile=None, no_prefix=False, procma if not program: raise Exception("No process manager found") - os.environ['PYTHONUNBUFFERED'] = "true" + os.environ["PYTHONUNBUFFERED"] = "true" if not no_dev: - os.environ['DEV_SERVER'] = "true" + os.environ["DEV_SERVER"] = "true" - command = [program, 'start'] + command = [program, "start"] if concurrency: - command.extend(['-c', concurrency]) + command.extend(["-c", concurrency]) if procfile: - command.extend(['-f', procfile]) + command.extend(["-f", procfile]) if no_prefix: - command.extend(['--no-prefix']) + command.extend(["--no-prefix"]) os.execv(program, command) -def migrate_site(site, bench_path='.'): - run_frappe_cmd('--site', site, 'migrate', bench_path=bench_path) +def migrate_site(site, bench_path="."): + run_frappe_cmd("--site", site, "migrate", bench_path=bench_path) -def backup_site(site, bench_path='.'): - run_frappe_cmd('--site', site, 'backup', bench_path=bench_path) +def backup_site(site, bench_path="."): + run_frappe_cmd("--site", site, "backup", bench_path=bench_path) -def backup_all_sites(bench_path='.'): +def backup_all_sites(bench_path="."): from bench.bench import Bench for site in Bench(bench_path).sites: backup_site(site, bench_path=bench_path) -def fix_prod_setup_perms(bench_path='.', frappe_user=None): +def fix_prod_setup_perms(bench_path=".", frappe_user=None): from glob import glob from bench.bench import Bench - frappe_user = frappe_user or Bench(bench_path).conf.get('frappe_user') + frappe_user = frappe_user or Bench(bench_path).conf.get("frappe_user") if not frappe_user: print("frappe user not set") @@ -154,15 +177,15 @@ def fix_prod_setup_perms(bench_path='.', frappe_user=None): def setup_fonts(): - fonts_path = os.path.join('/tmp', 'fonts') + fonts_path = os.path.join("/tmp", "fonts") - if os.path.exists('/etc/fonts_backup'): + if os.path.exists("/etc/fonts_backup"): return - exec_cmd("git clone https://github.com/frappe/fonts.git", cwd='/tmp') - os.rename('/etc/fonts', '/etc/fonts_backup') - os.rename('/usr/share/fonts', '/usr/share/fonts_backup') - os.rename(os.path.join(fonts_path, 'etc_fonts'), '/etc/fonts') - os.rename(os.path.join(fonts_path, 'usr_share_fonts'), '/usr/share/fonts') + exec_cmd("git clone https://github.com/frappe/fonts.git", cwd="/tmp") + os.rename("/etc/fonts", "/etc/fonts_backup") + os.rename("/usr/share/fonts", "/usr/share/fonts_backup") + os.rename(os.path.join(fonts_path, "etc_fonts"), "/etc/fonts") + os.rename(os.path.join(fonts_path, "usr_share_fonts"), "/usr/share/fonts") shutil.rmtree(fonts_path) exec_cmd("fc-cache -fv") diff --git a/bench/utils/translation.py b/bench/utils/translation.py index d4420eac..ad3a202d 100644 --- a/bench/utils/translation.py +++ b/bench/utils/translation.py @@ -10,7 +10,7 @@ def update_translations_p(args): try: update_translations(*args) except requests.exceptions.HTTPError: - print('Download failed for', args[0], args[1]) + print("Download failed for", args[0], args[1]) def download_translations_p(): @@ -19,7 +19,7 @@ def download_translations_p(): pool = multiprocessing.Pool(multiprocessing.cpu_count()) langs = get_langs() - apps = ('frappe', 'erpnext') + apps = ("frappe", "erpnext") args = list(itertools.product(apps, langs)) pool.map(update_translations_p, args) @@ -27,32 +27,32 @@ def download_translations_p(): def download_translations(): langs = get_langs() - apps = ('frappe', 'erpnext') + apps = ("frappe", "erpnext") for app, lang in itertools.product(apps, langs): update_translations(app, lang) def get_langs(): - lang_file = 'apps/frappe/frappe/geo/languages.json' + lang_file = "apps/frappe/frappe/geo/languages.json" with open(lang_file) as f: langs = json.loads(f.read()) - return [d['code'] for d in langs] + return [d["code"] for d in langs] def update_translations(app, lang): import requests - translations_dir = os.path.join('apps', app, app, 'translations') - csv_file = os.path.join(translations_dir, lang + '.csv') + translations_dir = os.path.join("apps", app, app, "translations") + csv_file = os.path.join(translations_dir, lang + ".csv") url = f"https://translate.erpnext.com/files/{app}-{lang}.csv" r = requests.get(url, stream=True) r.raise_for_status() - with open(csv_file, 'wb') as f: + with open(csv_file, "wb") as f: for chunk in r.iter_content(chunk_size=1024): # filter out keep-alive new chunks if chunk: f.write(chunk) f.flush() - print('downloaded for', app, lang) + print("downloaded for", app, lang)