From e12f53d1a56d598d90307bdfa755db02cc37a30c Mon Sep 17 00:00:00 2001 From: Francisco Roldan Date: Mon, 30 Aug 2021 12:35:53 -0300 Subject: [PATCH 01/91] feat: added option to provide custom queues --- bench/config/supervisor.py | 1 + bench/config/templates/supervisor.conf | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 8c036624..9aa1a2be 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -42,6 +42,7 @@ def generate_supervisor_config(bench_path, user=None, yes=False, skip_redis=Fals "background_workers": config.get('background_workers') or 1, "bench_cmd": which('bench'), "skip_redis": skip_redis, + "custom_queues": config.get("custom_queues", {}), }) conf_path = os.path.join(bench_path, 'config', 'supervisor.conf') diff --git a/bench/config/templates/supervisor.conf b/bench/config/templates/supervisor.conf index d24b1cb6..06872b00 100644 --- a/bench/config/templates/supervisor.conf +++ b/bench/config/templates/supervisor.conf @@ -65,6 +65,22 @@ killasgroup=true numprocs={{ background_workers }} process_name=%(program_name)s-%(process_num)d +{% for queue_name, queue_details in custom_queues.items() %} +[program:{{ bench_name }}-frappe-{{ queue_name }}-worker] +command={{ bench_cmd }} worker --queue {{ queue_name }} +priority=4 +autostart=true +autorestart=true +stdout_logfile={{ bench_dir }}/logs/worker.log +stderr_logfile={{ bench_dir }}/logs/worker.error.log +user={{ user }} +stopwaitsecs={{ queue_details["timeout"] }} +directory={{ bench_dir }} +killasgroup=true +numprocs={{ queue_details["workers"] or background_workers }} +process_name=%(program_name)s-%(process_num)d +{% endfor %} + {% else %} [program:{{ bench_name }}-frappe-workerbeat] command={{ bench_dir }}/env/bin/python -m frappe.celery_app beat -s beat.schedule From 75f84625ad4b19f74bffa989c878ecda07b5c68b Mon Sep 17 00:00:00 2001 From: Francisco Roldan Date: Tue, 14 Sep 2021 08:42:06 -0300 Subject: [PATCH 02/91] fix: rename key --- bench/config/supervisor.py | 2 +- bench/config/templates/supervisor.conf | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 9aa1a2be..cfbaebbd 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -42,7 +42,7 @@ def generate_supervisor_config(bench_path, user=None, yes=False, skip_redis=Fals "background_workers": config.get('background_workers') or 1, "bench_cmd": which('bench'), "skip_redis": skip_redis, - "custom_queues": config.get("custom_queues", {}), + "workers": config.get("workers", {}), }) conf_path = os.path.join(bench_path, 'config', 'supervisor.conf') diff --git a/bench/config/templates/supervisor.conf b/bench/config/templates/supervisor.conf index 06872b00..085cc2cf 100644 --- a/bench/config/templates/supervisor.conf +++ b/bench/config/templates/supervisor.conf @@ -65,19 +65,19 @@ killasgroup=true numprocs={{ background_workers }} process_name=%(program_name)s-%(process_num)d -{% for queue_name, queue_details in custom_queues.items() %} -[program:{{ bench_name }}-frappe-{{ queue_name }}-worker] -command={{ bench_cmd }} worker --queue {{ queue_name }} +{% for worker_name, worker_details in workers.items() %} +[program:{{ bench_name }}-frappe-{{ worker_name }}-worker] +command={{ bench_cmd }} worker --queue {{ worker_name }} priority=4 autostart=true autorestart=true stdout_logfile={{ bench_dir }}/logs/worker.log stderr_logfile={{ bench_dir }}/logs/worker.error.log user={{ user }} -stopwaitsecs={{ queue_details["timeout"] }} +stopwaitsecs={{ worker_details["timeout"] }} directory={{ bench_dir }} killasgroup=true -numprocs={{ queue_details["workers"] or background_workers }} +numprocs={{ worker_details["background_workers"] or background_workers }} process_name=%(program_name)s-%(process_num)d {% endfor %} From 641181260e698073c9df6a68ae8ef75c40769eaa Mon Sep 17 00:00:00 2001 From: kevgeni <90869237+kevgeni@users.noreply.github.com> Date: Fri, 17 Sep 2021 05:59:14 -0400 Subject: [PATCH 03/91] fix: Allow git url with custom username (#1200) Allow git url with custom user, and not just the git user. --- bench/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/app.py b/bench/app.py index 97cbe6a8..9fe122ab 100755 --- a/bench/app.py +++ b/bench/app.py @@ -53,7 +53,7 @@ def write_appstxt(apps, bench_path='.'): def is_git_url(url): # modified to allow without the tailing .git from https://github.com/jonschlinkert/is-git-url.git - pattern = r"(?:git|ssh|https?|git@[-\w.]+):(\/\/)?(.*?)(\.git)?(\/?|\#[-\d\w._]+?)$" + pattern = r"(?:git|ssh|https?|\w*@[-\w.]+):(\/\/)?(.*?)(\.git)?(\/?|\#[-\d\w._]+?)$" return bool(re.match(pattern, url)) def get_excluded_apps(bench_path='.'): From 593b80eec96502901068bf5a7c734f48b71e5d31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Rold=C3=A1n?= Date: Thu, 7 Oct 2021 03:50:36 -0300 Subject: [PATCH 04/91] feat: Add custom workers in Procfile (#1201) * feat: updated procfile * fix: add missing context * fix: add missing context --- bench/config/procfile.py | 3 ++- bench/config/templates/Procfile | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/bench/config/procfile.py b/bench/config/procfile.py index 50a45560..8a479b2e 100755 --- a/bench/config/procfile.py +++ b/bench/config/procfile.py @@ -23,7 +23,8 @@ def setup_procfile(bench_path, yes=False, skip_redis=False): use_rq=use_rq(bench_path), webserver_port=config.get('webserver_port'), CI=os.environ.get('CI'), - skip_redis=skip_redis) + skip_redis=skip_redis, + workers=config.get("workers", {})) with open(procfile_path, 'w') as f: f.write(procfile) diff --git a/bench/config/templates/Procfile b/bench/config/templates/Procfile index f810506e..d9391087 100644 --- a/bench/config/templates/Procfile +++ b/bench/config/templates/Procfile @@ -14,6 +14,9 @@ schedule: bench schedule worker_short: bench worker --queue short 1>> logs/worker.log 2>> logs/worker.error.log worker_long: bench worker --queue long 1>> logs/worker.log 2>> logs/worker.error.log worker_default: bench worker --queue default 1>> logs/worker.log 2>> logs/worker.error.log +{% for worker_name, worker_details in workers.items() %} +worker_{{ worker_name }}: bench worker --queue {{ worker_name }} 1>> logs/worker.log 2>> logs/worker.error.log +{% endfor %} {% else %} workerbeat: sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app beat -s scheduler.schedule' worker: sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app worker -n jobs@%h -Ofair --soft-time-limit 360 --time-limit 390' From 7ad4813cc3a308bd231fb872f032261ca1383ee5 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 12 Oct 2021 21:47:20 +0530 Subject: [PATCH 05/91] build: Loose-er dependencies * Replaced one of the = with ~ to allow any patch releases to be compatible * Removed pinning from requests, honcho & Click since we're using the more basic APIs * Made these changes so that it's easier to install bench globally :') --- requirements.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/requirements.txt b/requirements.txt index 66c2c947..e84b98c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ -Click==7.0 -GitPython==2.1.15 -honcho==1.0.1 -Jinja2==2.11.3 -python-crontab==2.4.0 -requests==2.22.0 -semantic-version==2.8.2 +Click +GitPython~=2.1.15 +honcho +Jinja2~=2.11.3 +python-crontab~=2.4.0 +requests +semantic-version~=2.8.2 setuptools virtualenv From 35ec3dae4c7c8fdb3756be4fe76a11c4fe709c64 Mon Sep 17 00:00:00 2001 From: fossabot Date: Sat, 21 Aug 2021 09:20:08 -0700 Subject: [PATCH 06/91] docs: Add license scan report and status Signed off by: fossabot --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 16fdc2b1..797b1eb5 100755 --- a/README.md +++ b/README.md @@ -6,6 +6,8 @@ Bench is a command-line utility that helps you to install, update, and manage multiple sites for Frappe/ERPNext applications on [*nix systems](https://en.wikipedia.org/wiki/Unix-like) for development and production. ## Table of Contents +[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Ffrappe%2Fbench.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2Ffrappe%2Fbench?ref=badge_shield) + - [Installation](#installation) - [Containerized Installation](#containerized-installation) @@ -20,6 +22,9 @@ Bench is a command-line utility that helps you to install, update, and manage mu - [License](#license) + +[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Ffrappe%2Fbench.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2Ffrappe%2Fbench?ref=badge_large) + ## Installation A typical bench setup provides two types of environments — Development and Production. @@ -238,4 +243,4 @@ To confirm the switch, check the output of `bench src`. It should change from so ## License -This repository has been released under the [GNU GPLv3 License](LICENSE). +This repository has been released under the [GNU GPLv3 License](LICENSE). \ No newline at end of file From 8106dde852c04b8372b5dd2cba965bc3bac4088e Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 13 Oct 2021 20:24:39 +0530 Subject: [PATCH 07/91] chore: Add TravisCI badge to README --- README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 797b1eb5..e7cf4c2d 100755 --- a/README.md +++ b/README.md @@ -5,9 +5,14 @@ Bench is a command-line utility that helps you to install, update, and manage multiple sites for Frappe/ERPNext applications on [*nix systems](https://en.wikipedia.org/wiki/Unix-like) for development and production. -## Table of Contents -[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Ffrappe%2Fbench.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2Ffrappe%2Fbench?ref=badge_shield) +
+[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Ffrappe%2Fbench.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2Ffrappe%2Fbench?ref=badge_shield) +[![CI Status](https://app.travis-ci.com/frappe/bench.svg?branch=develop)](https://app.travis-ci.com/github/frappe/bench) + +
+ +## Table of Contents - [Installation](#installation) - [Containerized Installation](#containerized-installation) @@ -22,9 +27,6 @@ Bench is a command-line utility that helps you to install, update, and manage mu - [License](#license) - -[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Ffrappe%2Fbench.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2Ffrappe%2Fbench?ref=badge_large) - ## Installation A typical bench setup provides two types of environments — Development and Production. From 9f74ec0adb3616104a7004b16fec1c9d25cca834 Mon Sep 17 00:00:00 2001 From: Pruthvi Patel Date: Mon, 18 Oct 2021 16:29:05 +0530 Subject: [PATCH 08/91] feat: add `live_reload` to default_config --- bench/config/common_site_config.py | 3 ++- bench/patches/v5/set_live_reload_config.py | 5 +++++ patches.txt | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 bench/patches/v5/set_live_reload_config.py diff --git a/bench/config/common_site_config.py b/bench/config/common_site_config.py index 8f64f541..c8d13db3 100644 --- a/bench/config/common_site_config.py +++ b/bench/config/common_site_config.py @@ -13,7 +13,8 @@ default_config = { 'frappe_user': getpass.getuser(), 'shallow_clone': True, 'background_workers': 1, - 'use_redis_auth': False + 'use_redis_auth': False, + 'live_reload': True } def make_config(bench_path): diff --git a/bench/patches/v5/set_live_reload_config.py b/bench/patches/v5/set_live_reload_config.py new file mode 100644 index 00000000..4538a6df --- /dev/null +++ b/bench/patches/v5/set_live_reload_config.py @@ -0,0 +1,5 @@ +from bench.config.common_site_config import update_config + + +def execute(bench_path): + update_config({'live_reload': True}, bench_path) diff --git a/patches.txt b/patches.txt index 8f127fe4..dcae0b8a 100644 --- a/patches.txt +++ b/patches.txt @@ -1 +1,2 @@ bench.patches.v3.deprecate_old_config +bench.patches.v5.set_live_reload_config From a28a0d0d2389fcb7268e67ac56b6af74c93501af Mon Sep 17 00:00:00 2001 From: Pruthvi Patel Date: Mon, 18 Oct 2021 18:21:24 +0530 Subject: [PATCH 09/91] fix: correct patches.txt --- bench/patches/patches.txt | 1 + patches.txt | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bench/patches/patches.txt b/bench/patches/patches.txt index 55a65019..da51be2e 100644 --- a/bench/patches/patches.txt +++ b/bench/patches/patches.txt @@ -6,3 +6,4 @@ bench.patches.v4.update_socketio bench.patches.v4.install_yarn #2 bench.patches.v5.fix_user_permissions bench.patches.v5.fix_backup_cronjob +bench.patches.v5.set_live_reload_config \ No newline at end of file diff --git a/patches.txt b/patches.txt index dcae0b8a..8ae25fd2 100644 --- a/patches.txt +++ b/patches.txt @@ -1,2 +1 @@ -bench.patches.v3.deprecate_old_config -bench.patches.v5.set_live_reload_config +bench.patches.v3.deprecate_old_config \ No newline at end of file From 85500b7751756b8a17304268f5514a0385991cc8 Mon Sep 17 00:00:00 2001 From: Pruthvi Patel Date: Tue, 19 Oct 2021 19:50:50 +0530 Subject: [PATCH 10/91] fix: remove unwanted `patches.txt` --- patches.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 patches.txt diff --git a/patches.txt b/patches.txt deleted file mode 100644 index 8ae25fd2..00000000 --- a/patches.txt +++ /dev/null @@ -1 +0,0 @@ -bench.patches.v3.deprecate_old_config \ No newline at end of file From ce00798b5d0c6506b4164999fd8c286dcd5b7f88 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 20 Oct 2021 00:33:34 +0530 Subject: [PATCH 11/91] feat: App Meta Base Class * Less messier way of setting up and managing pre-fetched app meta * Added support to fetch apps using the GitHub org/repo pattern * Can be used to get apps and easily handle dependencies through version identifiers --- bench/__init__.py | 2 + bench/app.py | 107 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 108 insertions(+), 1 deletion(-) diff --git a/bench/__init__.py b/bench/__init__.py index 19d73b97..ab9281b2 100644 --- a/bench/__init__.py +++ b/bench/__init__.py @@ -1,6 +1,8 @@ VERSION = "5.0.0-dev" PROJECT_NAME = "frappe-bench" FRAPPE_VERSION = None +current_path = None +updated_path = None def set_frappe_version(bench_path='.'): diff --git a/bench/app.py b/bench/app.py index 9fe122ab..24c8c81e 100755 --- a/bench/app.py +++ b/bench/app.py @@ -13,7 +13,7 @@ from setuptools.config import read_configuration # imports - module imports import bench -from bench.utils import color, CommandFailedError, build_assets, check_git_for_shallow_clone, exec_cmd, get_cmd_output, get_frappe, restart_supervisor_processes, restart_systemd_processes, run_frappe_cmd +from bench.utils import color, CommandFailedError, build_assets, check_git_for_shallow_clone, exec_cmd, get_cmd_output, get_frappe, is_bench_directory, restart_supervisor_processes, restart_systemd_processes, run_frappe_cmd logger = logging.getLogger(bench.PROJECT_NAME) @@ -22,6 +22,111 @@ logger = logging.getLogger(bench.PROJECT_NAME) class InvalidBranchException(Exception): pass class InvalidRemoteException(Exception): pass + +def find_org(org_repo): + import requests + + org_repo = org_repo[0] + + for org in ["frappe", "erpnext"]: + res = requests.head(f'https://api.github.com/repos/{org}/{org_repo}') + if res.ok: + return org, org_repo + + raise InvalidRemoteException + + +def fetch_details_from_tag(_tag): + if not _tag: + raise Exception("Tag is not provided") + + app_tag = _tag.split("@") + org_repo = app_tag[0].split("/") + + try: + repo, tag = app_tag + except ValueError: + repo, tag = app_tag + [None] + + try: + org, repo = org_repo + except ValueError: + org, repo = find_org(org_repo) + + return org, repo, tag + + +class App: + def __init__(self, name: str, branch : str = None): + """ + name (str): This could look something like + 1. https://github.com/frappe/healthcare.git + 2. git@github.com:frappe/healthcare.git + 3. frappe/healthcare@develop + 4. healthcare + 5. healthcare@develop, healthcare@v13.12.1 + + References for Version Identifiers: + * https://www.python.org/dev/peps/pep-0440/#version-specifiers + * https://docs.npmjs.com/about-semantic-versioning + + class Healthcare(AppConfig): + dependencies = [{"frappe/erpnext": "~13.17.0"}] + """ + self.name = name + self.remote_server = "github.com" + self.use_ssh = False + self.branch = branch + self.setup_details() + + def setup_details(self): + # fetch meta for repo on mounted disk + if os.path.exists(self.name): + self._setup_details_from_mounted_disk() + + # fetch meta for repo from remote git server - traditional get-app url + elif is_git_url(self.name): + self._setup_details_from_git_url() + + # fetch meta from new styled name tags & first party apps on github + else: + self._setup_details_from_name_tag() + + def _setup_details_from_name_tag(self): + self.org, self.repo, self.tag = fetch_details_from_tag(self.name) + + def _setup_details_from_mounted_disk(self): + return self.__setup_details_from_git(remote=False) + + def _setup_details_from_git_url(self): + return self.__setup_details_from_git(remote=True) + + def __setup_details_from_git(self, remote=True): + if not remote: + raise NotImplementedError + + if self.use_ssh: + self.org, _repo = self.name.split(":")[1].split("/") + else: + self.org, _repo = self.name.split("/")[-2:] + + self.tag = self.branch + self.repo = _repo.split(".")[0] + + @property + def url(self): + if self.use_ssh: + return self.get_ssh_url() + + return self.get_http_url() + + def get_http_url(self): + return f"https://{self.remote_server}/{self.org}/{self.repo}.git" + + def get_ssh_url(self): + return f"git@{self.remote_server}:{self.org}/{self.repo}.git" + + class MajorVersionUpgradeException(Exception): def __init__(self, message, upstream_version, local_version): super(MajorVersionUpgradeException, self).__init__(message) From 906ad5a09936bd36af867cdf52b640f39344470f Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 20 Oct 2021 00:43:46 +0530 Subject: [PATCH 12/91] feat: bench drop {bench_path} * Drops bench if no sites' folders found * Tries to drop crontab entry made for backups --- bench/app.py | 20 ++++++++++++++++++++ bench/cli.py | 5 ++++- bench/commands/__init__.py | 3 ++- bench/commands/make.py | 7 +++++++ bench/utils.py | 15 +++++++++++++++ 5 files changed, 48 insertions(+), 2 deletions(-) diff --git a/bench/app.py b/bench/app.py index 24c8c81e..c2a13514 100755 --- a/bench/app.py +++ b/bench/app.py @@ -188,6 +188,26 @@ def remove_from_excluded_apps_txt(app, bench_path='.'): apps.remove(app) return write_excluded_apps_txt(apps, bench_path=bench_path) +def drop_bench(bench_path): + if not os.path.exists(bench_path): + print(f"Bench {bench_path} does not exist") + return + + import shutil + from bench.utils import remove_backups_crontab + + sites_exist = [ + x for x in os.listdir(os.path.join(bench_path, 'sites')) if x not in ('assets', 'apps.txt', 'common_site_config.json') + ] + if sites_exist: + raise Exception("Cannot remove non-empty bench directory") + remove_backups_crontab(bench_path) + shutil.rmtree(bench_path) + print('Bench dropped') + +def get_bench_name(git_url, bench_path): + return "temp-bench" + def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=False, restart_bench=True, overwrite=False): import requests import shutil diff --git a/bench/cli.py b/bench/cli.py index f194349c..01fe5eb7 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -65,7 +65,7 @@ def cli(): not is_bench_directory() and not cmd_requires_root() and len(sys.argv) > 1 - and sys.argv[1] not in ("init", "find", "src") + and sys.argv[1] not in ("init", "find", "src", "drop", "get", "get-app") ): log("Command not being executed in bench directory", level=3) @@ -100,6 +100,7 @@ def cli(): if return_code: logger.warning(f"{command} executed with exit code {return_code}") if isinstance(e, Exception): + click.secho(f"ERROR: {e}", fg="red") raise e finally: try: @@ -209,6 +210,8 @@ def change_working_directory(): """Allows bench commands to be run from anywhere inside a bench directory""" cur_dir = os.path.abspath(".") bench_path = find_parent_bench(cur_dir) + bench.current_path = os.getcwd() + bench.updated_path = bench_path if bench_path: os.chdir(bench_path) diff --git a/bench/commands/__init__.py b/bench/commands/__init__.py index b5ba5a4f..51ec037e 100755 --- a/bench/commands/__init__.py +++ b/bench/commands/__init__.py @@ -17,8 +17,9 @@ def bench_command(bench_path='.'): bench.set_frappe_version(bench_path=bench_path) -from bench.commands.make import init, get_app, new_app, remove_app, exclude_app_for_update, include_app_for_update, pip +from bench.commands.make import init, drop, get_app, new_app, remove_app, exclude_app_for_update, include_app_for_update, pip bench_command.add_command(init) +bench_command.add_command(drop) bench_command.add_command(get_app) bench_command.add_command(new_app) bench_command.add_command(remove_app) diff --git a/bench/commands/make.py b/bench/commands/make.py index 20c93659..a87ee48a 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -50,6 +50,13 @@ def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, c shutil.rmtree(path) +@click.command('drop') +@click.argument('path') +def drop(path): + from bench.app import drop_bench + drop_bench(path) + + @click.command('get-app', help='Clone an app from the internet or filesystem and set it up in your bench') @click.argument('name', nargs=-1) # Dummy argument for backward compatibility @click.argument('git-url') diff --git a/bench/utils.py b/bench/utils.py index 99f7870b..4bdba379 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -415,6 +415,21 @@ def setup_backups(bench_path='.'): system_crontab.write() +def remove_backups_crontab(bench_path='.'): + from crontab import CronTab, CronItem + from bench.config.common_site_config import get_config + logger.log('removing backup cronjob') + + bench_dir = os.path.abspath(bench_path) + user = get_config(bench_path=bench_dir).get('frappe_user') + logfile = os.path.join(bench_dir, 'logs', 'backup.log') + system_crontab = CronTab(user=user) + backup_command = f"cd {bench_dir} && {sys.argv[0]} --verbose --site all backup" + job_command = f"{backup_command} >> {logfile} 2>&1" + + system_crontab.remove_all(command=job_command) + + def setup_sudoers(user): if not os.path.exists('/etc/sudoers.d'): os.makedirs('/etc/sudoers.d') From b75d3524548713f87fc8b8aa2791930563575507 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 20 Oct 2021 00:45:05 +0530 Subject: [PATCH 13/91] feat: Aliased Commands * Added alias for get-app as get * Now run `bench get erpnext` to install ERPNext on your bench --- bench/commands/__init__.py | 25 +++++++++++++++++++++++-- bench/commands/make.py | 2 +- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/bench/commands/__init__.py b/bench/commands/__init__.py index 51ec037e..547bf08c 100755 --- a/bench/commands/__init__.py +++ b/bench/commands/__init__.py @@ -1,5 +1,5 @@ import click - +from click.core import _check_multicommand def print_bench_version(ctx, param, value): """Prints current bench version""" @@ -10,7 +10,28 @@ def print_bench_version(ctx, param, value): click.echo(bench.VERSION) ctx.exit() -@click.group() +class MultiCommandGroup(click.Group): + def add_command(self, cmd, name=None): + """Registers another :class:`Command` with this group. If the name + is not provided, the name of the command is used. + + Note: This is a custom Group that allows passing a list of names for + the command name. + """ + name = name or cmd.name + if name is None: + raise TypeError('Command has no name.') + _check_multicommand(self, name, cmd, register=True) + + try: + self.commands[name] = cmd + except TypeError: + if isinstance(name, list): + for _name in name: + self.commands[_name] = cmd + + +@click.group(cls=MultiCommandGroup) @click.option('--version', is_flag=True, is_eager=True, callback=print_bench_version, expose_value=False) def bench_command(bench_path='.'): import bench diff --git a/bench/commands/make.py b/bench/commands/make.py index a87ee48a..f6092af2 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -57,7 +57,7 @@ def drop(path): drop_bench(path) -@click.command('get-app', help='Clone an app from the internet or filesystem and set it up in your bench') +@click.command(['get', 'get-app'], help='Clone an app from the internet or filesystem and set it up in your bench') @click.argument('name', nargs=-1) # Dummy argument for backward compatibility @click.argument('git-url') @click.option('--branch', default=None, help="branch to checkout") From 38382b84fb1103b0ea6308f8c09e4a7067562ffc Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 20 Oct 2021 00:46:25 +0530 Subject: [PATCH 14/91] feat: Initialize bench on bench get(-app) if not created Why bother creating a bench well in advance? If someone wants to install your app, they should be able to with a single command. For instance, you can run simply `bench get healthcare` in any directory and bench will be setup automatically --- bench/app.py | 61 +++++++++++++++++++++++++++++--------------------- bench/utils.py | 3 +-- 2 files changed, 36 insertions(+), 28 deletions(-) diff --git a/bench/app.py b/bench/app.py index c2a13514..485d5aaa 100755 --- a/bench/app.py +++ b/bench/app.py @@ -206,56 +206,65 @@ def drop_bench(bench_path): print('Bench dropped') def get_bench_name(git_url, bench_path): - return "temp-bench" + if os.path.exists(git_url): + guessed_app_name = os.path.basename(git_url) + else: + app = App(git_url) + guessed_app_name = f"{app.org}_{app.repo}" + + return os.path.join(bench_path, f"{guessed_app_name}-bench") def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=False, restart_bench=True, overwrite=False): - import requests import shutil - if not os.path.exists(git_url): - if not is_git_url(git_url): - orgs = ['frappe', 'erpnext'] - for org in orgs: - url = f'https://api.github.com/repos/{org}/{git_url}' - res = requests.get(url) - if res.ok: - data = res.json() - if 'name' in data: - if git_url == data['name']: - git_url = f'https://github.com/{org}/{git_url}' - break - else: - bench.utils.log(f"App {git_url} not found", level=2) - sys.exit(1) + if not is_bench_directory(bench_path): + bench_name = get_bench_name(git_url, bench_path) + from bench.commands.make import init - # Gets repo name from URL - repo_name = git_url.rstrip('/').rsplit('/', 1)[1].rsplit('.', 1)[0] + click.get_current_context().invoke(init, path=bench_name) + bench_path = bench_name + + if not os.path.exists(git_url): + app = App(git_url) + + git_url = app.url + repo_name = app.repo shallow_clone = '--depth 1' if check_git_for_shallow_clone() else '' - branch = f'--branch {branch}' if branch else '' + branch = f'--branch {app.tag}' if app.tag else '' + else: git_url = os.path.abspath(git_url) _, repo_name = os.path.split(git_url) shallow_clone = '' branch = f'--branch {branch}' if branch else '' - if os.path.isdir(os.path.join(bench_path, 'apps', repo_name)): + cloned_path = os.path.join(bench_path, 'apps', repo_name) + + if os.path.isdir(cloned_path): # application directory already exists # prompt user to overwrite it if overwrite or click.confirm(f'''A directory for the application "{repo_name}" already exists. Do you want to continue and overwrite it?'''): - shutil.rmtree(os.path.join(bench_path, 'apps', repo_name)) + shutil.rmtree(cloned_path) elif click.confirm('''Do you want to reinstall the existing application?''', abort=True): app_name = get_app_name(bench_path, repo_name) - install_app(app=app_name, bench_path=bench_path, verbose=verbose, skip_assets=skip_assets) + install_app( + app=app_name, bench_path=bench_path, verbose=verbose, skip_assets=skip_assets + ) sys.exit() print(f'\n{color.yellow}Getting {repo_name}{color.nc}') logger.log(f'Getting app {repo_name}') - exec_cmd(f"git clone {git_url} {branch} {shallow_clone} --origin upstream", - cwd=os.path.join(bench_path, 'apps')) + + exec_cmd( + f"git clone {git_url} {branch} {shallow_clone} --origin upstream", + cwd=os.path.join(bench_path, 'apps') + ) app_name = get_app_name(bench_path, repo_name) - install_app(app=app_name, bench_path=bench_path, verbose=verbose, skip_assets=skip_assets) + install_app( + app=app_name, bench_path=bench_path, verbose=verbose, skip_assets=skip_assets + ) def get_app_name(bench_path, repo_name): diff --git a/bench/utils.py b/bench/utils.py index 4bdba379..bd101261 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -158,8 +158,7 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, if clone_from: clone_apps_from(bench_path=path, clone_from=clone_from, update_app=not clone_without_update) else: - if not frappe_path: - frappe_path = 'https://github.com/frappe/frappe.git' + frappe_path = frappe_path or 'https://github.com/frappe/frappe.git' get_app(frappe_path, branch=frappe_branch, bench_path=path, skip_assets=True, verbose=verbose) From 1865eeeb6d6d5e20f42f5e623089af68ce253850 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 20 Oct 2021 19:33:16 +0530 Subject: [PATCH 15/91] feat(minor): App Meta handling on disk repos --- bench/app.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/bench/app.py b/bench/app.py index 485d5aaa..11604e5c 100755 --- a/bench/app.py +++ b/bench/app.py @@ -75,6 +75,7 @@ class App: """ self.name = name self.remote_server = "github.com" + self.on_disk = False self.use_ssh = False self.branch = branch self.setup_details() @@ -82,6 +83,7 @@ class App: def setup_details(self): # fetch meta for repo on mounted disk if os.path.exists(self.name): + self.on_disk = True self._setup_details_from_mounted_disk() # fetch meta for repo from remote git server - traditional get-app url @@ -96,15 +98,12 @@ class App: self.org, self.repo, self.tag = fetch_details_from_tag(self.name) def _setup_details_from_mounted_disk(self): - return self.__setup_details_from_git(remote=False) + self.org, self.repo, self.tag = os.path.split(self.name)[-2:] + [self.branch] def _setup_details_from_git_url(self): - return self.__setup_details_from_git(remote=True) - - def __setup_details_from_git(self, remote=True): - if not remote: - raise NotImplementedError + return self.__setup_details_from_git() + def __setup_details_from_git(self): if self.use_ssh: self.org, _repo = self.name.split(":")[1].split("/") else: @@ -115,6 +114,9 @@ class App: @property def url(self): + if self.on_disk: + return os.path.abspath(self.name) + if self.use_ssh: return self.get_ssh_url() From dafddaef6d9af025745d9091739c93adae1c713d Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 20 Oct 2021 19:34:00 +0530 Subject: [PATCH 16/91] refactor: bench get-app * Added app dependencies resolving using `required_apps` * Simplified workflow + docstring to explain what's goin on xD * Get rid of redundant restart_bench kwarg from get_app --- bench/app.py | 94 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 58 insertions(+), 36 deletions(-) diff --git a/bench/app.py b/bench/app.py index 11604e5c..d28c3550 100755 --- a/bench/app.py +++ b/bench/app.py @@ -216,58 +216,80 @@ def get_bench_name(git_url, bench_path): return os.path.join(bench_path, f"{guessed_app_name}-bench") -def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=False, restart_bench=True, overwrite=False): - import shutil +def setup_app_dependencies(repo_name, bench_path='.', branch=None): + # branch kwarg is somewhat of a hack here; since we're assuming the same branches for all apps + # for eg: if you're installing erpnext@develop, you'll want frappe@develop and healthcare@develop too + import glob + + apps_path = os.path.join(os.path.abspath(bench_path), 'apps') + files = glob.glob(os.path.join(apps_path, repo_name, '**', 'hooks.py')) + + if files: + lines = [x for x in open(files[0]).read().split('\n') if x.strip().startswith('required_apps')] + if lines: + required_apps = eval(lines[0].strip('required_apps').strip().lstrip('=').strip()) + # TODO: when the time comes, add version check here + for app in required_apps: + if app not in get_apps(bench_path=bench_path): + get_app(app, bench_path=bench_path, branch=branch) + +def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=False, overwrite=False): + """bench get-app clones a Frappe App from remote (GitHub or any other git server), + and installs it on the current bench. This also resolves dependencies based on the + apps' required_apps defined in the hooks.py file. + + If the bench_path is not a bench directory, a new bench is created named using the + git_url parameter. + """ + app = App(git_url, branch=branch) + git_url = app.url + repo_name = app.repo + branch = app.tag if not is_bench_directory(bench_path): - bench_name = get_bench_name(git_url, bench_path) + bench_path = get_bench_name(git_url, bench_path) from bench.commands.make import init - - click.get_current_context().invoke(init, path=bench_name) - bench_path = bench_name - - if not os.path.exists(git_url): - app = App(git_url) - - git_url = app.url - repo_name = app.repo - shallow_clone = '--depth 1' if check_git_for_shallow_clone() else '' - branch = f'--branch {app.tag}' if app.tag else '' - - else: - git_url = os.path.abspath(git_url) - _, repo_name = os.path.split(git_url) - shallow_clone = '' - branch = f'--branch {branch}' if branch else '' + click.get_current_context().invoke(init, path=bench_path, frappe_branch=branch) cloned_path = os.path.join(bench_path, 'apps', repo_name) + dir_already_exists = os.path.isdir(cloned_path) + to_clone = not dir_already_exists + to_resolve_dependencies = True + to_install = True - if os.path.isdir(cloned_path): + if dir_already_exists: # application directory already exists # prompt user to overwrite it if overwrite or click.confirm(f'''A directory for the application "{repo_name}" already exists. Do you want to continue and overwrite it?'''): + import shutil shutil.rmtree(cloned_path) + to_clone = True elif click.confirm('''Do you want to reinstall the existing application?''', abort=True): - app_name = get_app_name(bench_path, repo_name) - install_app( - app=app_name, bench_path=bench_path, verbose=verbose, skip_assets=skip_assets - ) - sys.exit() + pass - print(f'\n{color.yellow}Getting {repo_name}{color.nc}') - logger.log(f'Getting app {repo_name}') + if to_clone: + fetch_txt = f"Getting {repo_name}" + click.secho(fetch_txt, fg="yellow") + logger.log(fetch_txt) - exec_cmd( - f"git clone {git_url} {branch} {shallow_clone} --origin upstream", - cwd=os.path.join(bench_path, 'apps') - ) + git_branch = f'--branch {app.tag}' if app.tag else '' + shallow_clone = '--depth 1' if check_git_for_shallow_clone() else '' + exec_cmd( + f"git clone {git_url} {git_branch} {shallow_clone} --origin upstream", + cwd=os.path.join(bench_path, 'apps') + ) - app_name = get_app_name(bench_path, repo_name) - install_app( - app=app_name, bench_path=bench_path, verbose=verbose, skip_assets=skip_assets - ) + if to_resolve_dependencies: + setup_app_dependencies( + repo_name=repo_name, bench_path=bench_path, branch=branch + ) + if to_install: + app_name = get_app_name(bench_path, repo_name) + install_app( + app=app_name, bench_path=bench_path, verbose=verbose, skip_assets=skip_assets + ) def get_app_name(bench_path, repo_name): app_name = None From c060ce7b5425d3c3916af881f2a983194ae18374 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 20 Oct 2021 19:43:10 +0530 Subject: [PATCH 17/91] refactor(minor): bench install-app Getting rid of the dependency on the manual coloring of all prints using color --- bench/app.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/bench/app.py b/bench/app.py index d28c3550..73f6abc6 100755 --- a/bench/app.py +++ b/bench/app.py @@ -13,7 +13,7 @@ from setuptools.config import read_configuration # imports - module imports import bench -from bench.utils import color, CommandFailedError, build_assets, check_git_for_shallow_clone, exec_cmd, get_cmd_output, get_frappe, is_bench_directory, restart_supervisor_processes, restart_systemd_processes, run_frappe_cmd +from bench.utils import CommandFailedError, build_assets, check_git_for_shallow_clone, exec_cmd, get_cmd_output, get_frappe, is_bench_directory, restart_supervisor_processes, restart_systemd_processes, run_frappe_cmd logger = logging.getLogger(bench.PROJECT_NAME) @@ -322,12 +322,14 @@ def new_app(app, bench_path='.'): def install_app(app, bench_path=".", verbose=False, no_cache=False, restart_bench=True, skip_assets=False): + from bench.utils import get_env_cmd from bench.config.common_site_config import get_config - print(f'\n{color.yellow}Installing {app}{color.nc}') - logger.log(f"installing {app}") + install_text = f'Installing {app}' + click.secho(install_text, fg="yellow") + logger.log(install_text) - python_path = os.path.join(bench_path, "env", "bin", "python") + python_path = get_env_cmd("python", bench_path=bench_path) quiet_flag = "-q" if not verbose else "" app_path = os.path.join(bench_path, "apps", app) cache_flag = "--no-cache-dir" if no_cache else "" From 01442ba1500a8f6e98dd214084ddbe97335da09c Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 21 Oct 2021 15:28:55 +0530 Subject: [PATCH 18/91] chore: Add compatibility & license shields to README --- README.md | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e7cf4c2d..43f209c4 100755 --- a/README.md +++ b/README.md @@ -6,10 +6,21 @@ Bench is a command-line utility that helps you to install, update, and manage multiple sites for Frappe/ERPNext applications on [*nix systems](https://en.wikipedia.org/wiki/Unix-like) for development and production.
- -[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Ffrappe%2Fbench.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2Ffrappe%2Fbench?ref=badge_shield) -[![CI Status](https://app.travis-ci.com/frappe/bench.svg?branch=develop)](https://app.travis-ci.com/github/frappe/bench) - + + + + + + + + + + + + + + +
## Table of Contents From c5a38b3647bc9e6b890cfaed5c178c83bf5ffc97 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 21 Oct 2021 16:36:03 +0530 Subject: [PATCH 19/91] refactor: Exceptions, Setup & Styles --- bench/app.py | 15 ++------ bench/commands/setup.py | 4 +- bench/config/common_site_config.py | 2 +- bench/config/lets_encrypt.py | 3 +- bench/config/production_setup.py | 4 +- bench/exceptions.py | 11 ++++++ bench/utils.py | 59 ++++++++++++++++++++---------- 7 files changed, 60 insertions(+), 38 deletions(-) create mode 100644 bench/exceptions.py diff --git a/bench/app.py b/bench/app.py index 73f6abc6..9d6013ae 100755 --- a/bench/app.py +++ b/bench/app.py @@ -13,16 +13,13 @@ from setuptools.config import read_configuration # imports - module imports import bench -from bench.utils import CommandFailedError, build_assets, check_git_for_shallow_clone, exec_cmd, get_cmd_output, get_frappe, is_bench_directory, restart_supervisor_processes, restart_systemd_processes, run_frappe_cmd +from bench.exceptions import InvalidRemoteException, InvalidBranchException, CommandFailedError +from bench.utils import build_assets, check_git_for_shallow_clone, exec_cmd, get_cmd_output, is_bench_directory, restart_supervisor_processes, restart_systemd_processes, run_frappe_cmd logger = logging.getLogger(bench.PROJECT_NAME) -class InvalidBranchException(Exception): pass -class InvalidRemoteException(Exception): pass - - def find_org(org_repo): import requests @@ -129,12 +126,6 @@ class App: return f"git@{self.remote_server}:{self.org}/{self.repo}.git" -class MajorVersionUpgradeException(Exception): - def __init__(self, message, upstream_version, local_version): - super(MajorVersionUpgradeException, self).__init__(message) - self.upstream_version = upstream_version - self.local_version = local_version - def get_apps(bench_path='.'): try: with open(os.path.join(bench_path, 'sites', 'apps.txt')) as f: @@ -594,7 +585,7 @@ def get_repo_dir(app, bench_path='.'): def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrade=True): import git import importlib - from bench.utils import update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets, post_upgrade + from bench.utils import update_requirements, update_node_packages, backup_all_sites, patch_sites, post_upgrade apps_dir = os.path.join(bench_path, 'apps') version_upgrade = (False,) diff --git a/bench/commands/setup.py b/bench/commands/setup.py index 73b3eb3a..926a25bc 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -198,8 +198,8 @@ def setup_manager(yes=False, port=23624, domain=None): @click.command("config", help="Generate or over-write sites/common_site_config.json") def setup_config(): - from bench.config.common_site_config import make_config - make_config(".") + from bench.config.common_site_config import setup_config + setup_config(".") @click.command("add-domain", help="Add a custom domain to a particular site") diff --git a/bench/config/common_site_config.py b/bench/config/common_site_config.py index c8d13db3..be011d87 100644 --- a/bench/config/common_site_config.py +++ b/bench/config/common_site_config.py @@ -17,7 +17,7 @@ default_config = { 'live_reload': True } -def make_config(bench_path): +def setup_config(bench_path): make_pid_folder(bench_path) bench_config = get_config(bench_path) bench_config.update(default_config) diff --git a/bench/config/lets_encrypt.py b/bench/config/lets_encrypt.py index 38d52921..41da4f44 100755 --- a/bench/config/lets_encrypt.py +++ b/bench/config/lets_encrypt.py @@ -10,7 +10,8 @@ from bench.config.common_site_config import get_config from bench.config.nginx import make_nginx_conf from bench.config.production_setup import service from bench.config.site_config import get_domains, remove_domain, update_site_config -from bench.utils import CommandFailedError, exec_cmd, update_common_site_config +from bench.utils import exec_cmd, update_common_site_config +from bench.exceptions import CommandFailedError def setup_letsencrypt(site, custom_domain, bench_path, interactive): diff --git a/bench/config/production_setup.py b/bench/config/production_setup.py index 07f39e4b..559a9ef8 100755 --- a/bench/config/production_setup.py +++ b/bench/config/production_setup.py @@ -9,8 +9,8 @@ from bench.config.common_site_config import get_config from bench.config.nginx import make_nginx_conf from bench.config.supervisor import generate_supervisor_config, update_supervisord_config from bench.config.systemd import generate_systemd_config -from bench.utils import CommandFailedError, exec_cmd, which, fix_prod_setup_perms, get_bench_name, get_cmd_output, log - +from bench.utils import exec_cmd, which, fix_prod_setup_perms, get_bench_name, get_cmd_output, log +from bench.exceptions import CommandFailedError logger = logging.getLogger(bench.PROJECT_NAME) diff --git a/bench/exceptions.py b/bench/exceptions.py new file mode 100644 index 00000000..a0bb8d80 --- /dev/null +++ b/bench/exceptions.py @@ -0,0 +1,11 @@ +class InvalidBranchException(Exception): + pass + +class InvalidRemoteException(Exception): + pass + +class PatchError(Exception): + pass + +class CommandFailedError(Exception): + pass diff --git a/bench/utils.py b/bench/utils.py index bd101261..d37dbba9 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -16,14 +16,9 @@ import click # imports - module imports import bench +from bench.exceptions import PatchError -class PatchError(Exception): - pass - -class CommandFailedError(Exception): - pass - logger = logging.getLogger(bench.PROJECT_NAME) bench_cache_file = '.bench.cmd' folders_in_bench = ('apps', 'sites', 'config', 'logs', 'config/pids') @@ -122,17 +117,7 @@ def pause_exec(seconds=10): print(" " * 40, end="\r") -def init(path, apps_path=None, no_procfile=False, no_backups=False, - frappe_path=None, frappe_branch=None, verbose=False, clone_from=None, - skip_redis_config_generation=False, clone_without_update=False, ignore_exist=False, skip_assets=False, - python='python3'): - """Initialize a new bench directory""" - from bench.app import get_app, install_apps_from_path - from bench.config import redis - from bench.config.common_site_config import make_config - from bench.config.procfile import setup_procfile - from bench.patches import set_all_patches_executed - +def setup_bench_directory(path, ignore_exist=False): if os.path.exists(path) and not ignore_exist: log(f'Path {path} already exists!') sys.exit(0) @@ -149,11 +134,42 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, if e.errno == errno.EEXIST: pass + +def init(path, apps_path=None, no_procfile=False, no_backups=False, + frappe_path=None, frappe_branch=None, verbose=False, clone_from=None, + skip_redis_config_generation=False, clone_without_update=False, ignore_exist=False, skip_assets=False, + python='python3'): + """Initialize a new bench directory + + 1. create a bench directory in the given path + 2. setup logging for the bench + 3. setup env for the bench + 4. setup config for the bench + 5. clone frappe + 6. install python & node dependencies + 7. build assets + 8. setup redi + 9. setup procfile + 10. setup backups crontab + 11. setup patches.txt for bench + """ + + # Use print("\033c", end="") to clear entire screen after each step and re-render each list + # another way => https://stackoverflow.com/a/44591228/10309266 + + from bench.app import get_app, install_apps_from_path + from bench.config import redis + from bench.config.common_site_config import setup_config + from bench.config.procfile import setup_procfile + from bench.patches import set_all_patches_executed + + setup_bench_directory(path=path, ignore_exist=ignore_exist) + setup_logging(bench_path=path) setup_env(bench_path=path, python=python) - make_config(path) + setup_config(path) if clone_from: clone_apps_from(bench_path=path, clone_from=clone_from, update_app=not clone_without_update) @@ -177,6 +193,7 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, if not no_procfile: setup_procfile(path, skip_redis=skip_redis_config_generation) + if not no_backups: setup_backups(bench_path=path) @@ -283,8 +300,10 @@ To avoid seeing this warning, set shallow_clone to false in your common_site_con def copy_patches_txt(bench_path): import shutil - shutil.copy(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'patches', 'patches.txt'), - os.path.join(bench_path, 'patches.txt')) + shutil.copy( + os.path.join(os.path.dirname(os.path.abspath(__file__)), 'patches', 'patches.txt'), + os.path.join(bench_path, 'patches.txt') + ) def clone_apps_from(bench_path, clone_from, update_app=True): From b59379c5a962b36ae622b1c865427f0cad07d123 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 30 Oct 2021 02:06:10 +0530 Subject: [PATCH 20/91] refactor(minor): AppMeta class * Fix tuple addition * Rename App to AppMeta - it's more apt given how Bench classes are being structured now --- bench/app.py | 52 ++++++++++---------------------------------------- bench/utils.py | 33 ++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 42 deletions(-) diff --git a/bench/app.py b/bench/app.py index 9d6013ae..3a565809 100755 --- a/bench/app.py +++ b/bench/app.py @@ -13,47 +13,16 @@ from setuptools.config import read_configuration # imports - module imports import bench +from bench.bench import Bench from bench.exceptions import InvalidRemoteException, InvalidBranchException, CommandFailedError -from bench.utils import build_assets, check_git_for_shallow_clone, exec_cmd, get_cmd_output, is_bench_directory, restart_supervisor_processes, restart_systemd_processes, run_frappe_cmd +from bench.utils import exec_cmd, get_cmd_output, is_bench_directory, run_frappe_cmd +from bench.utils import build_assets, check_git_for_shallow_clone, fetch_details_from_tag, restart_supervisor_processes, restart_systemd_processes logger = logging.getLogger(bench.PROJECT_NAME) -def find_org(org_repo): - import requests - - org_repo = org_repo[0] - - for org in ["frappe", "erpnext"]: - res = requests.head(f'https://api.github.com/repos/{org}/{org_repo}') - if res.ok: - return org, org_repo - - raise InvalidRemoteException - - -def fetch_details_from_tag(_tag): - if not _tag: - raise Exception("Tag is not provided") - - app_tag = _tag.split("@") - org_repo = app_tag[0].split("/") - - try: - repo, tag = app_tag - except ValueError: - repo, tag = app_tag + [None] - - try: - org, repo = org_repo - except ValueError: - org, repo = find_org(org_repo) - - return org, repo, tag - - -class App: +class AppMeta: def __init__(self, name: str, branch : str = None): """ name (str): This could look something like @@ -95,7 +64,7 @@ class App: self.org, self.repo, self.tag = fetch_details_from_tag(self.name) def _setup_details_from_mounted_disk(self): - self.org, self.repo, self.tag = os.path.split(self.name)[-2:] + [self.branch] + self.org, self.repo, self.tag = os.path.split(self.name)[-2:] + (self.branch,) def _setup_details_from_git_url(self): return self.__setup_details_from_git() @@ -126,12 +95,11 @@ class App: return f"git@{self.remote_server}:{self.org}/{self.repo}.git" -def get_apps(bench_path='.'): - try: - with open(os.path.join(bench_path, 'sites', 'apps.txt')) as f: - return f.read().strip().split('\n') - except IOError: - return [] +class App(AppMeta): + def __init__(self, name: str, branch : str = None, bench=None): + super().__init__(name, branch) + self.bench = bench + def add_to_appstxt(app, bench_path='.'): apps = get_apps(bench_path=bench_path) diff --git a/bench/utils.py b/bench/utils.py index d37dbba9..ad0b3be4 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -1141,3 +1141,36 @@ def clear_command_cache(bench_path='.'): os.remove(bench_cache_file) else: print("Bench command cache doesn't exist in this folder!") + + +def find_org(org_repo): + import requests + + org_repo = org_repo[0] + + for org in ["frappe", "erpnext"]: + res = requests.head(f'https://api.github.com/repos/{org}/{org_repo}') + if res.ok: + return org, org_repo + + raise InvalidRemoteException + + +def fetch_details_from_tag(_tag): + if not _tag: + raise Exception("Tag is not provided") + + app_tag = _tag.split("@") + org_repo = app_tag[0].split("/") + + try: + repo, tag = app_tag + except ValueError: + repo, tag = app_tag + [None] + + try: + org, repo = org_repo + except ValueError: + org, repo = find_org(org_repo) + + return org, repo, tag \ No newline at end of file From 153546afd77212a564ff1c64794b0226f4a3c216 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 15 Oct 2021 03:12:19 +0530 Subject: [PATCH 21/91] refactor: Simplify Bench with OOP Goals: - Commonify bench operations in a way that intuitive - Get rid of the multiple duplicate functions because it's so hard to understand flow in the codebase ;) - Eliminate the need to guess, re-guess and pass bench info in each function that needs to do literally anything - Increase my happiness index because I just realised that I'm the top contributor of this project and I'd like to make it my own. - Adopt the principles of Least Surprise & The Principle of The Bigger Smile (of DHH) [inspired by ruby & ror] Changes: - The bench module has Bench and the action classes that can be accessed through the bench object - Used the Bench class to access properties like sites, apps & run (to execute commands in context) to reduce effort and clutter - Style improvements & minor changes included --- bench/app.py | 43 ++++--- bench/bench.py | 203 +++++++++++++++++++++++++++++++ bench/cli.py | 4 +- bench/commands/config.py | 5 +- bench/commands/git.py | 5 +- bench/commands/setup.py | 15 +-- bench/commands/utils.py | 14 ++- bench/config/lets_encrypt.py | 6 +- bench/config/nginx.py | 22 ++-- bench/config/procfile.py | 4 +- bench/config/production_setup.py | 16 ++- bench/config/redis.py | 4 +- bench/config/site_config.py | 6 +- bench/config/supervisor.py | 5 +- bench/config/systemd.py | 5 +- bench/utils.py | 27 ++-- 16 files changed, 297 insertions(+), 87 deletions(-) create mode 100644 bench/bench.py diff --git a/bench/app.py b/bench/app.py index 3a565809..0fb080eb 100755 --- a/bench/app.py +++ b/bench/app.py @@ -102,13 +102,15 @@ class App(AppMeta): def add_to_appstxt(app, bench_path='.'): - apps = get_apps(bench_path=bench_path) + apps = Bench(bench_path).apps + if app not in apps: apps.append(app) return write_appstxt(apps, bench_path=bench_path) def remove_from_appstxt(app, bench_path='.'): - apps = get_apps(bench_path=bench_path) + apps = Bench(bench_path).apps + if app in apps: apps.remove(app) return write_appstxt(apps, bench_path=bench_path) @@ -189,7 +191,7 @@ def setup_app_dependencies(repo_name, bench_path='.', branch=None): required_apps = eval(lines[0].strip('required_apps').strip().lstrip('=').strip()) # TODO: when the time comes, add version check here for app in required_apps: - if app not in get_apps(bench_path=bench_path): + if app not in Bench(bench_path).apps: get_app(app, bench_path=bench_path, branch=branch) def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=False, overwrite=False): @@ -219,12 +221,14 @@ def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=Fal if dir_already_exists: # application directory already exists # prompt user to overwrite it - if overwrite or click.confirm(f'''A directory for the application "{repo_name}" already exists. -Do you want to continue and overwrite it?'''): + if overwrite or click.confirm( + f"A directory for the application '{repo_name}' already exists." + "Do you want to continue and overwrite it?" + ): import shutil shutil.rmtree(cloned_path) to_clone = True - elif click.confirm('''Do you want to reinstall the existing application?''', abort=True): + elif click.confirm("Do you want to reinstall the existing application?", abort=True): pass if to_clone: @@ -282,7 +286,6 @@ def new_app(app, bench_path='.'): def install_app(app, bench_path=".", verbose=False, no_cache=False, restart_bench=True, skip_assets=False): from bench.utils import get_env_cmd - from bench.config.common_site_config import get_config install_text = f'Installing {app}' click.secho(install_text, fg="yellow") @@ -300,7 +303,7 @@ def install_app(app, bench_path=".", verbose=False, no_cache=False, restart_benc add_to_appstxt(app, bench_path=bench_path) - conf = get_config(bench_path=bench_path) + conf = Bench(bench_path).conf if conf.get("developer_mode"): from bench.utils import install_python_dev_dependencies @@ -318,13 +321,13 @@ def install_app(app, bench_path=".", verbose=False, no_cache=False, restart_benc def remove_app(app, bench_path='.'): import shutil - from bench.config.common_site_config import get_config + bench = Bench(bench_path) app_path = os.path.join(bench_path, 'apps', app) py = os.path.join(bench_path, 'env', 'bin', 'python') # validate app removal - if app not in get_apps(bench_path): + if app not in bench.apps: print(f"No app named {app}") sys.exit(1) @@ -337,9 +340,10 @@ def remove_app(app, bench_path='.'): # re-build assets and restart processes run_frappe_cmd("build", bench_path=bench_path) - if get_config(bench_path).get('restart_supervisor_on_update'): + + if bench.conf.get('restart_supervisor_on_update'): restart_supervisor_processes(bench_path=bench_path) - if get_config(bench_path).get('restart_systemd_on_update'): + if bench.conf.get('restart_systemd_on_update'): restart_systemd_processes(bench_path=bench_path) @@ -388,11 +392,10 @@ def check_app_installed_legacy(app, bench_path="."): def pull_apps(apps=None, bench_path='.', reset=False): '''Check all apps if there no local changes, pull''' - from bench.config.common_site_config import get_config + bench = Bench(bench_path) + rebase = '--rebase' if bench.conf.get('rebase_on_pull') else '' + apps = apps or bench.apps - rebase = '--rebase' if get_config(bench_path).get('rebase_on_pull') else '' - - apps = apps or get_apps(bench_path=bench_path) # check for local changes if not reset: for app in apps: @@ -432,7 +435,7 @@ Here are your choices: print(f"Skipping pull for app {app}, since remote doesn't exist, and adding it to excluded apps") continue - if not get_config(bench_path).get('shallow_clone') or not reset: + if not bench.conf.get('shallow_clone') or not reset: is_shallow = os.path.exists(os.path.join(app_dir, ".git", "shallow")) if is_shallow: s = " to safely pull remote changes." if not reset else "" @@ -443,7 +446,7 @@ Here are your choices: logger.log(f'pulling {app}') if reset: reset_cmd = f"git reset --hard {remote}/{branch}" - if get_config(bench_path).get('shallow_clone'): + if bench.conf.get('shallow_clone'): exec_cmd(f"git fetch --depth=1 --no-tags {remote} {branch}", cwd=app_dir) exec_cmd(reset_cmd, cwd=app_dir) @@ -638,7 +641,9 @@ def get_apps_json(path): return json.load(f) def validate_branch(): - installed_apps = set(get_apps()) + apps = Bench(".").apps + + installed_apps = set(apps) check_apps = set(['frappe', 'erpnext']) intersection_apps = installed_apps.intersection(check_apps) diff --git a/bench/bench.py b/bench/bench.py new file mode 100644 index 00000000..2d616289 --- /dev/null +++ b/bench/bench.py @@ -0,0 +1,203 @@ +import os +import shutil +import sys +import logging +from typing import MutableSequence + +import bench +from bench.utils import remove_backups_crontab, folders_in_bench, get_venv_path, exec_cmd, get_env_cmd +from bench.config.common_site_config import setup_config + + +logger = logging.getLogger(bench.PROJECT_NAME) + + +class Base: + def run(self, cmd): + return exec_cmd(cmd, cwd=self.cwd) + + +class Bench(Base): + def __init__(self, path): + self.name = path + self.cwd = os.path.abspath(path) + self.exists = os.path.exists(self.name) + self.setup = BenchSetup(self) + self.teardown = BenchTearDown(self) + self.apps = BenchApps(self) + + @property + def sites(self): + return [ + path for path in os.listdir(os.path.join(self.name, 'sites')) + if os.path.exists( + os.path.join("sites", path, "site_config.json") + ) + ] + + @property + def conf(self): + from bench.config.common_site_config import get_config + return get_config(self.name) + + def init(self): + self.setup.dirs() + self.setup.env() + self.setup.backups() + + def drop(self): + self.teardown.backups() + self.teardown.dirs() + + def get_app(self, app, version=None): + pass + + def drop_app(self, app, version=None): + pass + + def install(self, app, branch=None): + from bench.app import App + + app = App(app, branch=branch) + + # get app? + # install app to env + # add to apps.txt + return + + def uninstall(self, app): + # remove from apps.txt + # uninstall app from env + # remove app? + return + + +class BenchApps(MutableSequence): + def __init__(self, bench : Bench): + self.bench = bench + self.initialize_apps() + + def initialize_apps(self): + try: + self.apps = open( + os.path.join(self.bench.name, "sites", "apps.txt") + ).read().splitlines() + except FileNotFoundError: + self.apps = [] + + def __getitem__(self, key): + ''' retrieves an item by its index, key''' + return self.apps[key] + + def __setitem__(self, key, value): + ''' set the item at index, key, to value ''' + # should probably not be allowed + # self.apps[key] = value + raise NotImplementedError + + def __delitem__(self, key): + ''' removes the item at index, key ''' + # TODO: uninstall and delete app from bench + del self.apps[key] + + def __len__(self): + return len(self.apps) + + def insert(self, key, value): + ''' add an item, value, at index, key. ''' + # TODO: fetch and install app to bench + self.apps.insert(key, value) + + def __repr__(self): + return self.__str__() + + def __str__(self): + return str([x for x in self.apps]) + + +class BenchSetup(Base): + def __init__(self, bench : Bench): + self.bench = bench + self.cwd = self.bench.cwd + + def dirs(self): + os.makedirs(self.bench.name, exist_ok=True) + + for dirname in folders_in_bench: + os.makedirs(os.path.join(self.bench.name, dirname), exist_ok=True) + + def env(self, python="python3"): + """Setup env folder + - create env if not exists + - upgrade env pip + - install frappe python dependencies + """ + frappe = os.path.join(self.bench.name, "apps", "frappe") + env_python = get_env_cmd("python", bench_path=self.bench.name) + virtualenv = get_venv_path() + + if not os.path.exists(env_python): + self.run(f"{virtualenv} -q env -p {python}") + + self.run(f"{env_python} -m pip install -q -U pip") + + if os.path.exists(frappe): + self.run(f"{env_python} -m pip install -q -U -e {frappe}") + + def config(self, redis=True, procfile=True): + """Setup config folder + - create pids folder + - generate sites/common_site_config.json + """ + setup_config(self.bench.name) + + if redis: + from bench.config.redis import generate_config + generate_config(self.bench.name) + + if procfile: + from bench.config.procfile import setup_procfile + setup_procfile(self.bench.name, skip_redis=not redis) + + def logging(self): + from bench.utils import setup_logging + return setup_logging(bench_path=self.bench.name) + + def patches(self): + import shutil + + shutil.copy( + os.path.join(os.path.dirname(os.path.abspath(__file__)), 'patches', 'patches.txt'), + os.path.join(self.bench.name, 'patches.txt') + ) + + def backups(self): + # TODO: to something better for logging data? - maybe a wrapper that auto-logs with more context + logger.log('setting up backups') + + from crontab import CronTab + + bench_dir = os.path.abspath(self.bench.name) + user = self.bench.conf.get('frappe_user') + logfile = os.path.join(bench_dir, 'logs', 'backup.log') + system_crontab = CronTab(user=user) + backup_command = f"cd {bench_dir} && {sys.argv[0]} --verbose --site all backup" + job_command = f"{backup_command} >> {logfile} 2>&1" + + if job_command not in str(system_crontab): + job = system_crontab.new(command=job_command, comment="bench auto backups set for every 6 hours") + job.every(6).hours() + system_crontab.write() + + logger.log('backups were set up') + + +class BenchTearDown: + def __init__(self, bench): + self.bench = bench + + def backups(self): + remove_backups_crontab(self.bench.name) + + def dirs(self): + shutil.rmtree(self.bench.name) diff --git a/bench/cli.py b/bench/cli.py index 01fe5eb7..7d35879c 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -10,7 +10,7 @@ import click # imports - module imports import bench -from bench.app import get_apps +from bench.bench import Bench from bench.commands import bench_command from bench.config.common_site_config import get_config from bench.utils import ( @@ -87,7 +87,7 @@ def cli(): if sys.argv[1] in get_frappe_commands(): frappe_cmd() - if sys.argv[1] in get_apps(): + if sys.argv[1] in Bench(".").apps: app_cmd() if not (len(sys.argv) > 1 and sys.argv[1] == "src"): diff --git a/bench/commands/config.py b/bench/commands/config.py index b3691941..c3376f71 100644 --- a/bench/commands/config.py +++ b/bench/commands/config.py @@ -1,5 +1,5 @@ # imports - module imports -from bench.config.common_site_config import update_config, get_config, put_config +from bench.config.common_site_config import update_config, put_config # imports - third party imports import click @@ -68,7 +68,8 @@ def set_common_config(configs): @click.command('remove-common-config', help='Remove specific keys from current bench\'s common config') @click.argument('keys', nargs=-1) def remove_common_config(keys): - common_site_config = get_config('.') + from bench.bench import Bench + common_site_config = Bench('.').conf for key in keys: if key in common_site_config: del common_site_config[key] diff --git a/bench/commands/git.py b/bench/commands/git.py index 41f05d24..19aa4d36 100644 --- a/bench/commands/git.py +++ b/bench/commands/git.py @@ -3,7 +3,8 @@ import os import subprocess # imports - module imports -from bench.app import get_repo_dir, get_apps, get_remote +from bench.bench import Bench +from bench.app import get_repo_dir, get_remote from bench.utils import set_git_remote_url # imports - third party imports @@ -25,7 +26,7 @@ def remote_reset_url(app): @click.command('remote-urls', help="Show apps remote url") def remote_urls(): - for app in get_apps(): + for app in Bench(".").apps: repo_dir = get_repo_dir(app) if os.path.exists(os.path.join(repo_dir, '.git')): diff --git a/bench/commands/setup.py b/bench/commands/setup.py index 926a25bc..9488cae9 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -70,13 +70,15 @@ def setup_production(user, yes=False): @click.command("backups", help="Add cronjob for bench backups") def setup_backups(): - bench.utils.setup_backups() + from bench.bench import Bench + Bench(".").setup.backups() @click.command("env", help="Setup virtualenv for bench") @click.option("--python", type = str, default = "python3", help = "Path to Python Executable.") def setup_env(python="python3"): - bench.utils.setup_env(python=python) + from bench.bench import Bench + return Bench(".").setup.env(python=python) @click.command("firewall", help="Setup firewall for system") @@ -162,8 +164,7 @@ def setup_requirements(node=False, python=False, dev=False): @click.option("--port", help="Port on which you want to run bench manager", default=23624) @click.option("--domain", help="Domain on which you want to run bench manager") def setup_manager(yes=False, port=23624, domain=None): - from bench.utils import get_sites - from bench.config.common_site_config import get_config + from bench.bench import Bench from bench.config.nginx import make_bench_manager_nginx_conf create_new_site = True @@ -182,15 +183,15 @@ def setup_manager(yes=False, port=23624, domain=None): exec_cmd("bench --site bench-manager.local install-app bench_manager") bench_path = "." - conf = get_config(bench_path) + bench = Bench(bench_path) - if conf.get("restart_supervisor_on_update") or conf.get("restart_systemd_on_update"): + if bench.conf.get("restart_supervisor_on_update") or bench.conf.get("restart_systemd_on_update"): # implicates a production setup or so I presume if not domain: print("Please specify the site name on which you want to host bench-manager using the 'domain' flag") sys.exit(1) - if domain not in get_sites(bench_path): + if domain not in bench.sites: raise Exception("No such site") make_bench_manager_nginx_conf(bench_path, yes=yes, port=port, domain=domain) diff --git a/bench/commands/utils.py b/bench/commands/utils.py index 3d73e940..0b2b002f 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -22,10 +22,13 @@ def start(no_dev, concurrency, procfile, no_prefix): @click.option('--systemd', is_flag=True, default=False) def restart(web, supervisor, systemd): from bench.utils import restart_supervisor_processes, restart_systemd_processes - from bench.config.common_site_config import get_config - if get_config('.').get('restart_supervisor_on_update') or supervisor: + from bench.bench import Bench + + bench = Bench(".") + + if bench.conf.get('restart_supervisor_on_update') or supervisor: restart_supervisor_processes(bench_path='.', web_workers=web) - if get_config('.').get('restart_systemd_on_update') or systemd: + if bench.conf.get('restart_systemd_on_update') or systemd: restart_systemd_processes(bench_path='.', web_workers=web) @@ -114,8 +117,9 @@ def renew_lets_encrypt(): @click.command('backup', help="Backup single site") @click.argument('site') def backup_site(site): - from bench.utils import get_sites, backup_site - if site not in get_sites(bench_path='.'): + from bench.bench import Bench + from bench.utils import backup_site + if site not in Bench(".").sites: print(f'Site `{site}` not found') sys.exit(1) backup_site(site, bench_path='.') diff --git a/bench/config/lets_encrypt.py b/bench/config/lets_encrypt.py index 41da4f44..c69b6121 100755 --- a/bench/config/lets_encrypt.py +++ b/bench/config/lets_encrypt.py @@ -6,10 +6,10 @@ import click # imports - module imports import bench -from bench.config.common_site_config import get_config from bench.config.nginx import make_nginx_conf from bench.config.production_setup import service from bench.config.site_config import get_domains, remove_domain, update_site_config +from bench.bench import Bench from bench.utils import exec_cmd, update_common_site_config from bench.exceptions import CommandFailedError @@ -37,7 +37,7 @@ def setup_letsencrypt(site, custom_domain, bench_path, interactive): 'Do you want to continue?', abort=True) - if not get_config(bench_path).get("dns_multitenant"): + if not Bench(bench_path).conf.get("dns_multitenant"): print("You cannot setup SSL without DNS Multitenancy") return @@ -151,7 +151,7 @@ def setup_wildcard_ssl(domain, email, bench_path, exclude_base_domain): return domain_list - if not get_config(bench_path).get("dns_multitenant"): + if not Bench(bench_path).conf.get("dns_multitenant"): print("You cannot setup SSL without DNS Multitenancy") return diff --git a/bench/config/nginx.py b/bench/config/nginx.py index 074dd56e..53685e60 100644 --- a/bench/config/nginx.py +++ b/bench/config/nginx.py @@ -9,7 +9,8 @@ import click # imports - module imports import bench -from bench.utils import get_bench_name, get_sites +from bench.bench import Bench +from bench.utils import get_bench_name def make_nginx_conf(bench_path, yes=False): @@ -23,7 +24,7 @@ def make_nginx_conf(bench_path, yes=False): bench_path = os.path.abspath(bench_path) sites_path = os.path.join(bench_path, "sites") - config = bench.config.common_site_config.get_config(bench_path) + config = Bench(bench_path).conf sites = prepare_sites(config, bench_path) bench_name = get_bench_name(bench_path) @@ -56,13 +57,12 @@ def make_nginx_conf(bench_path, yes=False): def make_bench_manager_nginx_conf(bench_path, yes=False, port=23624, domain=None): from bench.config.site_config import get_site_config - from bench.config.common_site_config import get_config template = bench.config.env().get_template('bench_manager_nginx.conf') bench_path = os.path.abspath(bench_path) sites_path = os.path.join(bench_path, "sites") - config = get_config(bench_path) + config = Bench(bench_path).conf site_config = get_site_config(domain, bench_path=bench_path) bench_name = get_bench_name(bench_path) @@ -182,18 +182,20 @@ def prepare_sites(config, bench_path): return sites def get_sites_with_config(bench_path): - from bench.config.common_site_config import get_config + from bench.bench import Bench from bench.config.site_config import get_site_config - sites = get_sites(bench_path=bench_path) - dns_multitenant = get_config(bench_path).get('dns_multitenant') + bench = Bench(bench_path) + sites = bench.sites + conf = bench.conf + dns_multitenant = conf.get('dns_multitenant') ret = [] for site in sites: try: site_config = get_site_config(site, bench_path=bench_path) except Exception as e: - strict_nginx = get_config(bench_path).get('strict_nginx') + strict_nginx = conf.get('strict_nginx') if strict_nginx: print(f"\n\nERROR: The site config for the site {site} is broken.", "If you want this command to pass, instead of just throwing an error,", @@ -236,8 +238,8 @@ def use_wildcard_certificate(bench_path, ret): "ssl_certificate_key": "/path/to/erpnext.com.key" } ''' - from bench.config.common_site_config import get_config - config = get_config(bench_path=bench_path) + from bench.bench import Bench + config = Bench(bench_path).conf wildcard = config.get('wildcard') if not wildcard: diff --git a/bench/config/procfile.py b/bench/config/procfile.py index 8a479b2e..a6bb64ec 100755 --- a/bench/config/procfile.py +++ b/bench/config/procfile.py @@ -7,12 +7,12 @@ import click # imports - module imports import bench from bench.app import use_rq -from bench.config.common_site_config import get_config from bench.utils import which +from bench.bench import Bench def setup_procfile(bench_path, yes=False, skip_redis=False): - config = get_config(bench_path=bench_path) + config = Bench(bench_path).conf procfile_path = os.path.join(bench_path, 'Procfile') if not yes and os.path.exists(procfile_path): click.confirm('A Procfile already exists and this will overwrite it. Do you want to continue?', diff --git a/bench/config/production_setup.py b/bench/config/production_setup.py index 559a9ef8..10ecc191 100755 --- a/bench/config/production_setup.py +++ b/bench/config/production_setup.py @@ -5,10 +5,10 @@ import sys # imports - module imports import bench -from bench.config.common_site_config import get_config from bench.config.nginx import make_nginx_conf from bench.config.supervisor import generate_supervisor_config, update_supervisord_config from bench.config.systemd import generate_systemd_config +from bench.bench import Bench from bench.utils import exec_cmd, which, fix_prod_setup_perms, get_bench_name, get_cmd_output, log from bench.exceptions import CommandFailedError @@ -30,10 +30,13 @@ def setup_production_prerequisites(): def setup_production(user, bench_path='.', yes=False): print("Setting Up prerequisites...") setup_production_prerequisites() - if get_config(bench_path).get('restart_supervisor_on_update') and get_config(bench_path).get('restart_systemd_on_update'): + + conf = Bench(bench_path).conf + + if conf.get('restart_supervisor_on_update') and conf.get('restart_systemd_on_update'): raise Exception("You cannot use supervisor and systemd at the same time. Modify your common_site_config accordingly." ) - if get_config(bench_path).get('restart_systemd_on_update'): + if conf.get('restart_systemd_on_update'): print("Setting Up systemd...") generate_systemd_config(bench_path=bench_path, user=user, yes=yes) else: @@ -50,7 +53,7 @@ def setup_production(user, bench_path='.', yes=False): nginx_conf = f'/etc/nginx/conf.d/{bench_name}.conf' print("Setting Up symlinks and reloading services...") - if get_config(bench_path).get('restart_supervisor_on_update'): + if conf.get('restart_supervisor_on_update'): supervisor_conf_extn = "ini" if is_centos7() else "conf" supervisor_conf = os.path.join(get_supervisor_confdir(), f'{bench_name}.{supervisor_conf_extn}') @@ -61,7 +64,7 @@ def setup_production(user, bench_path='.', yes=False): if not os.path.islink(nginx_conf): os.symlink(os.path.abspath(os.path.join(bench_path, 'config', 'nginx.conf')), nginx_conf) - if get_config(bench_path).get('restart_supervisor_on_update'): + if conf.get('restart_supervisor_on_update'): reload_supervisor() if os.environ.get('NO_SERVICE_RESTART'): @@ -72,6 +75,7 @@ def setup_production(user, bench_path='.', yes=False): def disable_production(bench_path='.'): bench_name = get_bench_name(bench_path) + conf = Bench(bench_path).conf # supervisorctl supervisor_conf_extn = "ini" if is_centos7() else "conf" @@ -80,7 +84,7 @@ def disable_production(bench_path='.'): if os.path.islink(supervisor_conf): os.unlink(supervisor_conf) - if get_config(bench_path).get('restart_supervisor_on_update'): + if conf.get('restart_supervisor_on_update'): reload_supervisor() # nginx diff --git a/bench/config/redis.py b/bench/config/redis.py index 43d99770..5d509af3 100644 --- a/bench/config/redis.py +++ b/bench/config/redis.py @@ -5,13 +5,13 @@ import subprocess # imports - module imports import bench -from bench.config.common_site_config import get_config def generate_config(bench_path): from urllib.parse import urlparse + from bench.bench import Bench - config = get_config(bench_path) + config = Bench(bench_path).conf redis_version = get_redis_version() ports = {} diff --git a/bench/config/site_config.py b/bench/config/site_config.py index cbd759b3..bb4cea74 100644 --- a/bench/config/site_config.py +++ b/bench/config/site_config.py @@ -3,9 +3,6 @@ import json import os from collections import defaultdict -# imports - module imports -from bench.utils import get_sites - def get_site_config(site, bench_path='.'): config_path = os.path.join(bench_path, 'sites', site, 'site_config.json') @@ -35,8 +32,9 @@ def set_ssl_certificate_key(site, ssl_certificate_key, bench_path='.', gen_confi def set_site_config_nginx_property(site, config, bench_path='.', gen_config=True): from bench.config.nginx import make_nginx_conf + from bench.bench import Bench - if site not in get_sites(bench_path=bench_path): + if site not in Bench(bench_path).sites: raise Exception("No such site") update_site_config(site, config, bench_path=bench_path) if gen_config: diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index cfbaebbd..8db76c3a 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -7,7 +7,8 @@ import os import bench from bench.app import use_rq from bench.utils import get_bench_name, which -from bench.config.common_site_config import get_config, update_config, get_gunicorn_workers +from bench.bench import Bench +from bench.config.common_site_config import update_config, get_gunicorn_workers # imports - third party imports import click @@ -21,8 +22,8 @@ def generate_supervisor_config(bench_path, user=None, yes=False, skip_redis=Fals if not user: user = getpass.getuser() + config = Bench(bench_path).conf template = bench.config.env().get_template('supervisor.conf') - config = get_config(bench_path=bench_path) bench_dir = os.path.abspath(bench_path) config = template.render(**{ diff --git a/bench/config/systemd.py b/bench/config/systemd.py index a25f6e51..60053e54 100644 --- a/bench/config/systemd.py +++ b/bench/config/systemd.py @@ -8,7 +8,8 @@ import click # imports - module imports import bench from bench.app import use_rq -from bench.config.common_site_config import get_config, get_gunicorn_workers, update_config +from bench.bench import Bench +from bench.config.common_site_config import get_gunicorn_workers, update_config from bench.utils import exec_cmd, which, get_bench_name @@ -19,7 +20,7 @@ def generate_systemd_config(bench_path, user=None, yes=False, if not user: user = getpass.getuser() - config = get_config(bench_path=bench_path) + config = Bench(bench_path).conf bench_dir = os.path.abspath(bench_path) bench_name = get_bench_name(bench_path) diff --git a/bench/utils.py b/bench/utils.py index ad0b3be4..ef54ec4e 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -395,7 +395,10 @@ def setup_socketio(bench_path='.'): def patch_sites(bench_path='.'): - for site in get_sites(bench_path=bench_path): + from bench.bench import Bench + bench = Bench(bench_path) + + for site in bench.sites: try: migrate_site(site, bench_path=bench_path) except subprocess.CalledProcessError: @@ -417,21 +420,7 @@ def get_sites(bench_path='.'): def setup_backups(bench_path='.'): from crontab import CronTab - from bench.config.common_site_config import get_config - logger.log('setting up backups') - - bench_dir = os.path.abspath(bench_path) - user = get_config(bench_path=bench_dir).get('frappe_user') - logfile = os.path.join(bench_dir, 'logs', 'backup.log') - system_crontab = CronTab(user=user) - backup_command = f"cd {bench_dir} && {sys.argv[0]} --verbose --site all backup" - job_command = f"{backup_command} >> {logfile} 2>&1" - - if job_command not in str(system_crontab): - job = system_crontab.new(command=job_command, comment="bench auto backups set for every 6 hours") - job.every(6).hours() - system_crontab.write() - + from bench.bench import Bench def remove_backups_crontab(bench_path='.'): from crontab import CronTab, CronItem @@ -439,7 +428,7 @@ def remove_backups_crontab(bench_path='.'): logger.log('removing backup cronjob') bench_dir = os.path.abspath(bench_path) - user = get_config(bench_path=bench_dir).get('frappe_user') + user = Bench(bench_dir).conf.get('frappe_user') logfile = os.path.join(bench_dir, 'logs', 'backup.log') system_crontab = CronTab(user=user) backup_command = f"cd {bench_dir} && {sys.argv[0]} --verbose --site all backup" @@ -540,8 +529,8 @@ def get_git_version(): def check_git_for_shallow_clone(): - from bench.config.common_site_config import get_config - config = get_config('.') + from bench.bench import Bench + config = Bench('.').conf if config: if config.get('release_bench'): From 984959beb8e2c70d1e04fdbd7f68b0b7cf41dee7 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 30 Oct 2021 02:32:04 +0530 Subject: [PATCH 22/91] refactor: Adoption of new style Bench notations * Updated multiple functions * Updated flow & re-ordered steps of `bench init` - tested with multiple args * Updated exceptions --- bench/app.py | 21 +---- bench/commands/make.py | 29 +++++-- bench/exceptions.py | 6 ++ bench/patches/__init__.py | 8 -- bench/utils.py | 168 +++++++++++++------------------------- 5 files changed, 90 insertions(+), 142 deletions(-) diff --git a/bench/app.py b/bench/app.py index 0fb080eb..b823b94d 100755 --- a/bench/app.py +++ b/bench/app.py @@ -151,24 +151,7 @@ def remove_from_excluded_apps_txt(app, bench_path='.'): apps.remove(app) return write_excluded_apps_txt(apps, bench_path=bench_path) -def drop_bench(bench_path): - if not os.path.exists(bench_path): - print(f"Bench {bench_path} does not exist") - return - - import shutil - from bench.utils import remove_backups_crontab - - sites_exist = [ - x for x in os.listdir(os.path.join(bench_path, 'sites')) if x not in ('assets', 'apps.txt', 'common_site_config.json') - ] - if sites_exist: - raise Exception("Cannot remove non-empty bench directory") - remove_backups_crontab(bench_path) - shutil.rmtree(bench_path) - print('Bench dropped') - -def get_bench_name(git_url, bench_path): +def generate_bench_name(git_url, bench_path): if os.path.exists(git_url): guessed_app_name = os.path.basename(git_url) else: @@ -208,7 +191,7 @@ def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=Fal branch = app.tag if not is_bench_directory(bench_path): - bench_path = get_bench_name(git_url, bench_path) + bench_path = generate_bench_name(git_url, bench_path) from bench.commands.make import init click.get_current_context().invoke(init, path=bench_path, frappe_branch=branch) diff --git a/bench/commands/make.py b/bench/commands/make.py index f6092af2..b016affb 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -17,27 +17,31 @@ import click @click.option('--skip-assets',is_flag=True, default=False, help="Do not build assets") @click.option('--verbose',is_flag=True, help="Verbose output during install") def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, clone_from, verbose, skip_redis_config_generation, clone_without_update, ignore_exist=False, skip_assets=False, python='python3'): + import os from bench.utils import init, log + if not ignore_exist and os.path.exists(path): + log(f"Bench instance already exists at {path}", level=2) + return + try: init( path, - apps_path=apps_path, + apps_path=apps_path, # can be used from --config flag? Maybe config file could have more info? no_procfile=no_procfile, no_backups=no_backups, frappe_path=frappe_path, frappe_branch=frappe_branch, - verbose=verbose, clone_from=clone_from, skip_redis_config_generation=skip_redis_config_generation, clone_without_update=clone_without_update, - ignore_exist=ignore_exist, skip_assets=skip_assets, python=python, + verbose=verbose, ) log(f'Bench {path} initialized', level=1) except SystemExit: - pass + raise except Exception as e: import os, shutil, time # add a sleep here so that the traceback of other processes doesnt overlap with the prompts @@ -53,8 +57,21 @@ def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, c @click.command('drop') @click.argument('path') def drop(path): - from bench.app import drop_bench - drop_bench(path) + from bench.bench import Bench + from bench.exceptions import BenchNotFoundError, ValidationError + + bench = Bench(path) + + if not bench.exists: + raise BenchNotFoundError(f"Bench {bench.name} does not exist") + + if bench.sites: + raise ValidationError("Cannot remove non-empty bench directory") + + bench.drop() + + print('Bench dropped') + @click.command(['get', 'get-app'], help='Clone an app from the internet or filesystem and set it up in your bench') diff --git a/bench/exceptions.py b/bench/exceptions.py index a0bb8d80..b3f206c7 100644 --- a/bench/exceptions.py +++ b/bench/exceptions.py @@ -9,3 +9,9 @@ class PatchError(Exception): class CommandFailedError(Exception): pass + +class BenchNotFoundError(Exception): + pass + +class ValidationError(Exception): + pass diff --git a/bench/patches/__init__.py b/bench/patches/__init__.py index b80170a1..5a504737 100644 --- a/bench/patches/__init__.py +++ b/bench/patches/__init__.py @@ -29,11 +29,3 @@ def run(bench_path): # end with an empty line f.write('\n') - -def set_all_patches_executed(bench_path): - source_patch_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'patches.txt') - target_patch_file = os.path.join(os.path.abspath(bench_path), 'patches.txt') - - with open(target_patch_file, 'w') as tf: - with open(source_patch_file, 'r') as sf: - tf.write(sf.read()) diff --git a/bench/utils.py b/bench/utils.py index ef54ec4e..44b18333 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - # imports - standard imports import grp import itertools @@ -10,14 +7,14 @@ import os import pwd import subprocess import sys +from shlex import split # imports - third party imports import click # imports - module imports import bench -from bench.exceptions import PatchError - +from bench.exceptions import InvalidRemoteException, PatchError logger = logging.getLogger(bench.PROJECT_NAME) bench_cache_file = '.bench.cmd' @@ -116,88 +113,56 @@ def pause_exec(seconds=10): print(" " * 40, end="\r") - -def setup_bench_directory(path, ignore_exist=False): - if os.path.exists(path) and not ignore_exist: - log(f'Path {path} already exists!') - sys.exit(0) - elif not os.path.exists(path): - # only create dir if it does not exist - os.makedirs(path) - - for dirname in folders_in_bench: - try: - os.makedirs(os.path.join(path, dirname)) - except OSError as e: - import errno - - if e.errno == errno.EEXIST: - pass - - def init(path, apps_path=None, no_procfile=False, no_backups=False, frappe_path=None, frappe_branch=None, verbose=False, clone_from=None, - skip_redis_config_generation=False, clone_without_update=False, ignore_exist=False, skip_assets=False, + skip_redis_config_generation=False, clone_without_update=False, skip_assets=False, python='python3'): """Initialize a new bench directory - 1. create a bench directory in the given path - 2. setup logging for the bench - 3. setup env for the bench - 4. setup config for the bench - 5. clone frappe - 6. install python & node dependencies - 7. build assets - 8. setup redi - 9. setup procfile - 10. setup backups crontab - 11. setup patches.txt for bench + * create a bench directory in the given path + * setup logging for the bench + * setup env for the bench + * setup config (dir/pids/redis/procfile) for the bench + * setup patches.txt for bench + * clone & install frappe + * install python & node dependencies + * build assets + * setup backups crontab """ # Use print("\033c", end="") to clear entire screen after each step and re-render each list # another way => https://stackoverflow.com/a/44591228/10309266 from bench.app import get_app, install_apps_from_path - from bench.config import redis - from bench.config.common_site_config import setup_config - from bench.config.procfile import setup_procfile - from bench.patches import set_all_patches_executed + from bench.bench import Bench - setup_bench_directory(path=path, ignore_exist=ignore_exist) + bench = Bench(path) - setup_logging(bench_path=path) - - setup_env(bench_path=path, python=python) - - setup_config(path) + bench.setup.dirs() + bench.setup.logging() + bench.setup.env(python=python) + bench.setup.config(redis=not skip_redis_config_generation, procfile=not no_procfile) + bench.setup.patches() + # local apps if clone_from: clone_apps_from(bench_path=path, clone_from=clone_from, update_app=not clone_without_update) + + # remote apps else: frappe_path = frappe_path or 'https://github.com/frappe/frappe.git' get_app(frappe_path, branch=frappe_branch, bench_path=path, skip_assets=True, verbose=verbose) + # fetch remote apps using config file - deprecate this! if apps_path: install_apps_from_path(apps_path, bench_path=path) - if not skip_assets: - update_node_packages(bench_path=path) - - set_all_patches_executed(bench_path=path) if not skip_assets: build_assets(bench_path=path) - if not skip_redis_config_generation: - redis.generate_config(path) - - if not no_procfile: - setup_procfile(path, skip_redis=skip_redis_config_generation) - if not no_backups: - setup_backups(bench_path=path) - - copy_patches_txt(path) + bench.setup.backups() def update(pull=False, apps=None, patch=False, build=False, requirements=False, backup=True, compile=True, @@ -206,11 +171,13 @@ def update(pull=False, apps=None, patch=False, build=False, requirements=False, import re from bench import patches from bench.app import is_version_upgrade, pull_apps, validate_branch - from bench.config.common_site_config import get_config, update_config + from bench.config.common_site_config import update_config + from bench.bench import Bench bench_path = os.path.abspath(".") + bench = Bench(bench_path) patches.run(bench_path=bench_path) - conf = get_config(bench_path) + conf = bench.conf if apps and not pull: apps = [] @@ -297,15 +264,6 @@ To avoid seeing this warning, set shallow_clone to false in your common_site_con print("_" * 80 + "\nBench: Deployment tool for Frappe and Frappe Applications (https://frappe.io/bench).\nOpen source depends on your contributions, so do give back by submitting bug reports, patches and fixes and be a part of the community :)") -def copy_patches_txt(bench_path): - import shutil - - shutil.copy( - os.path.join(os.path.dirname(os.path.abspath(__file__)), 'patches', 'patches.txt'), - os.path.join(bench_path, 'patches.txt') - ) - - def clone_apps_from(bench_path, clone_from, update_app=True): from bench.app import install_app print(f'Copying apps from {clone_from}...') @@ -409,22 +367,13 @@ def build_assets(bench_path='.', app=None): command = 'bench build' if app: command += f' --app {app}' - exec_cmd(command, cwd=bench_path) + exec_cmd(command, cwd=bench_path, env={"BENCH_DEVELOPER": "1"}) -def get_sites(bench_path='.'): - sites_path = os.path.join(bench_path, 'sites') - sites = (site for site in os.listdir(sites_path) if os.path.exists(os.path.join(sites_path, site, 'site_config.json'))) - return sites - - -def setup_backups(bench_path='.'): +def remove_backups_crontab(bench_path='.'): from crontab import CronTab from bench.bench import Bench -def remove_backups_crontab(bench_path='.'): - from crontab import CronTab, CronItem - from bench.config.common_site_config import get_config logger.log('removing backup cronjob') bench_dir = os.path.abspath(bench_path) @@ -557,13 +506,15 @@ def get_cmd_output(cmd, cwd='.', _raise=True): def restart_supervisor_processes(bench_path='.', web_workers=False): - from bench.config.common_site_config import get_config - conf = get_config(bench_path=bench_path) + from bench.bench import Bench + + bench = Bench(bench_path) + conf = bench.conf + cmd = conf.get('supervisor_restart_cmd') bench_name = get_bench_name(bench_path) - cmd = conf.get('supervisor_restart_cmd') if cmd: - exec_cmd(cmd, cwd=bench_path) + bench.run(cmd) else: supervisor_status = get_cmd_output('supervisorctl status', cwd=bench_path) @@ -583,7 +534,7 @@ def restart_supervisor_processes(bench_path='.', web_workers=False): else: group = 'frappe:' - exec_cmd(f'supervisorctl restart {group}', cwd=bench_path) + bench.run(f"supervisorctl restart {group}") def restart_systemd_processes(bench_path='.', web_workers=False): @@ -592,35 +543,31 @@ def restart_systemd_processes(bench_path='.', web_workers=False): exec_cmd(f'sudo systemctl start -- $(systemctl show -p Requires {bench_name}.target | cut -d= -f2)') -def set_default_site(site, bench_path='.'): - if site not in get_sites(bench_path=bench_path): - raise Exception("Site not in bench") - exec_cmd(f"{get_frappe(bench_path)} --use {site}", cwd=os.path.join(bench_path, 'sites')) - - def update_env_pip(bench_path): env_py = get_env_cmd("python") exec_cmd(f"{env_py} -m pip install -q -U pip") def update_requirements(bench_path='.'): - from bench.app import get_apps, install_app + from bench.app import install_app + from bench.bench import Bench + print('Installing applications...') update_env_pip(bench_path) - for app in get_apps(): + for app in Bench(bench_path).apps: install_app(app, bench_path=bench_path, skip_assets=True, restart_bench=False) def update_python_packages(bench_path='.'): - from bench.app import get_apps + from bench.bench import Bench env_py = get_env_cmd("python") print('Updating Python libraries...') update_env_pip(bench_path) - for app in get_apps(): - print(f'\n{color.yellow}Installing python dependencies for {app}{color.nc}') + for app in Bench(bench_path).apps: + click.secho(f"\nInstalling python dependencies for {app}", fg="yellow") app_path = os.path.join(bench_path, "apps", app) exec_cmd(f"{env_py} -m pip install -q -U -e {app_path}", cwd=bench_path) @@ -640,12 +587,12 @@ def update_node_packages(bench_path='.'): def install_python_dev_dependencies(bench_path='.', apps=None): - from bench.app import get_apps + from bench.bench import Bench if isinstance(apps, str): apps = [apps] elif apps is None: - apps = get_apps() + apps = Bench(bench_path).apps env_py = get_env_cmd("python") for app in apps: @@ -670,7 +617,7 @@ def update_yarn_packages(bench_path='.'): for app in os.listdir(apps_dir): app_path = os.path.join(apps_dir, app) if os.path.exists(os.path.join(app_path, 'package.json')): - print(f'\n{color.yellow}Installing node dependencies for {app}{color.nc}') + click.secho(f"\nInstalling node dependencies for {app}", fg="yellow") exec_cmd('yarn install', cwd=app_path) @@ -715,7 +662,9 @@ def backup_site(site, bench_path='.'): def backup_all_sites(bench_path='.'): - for site in get_sites(bench_path=bench_path): + from bench.bench import Bench + + for site in Bench(bench_path).sites: backup_site(site, bench_path=bench_path) @@ -779,10 +728,9 @@ def drop_privileges(uid_name='nobody', gid_name='nogroup'): def fix_prod_setup_perms(bench_path='.', frappe_user=None): from glob import glob - from bench.config.common_site_config import get_config + from bench.bench import Bench - if not frappe_user: - frappe_user = get_config(bench_path).get('frappe_user') + frappe_user = frappe_user or Bench(bench_path).conf.get('frappe_user') if not frappe_user: print("frappe user not set") @@ -961,9 +909,11 @@ def setup_fonts(): def set_git_remote_url(git_url, bench_path='.'): "Set app remote git url" + from bench.bench import Bench + app = git_url.rsplit('/', 1)[1].rsplit('.', 1)[0] - if app not in bench.app.get_apps(bench_path): + if app not in Bench(bench_path).apps: print(f"No app named {app}") sys.exit(1) @@ -1019,9 +969,9 @@ def find_benches(directory=None): def migrate_env(python, backup=False): import shutil from urllib.parse import urlparse - from bench.config.common_site_config import get_config - from bench.app import get_apps + from bench.bench import Bench + bench = Bench(".") nvenv = 'env' path = os.getcwd() python = which(python) @@ -1030,7 +980,7 @@ def migrate_env(python, backup=False): # Clear Cache before Bench Dies. try: - config = get_config(bench_path=os.getcwd()) + config = bench.conf rredis = urlparse(config['redis_cache']) redis = f"{which('redis-cli')} -p {rredis.port}" @@ -1065,7 +1015,7 @@ def migrate_env(python, backup=False): logger.log(f'Setting up a New Virtual {python} Environment') venv_creation = exec_cmd(f'{virtualenv} --python {python} {pvenv}') - apps = ' '.join([f"-e {os.path.join('apps', app)}" for app in get_apps()]) + apps = ' '.join([f"-e {os.path.join('apps', app)}" for app in bench.apps]) packages_setup = exec_cmd(f'{pvenv} -m pip install -q -U {apps}') logger.log(f'Migration Successful to {python}') From 616dff8fbbcede4a49cf4a96064a95081961cc41 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 30 Oct 2021 02:35:32 +0530 Subject: [PATCH 23/91] fix: Drop support for Frappe v4-5-6 ;) This was long time due. Removing software that won't be compatible with this version of Bench is a no-brainer. Sorry for not posting this in advance but the world has been moving too fast. Use `pip install frappe-bench --only-binary='all'` to install the latest supported version for your machine :D --- bench/cli.py | 7 ------- bench/utils.py | 32 +++++++++----------------------- 2 files changed, 9 insertions(+), 30 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index 7d35879c..48957f43 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -21,7 +21,6 @@ from bench.utils import ( generate_command_cache, get_cmd_output, get_env_cmd, - get_frappe, is_bench_directory, is_dist_editable, is_root, @@ -162,12 +161,6 @@ def change_uid(): sys.exit(1) -def old_frappe_cli(bench_path="."): - f = get_frappe(bench_path=bench_path) - os.chdir(os.path.join(bench_path, "sites")) - os.execv(f, [f] + sys.argv[2:]) - - def app_cmd(bench_path="."): f = get_env_cmd("python", bench_path=bench_path) os.chdir(os.path.join(bench_path, "sites")) diff --git a/bench/utils.py b/bench/utils.py index 44b18333..f6927cb9 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -92,14 +92,6 @@ def check_latest_version(): log(f"A newer version of bench is available: {local_version} → {pypi_version}") -def get_frappe(bench_path='.'): - frappe = get_env_cmd('frappe', bench_path=bench_path) - if not os.path.exists(frappe): - print('frappe app is not installed. Run the following command to install frappe') - print('bench get-app https://github.com/frappe/frappe.git') - return frappe - - def get_env_cmd(cmd, bench_path='.'): return os.path.abspath(os.path.join(bench_path, 'env', 'bin', cmd)) @@ -776,30 +768,24 @@ def validate_upgrade(from_ver, to_ver, bench_path='.'): def post_upgrade(from_ver, to_ver, bench_path='.'): - from bench.config.common_site_config import get_config from bench.config import redis from bench.config.supervisor import generate_supervisor_config from bench.config.nginx import make_nginx_conf - conf = get_config(bench_path=bench_path) + from bench.bench import Bench + + conf = Bench(bench_path).conf print("-" * 80 + f"Your bench was upgraded to version {to_ver}") if conf.get('restart_supervisor_on_update'): redis.generate_config(bench_path=bench_path) generate_supervisor_config(bench_path=bench_path) make_nginx_conf(bench_path=bench_path) - - if from_ver == 4 and to_ver == 5: - setup_backups(bench_path=bench_path) - - if from_ver <= 5 and to_ver == 6: - setup_socketio(bench_path=bench_path) - - message = """ -As you have setup your bench for production, you will have to reload configuration for nginx and supervisor. To complete the migration, please run the following commands -sudo service nginx restart -sudo supervisorctl reload - """.strip() - print(message) + print( + "As you have setup your bench for production, you will have to reload configuration for " + "nginx and supervisor. To complete the migration, please run the following commands:" + "\nsudo service nginx restart" + "\nsudo supervisorctl reload" + ) def update_translations_p(args): From 20560c97c4246b2480d7358c722bc9ad13606138 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 29 Sep 2021 06:12:19 +0530 Subject: [PATCH 24/91] feat: Skip warnings if envvar BENCH_DEVELOPER set --- bench/cli.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index 48957f43..4453c103 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -47,7 +47,7 @@ def cli(): change_uid() change_dir() - if ( + if not os.environ.get("BENCH_DEVELOPER") and ( is_dist_editable(bench.PROJECT_NAME) and len(sys.argv) > 1 and sys.argv[1] != "src" @@ -60,18 +60,17 @@ def cli(): level=3, ) + in_bench = is_bench_directory() + if ( - not is_bench_directory() + not in_bench and not cmd_requires_root() and len(sys.argv) > 1 and sys.argv[1] not in ("init", "find", "src", "drop", "get", "get-app") ): log("Command not being executed in bench directory", level=3) - if len(sys.argv) > 2 and sys.argv[1] == "frappe": - old_frappe_cli() - - elif len(sys.argv) > 1: + if in_bench and len(sys.argv) > 1: if sys.argv[1] == "--help": print(click.Context(bench_command).get_help()) print(get_frappe_help()) From ba289fba25248efa42773d969a47a63bb2cc9e98 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 29 Sep 2021 07:12:19 +0530 Subject: [PATCH 25/91] refactor(log): use click to colour everything A year (longer) ago, I added color to bench manually. I hadn't explored much of click back then. Had to figure out when we had to and we didnt have to colour stuff. At times the chars would just be ignored and rather printed to the terminal which was awkward...anyway, here, goodbyetoallthat :cheers: (hopefully) --- bench/utils.py | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/bench/utils.py b/bench/utils.py index f6927cb9..0cc34f77 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -22,15 +22,6 @@ folders_in_bench = ('apps', 'sites', 'config', 'logs', 'config/pids') sudoers_file = '/etc/sudoers.d/frappe' -class color: - nc = '\033[0m' - blue = '\033[94m' - green = '\033[92m' - yellow = '\033[93m' - red = '\033[91m' - silver = '\033[90m' - - def is_bench_directory(directory=os.path.curdir): is_bench = True @@ -43,22 +34,20 @@ def is_bench_directory(directory=os.path.curdir): def log(message, level=0): levels = { - 0: color.blue + 'INFO', # normal - 1: color.green + 'SUCCESS', # success - 2: color.red + 'ERROR', # fail - 3: color.yellow + 'WARN' # warn/suggest + 0: ("blue", "INFO"), # normal + 1: ("green", "SUCCESS"), # success + 2: ("red", "ERROR"), # fail + 3: ("yellow", "WARN") # warn/suggest } loggers = { 2: logger.error, 3: logger.warning } - - start_line = (levels.get(level) + ': ') if level in levels else '' + color, prefix = levels.get(level, levels[0]) level_logger = loggers.get(level, logger.info) - end_line = '\033[0m' level_logger(message) - print(start_line + message + end_line) + click.secho(f"{prefix}: {message}", fg=color) def safe_decode(string, encoding = 'utf-8'): From 93e5655570b7b262309c2f664a6afc01a251fbdd Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 30 Oct 2021 02:44:00 +0530 Subject: [PATCH 26/91] refactor: bench.utils.exec_cmd Refactoring the official way to run any commands through bench. Added a way to pass on envvars, and use the new log API w click ;) --- bench/utils.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/bench/utils.py b/bench/utils.py index 0cc34f77..03c46f07 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -282,14 +282,17 @@ def clone_apps_from(bench_path, clone_from, update_app=True): setup_app(app) -def exec_cmd(cmd, cwd='.'): - import shlex - print(f"{color.silver}$ {cmd}{color.nc}") +def exec_cmd(cmd, cwd='.', env=None): + if env: + env.update(os.environ.copy()) + + click.secho(f"$ {cmd}", fg='bright_black') + cwd_info = f"cd {cwd} && " if cwd != "." else "" cmd_log = f"{cwd_info}{cmd}" logger.debug(cmd_log) - cmd = shlex.split(cmd) - return_code = subprocess.call(cmd, cwd=cwd, universal_newlines=True) + cmd = split(cmd) + return_code = subprocess.call(cmd, cwd=cwd, universal_newlines=True, env=env) if return_code: logger.warning(f"{cmd_log} executed with exit code {return_code}") From 5c9ac457ddc089fa2659cbe626ee706c9d9124b7 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 1 Nov 2021 07:12:19 +0530 Subject: [PATCH 27/91] chore: Drop unused setup_env util This util was dropped in favour of Bench(path).setup.env() --- bench/utils.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/bench/utils.py b/bench/utils.py index 03c46f07..ddc4f6af 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -320,16 +320,6 @@ def get_venv_path(): return venv or log("virtualenv cannot be found", level=2) -def setup_env(bench_path='.', python='python3'): - frappe = os.path.join(bench_path, "apps", "frappe") - py = os.path.join(bench_path, "env", "bin", "python") - virtualenv = get_venv_path() - - exec_cmd(f'{virtualenv} -q env -p {python}', cwd=bench_path) - - if os.path.exists(frappe): - exec_cmd(f'{py} -m pip install -q -U -e {frappe}', cwd=bench_path) - def setup_socketio(bench_path='.'): exec_cmd("npm install socket.io redis express superagent cookie babel-core less chokidar \ From c32616d0f2bda170931f84452bef4b6731dc9a64 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 12 Nov 2021 17:51:20 +0530 Subject: [PATCH 28/91] refactor(utils): Re-arrange, clean-up * Cleaner APIs - got rid of redundant and repeated checks * Easier understanding on what's called where? ;) hopefully * Separation of utils to avoid confusion * Remove redundant utils * Use newer standard library APIs --- bench/utils.py | 1083 ------------------------------------ bench/utils/__init__.py | 400 +++++++++++++ bench/utils/app.py | 189 +++++++ bench/utils/bench.py | 550 ++++++++++++++++++ bench/utils/system.py | 189 +++++++ bench/utils/translation.py | 61 ++ 6 files changed, 1389 insertions(+), 1083 deletions(-) delete mode 100755 bench/utils.py create mode 100644 bench/utils/__init__.py create mode 100644 bench/utils/app.py create mode 100644 bench/utils/bench.py create mode 100644 bench/utils/system.py create mode 100644 bench/utils/translation.py diff --git a/bench/utils.py b/bench/utils.py deleted file mode 100755 index ddc4f6af..00000000 --- a/bench/utils.py +++ /dev/null @@ -1,1083 +0,0 @@ -# imports - standard imports -import grp -import itertools -import json -import logging -import os -import pwd -import subprocess -import sys -from shlex import split - -# imports - third party imports -import click - -# imports - module imports -import bench -from bench.exceptions import InvalidRemoteException, PatchError - -logger = logging.getLogger(bench.PROJECT_NAME) -bench_cache_file = '.bench.cmd' -folders_in_bench = ('apps', 'sites', 'config', 'logs', 'config/pids') -sudoers_file = '/etc/sudoers.d/frappe' - - -def is_bench_directory(directory=os.path.curdir): - is_bench = True - - for folder in folders_in_bench: - path = os.path.abspath(os.path.join(directory, folder)) - is_bench = is_bench and os.path.exists(path) - - return is_bench - - -def log(message, level=0): - levels = { - 0: ("blue", "INFO"), # normal - 1: ("green", "SUCCESS"), # success - 2: ("red", "ERROR"), # fail - 3: ("yellow", "WARN") # warn/suggest - } - loggers = { - 2: logger.error, - 3: logger.warning - } - color, prefix = levels.get(level, levels[0]) - level_logger = loggers.get(level, logger.info) - - level_logger(message) - click.secho(f"{prefix}: {message}", fg=color) - - -def safe_decode(string, encoding = 'utf-8'): - try: - string = string.decode(encoding) - except Exception: - pass - return string - - -def check_latest_version(): - if bench.VERSION.endswith("dev"): - return - - import requests - from semantic_version import Version - - try: - pypi_request = requests.get("https://pypi.org/pypi/frappe-bench/json") - except Exception: - # Exceptions thrown are defined in requests.exceptions - # ignore checking on all Exceptions - return - - if pypi_request.status_code == 200: - pypi_version_str = pypi_request.json().get('info').get('version') - pypi_version = Version(pypi_version_str) - local_version = Version(bench.VERSION) - - if pypi_version > local_version: - log(f"A newer version of bench is available: {local_version} → {pypi_version}") - - -def get_env_cmd(cmd, bench_path='.'): - return os.path.abspath(os.path.join(bench_path, 'env', 'bin', cmd)) - - -def pause_exec(seconds=10): - from time import sleep - - for i in range(seconds, 0, -1): - print(f"Will continue execution in {i} seconds...", end="\r") - sleep(1) - - print(" " * 40, end="\r") - -def init(path, apps_path=None, no_procfile=False, no_backups=False, - frappe_path=None, frappe_branch=None, verbose=False, clone_from=None, - skip_redis_config_generation=False, clone_without_update=False, skip_assets=False, - python='python3'): - """Initialize a new bench directory - - * create a bench directory in the given path - * setup logging for the bench - * setup env for the bench - * setup config (dir/pids/redis/procfile) for the bench - * setup patches.txt for bench - * clone & install frappe - * install python & node dependencies - * build assets - * setup backups crontab - """ - - # Use print("\033c", end="") to clear entire screen after each step and re-render each list - # another way => https://stackoverflow.com/a/44591228/10309266 - - from bench.app import get_app, install_apps_from_path - from bench.bench import Bench - - bench = Bench(path) - - bench.setup.dirs() - bench.setup.logging() - bench.setup.env(python=python) - bench.setup.config(redis=not skip_redis_config_generation, procfile=not no_procfile) - bench.setup.patches() - - # local apps - if clone_from: - clone_apps_from(bench_path=path, clone_from=clone_from, update_app=not clone_without_update) - - # remote apps - else: - frappe_path = frappe_path or 'https://github.com/frappe/frappe.git' - - get_app(frappe_path, branch=frappe_branch, bench_path=path, skip_assets=True, verbose=verbose) - - # fetch remote apps using config file - deprecate this! - if apps_path: - install_apps_from_path(apps_path, bench_path=path) - - if not skip_assets: - build_assets(bench_path=path) - - if not no_backups: - bench.setup.backups() - - -def update(pull=False, apps=None, patch=False, build=False, requirements=False, backup=True, compile=True, - force=False, reset=False, restart_supervisor=False, restart_systemd=False): - """command: bench update""" - import re - from bench import patches - from bench.app import is_version_upgrade, pull_apps, validate_branch - from bench.config.common_site_config import update_config - from bench.bench import Bench - - bench_path = os.path.abspath(".") - bench = Bench(bench_path) - patches.run(bench_path=bench_path) - conf = bench.conf - - if apps and not pull: - apps = [] - - clear_command_cache(bench_path='.') - - if conf.get('release_bench'): - print('Release bench detected, cannot update!') - sys.exit(1) - - if not (pull or patch or build or requirements): - pull, patch, build, requirements = True, True, True, True - - validate_branch() - version_upgrade = is_version_upgrade() - - if version_upgrade[0]: - if force: - log("""Force flag has been used for a major version change in Frappe and it's apps. -This will take significant time to migrate and might break custom apps.""", level=3) - else: - print(f"""This update will cause a major version change in Frappe/ERPNext from {version_upgrade[1]} to {version_upgrade[2]}. -This would take significant time to migrate and might break custom apps.""") - click.confirm('Do you want to continue?', abort=True) - - if not reset and conf.get('shallow_clone'): - log("""shallow_clone is set in your bench config. -However without passing the --reset flag, your repositories will be unshallowed. -To avoid this, cancel this operation and run `bench update --reset`. - -Consider the consequences of `git reset --hard` on your apps before you run that. -To avoid seeing this warning, set shallow_clone to false in your common_site_config.json - """, level=3) - pause_exec(seconds=10) - - if version_upgrade[0] or (not version_upgrade[0] and force): - validate_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) - conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 }) - update_config(conf, bench_path=bench_path) - - if backup: - print('Backing up sites...') - backup_all_sites(bench_path=bench_path) - - if apps: - apps = [app.strip() for app in re.split(",| ", apps) if app] - - if pull: - print('Updating apps source...') - pull_apps(apps=apps, bench_path=bench_path, reset=reset) - - if requirements: - print('Setting up requirements...') - update_requirements(bench_path=bench_path) - update_node_packages(bench_path=bench_path) - - if patch: - print('Patching sites...') - patch_sites(bench_path=bench_path) - - if build: - print('Building assets...') - build_assets(bench_path=bench_path) - - if version_upgrade[0] or (not version_upgrade[0] and force): - post_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) - - if pull and compile: - from compileall import compile_dir - - print('Compiling Python files...') - apps_dir = os.path.join(bench_path, 'apps') - compile_dir(apps_dir, quiet=1, rx=re.compile('.*node_modules.*')) - - if restart_supervisor or conf.get('restart_supervisor_on_update'): - restart_supervisor_processes(bench_path=bench_path) - - if restart_systemd or conf.get('restart_systemd_on_update'): - restart_systemd_processes(bench_path=bench_path) - - conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 }) - update_config(conf, bench_path=bench_path) - - print("_" * 80 + "\nBench: Deployment tool for Frappe and Frappe Applications (https://frappe.io/bench).\nOpen source depends on your contributions, so do give back by submitting bug reports, patches and fixes and be a part of the community :)") - - -def clone_apps_from(bench_path, clone_from, update_app=True): - from bench.app import install_app - print(f'Copying apps from {clone_from}...') - subprocess.check_output(['cp', '-R', os.path.join(clone_from, 'apps'), bench_path]) - - node_modules_path = os.path.join(clone_from, 'node_modules') - if os.path.exists(node_modules_path): - print(f'Copying node_modules from {clone_from}...') - subprocess.check_output(['cp', '-R', node_modules_path, bench_path]) - - def setup_app(app): - # run git reset --hard in each branch, pull latest updates and install_app - app_path = os.path.join(bench_path, 'apps', app) - - # remove .egg-ino - subprocess.check_output(['rm', '-rf', app + '.egg-info'], cwd=app_path) - - if update_app and os.path.exists(os.path.join(app_path, '.git')): - remotes = subprocess.check_output(['git', 'remote'], cwd=app_path).strip().split() - if 'upstream' in remotes: - remote = 'upstream' - else: - remote = remotes[0] - print(f'Cleaning up {app}') - branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], cwd=app_path).strip() - subprocess.check_output(['git', 'reset', '--hard'], cwd=app_path) - subprocess.check_output(['git', 'pull', '--rebase', remote, branch], cwd=app_path) - - install_app(app, bench_path, restart_bench=False) - - with open(os.path.join(clone_from, 'sites', 'apps.txt'), 'r') as f: - apps = f.read().splitlines() - - for app in apps: - setup_app(app) - - -def exec_cmd(cmd, cwd='.', env=None): - if env: - env.update(os.environ.copy()) - - click.secho(f"$ {cmd}", fg='bright_black') - - cwd_info = f"cd {cwd} && " if cwd != "." else "" - cmd_log = f"{cwd_info}{cmd}" - logger.debug(cmd_log) - cmd = split(cmd) - return_code = subprocess.call(cmd, cwd=cwd, universal_newlines=True, env=env) - if return_code: - logger.warning(f"{cmd_log} executed with exit code {return_code}") - - -def which(executable, raise_err=False): - from distutils.spawn import find_executable - - exec_ = find_executable(executable) - - if not exec_ and raise_err: - raise ValueError(f'{executable} not found.') - - return exec_ - - -def get_venv_path(): - venv = which('virtualenv') - - if not venv: - current_python = sys.executable - with open(os.devnull, "wb") as devnull: - is_venv_installed = not subprocess.call([current_python, "-m", "venv", "--help"], stdout=devnull) - if is_venv_installed: - venv = f"{current_python} -m venv" - - return venv or log("virtualenv cannot be found", level=2) - - -def setup_socketio(bench_path='.'): - exec_cmd("npm install socket.io redis express superagent cookie babel-core less chokidar \ - babel-cli babel-preset-es2015 babel-preset-es2016 babel-preset-es2017 babel-preset-babili", cwd=bench_path) - - -def patch_sites(bench_path='.'): - from bench.bench import Bench - bench = Bench(bench_path) - - for site in bench.sites: - try: - migrate_site(site, bench_path=bench_path) - except subprocess.CalledProcessError: - raise PatchError - - -def build_assets(bench_path='.', app=None): - command = 'bench build' - if app: - command += f' --app {app}' - exec_cmd(command, cwd=bench_path, env={"BENCH_DEVELOPER": "1"}) - - -def remove_backups_crontab(bench_path='.'): - from crontab import CronTab - from bench.bench import Bench - - logger.log('removing backup cronjob') - - bench_dir = os.path.abspath(bench_path) - user = Bench(bench_dir).conf.get('frappe_user') - logfile = os.path.join(bench_dir, 'logs', 'backup.log') - system_crontab = CronTab(user=user) - backup_command = f"cd {bench_dir} && {sys.argv[0]} --verbose --site all backup" - job_command = f"{backup_command} >> {logfile} 2>&1" - - system_crontab.remove_all(command=job_command) - - -def setup_sudoers(user): - if not os.path.exists('/etc/sudoers.d'): - os.makedirs('/etc/sudoers.d') - - set_permissions = False - if not os.path.exists('/etc/sudoers'): - set_permissions = True - - with open('/etc/sudoers', 'a') as f: - f.write('\n#includedir /etc/sudoers.d\n') - - if set_permissions: - os.chmod('/etc/sudoers', 0o440) - - template = bench.config.env().get_template('frappe_sudoers') - frappe_sudoers = template.render(**{ - 'user': user, - 'service': which('service'), - 'systemctl': which('systemctl'), - 'nginx': which('nginx'), - }) - frappe_sudoers = safe_decode(frappe_sudoers) - - with open(sudoers_file, 'w') as f: - f.write(frappe_sudoers) - - os.chmod(sudoers_file, 0o440) - log(f"Sudoers was set up for user {user}", level=1) - - -def setup_logging(bench_path='.'): - LOG_LEVEL = 15 - logging.addLevelName(LOG_LEVEL, "LOG") - def logv(self, message, *args, **kws): - if self.isEnabledFor(LOG_LEVEL): - self._log(LOG_LEVEL, message, args, **kws) - logging.Logger.log = logv - - if os.path.exists(os.path.join(bench_path, 'logs')): - log_file = os.path.join(bench_path, 'logs', 'bench.log') - hdlr = logging.FileHandler(log_file) - else: - hdlr = logging.NullHandler() - - logger = logging.getLogger(bench.PROJECT_NAME) - formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') - hdlr.setFormatter(formatter) - logger.addHandler(hdlr) - logger.setLevel(logging.DEBUG) - - return logger - - -def get_process_manager(): - for proc_man in ['honcho', 'foreman', 'forego']: - proc_man_path = which(proc_man) - if proc_man_path: - return proc_man_path - - -def start(no_dev=False, concurrency=None, procfile=None, no_prefix=False): - program = get_process_manager() - if not program: - raise Exception("No process manager found") - os.environ['PYTHONUNBUFFERED'] = "true" - if not no_dev: - os.environ['DEV_SERVER'] = "true" - - command = [program, 'start'] - if concurrency: - command.extend(['-c', concurrency]) - - if procfile: - command.extend(['-f', procfile]) - - if no_prefix: - command.extend(['--no-prefix']) - - os.execv(program, command) - - -def get_git_version(): - '''returns git version from `git --version` - extracts version number from string `get version 1.9.1` etc''' - version = get_cmd_output("git --version") - version = safe_decode(version) - version = version.strip().split()[2] - version = '.'.join(version.split('.')[0:2]) - return float(version) - - -def check_git_for_shallow_clone(): - from bench.bench import Bench - config = Bench('.').conf - - if config: - if config.get('release_bench'): - return False - - if not config.get('shallow_clone'): - return False - - git_version = get_git_version() - if git_version > 1.9: - return True - - -def get_cmd_output(cmd, cwd='.', _raise=True): - output = "" - try: - output = subprocess.check_output(cmd, cwd=cwd, shell=True, stderr=subprocess.PIPE).strip() - except subprocess.CalledProcessError as e: - if e.output: - output = e.output - elif _raise: - raise - return safe_decode(output) - - -def restart_supervisor_processes(bench_path='.', web_workers=False): - from bench.bench import Bench - - bench = Bench(bench_path) - conf = bench.conf - cmd = conf.get('supervisor_restart_cmd') - bench_name = get_bench_name(bench_path) - - if cmd: - bench.run(cmd) - - else: - supervisor_status = get_cmd_output('supervisorctl status', cwd=bench_path) - supervisor_status = safe_decode(supervisor_status) - - if web_workers and f'{bench_name}-web:' in supervisor_status: - group = f'{bench_name}-web:\t' - - elif f'{bench_name}-workers:' in supervisor_status: - group = f'{bench_name}-workers: {bench_name}-web:' - - # backward compatibility - elif f'{bench_name}-processes:' in supervisor_status: - group = f'{bench_name}-processes:' - - # backward compatibility - else: - group = 'frappe:' - - bench.run(f"supervisorctl restart {group}") - - -def restart_systemd_processes(bench_path='.', web_workers=False): - bench_name = get_bench_name(bench_path) - exec_cmd(f'sudo systemctl stop -- $(systemctl show -p Requires {bench_name}.target | cut -d= -f2)') - exec_cmd(f'sudo systemctl start -- $(systemctl show -p Requires {bench_name}.target | cut -d= -f2)') - - -def update_env_pip(bench_path): - env_py = get_env_cmd("python") - exec_cmd(f"{env_py} -m pip install -q -U pip") - - -def update_requirements(bench_path='.'): - from bench.app import install_app - from bench.bench import Bench - - print('Installing applications...') - - update_env_pip(bench_path) - - for app in Bench(bench_path).apps: - install_app(app, bench_path=bench_path, skip_assets=True, restart_bench=False) - - -def update_python_packages(bench_path='.'): - from bench.bench import Bench - env_py = get_env_cmd("python") - print('Updating Python libraries...') - - update_env_pip(bench_path) - for app in Bench(bench_path).apps: - click.secho(f"\nInstalling python dependencies for {app}", fg="yellow") - app_path = os.path.join(bench_path, "apps", app) - exec_cmd(f"{env_py} -m pip install -q -U -e {app_path}", cwd=bench_path) - - -def update_node_packages(bench_path='.'): - print('Updating node packages...') - from bench.app import get_develop_version - from distutils.version import LooseVersion - v = LooseVersion(get_develop_version('frappe', bench_path = bench_path)) - - # After rollup was merged, frappe_version = 10.1 - # if develop_verion is 11 and up, only then install yarn - if v < LooseVersion('11.x.x-develop'): - update_npm_packages(bench_path) - else: - update_yarn_packages(bench_path) - - -def install_python_dev_dependencies(bench_path='.', apps=None): - from bench.bench import Bench - - if isinstance(apps, str): - apps = [apps] - elif apps is None: - apps = Bench(bench_path).apps - - env_py = get_env_cmd("python") - for app in apps: - app_path = os.path.join(bench_path, "apps", app) - dev_requirements_path = os.path.join(app_path, "dev-requirements.txt") - - if os.path.exists(dev_requirements_path): - log(f'Installing python development dependencies for {app}') - exec_cmd(f"{env_py} -m pip install -q -r {dev_requirements_path}", cwd=bench_path) - else: - log(f'dev-requirements.txt not found in {app}', level=3) - - -def update_yarn_packages(bench_path='.'): - apps_dir = os.path.join(bench_path, 'apps') - - if not which('yarn'): - print("Please install yarn using below command and try again.") - print("`npm install -g yarn`") - return - - for app in os.listdir(apps_dir): - app_path = os.path.join(apps_dir, app) - if os.path.exists(os.path.join(app_path, 'package.json')): - click.secho(f"\nInstalling node dependencies for {app}", fg="yellow") - exec_cmd('yarn install', cwd=app_path) - - -def update_npm_packages(bench_path='.'): - apps_dir = os.path.join(bench_path, 'apps') - package_json = {} - - for app in os.listdir(apps_dir): - package_json_path = os.path.join(apps_dir, app, 'package.json') - - if os.path.exists(package_json_path): - with open(package_json_path, "r") as f: - app_package_json = json.loads(f.read()) - # package.json is usually a dict in a dict - for key, value in app_package_json.items(): - if not key in package_json: - package_json[key] = value - else: - if isinstance(value, dict): - package_json[key].update(value) - elif isinstance(value, list): - package_json[key].extend(value) - else: - package_json[key] = value - - if package_json is {}: - with open(os.path.join(os.path.dirname(__file__), 'package.json'), 'r') as f: - package_json = json.loads(f.read()) - - with open(os.path.join(bench_path, 'package.json'), 'w') as f: - f.write(json.dumps(package_json, indent=1, sort_keys=True)) - - exec_cmd('npm install', cwd=bench_path) - - -def migrate_site(site, bench_path='.'): - run_frappe_cmd('--site', site, 'migrate', bench_path=bench_path) - - -def backup_site(site, bench_path='.'): - run_frappe_cmd('--site', site, 'backup', bench_path=bench_path) - - -def backup_all_sites(bench_path='.'): - from bench.bench import Bench - - for site in Bench(bench_path).sites: - backup_site(site, bench_path=bench_path) - - -def is_root(): - return os.getuid() == 0 - - -def set_mariadb_host(host, bench_path='.'): - update_common_site_config({'db_host': host}, bench_path=bench_path) - - -def set_redis_cache_host(host, bench_path='.'): - update_common_site_config({'redis_cache': f"redis://{host}"}, bench_path=bench_path) - - -def set_redis_queue_host(host, bench_path='.'): - update_common_site_config({'redis_queue': f"redis://{host}"}, bench_path=bench_path) - - -def set_redis_socketio_host(host, bench_path='.'): - update_common_site_config({'redis_socketio': f"redis://{host}"}, bench_path=bench_path) - - -def update_common_site_config(ddict, bench_path='.'): - update_json_file(os.path.join(bench_path, 'sites', 'common_site_config.json'), ddict) - - -def update_json_file(filename, ddict): - if os.path.exists(filename): - with open(filename, 'r') as f: - content = json.load(f) - - else: - content = {} - - content.update(ddict) - with open(filename, 'w') as f: - json.dump(content, f, indent=1, sort_keys=True) - - -def drop_privileges(uid_name='nobody', gid_name='nogroup'): - # from http://stackoverflow.com/a/2699996 - if os.getuid() != 0: - # We're not root so, like, whatever dude - return - - # Get the uid/gid from the name - running_uid = pwd.getpwnam(uid_name).pw_uid - running_gid = grp.getgrnam(gid_name).gr_gid - - # Remove group privileges - os.setgroups([]) - - # Try setting the new uid/gid - os.setgid(running_gid) - os.setuid(running_uid) - - # Ensure a very conservative umask - os.umask(0o22) - - -def fix_prod_setup_perms(bench_path='.', frappe_user=None): - from glob import glob - from bench.bench import Bench - - frappe_user = frappe_user or Bench(bench_path).conf.get('frappe_user') - - if not frappe_user: - print("frappe user not set") - sys.exit(1) - - globs = ["logs/*", "config/*"] - for glob_name in globs: - for path in glob(glob_name): - uid = pwd.getpwnam(frappe_user).pw_uid - gid = grp.getgrnam(frappe_user).gr_gid - os.chown(path, uid, gid) - - -def run_frappe_cmd(*args, **kwargs): - from bench.cli import from_command_line - - bench_path = kwargs.get('bench_path', '.') - f = get_env_cmd('python', bench_path=bench_path) - sites_dir = os.path.join(bench_path, 'sites') - - is_async = False if from_command_line else True - if is_async: - stderr = stdout = subprocess.PIPE - else: - stderr = stdout = None - - p = subprocess.Popen((f, '-m', 'frappe.utils.bench_helper', 'frappe') + args, - cwd=sites_dir, stdout=stdout, stderr=stderr) - - if is_async: - return_code = print_output(p) - else: - return_code = p.wait() - - if return_code > 0: - sys.exit(return_code) - - -def validate_upgrade(from_ver, to_ver, bench_path='.'): - if to_ver >= 6: - if not which('npm') and not (which('node') or which('nodejs')): - raise Exception("Please install nodejs and npm") - - -def post_upgrade(from_ver, to_ver, bench_path='.'): - from bench.config import redis - from bench.config.supervisor import generate_supervisor_config - from bench.config.nginx import make_nginx_conf - from bench.bench import Bench - - conf = Bench(bench_path).conf - print("-" * 80 + f"Your bench was upgraded to version {to_ver}") - - if conf.get('restart_supervisor_on_update'): - redis.generate_config(bench_path=bench_path) - generate_supervisor_config(bench_path=bench_path) - make_nginx_conf(bench_path=bench_path) - print( - "As you have setup your bench for production, you will have to reload configuration for " - "nginx and supervisor. To complete the migration, please run the following commands:" - "\nsudo service nginx restart" - "\nsudo supervisorctl reload" - ) - - -def update_translations_p(args): - import requests - - try: - update_translations(*args) - except requests.exceptions.HTTPError: - print('Download failed for', args[0], args[1]) - - -def download_translations_p(): - import multiprocessing - - pool = multiprocessing.Pool(multiprocessing.cpu_count()) - - langs = get_langs() - apps = ('frappe', 'erpnext') - args = list(itertools.product(apps, langs)) - - pool.map(update_translations_p, args) - - -def download_translations(): - langs = get_langs() - apps = ('frappe', 'erpnext') - for app, lang in itertools.product(apps, langs): - update_translations(app, lang) - - -def get_langs(): - lang_file = 'apps/frappe/frappe/geo/languages.json' - with open(lang_file) as f: - langs = json.loads(f.read()) - return [d['code'] for d in langs] - - -def update_translations(app, lang): - import requests - - translations_dir = os.path.join('apps', app, app, 'translations') - csv_file = os.path.join(translations_dir, lang + '.csv') - url = f"https://translate.erpnext.com/files/{app}-{lang}.csv" - r = requests.get(url, stream=True) - r.raise_for_status() - - with open(csv_file, 'wb') as f: - for chunk in r.iter_content(chunk_size=1024): - # filter out keep-alive new chunks - if chunk: - f.write(chunk) - f.flush() - - print('downloaded for', app, lang) - - -def print_output(p): - from select import select - - while p.poll() is None: - readx = select([p.stdout.fileno(), p.stderr.fileno()], [], [])[0] - send_buffer = [] - for fd in readx: - if fd == p.stdout.fileno(): - while 1: - buf = p.stdout.read(1) - if not len(buf): - break - if buf == '\r' or buf == '\n': - send_buffer.append(buf) - log_line(''.join(send_buffer), 'stdout') - send_buffer = [] - else: - send_buffer.append(buf) - - if fd == p.stderr.fileno(): - log_line(p.stderr.readline(), 'stderr') - return p.poll() - - -def log_line(data, stream): - if stream == 'stderr': - return sys.stderr.write(data) - return sys.stdout.write(data) - - -def get_bench_name(bench_path): - return os.path.basename(os.path.abspath(bench_path)) - - -def setup_fonts(): - import shutil - - fonts_path = os.path.join('/tmp', 'fonts') - - if os.path.exists('/etc/fonts_backup'): - return - - exec_cmd("git clone https://github.com/frappe/fonts.git", cwd='/tmp') - os.rename('/etc/fonts', '/etc/fonts_backup') - os.rename('/usr/share/fonts', '/usr/share/fonts_backup') - os.rename(os.path.join(fonts_path, 'etc_fonts'), '/etc/fonts') - os.rename(os.path.join(fonts_path, 'usr_share_fonts'), '/usr/share/fonts') - shutil.rmtree(fonts_path) - exec_cmd("fc-cache -fv") - - -def set_git_remote_url(git_url, bench_path='.'): - "Set app remote git url" - from bench.bench import Bench - - app = git_url.rsplit('/', 1)[1].rsplit('.', 1)[0] - - if app not in Bench(bench_path).apps: - print(f"No app named {app}") - sys.exit(1) - - app_dir = bench.app.get_repo_dir(app, bench_path=bench_path) - if os.path.exists(os.path.join(app_dir, '.git')): - exec_cmd(f"git remote set-url upstream {git_url}", cwd=app_dir) - - -def run_playbook(playbook_name, extra_vars=None, tag=None): - if not which('ansible'): - print("Ansible is needed to run this command, please install it using 'pip install ansible'") - sys.exit(1) - args = ['ansible-playbook', '-c', 'local', playbook_name, '-vvvv'] - - if extra_vars: - args.extend(['-e', json.dumps(extra_vars)]) - - if tag: - args.extend(['-t', tag]) - - subprocess.check_call(args, cwd=os.path.join(bench.__path__[0], 'playbooks')) - - -def find_benches(directory=None): - if not directory: - directory = os.path.expanduser("~") - elif os.path.exists(directory): - directory = os.path.abspath(directory) - else: - log("Directory doesn't exist", level=2) - sys.exit(1) - - if is_bench_directory(directory): - if os.path.curdir == directory: - print("You are in a bench directory!") - else: - print(f"{directory} is a bench directory!") - return - - benches = [] - for sub in os.listdir(directory): - sub = os.path.join(directory, sub) - if os.path.isdir(sub) and not os.path.islink(sub): - if is_bench_directory(sub): - print(f"{sub} found!") - benches.append(sub) - else: - benches.extend(find_benches(sub)) - - return benches - - -def migrate_env(python, backup=False): - import shutil - from urllib.parse import urlparse - from bench.bench import Bench - - bench = Bench(".") - nvenv = 'env' - path = os.getcwd() - python = which(python) - virtualenv = which('virtualenv') - pvenv = os.path.join(path, nvenv) - - # Clear Cache before Bench Dies. - try: - config = bench.conf - rredis = urlparse(config['redis_cache']) - redis = f"{which('redis-cli')} -p {rredis.port}" - - logger.log('Clearing Redis Cache...') - exec_cmd(f'{redis} FLUSHALL') - logger.log('Clearing Redis DataBase...') - exec_cmd(f'{redis} FLUSHDB') - except: - logger.warning('Please ensure Redis Connections are running or Daemonized.') - - # Backup venv: restore using `virtualenv --relocatable` if needed - if backup: - from datetime import datetime - - parch = os.path.join(path, 'archived_envs') - if not os.path.exists(parch): - os.mkdir(parch) - - source = os.path.join(path, 'env') - target = parch - - logger.log('Backing up Virtual Environment') - stamp = datetime.now().strftime('%Y%m%d_%H%M%S') - dest = os.path.join(path, str(stamp)) - - os.rename(source, dest) - shutil.move(dest, target) - - # Create virtualenv using specified python - venv_creation, packages_setup = 1, 1 - try: - logger.log(f'Setting up a New Virtual {python} Environment') - venv_creation = exec_cmd(f'{virtualenv} --python {python} {pvenv}') - - apps = ' '.join([f"-e {os.path.join('apps', app)}" for app in bench.apps]) - packages_setup = exec_cmd(f'{pvenv} -m pip install -q -U {apps}') - - logger.log(f'Migration Successful to {python}') - except: - if venv_creation or packages_setup: - logger.warning('Migration Error') - - -def is_dist_editable(dist): - """Is distribution an editable install?""" - for path_item in sys.path: - egg_link = os.path.join(path_item, dist + '.egg-link') - if os.path.isfile(egg_link): - return True - return False - - -def find_parent_bench(path): - """Checks if parent directories are benches""" - if is_bench_directory(directory=path): - return path - - home_path = os.path.expanduser("~") - root_path = os.path.abspath(os.sep) - - if path not in {home_path, root_path}: - # NOTE: the os.path.split assumes that given path is absolute - parent_dir = os.path.split(path)[0] - return find_parent_bench(parent_dir) - - -def generate_command_cache(bench_path='.'): - """Caches all available commands (even custom apps) via Frappe - Default caching behaviour: generated the first time any command (for a specific bench directory) - """ - - python = get_env_cmd('python', bench_path=bench_path) - sites_path = os.path.join(bench_path, 'sites') - - if os.path.exists(bench_cache_file): - os.remove(bench_cache_file) - - try: - output = get_cmd_output(f"{python} -m frappe.utils.bench_helper get-frappe-commands", cwd=sites_path) - with open(bench_cache_file, 'w') as f: - json.dump(eval(output), f) - return json.loads(output) - - except subprocess.CalledProcessError as e: - if hasattr(e, "stderr"): - print(e.stderr.decode('utf-8')) - - return [] - - -def clear_command_cache(bench_path='.'): - """Clears commands cached - Default invalidation behaviour: destroyed on each run of `bench update` - """ - - if os.path.exists(bench_cache_file): - os.remove(bench_cache_file) - else: - print("Bench command cache doesn't exist in this folder!") - - -def find_org(org_repo): - import requests - - org_repo = org_repo[0] - - for org in ["frappe", "erpnext"]: - res = requests.head(f'https://api.github.com/repos/{org}/{org_repo}') - if res.ok: - return org, org_repo - - raise InvalidRemoteException - - -def fetch_details_from_tag(_tag): - if not _tag: - raise Exception("Tag is not provided") - - app_tag = _tag.split("@") - org_repo = app_tag[0].split("/") - - try: - repo, tag = app_tag - except ValueError: - repo, tag = app_tag + [None] - - try: - org, repo = org_repo - except ValueError: - org, repo = find_org(org_repo) - - return org, repo, tag \ No newline at end of file diff --git a/bench/utils/__init__.py b/bench/utils/__init__.py new file mode 100644 index 00000000..95b6c68b --- /dev/null +++ b/bench/utils/__init__.py @@ -0,0 +1,400 @@ +# imports - standard imports +import json +import logging +import os +import subprocess +import re +import sys +from glob import glob +from shlex import split + +# imports - third party imports +import click + +# imports - module imports +import bench +from bench.exceptions import InvalidRemoteException + + +logger = logging.getLogger(bench.PROJECT_NAME) +bench_cache_file = '.bench.cmd' +paths_in_app = ('hooks.py', 'modules.txt', 'patches.txt', 'public') +paths_in_bench = ('apps', 'sites', 'config', 'logs', 'config/pids') +sudoers_file = '/etc/sudoers.d/frappe' + + +def is_bench_directory(directory=os.path.curdir): + is_bench = True + + for folder in paths_in_bench: + path = os.path.abspath(os.path.join(directory, folder)) + is_bench = is_bench and os.path.exists(path) + + return is_bench + + +def is_frappe_app(directory): + is_frappe_app = True + + for folder in paths_in_app: + if not is_frappe_app: + break + + path = glob(os.path.join(directory, "**", folder)) + is_frappe_app = is_frappe_app and path + + return bool(is_frappe_app) + + +def log(message, level=0): + levels = { + 0: ("blue", "INFO"), # normal + 1: ("green", "SUCCESS"), # success + 2: ("red", "ERROR"), # fail + 3: ("yellow", "WARN") # warn/suggest + } + loggers = { + 2: logger.error, + 3: logger.warning + } + color, prefix = levels.get(level, levels[0]) + level_logger = loggers.get(level, logger.info) + + level_logger(message) + click.secho(f"{prefix}: {message}", fg=color) + + +def check_latest_version(): + if bench.VERSION.endswith("dev"): + return + + import requests + from semantic_version import Version + + try: + pypi_request = requests.get("https://pypi.org/pypi/frappe-bench/json") + except Exception: + # Exceptions thrown are defined in requests.exceptions + # ignore checking on all Exceptions + return + + if pypi_request.status_code == 200: + pypi_version_str = pypi_request.json().get('info').get('version') + pypi_version = Version(pypi_version_str) + local_version = Version(bench.VERSION) + + if pypi_version > local_version: + log(f"A newer version of bench is available: {local_version} → {pypi_version}") + + +def pause_exec(seconds=10): + from time import sleep + + for i in range(seconds, 0, -1): + print(f"Will continue execution in {i} seconds...", end="\r") + sleep(1) + + print(" " * 40, end="\r") + + +def exec_cmd(cmd, cwd='.', env=None): + if env: + env.update(os.environ.copy()) + + click.secho(f"$ {cmd}", fg='bright_black') + + cwd_info = f"cd {cwd} && " if cwd != "." else "" + cmd_log = f"{cwd_info}{cmd}" + logger.debug(cmd_log) + cmd = split(cmd) + return_code = subprocess.call(cmd, cwd=cwd, universal_newlines=True, env=env) + if return_code: + logger.warning(f"{cmd_log} executed with exit code {return_code}") + + +def which(executable, raise_err=False): + from shutil import which + + exec_ = which(executable) + + if not exec_ and raise_err: + raise ValueError(f'{executable} not found.') + + return exec_ + + +def setup_logging(bench_path='.'): + LOG_LEVEL = 15 + logging.addLevelName(LOG_LEVEL, "LOG") + def logv(self, message, *args, **kws): + if self.isEnabledFor(LOG_LEVEL): + self._log(LOG_LEVEL, message, args, **kws) + logging.Logger.log = logv + + if os.path.exists(os.path.join(bench_path, 'logs')): + log_file = os.path.join(bench_path, 'logs', 'bench.log') + hdlr = logging.FileHandler(log_file) + else: + hdlr = logging.NullHandler() + + logger = logging.getLogger(bench.PROJECT_NAME) + formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') + hdlr.setFormatter(formatter) + logger.addHandler(hdlr) + logger.setLevel(logging.DEBUG) + + return logger + + +def get_process_manager(): + for proc_man in ['honcho', 'foreman', 'forego']: + proc_man_path = which(proc_man) + if proc_man_path: + return proc_man_path + + +def get_git_version(): + '''returns git version from `git --version` + extracts version number from string `get version 1.9.1` etc''' + version = get_cmd_output("git --version") + version = version.strip().split()[2] + version = '.'.join(version.split('.')[0:2]) + return float(version) + + +def get_cmd_output(cmd, cwd='.', _raise=True): + output = "" + try: + output = subprocess.check_output(cmd, cwd=cwd, shell=True, stderr=subprocess.PIPE, encoding="utf-8").strip() + except subprocess.CalledProcessError as e: + if e.output: + output = e.output + elif _raise: + raise + return output + + +def is_root(): + return os.getuid() == 0 + + +def run_frappe_cmd(*args, **kwargs): + from bench.cli import from_command_line + + bench_path = kwargs.get('bench_path', '.') + f = get_env_cmd('python', bench_path=bench_path) + sites_dir = os.path.join(bench_path, 'sites') + + is_async = False if from_command_line else True + if is_async: + stderr = stdout = subprocess.PIPE + else: + stderr = stdout = None + + p = subprocess.Popen((f, '-m', 'frappe.utils.bench_helper', 'frappe') + args, + cwd=sites_dir, stdout=stdout, stderr=stderr) + + if is_async: + return_code = print_output(p) + else: + return_code = p.wait() + + if return_code > 0: + sys.exit(return_code) + + +def print_output(p): + from select import select + + while p.poll() is None: + readx = select([p.stdout.fileno(), p.stderr.fileno()], [], [])[0] + send_buffer = [] + for fd in readx: + if fd == p.stdout.fileno(): + while 1: + buf = p.stdout.read(1) + if not len(buf): + break + if buf == '\r' or buf == '\n': + send_buffer.append(buf) + log_line(''.join(send_buffer), 'stdout') + send_buffer = [] + else: + send_buffer.append(buf) + + if fd == p.stderr.fileno(): + log_line(p.stderr.readline(), 'stderr') + return p.poll() + + +def log_line(data, stream): + if stream == 'stderr': + return sys.stderr.write(data) + return sys.stdout.write(data) + + +def get_bench_name(bench_path): + return os.path.basename(os.path.abspath(bench_path)) + + +def set_git_remote_url(git_url, bench_path='.'): + "Set app remote git url" + from bench.bench import Bench + + app = git_url.rsplit('/', 1)[1].rsplit('.', 1)[0] + + if app not in Bench(bench_path).apps: + print(f"No app named {app}") + sys.exit(1) + + app_dir = bench.app.get_repo_dir(app, bench_path=bench_path) + if os.path.exists(os.path.join(app_dir, '.git')): + exec_cmd(f"git remote set-url upstream {git_url}", cwd=app_dir) + + +def run_playbook(playbook_name, extra_vars=None, tag=None): + if not which('ansible'): + print("Ansible is needed to run this command, please install it using 'pip install ansible'") + sys.exit(1) + args = ['ansible-playbook', '-c', 'local', playbook_name, '-vvvv'] + + if extra_vars: + args.extend(['-e', json.dumps(extra_vars)]) + + if tag: + args.extend(['-t', tag]) + + subprocess.check_call(args, cwd=os.path.join(bench.__path__[0], 'playbooks')) + + +def find_benches(directory=None): + if not directory: + directory = os.path.expanduser("~") + elif os.path.exists(directory): + directory = os.path.abspath(directory) + else: + log("Directory doesn't exist", level=2) + sys.exit(1) + + if is_bench_directory(directory): + if os.path.curdir == directory: + print("You are in a bench directory!") + else: + print(f"{directory} is a bench directory!") + return + + benches = [] + for sub in os.listdir(directory): + sub = os.path.join(directory, sub) + if os.path.isdir(sub) and not os.path.islink(sub): + if is_bench_directory(sub): + print(f"{sub} found!") + benches.append(sub) + else: + benches.extend(find_benches(sub)) + + return benches + + +def is_dist_editable(dist): + """Is distribution an editable install?""" + for path_item in sys.path: + egg_link = os.path.join(path_item, dist + '.egg-link') + if os.path.isfile(egg_link): + return True + return False + + +def find_parent_bench(path): + """Checks if parent directories are benches""" + if is_bench_directory(directory=path): + return path + + home_path = os.path.expanduser("~") + root_path = os.path.abspath(os.sep) + + if path not in {home_path, root_path}: + # NOTE: the os.path.split assumes that given path is absolute + parent_dir = os.path.split(path)[0] + return find_parent_bench(parent_dir) + + +def generate_command_cache(bench_path='.'): + """Caches all available commands (even custom apps) via Frappe + Default caching behaviour: generated the first time any command (for a specific bench directory) + """ + + python = get_env_cmd('python', bench_path=bench_path) + sites_path = os.path.join(bench_path, 'sites') + + if os.path.exists(bench_cache_file): + os.remove(bench_cache_file) + + try: + output = get_cmd_output(f"{python} -m frappe.utils.bench_helper get-frappe-commands", cwd=sites_path) + with open(bench_cache_file, 'w') as f: + json.dump(eval(output), f) + return json.loads(output) + + except subprocess.CalledProcessError as e: + if hasattr(e, "stderr"): + print(e.stderr.decode('utf-8')) + + return [] + + +def clear_command_cache(bench_path='.'): + """Clears commands cached + Default invalidation behaviour: destroyed on each run of `bench update` + """ + + if os.path.exists(bench_cache_file): + os.remove(bench_cache_file) + else: + print("Bench command cache doesn't exist in this folder!") + + +def find_org(org_repo): + import requests + + org_repo = org_repo[0] + + for org in ["frappe", "erpnext"]: + res = requests.head(f'https://api.github.com/repos/{org}/{org_repo}') + if res.ok: + return org, org_repo + + raise InvalidRemoteException + + +def fetch_details_from_tag(_tag): + if not _tag: + raise Exception("Tag is not provided") + + app_tag = _tag.split("@") + org_repo = app_tag[0].split("/") + + try: + repo, tag = app_tag + except ValueError: + repo, tag = app_tag + [None] + + try: + org, repo = org_repo + except ValueError: + org, repo = find_org(org_repo) + + return org, repo, tag + + +def is_git_url(url): + # modified to allow without the tailing .git from https://github.com/jonschlinkert/is-git-url.git + pattern = r"(?:git|ssh|https?|\w*@[-\w.]+):(\/\/)?(.*?)(\.git)?(\/?|\#[-\d\w._]+?)$" + return bool(re.match(pattern, url)) + + +# to avoid circular imports +from .app import * +from .bench import * +from .system import * +from .translation import * diff --git a/bench/utils/app.py b/bench/utils/app.py new file mode 100644 index 00000000..63bdf786 --- /dev/null +++ b/bench/utils/app.py @@ -0,0 +1,189 @@ +import os +import re +from setuptools.config import read_configuration +import bench +import sys +import subprocess +from bench.exceptions import InvalidRemoteException, InvalidBranchException, CommandFailedError + + +def is_version_upgrade(app='frappe', bench_path='.', branch=None): + upstream_version = get_upstream_version(app=app, branch=branch, bench_path=bench_path) + + if not upstream_version: + raise InvalidBranchException(f'Specified branch of app {app} is not in upstream remote') + + local_version = get_major_version(get_current_version(app, bench_path=bench_path)) + upstream_version = get_major_version(upstream_version) + + if upstream_version > local_version: + return (True, local_version, upstream_version) + + return (False, local_version, upstream_version) + + +def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrade=True): + import git + import importlib + from bench.utils import update_requirements, update_node_packages, backup_all_sites, patch_sites, post_upgrade + + apps_dir = os.path.join(bench_path, 'apps') + version_upgrade = (False,) + switched_apps = [] + + if not apps: + apps = [name for name in os.listdir(apps_dir) + if os.path.isdir(os.path.join(apps_dir, name))] + if branch=="v4.x.x": + apps.append('shopping_cart') + + for app in apps: + app_dir = os.path.join(apps_dir, app) + + if not os.path.exists(app_dir): + bench.utils.log(f"{app} does not exist!", level=2) + continue + + repo = git.Repo(app_dir) + unshallow_flag = os.path.exists(os.path.join(app_dir, ".git", "shallow")) + bench.utils.log(f"Fetching upstream {'unshallow ' if unshallow_flag else ''}for {app}") + + bench.utils.exec_cmd("git remote set-branches upstream '*'", cwd=app_dir) + bench.utils.exec_cmd(f"git fetch --all{' --unshallow' if unshallow_flag else ''} --quiet", cwd=app_dir) + + if check_upgrade: + version_upgrade = is_version_upgrade(app=app, bench_path=bench_path, branch=branch) + if version_upgrade[0] and not upgrade: + bench.utils.log(f"Switching to {branch} will cause upgrade from {version_upgrade[1]} to {version_upgrade[2]}. Pass --upgrade to confirm", level=2) + sys.exit(1) + + print("Switching for "+app) + bench.utils.exec_cmd(f"git checkout -f {branch}", cwd=app_dir) + + if str(repo.active_branch) == branch: + switched_apps.append(app) + else: + bench.utils.log(f"Switching branches failed for: {app}", level=2) + + if switched_apps: + bench.utils.log("Successfully switched branches for: " + ", ".join(switched_apps), level=1) + print('Please run `bench update --patch` to be safe from any differences in database schema') + + if version_upgrade[0] and upgrade: + update_requirements() + update_node_packages() + importlib.reload(bench.utils) + backup_all_sites() + patch_sites() + build_assets() + post_upgrade(version_upgrade[1], version_upgrade[2]) + + +def switch_to_branch(branch=None, apps=None, bench_path='.', upgrade=False): + switch_branch(branch, apps=apps, bench_path=bench_path, upgrade=upgrade) + +def switch_to_develop(apps=None, bench_path='.', upgrade=True): + switch_branch('develop', apps=apps, bench_path=bench_path, upgrade=upgrade) + +def get_version_from_string(contents, field='__version__'): + match = re.search(r"^(\s*%s\s*=\s*['\\\"])(.+?)(['\"])(?sm)" % field, contents) + return match.group(2) + +def get_major_version(version): + import semantic_version + + return semantic_version.Version(version).major + +def get_develop_version(app, bench_path='.'): + repo_dir = get_repo_dir(app, bench_path=bench_path) + with open(os.path.join(repo_dir, os.path.basename(repo_dir), 'hooks.py')) as f: + return get_version_from_string(f.read(), field='develop_version') + +def get_upstream_version(app, branch=None, bench_path='.'): + repo_dir = get_repo_dir(app, bench_path=bench_path) + if not branch: + branch = get_current_branch(app, bench_path=bench_path) + + try: + subprocess.call(f'git fetch --depth=1 --no-tags upstream {branch}', shell=True, cwd=repo_dir) + except CommandFailedError: + raise InvalidRemoteException(f'Failed to fetch from remote named upstream for {app}') + + try: + contents = subprocess.check_output(f'git show upstream/{branch}:{app}/__init__.py', + shell=True, cwd=repo_dir, stderr=subprocess.STDOUT) + contents = contents.decode('utf-8') + except subprocess.CalledProcessError as e: + if b"Invalid object" in e.output: + return None + else: + raise + return get_version_from_string(contents) + + +def get_current_frappe_version(bench_path='.'): + try: + return get_major_version(get_current_version('frappe', bench_path=bench_path)) + except IOError: + return 0 + +def get_current_branch(app, bench_path='.'): + repo_dir = get_repo_dir(app, bench_path=bench_path) + return get_cmd_output("basename $(git symbolic-ref -q HEAD)", cwd=repo_dir) + +def get_remote(app, bench_path='.'): + repo_dir = get_repo_dir(app, bench_path=bench_path) + contents = subprocess.check_output(['git', 'remote', '-v'], cwd=repo_dir, stderr=subprocess.STDOUT) + contents = contents.decode('utf-8') + if re.findall('upstream[\s]+', contents): + return 'upstream' + elif not contents: + # if contents is an empty string => remote doesn't exist + return False + else: + # get the first remote + return contents.splitlines()[0].split()[0] + + +def get_app_name(bench_path, repo_name): + app_name = None + apps_path = os.path.join(os.path.abspath(bench_path), 'apps') + config_path = os.path.join(apps_path, repo_name, 'setup.cfg') + if os.path.exists(config_path): + config = read_configuration(config_path) + app_name = config.get('metadata', {}).get('name') + + if not app_name: + # retrieve app name from setup.py as fallback + app_path = os.path.join(apps_path, repo_name, 'setup.py') + with open(app_path, 'rb') as f: + app_name = re.search(r'name\s*=\s*[\'"](.*)[\'"]', f.read().decode('utf-8')).group(1) + + if app_name and repo_name != app_name: + os.rename(os.path.join(apps_path, repo_name), os.path.join(apps_path, app_name)) + return app_name + + return repo_name + + +def get_current_version(app, bench_path='.'): + current_version = None + repo_dir = get_repo_dir(app, bench_path=bench_path) + config_path = os.path.join(repo_dir, "setup.cfg") + init_path = os.path.join(repo_dir, os.path.basename(repo_dir), '__init__.py') + setup_path = os.path.join(repo_dir, 'setup.py') + + try: + if os.path.exists(config_path): + config = read_configuration(config_path) + current_version = config.get("metadata", {}).get("version") + if not current_version: + with open(init_path) as f: + current_version = get_version_from_string(f.read()) + + except AttributeError: + # backward compatibility + with open(setup_path) as f: + current_version = get_version_from_string(f.read(), field='version') + + return current_version diff --git a/bench/utils/bench.py b/bench/utils/bench.py new file mode 100644 index 00000000..a9e106ea --- /dev/null +++ b/bench/utils/bench.py @@ -0,0 +1,550 @@ +# imports - standard imports +import json +import os +import re +import subprocess +import sys +from json.decoder import JSONDecodeError + +# imports - third party imports +import click + +# imports - module imports +from bench.exceptions import PatchError, ValidationError + +# TODO: Fix this +from bench.utils import * + + +def get_env_cmd(cmd, bench_path='.'): + return os.path.abspath(os.path.join(bench_path, 'env', 'bin', cmd)) + + +def get_venv_path(): + venv = which('virtualenv') + + if not venv: + current_python = sys.executable + with open(os.devnull, "wb") as devnull: + is_venv_installed = not subprocess.call([current_python, "-m", "venv", "--help"], stdout=devnull) + if is_venv_installed: + venv = f"{current_python} -m venv" + + return venv or log("virtualenv cannot be found", level=2) + + +def update_env_pip(bench_path): + env_py = get_env_cmd("python") + exec_cmd(f"{env_py} -m pip install -q -U pip") + + +def update_requirements(bench_path='.'): + from bench.app import install_app + from bench.bench import Bench + + bench = Bench(bench_path) + apps = [app for app in bench.apps if app not in bench.excluded_apps] + + print(f"Updating env pip...") + + update_env_pip(bench_path) + + print(f"Installing {len(apps)} applications...") + + for app in apps: + install_app(app, bench_path=bench_path, skip_assets=True, restart_bench=False) + + +def update_python_packages(bench_path='.'): + from bench.bench import Bench + + bench = Bench(bench_path) + env_py = get_env_cmd("python") + apps = [app for app in bench.apps if app not in bench.excluded_apps] + + print('Updating Python libraries...') + + update_env_pip(bench_path) + + for app in apps: + click.secho(f"\nInstalling python dependencies for {app}", fg="yellow") + app_path = os.path.join(bench_path, "apps", app) + bench.run(f"{env_py} -m pip install -q -U -e {app_path}") + + +def update_node_packages(bench_path='.'): + print('Updating node packages...') + from bench.app import get_develop_version + from distutils.version import LooseVersion + v = LooseVersion(get_develop_version('frappe', bench_path = bench_path)) + + # After rollup was merged, frappe_version = 10.1 + # if develop_verion is 11 and up, only then install yarn + if v < LooseVersion('11.x.x-develop'): + update_npm_packages(bench_path) + else: + update_yarn_packages(bench_path) + + +def install_python_dev_dependencies(bench_path='.', apps=None): + from bench.bench import Bench + + bench = Bench(bench_path) + + if isinstance(apps, str): + apps = [apps] + elif apps is None: + apps = [app for app in bench.apps if app not in bench.excluded_apps] + + env_py = get_env_cmd("python") + for app in apps: + app_path = os.path.join(bench_path, "apps", app) + dev_requirements_path = os.path.join(app_path, "dev-requirements.txt") + + if os.path.exists(dev_requirements_path): + log(f'Installing python development dependencies for {app}') + bench.run(f"{env_py} -m pip install -q -r {dev_requirements_path}") + + +def update_yarn_packages(bench_path='.'): + from bench.bench import Bench + + bench = Bench(bench_path) + apps = [app for app in bench.apps if app not in bench.excluded_apps] + apps_dir = os.path.join(bench.name, 'apps') + + # TODO: Check for stuff like this early on only?? + if not which('yarn'): + print("Please install yarn using below command and try again.") + print("`npm install -g yarn`") + return + + for app in apps: + app_path = os.path.join(apps_dir, app) + if os.path.exists(os.path.join(app_path, 'package.json')): + click.secho(f"\nInstalling node dependencies for {app}", fg="yellow") + bench.run('yarn install', cwd=app_path) + + +def update_npm_packages(bench_path='.'): + apps_dir = os.path.join(bench_path, 'apps') + package_json = {} + + for app in os.listdir(apps_dir): + package_json_path = os.path.join(apps_dir, app, 'package.json') + + if os.path.exists(package_json_path): + with open(package_json_path, "r") as f: + app_package_json = json.loads(f.read()) + # package.json is usually a dict in a dict + for key, value in app_package_json.items(): + if not key in package_json: + package_json[key] = value + else: + if isinstance(value, dict): + package_json[key].update(value) + elif isinstance(value, list): + package_json[key].extend(value) + else: + package_json[key] = value + + if package_json is {}: + with open(os.path.join(os.path.dirname(__file__), 'package.json'), 'r') as f: + package_json = json.loads(f.read()) + + with open(os.path.join(bench_path, 'package.json'), 'w') as f: + f.write(json.dumps(package_json, indent=1, sort_keys=True)) + + exec_cmd('npm install', cwd=bench_path) + + +def migrate_env(python, backup=False): + import shutil + from urllib.parse import urlparse + from bench.bench import Bench + + bench = Bench(".") + nvenv = 'env' + path = os.getcwd() + python = which(python) + virtualenv = which('virtualenv') + pvenv = os.path.join(path, nvenv) + + # Clear Cache before Bench Dies. + try: + config = bench.conf + rredis = urlparse(config['redis_cache']) + redis = f"{which('redis-cli')} -p {rredis.port}" + + logger.log('Clearing Redis Cache...') + exec_cmd(f'{redis} FLUSHALL') + logger.log('Clearing Redis DataBase...') + exec_cmd(f'{redis} FLUSHDB') + except: + logger.warning('Please ensure Redis Connections are running or Daemonized.') + + # Backup venv: restore using `virtualenv --relocatable` if needed + if backup: + from datetime import datetime + + parch = os.path.join(path, 'archived', 'envs') + if not os.path.exists(parch): + os.mkdir(parch) + + source = os.path.join(path, 'env') + target = parch + + logger.log('Backing up Virtual Environment') + stamp = datetime.now().strftime('%Y%m%d_%H%M%S') + dest = os.path.join(path, str(stamp)) + + os.rename(source, dest) + shutil.move(dest, target) + + # Create virtualenv using specified python + venv_creation, packages_setup = 1, 1 + try: + logger.log(f'Setting up a New Virtual {python} Environment') + venv_creation = exec_cmd(f'{virtualenv} --python {python} {pvenv}') + + apps = ' '.join([f"-e {os.path.join('apps', app)}" for app in bench.apps]) + packages_setup = exec_cmd(f'{pvenv} -m pip install -q -U {apps}') + + logger.log(f'Migration Successful to {python}') + except: + if venv_creation or packages_setup: + logger.warning('Migration Error') + +def validate_upgrade(from_ver, to_ver, bench_path='.'): + if to_ver >= 6: + if not which('npm') and not (which('node') or which('nodejs')): + raise Exception("Please install nodejs and npm") + + +def post_upgrade(from_ver, to_ver, bench_path='.'): + from bench.config import redis + from bench.config.supervisor import generate_supervisor_config + from bench.config.nginx import make_nginx_conf + from bench.bench import Bench + + conf = Bench(bench_path).conf + print("-" * 80 + f"Your bench was upgraded to version {to_ver}") + + if conf.get('restart_supervisor_on_update'): + redis.generate_config(bench_path=bench_path) + generate_supervisor_config(bench_path=bench_path) + make_nginx_conf(bench_path=bench_path) + print( + "As you have setup your bench for production, you will have to reload configuration for " + "nginx and supervisor. To complete the migration, please run the following commands:" + "\nsudo service nginx restart" + "\nsudo supervisorctl reload" + ) + +def patch_sites(bench_path='.'): + from bench.bench import Bench + from bench.utils.system import migrate_site + + bench = Bench(bench_path) + + for site in bench.sites: + try: + migrate_site(site, bench_path=bench_path) + except subprocess.CalledProcessError: + raise PatchError + +def restart_supervisor_processes(bench_path='.', web_workers=False): + from bench.bench import Bench + + bench = Bench(bench_path) + conf = bench.conf + cmd = conf.get('supervisor_restart_cmd') + bench_name = get_bench_name(bench_path) + + if cmd: + bench.run(cmd) + + else: + supervisor_status = get_cmd_output('supervisorctl status', cwd=bench_path) + + if web_workers and f'{bench_name}-web:' in supervisor_status: + group = f'{bench_name}-web:\t' + + elif f'{bench_name}-workers:' in supervisor_status: + group = f'{bench_name}-workers: {bench_name}-web:' + + # backward compatibility + elif f'{bench_name}-processes:' in supervisor_status: + group = f'{bench_name}-processes:' + + # backward compatibility + else: + group = 'frappe:' + + bench.run(f"supervisorctl restart {group}") + + +def restart_systemd_processes(bench_path='.', web_workers=False): + bench_name = get_bench_name(bench_path) + exec_cmd(f'sudo systemctl stop -- $(systemctl show -p Requires {bench_name}.target | cut -d= -f2)') + exec_cmd(f'sudo systemctl start -- $(systemctl show -p Requires {bench_name}.target | cut -d= -f2)') + + +def build_assets(bench_path='.', app=None): + command = 'bench build' + if app: + command += f' --app {app}' + exec_cmd(command, cwd=bench_path, env={"BENCH_DEVELOPER": "1"}) + +def update(pull=False, apps=None, patch=False, build=False, requirements=False, backup=True, compile=True, + force=False, reset=False, restart_supervisor=False, restart_systemd=False): + """command: bench update""" + import re + from bench import patches + from bench.utils import clear_command_cache, pause_exec, log + from bench.utils.bench import restart_supervisor_processes, restart_systemd_processes + from bench.app import pull_apps + from bench.utils.app import is_version_upgrade + from bench.config.common_site_config import update_config + from bench.bench import Bench + + bench_path = os.path.abspath(".") + bench = Bench(bench_path) + patches.run(bench_path=bench_path) + conf = bench.conf + + if apps and not pull: + apps = [] + + clear_command_cache(bench_path='.') + + if conf.get('release_bench'): + print('Release bench detected, cannot update!') + sys.exit(1) + + if not (pull or patch or build or requirements): + pull, patch, build, requirements = True, True, True, True + + validate_branch() + version_upgrade = is_version_upgrade() + + if version_upgrade[0]: + if force: + log("""Force flag has been used for a major version change in Frappe and it's apps. +This will take significant time to migrate and might break custom apps.""", level=3) + else: + print(f"""This update will cause a major version change in Frappe/ERPNext from {version_upgrade[1]} to {version_upgrade[2]}. +This would take significant time to migrate and might break custom apps.""") + click.confirm('Do you want to continue?', abort=True) + + if not reset and conf.get('shallow_clone'): + log("""shallow_clone is set in your bench config. +However without passing the --reset flag, your repositories will be unshallowed. +To avoid this, cancel this operation and run `bench update --reset`. + +Consider the consequences of `git reset --hard` on your apps before you run that. +To avoid seeing this warning, set shallow_clone to false in your common_site_config.json + """, level=3) + pause_exec(seconds=10) + + if version_upgrade[0] or (not version_upgrade[0] and force): + validate_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) + conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 }) + update_config(conf, bench_path=bench_path) + + if backup: + print('Backing up sites...') + backup_all_sites(bench_path=bench_path) + + if apps: + apps = [app.strip() for app in re.split(",| ", apps) if app] + + if pull: + print('Updating apps source...') + pull_apps(apps=apps, bench_path=bench_path, reset=reset) + + if requirements: + print('Setting up requirements...') + update_requirements(bench_path=bench_path) + update_node_packages(bench_path=bench_path) + + if patch: + print('Patching sites...') + patch_sites(bench_path=bench_path) + + if build: + print('Building assets...') + build_assets(bench_path=bench_path) + + if version_upgrade[0] or (not version_upgrade[0] and force): + post_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) + + if pull and compile: + from compileall import compile_dir + + print('Compiling Python files...') + apps_dir = os.path.join(bench_path, 'apps') + compile_dir(apps_dir, quiet=1, rx=re.compile('.*node_modules.*')) + + if restart_supervisor or conf.get('restart_supervisor_on_update'): + restart_supervisor_processes(bench_path=bench_path) + + if restart_systemd or conf.get('restart_systemd_on_update'): + restart_systemd_processes(bench_path=bench_path) + + conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 }) + update_config(conf, bench_path=bench_path) + + print("_" * 80 + "\nBench: Deployment tool for Frappe and Frappe Applications (https://frappe.io/bench).\nOpen source depends on your contributions, so do give back by submitting bug reports, patches and fixes and be a part of the community :)") + + +def clone_apps_from(bench_path, clone_from, update_app=True): + from bench.app import install_app + print(f'Copying apps from {clone_from}...') + subprocess.check_output(['cp', '-R', os.path.join(clone_from, 'apps'), bench_path]) + + node_modules_path = os.path.join(clone_from, 'node_modules') + if os.path.exists(node_modules_path): + print(f'Copying node_modules from {clone_from}...') + subprocess.check_output(['cp', '-R', node_modules_path, bench_path]) + + def setup_app(app): + # run git reset --hard in each branch, pull latest updates and install_app + app_path = os.path.join(bench_path, 'apps', app) + + # remove .egg-ino + subprocess.check_output(['rm', '-rf', app + '.egg-info'], cwd=app_path) + + if update_app and os.path.exists(os.path.join(app_path, '.git')): + remotes = subprocess.check_output(['git', 'remote'], cwd=app_path).strip().split() + if 'upstream' in remotes: + remote = 'upstream' + else: + remote = remotes[0] + print(f'Cleaning up {app}') + branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], cwd=app_path).strip() + subprocess.check_output(['git', 'reset', '--hard'], cwd=app_path) + subprocess.check_output(['git', 'pull', '--rebase', remote, branch], cwd=app_path) + + install_app(app, bench_path, restart_bench=False) + + with open(os.path.join(clone_from, 'sites', 'apps.txt'), 'r') as f: + apps = f.read().splitlines() + + for app in apps: + setup_app(app) + + +def remove_backups_crontab(bench_path='.'): + from crontab import CronTab + from bench.bench import Bench + + logger.log('removing backup cronjob') + + bench_dir = os.path.abspath(bench_path) + user = Bench(bench_dir).conf.get('frappe_user') + logfile = os.path.join(bench_dir, 'logs', 'backup.log') + system_crontab = CronTab(user=user) + backup_command = f"cd {bench_dir} && {sys.argv[0]} --verbose --site all backup" + job_command = f"{backup_command} >> {logfile} 2>&1" + + system_crontab.remove_all(command=job_command) + +def set_mariadb_host(host, bench_path='.'): + update_common_site_config({'db_host': host}, bench_path=bench_path) + + +def set_redis_cache_host(host, bench_path='.'): + update_common_site_config({'redis_cache': f"redis://{host}"}, bench_path=bench_path) + + +def set_redis_queue_host(host, bench_path='.'): + update_common_site_config({'redis_queue': f"redis://{host}"}, bench_path=bench_path) + + +def set_redis_socketio_host(host, bench_path='.'): + update_common_site_config({'redis_socketio': f"redis://{host}"}, bench_path=bench_path) + + +def update_common_site_config(ddict, bench_path='.'): + filename = os.path.join(bench_path, 'sites', 'common_site_config.json') + + if os.path.exists(filename): + with open(filename, 'r') as f: + content = json.load(f) + + else: + content = {} + + content.update(ddict) + with open(filename, 'w') as f: + json.dump(content, f, indent=1, sort_keys=True) + + +def validate_app_installed_on_sites(app, bench_path="."): + print("Checking if app installed on active sites...") + ret = check_app_installed(app, bench_path=bench_path) + + if ret is None: + check_app_installed_legacy(app, bench_path=bench_path) + else: + return ret + + +def check_app_installed(app, bench_path="."): + try: + out = subprocess.check_output( + ["bench", "--site", "all", "list-apps", "--format", "json"], + stderr=open(os.devnull, "wb"), + cwd=bench_path, + ).decode('utf-8') + except subprocess.CalledProcessError: + return None + + try: + apps_sites_dict = json.loads(out) + except JSONDecodeError: + return None + + for site, apps in apps_sites_dict.items(): + if app in apps: + raise ValidationError(f"Cannot remove, app is installed on site: {site}") + + +def check_app_installed_legacy(app, bench_path="."): + site_path = os.path.join(bench_path, 'sites') + + for site in os.listdir(site_path): + req_file = os.path.join(site_path, site, 'site_config.json') + if os.path.exists(req_file): + out = subprocess.check_output(["bench", "--site", site, "list-apps"], cwd=bench_path).decode('utf-8') + if re.search(r'\b' + app + r'\b', out): + print(f"Cannot remove, app is installed on site: {site}") + sys.exit(1) + +def validate_branch(): + from bench.bench import Bench + from bench.utils.app import get_current_branch + + apps = Bench(".").apps + + installed_apps = set(apps) + check_apps = set(['frappe', 'erpnext']) + intersection_apps = installed_apps.intersection(check_apps) + + for app in intersection_apps: + branch = get_current_branch(app) + + if branch == "master": + print("""'master' branch is renamed to 'version-11' since 'version-12' release. +As of January 2020, the following branches are +version Frappe ERPNext +11 version-11 version-11 +12 version-12 version-12 +13 version-13 version-13 +14 develop develop + +Please switch to new branches to get future updates. +To switch to your required branch, run the following commands: bench switch-to-branch [branch-name]""") + + sys.exit(1) diff --git a/bench/utils/system.py b/bench/utils/system.py new file mode 100644 index 00000000..8f64431b --- /dev/null +++ b/bench/utils/system.py @@ -0,0 +1,189 @@ +# imports - standard imports +import grp +import os +import pwd +import sys + +# imports - module imports +import bench + +# TODO: Fix this +import bench.utils +from bench.utils import * + + +def init(path, apps_path=None, no_procfile=False, no_backups=False, + frappe_path=None, frappe_branch=None, verbose=False, clone_from=None, + skip_redis_config_generation=False, clone_without_update=False, skip_assets=False, + python='python3'): + """Initialize a new bench directory + + * create a bench directory in the given path + * setup logging for the bench + * setup env for the bench + * setup config (dir/pids/redis/procfile) for the bench + * setup patches.txt for bench + * clone & install frappe + * install python & node dependencies + * build assets + * setup backups crontab + """ + + # Use print("\033c", end="") to clear entire screen after each step and re-render each list + # another way => https://stackoverflow.com/a/44591228/10309266 + + from bench.app import get_app, install_apps_from_path + from bench.bench import Bench + + bench = Bench(path) + + bench.setup.dirs() + bench.setup.logging() + bench.setup.env(python=python) + bench.setup.config(redis=not skip_redis_config_generation, procfile=not no_procfile) + bench.setup.patches() + + # local apps + if clone_from: + clone_apps_from(bench_path=path, clone_from=clone_from, update_app=not clone_without_update) + + # remote apps + else: + frappe_path = frappe_path or 'https://github.com/frappe/frappe.git' + + get_app(frappe_path, branch=frappe_branch, bench_path=path, skip_assets=True, verbose=verbose) + + # fetch remote apps using config file - deprecate this! + if apps_path: + install_apps_from_path(apps_path, bench_path=path) + + if not skip_assets: + build_assets(bench_path=path) + + if not no_backups: + bench.setup.backups() + +def setup_sudoers(user): + if not os.path.exists('/etc/sudoers.d'): + os.makedirs('/etc/sudoers.d') + + set_permissions = False + if not os.path.exists('/etc/sudoers'): + set_permissions = True + + with open('/etc/sudoers', 'a') as f: + f.write('\n#includedir /etc/sudoers.d\n') + + if set_permissions: + os.chmod('/etc/sudoers', 0o440) + + template = bench.config.env().get_template('frappe_sudoers') + frappe_sudoers = template.render(**{ + 'user': user, + 'service': which('service'), + 'systemctl': which('systemctl'), + 'nginx': which('nginx'), + }) + + with open(sudoers_file, 'w') as f: + f.write(frappe_sudoers) + + os.chmod(sudoers_file, 0o440) + log(f"Sudoers was set up for user {user}", level=1) + + +def start(no_dev=False, concurrency=None, procfile=None, no_prefix=False, procman=None): + if procman: + program = which(procman) + else: + program = get_process_manager() + + if not program: + raise Exception("No process manager found") + + os.environ['PYTHONUNBUFFERED'] = "true" + if not no_dev: + os.environ['DEV_SERVER'] = "true" + + command = [program, 'start'] + if concurrency: + command.extend(['-c', concurrency]) + + if procfile: + command.extend(['-f', procfile]) + + if no_prefix: + command.extend(['--no-prefix']) + + os.execv(program, command) + + +def migrate_site(site, bench_path='.'): + run_frappe_cmd('--site', site, 'migrate', bench_path=bench_path) + + +def backup_site(site, bench_path='.'): + run_frappe_cmd('--site', site, 'backup', bench_path=bench_path) + + +def backup_all_sites(bench_path='.'): + from bench.bench import Bench + + for site in Bench(bench_path).sites: + backup_site(site, bench_path=bench_path) + + +def drop_privileges(uid_name='nobody', gid_name='nogroup'): + # from http://stackoverflow.com/a/2699996 + if os.getuid() != 0: + # We're not root so, like, whatever dude + return + + # Get the uid/gid from the name + running_uid = pwd.getpwnam(uid_name).pw_uid + running_gid = grp.getgrnam(gid_name).gr_gid + + # Remove group privileges + os.setgroups([]) + + # Try setting the new uid/gid + os.setgid(running_gid) + os.setuid(running_uid) + + # Ensure a very conservative umask + os.umask(0o22) + + +def fix_prod_setup_perms(bench_path='.', frappe_user=None): + from glob import glob + from bench.bench import Bench + + frappe_user = frappe_user or Bench(bench_path).conf.get('frappe_user') + + if not frappe_user: + print("frappe user not set") + sys.exit(1) + + globs = ["logs/*", "config/*"] + for glob_name in globs: + for path in glob(glob_name): + uid = pwd.getpwnam(frappe_user).pw_uid + gid = grp.getgrnam(frappe_user).gr_gid + os.chown(path, uid, gid) + + +def setup_fonts(): + import shutil + + fonts_path = os.path.join('/tmp', 'fonts') + + if os.path.exists('/etc/fonts_backup'): + return + + exec_cmd("git clone https://github.com/frappe/fonts.git", cwd='/tmp') + os.rename('/etc/fonts', '/etc/fonts_backup') + os.rename('/usr/share/fonts', '/usr/share/fonts_backup') + os.rename(os.path.join(fonts_path, 'etc_fonts'), '/etc/fonts') + os.rename(os.path.join(fonts_path, 'usr_share_fonts'), '/usr/share/fonts') + shutil.rmtree(fonts_path) + exec_cmd("fc-cache -fv") diff --git a/bench/utils/translation.py b/bench/utils/translation.py new file mode 100644 index 00000000..8e3f1b06 --- /dev/null +++ b/bench/utils/translation.py @@ -0,0 +1,61 @@ +# imports - standard imports +import itertools +import json +import os + +# TODO: Fix this +from bench.utils import * + + +def update_translations_p(args): + import requests + + try: + update_translations(*args) + except requests.exceptions.HTTPError: + print('Download failed for', args[0], args[1]) + + +def download_translations_p(): + import multiprocessing + + pool = multiprocessing.Pool(multiprocessing.cpu_count()) + + langs = get_langs() + apps = ('frappe', 'erpnext') + args = list(itertools.product(apps, langs)) + + pool.map(update_translations_p, args) + + +def download_translations(): + langs = get_langs() + apps = ('frappe', 'erpnext') + for app, lang in itertools.product(apps, langs): + update_translations(app, lang) + + +def get_langs(): + lang_file = 'apps/frappe/frappe/geo/languages.json' + with open(lang_file) as f: + langs = json.loads(f.read()) + return [d['code'] for d in langs] + + +def update_translations(app, lang): + import requests + + translations_dir = os.path.join('apps', app, app, 'translations') + csv_file = os.path.join(translations_dir, lang + '.csv') + url = f"https://translate.erpnext.com/files/{app}-{lang}.csv" + r = requests.get(url, stream=True) + r.raise_for_status() + + with open(csv_file, 'wb') as f: + for chunk in r.iter_content(chunk_size=1024): + # filter out keep-alive new chunks + if chunk: + f.write(chunk) + f.flush() + + print('downloaded for', app, lang) From 1f11cf48471b264b8db24b50c5d691316b29251c Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 12 Nov 2021 17:55:10 +0530 Subject: [PATCH 29/91] chore: Use newer import paths * Although the utils refactor was done in such a way that there was no need of changing import paths, a cleanup was due. * Got rid of duplicated utils * Remove unused imports --- bench/app.py | 215 +++------------------------------------- bench/commands/setup.py | 8 +- 2 files changed, 17 insertions(+), 206 deletions(-) diff --git a/bench/app.py b/bench/app.py index b823b94d..97dabf7e 100755 --- a/bench/app.py +++ b/bench/app.py @@ -1,23 +1,21 @@ # imports - standard imports import json -from json.decoder import JSONDecodeError import logging import os import re +import shutil import subprocess import sys # imports - third party imports import click -from setuptools.config import read_configuration # imports - module imports import bench from bench.bench import Bench -from bench.exceptions import InvalidRemoteException, InvalidBranchException, CommandFailedError -from bench.utils import exec_cmd, get_cmd_output, is_bench_directory, run_frappe_cmd -from bench.utils import build_assets, check_git_for_shallow_clone, fetch_details_from_tag, restart_supervisor_processes, restart_systemd_processes - +from bench.utils import exec_cmd, is_bench_directory, run_frappe_cmd, is_git_url, fetch_details_from_tag +from bench.utils.app import get_app_name +from bench.utils.bench import get_env_cmd, build_assets, restart_supervisor_processes, restart_systemd_processes logger = logging.getLogger(bench.PROJECT_NAME) @@ -119,11 +117,6 @@ def write_appstxt(apps, bench_path='.'): with open(os.path.join(bench_path, 'sites', 'apps.txt'), 'w') as f: return f.write('\n'.join(apps)) -def is_git_url(url): - # modified to allow without the tailing .git from https://github.com/jonschlinkert/is-git-url.git - pattern = r"(?:git|ssh|https?|\w*@[-\w.]+):(\/\/)?(.*?)(\.git)?(\/?|\#[-\d\w._]+?)$" - return bool(re.match(pattern, url)) - def get_excluded_apps(bench_path='.'): try: with open(os.path.join(bench_path, 'sites', 'excluded_apps.txt')) as f: @@ -423,191 +416,33 @@ Here are your choices: if is_shallow: s = " to safely pull remote changes." if not reset else "" print(f"Unshallowing {app}{s}") - exec_cmd(f"git fetch {remote} --unshallow", cwd=app_dir) + bench.run(f"git fetch {remote} --unshallow", cwd=app_dir) branch = get_current_branch(app, bench_path=bench_path) logger.log(f'pulling {app}') if reset: reset_cmd = f"git reset --hard {remote}/{branch}" if bench.conf.get('shallow_clone'): - exec_cmd(f"git fetch --depth=1 --no-tags {remote} {branch}", - cwd=app_dir) - exec_cmd(reset_cmd, cwd=app_dir) - exec_cmd("git reflog expire --all", cwd=app_dir) - exec_cmd("git gc --prune=all", cwd=app_dir) + bench.run(f"git fetch --depth=1 --no-tags {remote} {branch}", cwd=app_dir) + bench.run(reset_cmd, cwd=app_dir) + bench.run("git reflog expire --all", cwd=app_dir) + bench.run("git gc --prune=all", cwd=app_dir) else: - exec_cmd("git fetch --all", cwd=app_dir) - exec_cmd(reset_cmd, cwd=app_dir) + bench.run("git fetch --all", cwd=app_dir) + bench.run(reset_cmd, cwd=app_dir) else: - exec_cmd(f"git pull {rebase} {remote} {branch}", cwd=app_dir) - exec_cmd('find . -name "*.pyc" -delete', cwd=app_dir) + bench.run(f"git pull {rebase} {remote} {branch}", cwd=app_dir) + bench.run('find . -name "*.pyc" -delete', cwd=app_dir) -def is_version_upgrade(app='frappe', bench_path='.', branch=None): - upstream_version = get_upstream_version(app=app, branch=branch, bench_path=bench_path) - - if not upstream_version: - raise InvalidBranchException(f'Specified branch of app {app} is not in upstream remote') - - local_version = get_major_version(get_current_version(app, bench_path=bench_path)) - upstream_version = get_major_version(upstream_version) - - if upstream_version > local_version: - return (True, local_version, upstream_version) - - return (False, local_version, upstream_version) - -def get_current_frappe_version(bench_path='.'): - try: - return get_major_version(get_current_version('frappe', bench_path=bench_path)) - except IOError: - return 0 - -def get_current_branch(app, bench_path='.'): - repo_dir = get_repo_dir(app, bench_path=bench_path) - return get_cmd_output("basename $(git symbolic-ref -q HEAD)", cwd=repo_dir) - -def get_remote(app, bench_path='.'): - repo_dir = get_repo_dir(app, bench_path=bench_path) - contents = subprocess.check_output(['git', 'remote', '-v'], cwd=repo_dir, stderr=subprocess.STDOUT) - contents = contents.decode('utf-8') - if re.findall('upstream[\s]+', contents): - return 'upstream' - elif not contents: - # if contents is an empty string => remote doesn't exist - return False - else: - # get the first remote - return contents.splitlines()[0].split()[0] - def use_rq(bench_path): bench_path = os.path.abspath(bench_path) celery_app = os.path.join(bench_path, 'apps', 'frappe', 'frappe', 'celery_app.py') return not os.path.exists(celery_app) -def get_current_version(app, bench_path='.'): - current_version = None - repo_dir = get_repo_dir(app, bench_path=bench_path) - config_path = os.path.join(repo_dir, "setup.cfg") - init_path = os.path.join(repo_dir, os.path.basename(repo_dir), '__init__.py') - setup_path = os.path.join(repo_dir, 'setup.py') - - try: - if os.path.exists(config_path): - config = read_configuration(config_path) - current_version = config.get("metadata", {}).get("version") - if not current_version: - with open(init_path) as f: - current_version = get_version_from_string(f.read()) - - except AttributeError: - # backward compatibility - with open(setup_path) as f: - current_version = get_version_from_string(f.read(), field='version') - - return current_version - -def get_develop_version(app, bench_path='.'): - repo_dir = get_repo_dir(app, bench_path=bench_path) - with open(os.path.join(repo_dir, os.path.basename(repo_dir), 'hooks.py')) as f: - return get_version_from_string(f.read(), field='develop_version') - -def get_upstream_version(app, branch=None, bench_path='.'): - repo_dir = get_repo_dir(app, bench_path=bench_path) - if not branch: - branch = get_current_branch(app, bench_path=bench_path) - - try: - subprocess.call(f'git fetch --depth=1 --no-tags upstream {branch}', shell=True, cwd=repo_dir) - except CommandFailedError: - raise InvalidRemoteException(f'Failed to fetch from remote named upstream for {app}') - - try: - contents = subprocess.check_output(f'git show upstream/{branch}:{app}/__init__.py', - shell=True, cwd=repo_dir, stderr=subprocess.STDOUT) - contents = contents.decode('utf-8') - except subprocess.CalledProcessError as e: - if b"Invalid object" in e.output: - return None - else: - raise - return get_version_from_string(contents) - def get_repo_dir(app, bench_path='.'): return os.path.join(bench_path, 'apps', app) -def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrade=True): - import git - import importlib - from bench.utils import update_requirements, update_node_packages, backup_all_sites, patch_sites, post_upgrade - - apps_dir = os.path.join(bench_path, 'apps') - version_upgrade = (False,) - switched_apps = [] - - if not apps: - apps = [name for name in os.listdir(apps_dir) - if os.path.isdir(os.path.join(apps_dir, name))] - if branch=="v4.x.x": - apps.append('shopping_cart') - - for app in apps: - app_dir = os.path.join(apps_dir, app) - - if not os.path.exists(app_dir): - bench.utils.log(f"{app} does not exist!", level=2) - continue - - repo = git.Repo(app_dir) - unshallow_flag = os.path.exists(os.path.join(app_dir, ".git", "shallow")) - bench.utils.log(f"Fetching upstream {'unshallow ' if unshallow_flag else ''}for {app}") - - bench.utils.exec_cmd("git remote set-branches upstream '*'", cwd=app_dir) - bench.utils.exec_cmd(f"git fetch --all{' --unshallow' if unshallow_flag else ''} --quiet", cwd=app_dir) - - if check_upgrade: - version_upgrade = is_version_upgrade(app=app, bench_path=bench_path, branch=branch) - if version_upgrade[0] and not upgrade: - bench.utils.log(f"Switching to {branch} will cause upgrade from {version_upgrade[1]} to {version_upgrade[2]}. Pass --upgrade to confirm", level=2) - sys.exit(1) - - print("Switching for "+app) - bench.utils.exec_cmd(f"git checkout -f {branch}", cwd=app_dir) - - if str(repo.active_branch) == branch: - switched_apps.append(app) - else: - bench.utils.log(f"Switching branches failed for: {app}", level=2) - - if switched_apps: - bench.utils.log("Successfully switched branches for: " + ", ".join(switched_apps), level=1) - print('Please run `bench update --patch` to be safe from any differences in database schema') - - if version_upgrade[0] and upgrade: - update_requirements() - update_node_packages() - importlib.reload(bench.utils) - backup_all_sites() - patch_sites() - build_assets() - post_upgrade(version_upgrade[1], version_upgrade[2]) - - -def switch_to_branch(branch=None, apps=None, bench_path='.', upgrade=False): - switch_branch(branch, apps=apps, bench_path=bench_path, upgrade=upgrade) - -def switch_to_develop(apps=None, bench_path='.', upgrade=True): - switch_branch('develop', apps=apps, bench_path=bench_path, upgrade=upgrade) - -def get_version_from_string(contents, field='__version__'): - match = re.search(r"^(\s*%s\s*=\s*['\\\"])(.+?)(['\"])(?sm)" % field, contents) - return match.group(2) - -def get_major_version(version): - import semantic_version - - return semantic_version.Version(version).major - def install_apps_from_path(path, bench_path='.'): apps = get_apps_json(path) for app in apps: @@ -622,27 +457,3 @@ def get_apps_json(path): with open(path) as f: return json.load(f) - -def validate_branch(): - apps = Bench(".").apps - - installed_apps = set(apps) - check_apps = set(['frappe', 'erpnext']) - intersection_apps = installed_apps.intersection(check_apps) - - for app in intersection_apps: - branch = get_current_branch(app) - - if branch == "master": - print("""'master' branch is renamed to 'version-11' since 'version-12' release. -As of January 2020, the following branches are -version Frappe ERPNext -11 version-11 version-11 -12 version-12 version-12 -13 version-13 version-13 -14 develop develop - -Please switch to new branches to get future updates. -To switch to your required branch, run the following commands: bench switch-to-branch [branch-name]""") - - sys.exit(1) diff --git a/bench/commands/setup.py b/bench/commands/setup.py index 9488cae9..137b16b7 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -140,19 +140,19 @@ def setup_socketio(): @click.option("--dev", help="Install optional python development dependencies", default=False, is_flag=True) def setup_requirements(node=False, python=False, dev=False): if not (node or python): - from bench.utils import update_requirements + from bench.utils.bench import update_requirements update_requirements() elif not node: - from bench.utils import update_python_packages + from bench.utils.bench import update_python_packages update_python_packages() elif not python: - from bench.utils import update_node_packages + from bench.utils.bench import update_node_packages update_node_packages() if dev: - from bench.utils import install_python_dev_dependencies + from bench.utils.bench import install_python_dev_dependencies install_python_dev_dependencies() if node: From 54f974138562bfaab51e1b97e2010cb1c48b4e14 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 11 Nov 2021 09:12:19 +0530 Subject: [PATCH 30/91] feat: New methods in the App Class Methods available in the App class: * get: Clone remote/local repos into set bench * remove: Uninstall from env & move app into `archived/apps` folder. Changed location from `archived_apps` * install: Install app's Frappe, Python & Node dependencies on set bench * uninstall: Uninstall Python package of the Frappe App. Does nothing else...just a `pip uninstall` --- bench/app.py | 88 ++++++++++++++++++++++++---------------------------- 1 file changed, 41 insertions(+), 47 deletions(-) diff --git a/bench/app.py b/bench/app.py index 97dabf7e..d6c4232a 100755 --- a/bench/app.py +++ b/bench/app.py @@ -94,10 +94,43 @@ class AppMeta: class App(AppMeta): - def __init__(self, name: str, branch : str = None, bench=None): + def __init__(self, name: str, branch : str = None, bench : Bench = None): super().__init__(name, branch) self.bench = bench + def get(self): + branch = f'--branch {self.tag}' if self.tag else '' + shallow = '--depth 1' if self.bench.shallow_clone else '' + + self.bench.run( + f"git clone {self.url} {branch} {shallow} --origin upstream", + cwd=os.path.join(self.bench.name, 'apps') + ) + + def remove(self): + shutil.move( + os.path.join("apps", self.repo), + os.path.join("archived", "apps", self.repo), + ) + + def install(self, skip_assets=False, verbose=False): + app_name = get_app_name(self.bench.name, self.repo) + + # TODO: this should go inside install_app only tho - issue: default/resolved branch + setup_app_dependencies( + repo_name=self.repo, bench_path=self.bench.name, branch=self.tag + ) + + install_app( + app=app_name, bench_path=self.bench.name, verbose=verbose, skip_assets=skip_assets + ) + + def uninstall(self): + env_python = get_env_cmd("python", bench_path=self.bench.name) + self.bench.run( + f"{env_python} -m pip uninstall -y {self.repo}" + ) + def add_to_appstxt(app, bench_path='.'): apps = Bench(bench_path).apps @@ -178,7 +211,8 @@ def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=Fal If the bench_path is not a bench directory, a new bench is created named using the git_url parameter. """ - app = App(git_url, branch=branch) + bench = Bench(bench_path) + app = App(git_url, branch=branch, bench=bench) git_url = app.url repo_name = app.repo branch = app.tag @@ -190,9 +224,6 @@ def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=Fal cloned_path = os.path.join(bench_path, 'apps', repo_name) dir_already_exists = os.path.isdir(cloned_path) - to_clone = not dir_already_exists - to_resolve_dependencies = True - to_install = True if dir_already_exists: # application directory already exists @@ -203,52 +234,15 @@ def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=Fal ): import shutil shutil.rmtree(cloned_path) - to_clone = True elif click.confirm("Do you want to reinstall the existing application?", abort=True): pass - if to_clone: - fetch_txt = f"Getting {repo_name}" - click.secho(fetch_txt, fg="yellow") - logger.log(fetch_txt) + fetch_txt = f"Getting {repo_name}" + click.secho(fetch_txt, fg="yellow") + logger.log(fetch_txt) - git_branch = f'--branch {app.tag}' if app.tag else '' - shallow_clone = '--depth 1' if check_git_for_shallow_clone() else '' - exec_cmd( - f"git clone {git_url} {git_branch} {shallow_clone} --origin upstream", - cwd=os.path.join(bench_path, 'apps') - ) - - if to_resolve_dependencies: - setup_app_dependencies( - repo_name=repo_name, bench_path=bench_path, branch=branch - ) - - if to_install: - app_name = get_app_name(bench_path, repo_name) - install_app( - app=app_name, bench_path=bench_path, verbose=verbose, skip_assets=skip_assets - ) - -def get_app_name(bench_path, repo_name): - app_name = None - apps_path = os.path.join(os.path.abspath(bench_path), 'apps') - config_path = os.path.join(apps_path, repo_name, 'setup.cfg') - if os.path.exists(config_path): - config = read_configuration(config_path) - app_name = config.get('metadata', {}).get('name') - - if not app_name: - # retrieve app name from setup.py as fallback - app_path = os.path.join(apps_path, repo_name, 'setup.py') - with open(app_path, 'rb') as f: - app_name = re.search(r'name\s*=\s*[\'"](.*)[\'"]', f.read().decode('utf-8')).group(1) - - if app_name and repo_name != app_name: - os.rename(os.path.join(apps_path, repo_name), os.path.join(apps_path, app_name)) - return app_name - - return repo_name + app.get() + app.install(verbose=verbose, skip_assets=skip_assets) def new_app(app, bench_path='.'): From 53e811fe0a2ac0f8f4adf174f1caa5aab7c2250e Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 11 Nov 2021 10:16:19 +0530 Subject: [PATCH 31/91] feat: New methods in Bench class New methods added in Bench class: * excluded_apps property * shallow_clone property * apps property - check if frappe app & sort * sync: Sync the apps.txt file from the installed set of Frappe apps * build: Build bench assets * reload: Reload the Frappe processes - supervisor restart New methods added in BenchApps class: * add/append: Adding an App object to this, installs the app on the Bench * remove: Removing an App from here, uninstalls and archives the app --- bench/app.py | 76 ++----------------------------------- bench/bench.py | 101 ++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 87 insertions(+), 90 deletions(-) diff --git a/bench/app.py b/bench/app.py index d6c4232a..a4b318d8 100755 --- a/bench/app.py +++ b/bench/app.py @@ -289,87 +289,18 @@ def install_app(app, bench_path=".", verbose=False, no_cache=False, restart_benc restart_systemd_processes(bench_path=bench_path) -def remove_app(app, bench_path='.'): - import shutil - - bench = Bench(bench_path) - app_path = os.path.join(bench_path, 'apps', app) - py = os.path.join(bench_path, 'env', 'bin', 'python') - - # validate app removal - if app not in bench.apps: - print(f"No app named {app}") - sys.exit(1) - - validate_app_installed_on_sites(app, bench_path=bench_path) - - # remove app from bench - exec_cmd("{0} -m pip uninstall -y {1}".format(py, app), cwd=bench_path) - remove_from_appstxt(app, bench_path) - shutil.rmtree(app_path) - - # re-build assets and restart processes - run_frappe_cmd("build", bench_path=bench_path) - - if bench.conf.get('restart_supervisor_on_update'): - restart_supervisor_processes(bench_path=bench_path) - if bench.conf.get('restart_systemd_on_update'): - restart_systemd_processes(bench_path=bench_path) - - -def validate_app_installed_on_sites(app, bench_path="."): - print("Checking if app installed on active sites...") - ret = check_app_installed(app, bench_path=bench_path) - - if ret is None: - check_app_installed_legacy(app, bench_path=bench_path) - else: - return ret - - -def check_app_installed(app, bench_path="."): - try: - out = subprocess.check_output( - ["bench", "--site", "all", "list-apps", "--format", "json"], - stderr=open(os.devnull, "wb"), - cwd=bench_path, - ).decode('utf-8') - except subprocess.CalledProcessError: - return None - - try: - apps_sites_dict = json.loads(out) - except JSONDecodeError: - return None - - for site, apps in apps_sites_dict.items(): - if app in apps: - print("Cannot remove, app is installed on site: {0}".format(site)) - sys.exit(1) - - -def check_app_installed_legacy(app, bench_path="."): - site_path = os.path.join(bench_path, 'sites') - - for site in os.listdir(site_path): - req_file = os.path.join(site_path, site, 'site_config.json') - if os.path.exists(req_file): - out = subprocess.check_output(["bench", "--site", site, "list-apps"], cwd=bench_path).decode('utf-8') - if re.search(r'\b' + app + r'\b', out): - print(f"Cannot remove, app is installed on site: {site}") - sys.exit(1) - - def pull_apps(apps=None, bench_path='.', reset=False): '''Check all apps if there no local changes, pull''' + from bench.utils.app import get_remote, get_current_branch + bench = Bench(bench_path) rebase = '--rebase' if bench.conf.get('rebase_on_pull') else '' apps = apps or bench.apps + excluded_apps = bench.excluded_apps # check for local changes if not reset: for app in apps: - excluded_apps = get_excluded_apps() if app in excluded_apps: print(f"Skipping reset for app {app}") continue @@ -391,7 +322,6 @@ Here are your choices: wait for them to be merged in the core.''') sys.exit(1) - excluded_apps = get_excluded_apps() for app in apps: if app in excluded_apps: print(f"Skipping pull for app {app}") diff --git a/bench/bench.py b/bench/bench.py index 2d616289..53ca2b18 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -1,31 +1,67 @@ import os import shutil import sys +import typing import logging from typing import MutableSequence import bench -from bench.utils import remove_backups_crontab, folders_in_bench, get_venv_path, exec_cmd, get_env_cmd +from bench.exceptions import ValidationError from bench.config.common_site_config import setup_config +from bench.utils import paths_in_bench, get_venv_path, exec_cmd, get_env_cmd, is_frappe_app, get_git_version, run_frappe_cmd +from bench.utils.bench import validate_app_installed_on_sites, restart_supervisor_processes, restart_systemd_processes, remove_backups_crontab +if typing.TYPE_CHECKING: + from bench.app import App + logger = logging.getLogger(bench.PROJECT_NAME) class Base: - def run(self, cmd): - return exec_cmd(cmd, cwd=self.cwd) + def run(self, cmd, cwd=None): + return exec_cmd(cmd, cwd=cwd or self.cwd) -class Bench(Base): +class Validator: + def validate_app_uninstall(self, app): + if app not in self.apps: + raise ValidationError(f"No app named {app}") + validate_app_installed_on_sites(app, bench_path=self.name) + + +class Bench(Base, Validator): def __init__(self, path): self.name = path self.cwd = os.path.abspath(path) self.exists = os.path.exists(self.name) + self.setup = BenchSetup(self) self.teardown = BenchTearDown(self) self.apps = BenchApps(self) + self.apps_txt = os.path.join(self.name, 'sites', 'apps.txt') + self.excluded_apps_txt = os.path.join(self.name, 'sites', 'excluded_apps.txt') + + @property + def shallow_clone(self): + config = self.conf + + if config: + if config.get('release_bench') or not config.get('shallow_clone'): + return False + + if get_git_version() > 1.9: + return True + + @property + def excluded_apps(self): + try: + with open(self.excluded_apps_txt) as f: + return f.read().strip().split('\n') + except Exception: + return [] + @property def sites(self): return [ @@ -59,17 +95,33 @@ class Bench(Base): from bench.app import App app = App(app, branch=branch) - - # get app? - # install app to env - # add to apps.txt - return + self.apps.append(app) + self.sync() def uninstall(self, app): - # remove from apps.txt - # uninstall app from env - # remove app? - return + from bench.app import App + + self.validate_app_uninstall(app) + self.apps.remove(App(app, bench=self)) + self.sync() + self.build() + self.reload() + + def build(self): + # build assets & stuff + run_frappe_cmd("build", bench_path=self.name) + + def reload(self): + conf = self.conf + if conf.get('restart_supervisor_on_update'): + restart_supervisor_processes(bench_path=self.name) + if conf.get('restart_systemd_on_update'): + restart_systemd_processes(bench_path=self.name) + + def sync(self): + self.apps.initialize_apps() + with open(self.apps_txt, "w") as f: + return f.write("\n".join(self.apps)) class BenchApps(MutableSequence): @@ -79,9 +131,10 @@ class BenchApps(MutableSequence): def initialize_apps(self): try: - self.apps = open( - os.path.join(self.bench.name, "sites", "apps.txt") - ).read().splitlines() + self.apps = [x for x in os.listdir( + os.path.join(self.bench.name, "apps") + ) if is_frappe_app(os.path.join(self.bench.name, "apps", x))] + self.apps.sort() except FileNotFoundError: self.apps = [] @@ -108,6 +161,20 @@ class BenchApps(MutableSequence): # TODO: fetch and install app to bench self.apps.insert(key, value) + def add(self, app: "App"): + app.get() + app.install() + super().append(app.repo) + self.apps.sort() + + def remove(self, app: "App"): + app.uninstall() + app.remove() + super().remove(app.repo) + + def append(self, app : "App"): + return self.add(app) + def __repr__(self): return self.__str__() @@ -123,7 +190,7 @@ class BenchSetup(Base): def dirs(self): os.makedirs(self.bench.name, exist_ok=True) - for dirname in folders_in_bench: + for dirname in paths_in_bench: os.makedirs(os.path.join(self.bench.name, dirname), exist_ok=True) def env(self, python="python3"): From d04c7feab7ce7e9fb1202d913886f4f51b196e1c Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 12 Nov 2021 18:13:01 +0530 Subject: [PATCH 32/91] refactor(minor): bench init Rename --frappe-branch flag to --version, both are allowed though. Cause of rename: Makes more sense as version rather than frappe_branch. This is because you can pass any git reference instead of just branches, including version tags. --- bench/commands/make.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bench/commands/make.py b/bench/commands/make.py index b016affb..5d96e487 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -4,11 +4,11 @@ import click @click.command('init', help='Initialize a new bench instance in the specified path') @click.argument('path') -@click.option('--python', type = str, default = 'python3', help = 'Path to Python Executable.') +@click.option('--version', '--frappe-branch', 'frappe_branch', default=None, help="Clone a particular branch of frappe") @click.option('--ignore-exist', is_flag = True, default = False, help = "Ignore if Bench instance exists.") +@click.option('--python', type = str, default = 'python3', help = 'Path to Python Executable.') @click.option('--apps_path', default=None, help="path to json files with apps to install after init") @click.option('--frappe-path', default=None, help="path to frappe repo") -@click.option('--frappe-branch', default=None, help="Clone a particular branch of frappe") @click.option('--clone-from', default=None, help="copy repos from path") @click.option('--clone-without-update', is_flag=True, help="copy repos from path without update") @click.option('--no-procfile', is_flag=True, help="Do not create a Procfile") From bae338ad0babbea3e015d4a1bcf555fcfc20e0cb Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 12 Nov 2021 18:15:51 +0530 Subject: [PATCH 33/91] feat(minor): Any procman in bench start Using the `-m` flag, we can use any process manager instead of the standard honcho. For instance, you've installed overmind on your machine and are married to it, just run `bench start -m overmind` :P --- bench/commands/utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bench/commands/utils.py b/bench/commands/utils.py index 0b2b002f..5ab0e3d9 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -11,9 +11,10 @@ import click @click.option('--no-prefix', is_flag=True, default=False, help="Hide process name from bench start log") @click.option('--concurrency', '-c', type=str) @click.option('--procfile', '-p', type=str) -def start(no_dev, concurrency, procfile, no_prefix): - from bench.utils import start - start(no_dev=no_dev, concurrency=concurrency, procfile=procfile, no_prefix=no_prefix) +@click.option('--man', '-m', help="Process Manager of your choice ;)") +def start(no_dev, concurrency, procfile, no_prefix, man): + from bench.utils.system import start + start(no_dev=no_dev, concurrency=concurrency, procfile=procfile, no_prefix=no_prefix, procman=man) @click.command('restart', help="Restart supervisor processes or systemd units") From e1ed4d4e534fed30d0d6e676e8cc8e01c5794509 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 10 Nov 2021 10:16:19 +0530 Subject: [PATCH 34/91] refactor(minor): Use Bench object to remove app --- bench/commands/make.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bench/commands/make.py b/bench/commands/make.py index 5d96e487..53908e9e 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -96,8 +96,9 @@ def new_app(app_name): @click.command('remove-app', help='Completely remove app from bench and re-build assets if not installed on any site') @click.argument('app-name') def remove_app(app_name): - from bench.app import remove_app - remove_app(app_name) + from bench.bench import Bench + bench = Bench(".") + bench.uninstall(app_name) @click.command('exclude-app', help='Exclude app from updating') From 89fdd1c5accdd791a8823ba8a799d442b6368fad Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 10 Nov 2021 09:16:19 +0530 Subject: [PATCH 35/91] chore: Deprecate bench setup socketio --- bench/commands/setup.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bench/commands/setup.py b/bench/commands/setup.py index 137b16b7..d049b953 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -129,10 +129,9 @@ def setup_procfile(): bench.config.procfile.setup_procfile(".") -@click.command("socketio", help="Setup node dependencies for socketio server") +@click.command("socketio", help="[DEPRECATED] Setup node dependencies for socketio server") def setup_socketio(): - bench.utils.setup_socketio() - + return @click.command("requirements", help="Setup Python and Node dependencies") @click.option("--node", help="Update only Node packages", default=False, is_flag=True) From ee6a967a205addf57b1e233d2451c8a191b28cec Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 12 Nov 2021 18:33:23 +0530 Subject: [PATCH 36/91] fix: import path for util --- bench/__init__.py | 2 +- bench/app.py | 18 +++++++++++++++--- bench/commands/git.py | 3 ++- bench/patches/v3/celery_to_rq.py | 2 +- bench/tests/test_base.py | 10 ++++++---- bench/utils/__init__.py | 2 +- bench/utils/app.py | 2 ++ bench/utils/system.py | 1 + 8 files changed, 29 insertions(+), 11 deletions(-) diff --git a/bench/__init__.py b/bench/__init__.py index ab9281b2..e2f6fd6d 100644 --- a/bench/__init__.py +++ b/bench/__init__.py @@ -6,7 +6,7 @@ updated_path = None def set_frappe_version(bench_path='.'): - from .app import get_current_frappe_version + from .utils.app import get_current_frappe_version global FRAPPE_VERSION if not FRAPPE_VERSION: FRAPPE_VERSION = get_current_frappe_version(bench_path=bench_path) \ No newline at end of file diff --git a/bench/app.py b/bench/app.py index a4b318d8..b1a0f1e3 100755 --- a/bench/app.py +++ b/bench/app.py @@ -6,20 +6,22 @@ import re import shutil import subprocess import sys +import typing # imports - third party imports import click # imports - module imports import bench -from bench.bench import Bench from bench.utils import exec_cmd, is_bench_directory, run_frappe_cmd, is_git_url, fetch_details_from_tag -from bench.utils.app import get_app_name from bench.utils.bench import get_env_cmd, build_assets, restart_supervisor_processes, restart_systemd_processes logger = logging.getLogger(bench.PROJECT_NAME) +if typing.TYPE_CHECKING: + from bench.bench import Bench + class AppMeta: def __init__(self, name: str, branch : str = None): """ @@ -94,7 +96,7 @@ class AppMeta: class App(AppMeta): - def __init__(self, name: str, branch : str = None, bench : Bench = None): + def __init__(self, name: str, branch : str = None, bench : "Bench" = None): super().__init__(name, branch) self.bench = bench @@ -114,6 +116,8 @@ class App(AppMeta): ) def install(self, skip_assets=False, verbose=False): + from bench.utils.app import get_app_name + app_name = get_app_name(self.bench.name, self.repo) # TODO: this should go inside install_app only tho - issue: default/resolved branch @@ -133,6 +137,8 @@ class App(AppMeta): def add_to_appstxt(app, bench_path='.'): + from bench.bench import Bench + apps = Bench(bench_path).apps if app not in apps: @@ -140,6 +146,8 @@ def add_to_appstxt(app, bench_path='.'): return write_appstxt(apps, bench_path=bench_path) def remove_from_appstxt(app, bench_path='.'): + from bench.bench import Bench + apps = Bench(bench_path).apps if app in apps: @@ -211,6 +219,8 @@ def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=Fal If the bench_path is not a bench directory, a new bench is created named using the git_url parameter. """ + from bench.bench import Bench + bench = Bench(bench_path) app = App(git_url, branch=branch, bench=bench) git_url = app.url @@ -255,6 +265,7 @@ def new_app(app, bench_path='.'): def install_app(app, bench_path=".", verbose=False, no_cache=False, restart_bench=True, skip_assets=False): + from bench.bench import Bench from bench.utils import get_env_cmd install_text = f'Installing {app}' @@ -291,6 +302,7 @@ def install_app(app, bench_path=".", verbose=False, no_cache=False, restart_benc def pull_apps(apps=None, bench_path='.', reset=False): '''Check all apps if there no local changes, pull''' + from bench.bench import Bench from bench.utils.app import get_remote, get_current_branch bench = Bench(bench_path) diff --git a/bench/commands/git.py b/bench/commands/git.py index 19aa4d36..a0a26ccb 100644 --- a/bench/commands/git.py +++ b/bench/commands/git.py @@ -4,8 +4,9 @@ import subprocess # imports - module imports from bench.bench import Bench -from bench.app import get_repo_dir, get_remote +from bench.app import get_repo_dir from bench.utils import set_git_remote_url +from bench.utils.app import get_remote # imports - third party imports import click diff --git a/bench/patches/v3/celery_to_rq.py b/bench/patches/v3/celery_to_rq.py index 035730a6..29b2af2e 100644 --- a/bench/patches/v3/celery_to_rq.py +++ b/bench/patches/v3/celery_to_rq.py @@ -1,7 +1,7 @@ import click, os from bench.config.procfile import setup_procfile from bench.config.supervisor import generate_supervisor_config -from bench.app import get_current_frappe_version, get_current_branch +from bench.utils.app import get_current_frappe_version, get_current_branch def execute(bench_path): frappe_branch = get_current_branch('frappe', bench_path) diff --git a/bench/tests/test_base.py b/bench/tests/test_base.py index 298d081c..ad884ba5 100644 --- a/bench/tests/test_base.py +++ b/bench/tests/test_base.py @@ -11,6 +11,7 @@ import unittest # imports - module imports import bench import bench.utils +from bench.bench import Bench if sys.version_info.major == 2: FRAPPE_BRANCH = "version-12" @@ -25,15 +26,16 @@ class TestBenchBase(unittest.TestCase): def tearDown(self): for bench_name in self.benches: bench_path = os.path.join(self.benches_path, bench_name) + bench = Bench(bench_path) mariadb_password = "travis" if os.environ.get("CI") else getpass.getpass(prompt="Enter MariaDB root Password: ") - if os.path.exists(bench_path): - sites = bench.utils.get_sites(bench_path=bench_path) - for site in sites: + + if bench.exists: + for site in bench.sites: subprocess.call(["bench", "drop-site", site, "--force", "--no-backup", "--root-password", mariadb_password], cwd=bench_path) shutil.rmtree(bench_path, ignore_errors=True) def assert_folders(self, bench_name): - for folder in bench.utils.folders_in_bench: + for folder in bench.utils.paths_in_bench: self.assert_exists(bench_name, folder) self.assert_exists(bench_name, "apps", "frappe") diff --git a/bench/utils/__init__.py b/bench/utils/__init__.py index 95b6c68b..b52640b1 100644 --- a/bench/utils/__init__.py +++ b/bench/utils/__init__.py @@ -338,7 +338,7 @@ def generate_command_cache(bench_path='.'): except subprocess.CalledProcessError as e: if hasattr(e, "stderr"): - print(e.stderr.decode('utf-8')) + print(e.stderr) return [] diff --git a/bench/utils/app.py b/bench/utils/app.py index 63bdf786..64d6bf9b 100644 --- a/bench/utils/app.py +++ b/bench/utils/app.py @@ -5,6 +5,7 @@ import bench import sys import subprocess from bench.exceptions import InvalidRemoteException, InvalidBranchException, CommandFailedError +from bench.app import get_repo_dir def is_version_upgrade(app='frappe', bench_path='.', branch=None): @@ -26,6 +27,7 @@ def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrad import git import importlib from bench.utils import update_requirements, update_node_packages, backup_all_sites, patch_sites, post_upgrade + from bench.utils.bench import build_assets apps_dir = os.path.join(bench_path, 'apps') version_upgrade = (False,) diff --git a/bench/utils/system.py b/bench/utils/system.py index 8f64431b..16840513 100644 --- a/bench/utils/system.py +++ b/bench/utils/system.py @@ -10,6 +10,7 @@ import bench # TODO: Fix this import bench.utils from bench.utils import * +from bench.utils.bench import build_assets def init(path, apps_path=None, no_procfile=False, no_backups=False, From 71d178dde952742a02fb19bfac812c6db8ef9d9d Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 12 Nov 2021 20:30:53 +0530 Subject: [PATCH 37/91] fix: sync apps file in init, imports and no warn in CI --- bench/app.py | 3 ++- bench/cli.py | 3 ++- bench/commands/update.py | 2 +- bench/utils/system.py | 11 +++++------ 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/bench/app.py b/bench/app.py index b1a0f1e3..5872189a 100755 --- a/bench/app.py +++ b/bench/app.py @@ -203,7 +203,8 @@ def setup_app_dependencies(repo_name, bench_path='.', branch=None): files = glob.glob(os.path.join(apps_path, repo_name, '**', 'hooks.py')) if files: - lines = [x for x in open(files[0]).read().split('\n') if x.strip().startswith('required_apps')] + with open(files[0]) as f: + lines = [x for x in f.read().split('\n') if x.strip().startswith('required_apps')] if lines: required_apps = eval(lines[0].strip('required_apps').strip().lstrip('=').strip()) # TODO: when the time comes, add version check here diff --git a/bench/cli.py b/bench/cli.py index 4453c103..66b4d21d 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -37,6 +37,7 @@ def cli(): global from_command_line from_command_line = True command = " ".join(sys.argv) + is_envvar_warn_set = not (os.environ.get("BENCH_DEVELOPER") or os.environ.get("CI")) change_working_directory() logger = setup_logging() @@ -47,7 +48,7 @@ def cli(): change_uid() change_dir() - if not os.environ.get("BENCH_DEVELOPER") and ( + if is_envvar_warn_set and ( is_dist_editable(bench.PROJECT_NAME) and len(sys.argv) > 1 and sys.argv[1] != "src" diff --git a/bench/commands/update.py b/bench/commands/update.py index 16ffefbf..2ebbf4ee 100755 --- a/bench/commands/update.py +++ b/bench/commands/update.py @@ -37,7 +37,7 @@ def retry_upgrade(version): @click.argument('apps', nargs=-1) @click.option('--upgrade',is_flag=True) def switch_to_branch(branch, apps, upgrade=False): - from bench.app import switch_to_branch + from bench.utils.app import switch_to_branch switch_to_branch(branch=branch, apps=list(apps), upgrade=upgrade) diff --git a/bench/utils/system.py b/bench/utils/system.py index 16840513..e795871b 100644 --- a/bench/utils/system.py +++ b/bench/utils/system.py @@ -2,15 +2,14 @@ import grp import os import pwd +import shutil import sys # imports - module imports import bench +from bench.utils import exec_cmd, get_process_manager, log, run_frappe_cmd, sudoers_file, which +from bench.utils.bench import build_assets, clone_apps_from -# TODO: Fix this -import bench.utils -from bench.utils import * -from bench.utils.bench import build_assets def init(path, apps_path=None, no_procfile=False, no_backups=False, @@ -58,6 +57,8 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, if apps_path: install_apps_from_path(apps_path, bench_path=path) + bench.sync() + if not skip_assets: build_assets(bench_path=path) @@ -174,8 +175,6 @@ def fix_prod_setup_perms(bench_path='.', frappe_user=None): def setup_fonts(): - import shutil - fonts_path = os.path.join('/tmp', 'fonts') if os.path.exists('/etc/fonts_backup'): From 4d1ddc472233e5e8261bb41bafd8cc1482e7823d Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 12 Nov 2021 22:13:27 +0530 Subject: [PATCH 38/91] refactor: Bench.sync => Bench.apps.sync * Added missing imports * Fixed broken import paths --- bench/bench.py | 14 +++++++------- bench/commands/update.py | 2 +- bench/utils/bench.py | 10 ++++++---- bench/utils/system.py | 2 +- bench/utils/translation.py | 3 --- 5 files changed, 15 insertions(+), 16 deletions(-) diff --git a/bench/bench.py b/bench/bench.py index 53ca2b18..9153dc52 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -96,14 +96,14 @@ class Bench(Base, Validator): app = App(app, branch=branch) self.apps.append(app) - self.sync() + self.apps.sync() def uninstall(self, app): from bench.app import App self.validate_app_uninstall(app) self.apps.remove(App(app, bench=self)) - self.sync() + self.apps.sync() self.build() self.reload() @@ -118,17 +118,17 @@ class Bench(Base, Validator): if conf.get('restart_systemd_on_update'): restart_systemd_processes(bench_path=self.name) - def sync(self): - self.apps.initialize_apps() - with open(self.apps_txt, "w") as f: - return f.write("\n".join(self.apps)) - class BenchApps(MutableSequence): def __init__(self, bench : Bench): self.bench = bench self.initialize_apps() + def sync(self): + self.initialize_apps() + with open(self.bench.apps_txt, "w") as f: + return f.write("\n".join(self.apps)) + def initialize_apps(self): try: self.apps = [x for x in os.listdir( diff --git a/bench/commands/update.py b/bench/commands/update.py index 2ebbf4ee..1e1979cb 100755 --- a/bench/commands/update.py +++ b/bench/commands/update.py @@ -44,5 +44,5 @@ def switch_to_branch(branch, apps, upgrade=False): @click.command('switch-to-develop') def switch_to_develop(upgrade=False): "Switch frappe and erpnext to develop branch" - from bench.app import switch_to_develop + from bench.utils.app import switch_to_develop switch_to_develop(apps=['frappe', 'erpnext']) diff --git a/bench/utils/bench.py b/bench/utils/bench.py index a9e106ea..0062ce99 100644 --- a/bench/utils/bench.py +++ b/bench/utils/bench.py @@ -1,5 +1,6 @@ # imports - standard imports import json +import logging import os import re import subprocess @@ -8,12 +9,13 @@ from json.decoder import JSONDecodeError # imports - third party imports import click +import bench # imports - module imports +from bench.utils import which, log, exec_cmd, get_bench_name, get_cmd_output from bench.exceptions import PatchError, ValidationError -# TODO: Fix this -from bench.utils import * +logger = logging.getLogger(bench.PROJECT_NAME) def get_env_cmd(cmd, bench_path='.'): @@ -74,7 +76,7 @@ def update_python_packages(bench_path='.'): def update_node_packages(bench_path='.'): print('Updating node packages...') - from bench.app import get_develop_version + from bench.utils.app import get_develop_version from distutils.version import LooseVersion v = LooseVersion(get_develop_version('frappe', bench_path = bench_path)) @@ -302,7 +304,7 @@ def update(pull=False, apps=None, patch=False, build=False, requirements=False, import re from bench import patches from bench.utils import clear_command_cache, pause_exec, log - from bench.utils.bench import restart_supervisor_processes, restart_systemd_processes + from bench.utils.bench import restart_supervisor_processes, restart_systemd_processes, backup_all_sites from bench.app import pull_apps from bench.utils.app import is_version_upgrade from bench.config.common_site_config import update_config diff --git a/bench/utils/system.py b/bench/utils/system.py index e795871b..33e94cce 100644 --- a/bench/utils/system.py +++ b/bench/utils/system.py @@ -57,7 +57,7 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, if apps_path: install_apps_from_path(apps_path, bench_path=path) - bench.sync() + bench.apps.sync() if not skip_assets: build_assets(bench_path=path) diff --git a/bench/utils/translation.py b/bench/utils/translation.py index 8e3f1b06..d4420eac 100644 --- a/bench/utils/translation.py +++ b/bench/utils/translation.py @@ -3,9 +3,6 @@ import itertools import json import os -# TODO: Fix this -from bench.utils import * - def update_translations_p(args): import requests From a2ecb398dde15deb9b9cce1f6c88248da575151e Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 12 Nov 2021 22:14:30 +0530 Subject: [PATCH 39/91] ci: Use Nodejs version 14 for Tests --- .travis.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 67c7f56b..f5f6d6e4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -76,17 +76,20 @@ matrix: script: sudo python $TRAVIS_BUILD_DIR/install.py --user travis --run-travis --production --verbose install: - - pip install urllib3 pyOpenSSL ndg-httpsclient pyasn1 + - pip3 install urllib3 pyOpenSSL ndg-httpsclient pyasn1 - if [ $TEST == "bench" ];then - wget -q -O /tmp/wkhtmltox.tar.xz https://github.com/frappe/wkhtmltopdf/raw/master/wkhtmltox-0.12.3_linux-generic-amd64.tar.xz; - tar -xf /tmp/wkhtmltox.tar.xz -C /tmp; - sudo mv /tmp/wkhtmltox/bin/wkhtmltopdf /usr/local/bin/wkhtmltopdf; - sudo chmod o+x /usr/local/bin/wkhtmltopdf; + wget -q -O /tmp/wkhtmltox.tar.xz https://github.com/frappe/wkhtmltopdf/raw/master/wkhtmltox-0.12.3_linux-generic-amd64.tar.xz; + tar -xf /tmp/wkhtmltox.tar.xz -C /tmp; + sudo mv /tmp/wkhtmltox/bin/wkhtmltopdf /usr/local/bin/wkhtmltopdf; + sudo chmod o+x /usr/local/bin/wkhtmltopdf; + + nvm install 14; + nvm use 14; mkdir -p ~/.bench; cp -r $TRAVIS_BUILD_DIR/* ~/.bench; - pip install -q -U -e ~/.bench; + pip3 install -q -U -e ~/.bench; sudo pip install -q -U -e ~/.bench; mysql -u root -e "SET GLOBAL character_set_server = 'utf8mb4'"; From e08a12477ddbd1cfac08cdd8e95c7251816d4dce Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 13 Nov 2021 00:19:12 +0530 Subject: [PATCH 40/91] chore: Get rid of import * utils --- bench/app.py | 4 ++-- bench/bench.py | 11 ++++------- bench/commands/install.py | 3 ++- bench/commands/make.py | 5 +++-- bench/commands/update.py | 4 ++-- bench/commands/utils.py | 18 +++++++++--------- bench/config/lets_encrypt.py | 3 ++- bench/config/production_setup.py | 3 ++- bench/release.py | 2 ++ bench/utils/__init__.py | 26 ++++++++++++++++++++------ bench/utils/system.py | 21 --------------------- 11 files changed, 48 insertions(+), 52 deletions(-) diff --git a/bench/app.py b/bench/app.py index 5872189a..d3aee384 100755 --- a/bench/app.py +++ b/bench/app.py @@ -267,7 +267,7 @@ def new_app(app, bench_path='.'): def install_app(app, bench_path=".", verbose=False, no_cache=False, restart_bench=True, skip_assets=False): from bench.bench import Bench - from bench.utils import get_env_cmd + from bench.utils.bench import get_env_cmd install_text = f'Installing {app}' click.secho(install_text, fg="yellow") @@ -288,7 +288,7 @@ def install_app(app, bench_path=".", verbose=False, no_cache=False, restart_benc conf = Bench(bench_path).conf if conf.get("developer_mode"): - from bench.utils import install_python_dev_dependencies + from bench.utils.bench import install_python_dev_dependencies install_python_dev_dependencies(apps=app) if not skip_assets: diff --git a/bench/bench.py b/bench/bench.py index 9153dc52..e1db12b6 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -1,18 +1,17 @@ import os import shutil import sys -import typing import logging -from typing import MutableSequence +from typing import MutableSequence, TYPE_CHECKING import bench from bench.exceptions import ValidationError from bench.config.common_site_config import setup_config -from bench.utils import paths_in_bench, get_venv_path, exec_cmd, get_env_cmd, is_frappe_app, get_git_version, run_frappe_cmd -from bench.utils.bench import validate_app_installed_on_sites, restart_supervisor_processes, restart_systemd_processes, remove_backups_crontab +from bench.utils import paths_in_bench, exec_cmd, is_frappe_app, get_git_version, run_frappe_cmd +from bench.utils.bench import validate_app_installed_on_sites, restart_supervisor_processes, restart_systemd_processes, remove_backups_crontab, get_venv_path, get_env_cmd -if typing.TYPE_CHECKING: +if TYPE_CHECKING: from bench.app import App logger = logging.getLogger(bench.PROJECT_NAME) @@ -231,8 +230,6 @@ class BenchSetup(Base): return setup_logging(bench_path=self.bench.name) def patches(self): - import shutil - shutil.copy( os.path.join(os.path.dirname(os.path.abspath(__file__)), 'patches', 'patches.txt'), os.path.join(self.bench.name, 'patches.txt') diff --git a/bench/commands/install.py b/bench/commands/install.py index e1fc93bb..132d2255 100644 --- a/bench/commands/install.py +++ b/bench/commands/install.py @@ -1,5 +1,6 @@ # imports - module imports -from bench.utils import run_playbook, setup_sudoers +from bench.utils import run_playbook +from bench.utils.system import setup_sudoers # imports - third party imports import click diff --git a/bench/commands/make.py b/bench/commands/make.py index 53908e9e..618af9ad 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -18,7 +18,8 @@ import click @click.option('--verbose',is_flag=True, help="Verbose output during install") def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, clone_from, verbose, skip_redis_config_generation, clone_without_update, ignore_exist=False, skip_assets=False, python='python3'): import os - from bench.utils import init, log + from bench.utils import log + from bench.utils.system import init if not ignore_exist and os.path.exists(path): log(f"Bench instance already exists at {path}", level=2) @@ -122,6 +123,6 @@ def include_app_for_update(app_name): def pip(ctx, args): "Run pip commands in bench env" import os - from bench.utils import get_env_cmd + from bench.utils.bench import get_env_cmd env_py = get_env_cmd('python') os.execv(env_py, (env_py, '-m', 'pip') + args) diff --git a/bench/commands/update.py b/bench/commands/update.py index 1e1979cb..5d97d1d8 100755 --- a/bench/commands/update.py +++ b/bench/commands/update.py @@ -3,7 +3,7 @@ import click # imports - module imports from bench.app import pull_apps -from bench.utils import post_upgrade, patch_sites, build_assets +from bench.utils.bench import post_upgrade, patch_sites, build_assets @click.command('update', help="Performs an update operation on current bench. Without any flags will backup, pull, setup requirements, build, run patches and restart bench. Using specific flags will only do certain tasks instead of all") @@ -19,7 +19,7 @@ from bench.utils import post_upgrade, patch_sites, build_assets @click.option('--force', is_flag=True, help="Forces major version upgrades") @click.option('--reset', is_flag=True, help="Hard resets git branch's to their new states overriding any changes and overriding rebase on pull") def update(pull, apps, patch, build, requirements, restart_supervisor, restart_systemd, no_backup, no_compile, force, reset): - from bench.utils import update + from bench.utils.bench import update update(pull=pull, apps=apps, patch=patch, build=build, requirements=requirements, restart_supervisor=restart_supervisor, restart_systemd=restart_systemd, backup=not no_backup, compile=not no_compile, force=force, reset=reset) diff --git a/bench/commands/utils.py b/bench/commands/utils.py index 5ab0e3d9..a5beb614 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -22,8 +22,8 @@ def start(no_dev, concurrency, procfile, no_prefix, man): @click.option('--supervisor', is_flag=True, default=False) @click.option('--systemd', is_flag=True, default=False) def restart(web, supervisor, systemd): - from bench.utils import restart_supervisor_processes, restart_systemd_processes from bench.bench import Bench + from bench.utils.bench import restart_supervisor_processes, restart_systemd_processes bench = Bench(".") @@ -68,7 +68,7 @@ def set_url_root(site, url_root): @click.command('set-mariadb-host', help="Set MariaDB host for bench") @click.argument('host') def set_mariadb_host(host): - from bench.utils import set_mariadb_host + from bench.utils.bench import set_mariadb_host set_mariadb_host(host) @@ -78,7 +78,7 @@ def set_redis_cache_host(host): """ Usage: bench set-redis-cache-host localhost:6379/1 """ - from bench.utils import set_redis_cache_host + from bench.utils.bench import set_redis_cache_host set_redis_cache_host(host) @@ -88,7 +88,7 @@ def set_redis_queue_host(host): """ Usage: bench set-redis-queue-host localhost:6379/2 """ - from bench.utils import set_redis_queue_host + from bench.utils.bench import set_redis_queue_host set_redis_queue_host(host) @@ -98,14 +98,14 @@ def set_redis_socketio_host(host): """ Usage: bench set-redis-socketio-host localhost:6379/3 """ - from bench.utils import set_redis_socketio_host + from bench.utils.bench import set_redis_socketio_host set_redis_socketio_host(host) @click.command('download-translations', help="Download latest translations") def download_translations(): - from bench.utils import download_translations_p + from bench.utils.translation import download_translations_p download_translations_p() @@ -119,7 +119,7 @@ def renew_lets_encrypt(): @click.argument('site') def backup_site(site): from bench.bench import Bench - from bench.utils import backup_site + from bench.utils.system import backup_site if site not in Bench(".").sites: print(f'Site `{site}` not found') sys.exit(1) @@ -128,7 +128,7 @@ def backup_site(site): @click.command('backup-all-sites', help="Backup all sites in current bench") def backup_all_sites(): - from bench.utils import backup_all_sites + from bench.utils.system import backup_all_sites backup_all_sites(bench_path='.') @@ -178,7 +178,7 @@ def find_benches(location): @click.argument('python', type=str) @click.option('--no-backup', 'backup', is_flag=True, default=True) def migrate_env(python, backup=True): - from bench.utils import migrate_env + from bench.utils.bench import migrate_env migrate_env(python=python, backup=backup) diff --git a/bench/config/lets_encrypt.py b/bench/config/lets_encrypt.py index c69b6121..45a61308 100755 --- a/bench/config/lets_encrypt.py +++ b/bench/config/lets_encrypt.py @@ -10,7 +10,8 @@ from bench.config.nginx import make_nginx_conf from bench.config.production_setup import service from bench.config.site_config import get_domains, remove_domain, update_site_config from bench.bench import Bench -from bench.utils import exec_cmd, update_common_site_config +from bench.utils import exec_cmd +from bench.utils.bench import update_common_site_config from bench.exceptions import CommandFailedError diff --git a/bench/config/production_setup.py b/bench/config/production_setup.py index 10ecc191..7632f830 100755 --- a/bench/config/production_setup.py +++ b/bench/config/production_setup.py @@ -9,7 +9,8 @@ from bench.config.nginx import make_nginx_conf from bench.config.supervisor import generate_supervisor_config, update_supervisord_config from bench.config.systemd import generate_systemd_config from bench.bench import Bench -from bench.utils import exec_cmd, which, fix_prod_setup_perms, get_bench_name, get_cmd_output, log +from bench.utils import exec_cmd, which, get_bench_name, get_cmd_output, log +from bench.utils.system import fix_prod_setup_perms from bench.exceptions import CommandFailedError logger = logging.getLogger(bench.PROJECT_NAME) diff --git a/bench/release.py b/bench/release.py index 98911cd7..d236e1c4 100755 --- a/bench/release.py +++ b/bench/release.py @@ -7,6 +7,8 @@ import git import getpass import re from time import sleep + +from bench.exceptions import ValidationError from .config.common_site_config import get_config import click diff --git a/bench/utils/__init__.py b/bench/utils/__init__.py index b52640b1..dc51e446 100644 --- a/bench/utils/__init__.py +++ b/bench/utils/__init__.py @@ -13,7 +13,7 @@ import click # imports - module imports import bench -from bench.exceptions import InvalidRemoteException +from bench.exceptions import InvalidRemoteException, ValidationError logger = logging.getLogger(bench.PROJECT_NAME) @@ -393,8 +393,22 @@ def is_git_url(url): return bool(re.match(pattern, url)) -# to avoid circular imports -from .app import * -from .bench import * -from .system import * -from .translation import * +def drop_privileges(uid_name='nobody', gid_name='nogroup'): + # from http://stackoverflow.com/a/2699996 + if os.getuid() != 0: + # We're not root so, like, whatever dude + return + + # Get the uid/gid from the name + running_uid = pwd.getpwnam(uid_name).pw_uid + running_gid = grp.getgrnam(gid_name).gr_gid + + # Remove group privileges + os.setgroups([]) + + # Try setting the new uid/gid + os.setgid(running_gid) + os.setuid(running_uid) + + # Ensure a very conservative umask + os.umask(0o22) diff --git a/bench/utils/system.py b/bench/utils/system.py index 33e94cce..d0a2c428 100644 --- a/bench/utils/system.py +++ b/bench/utils/system.py @@ -135,27 +135,6 @@ def backup_all_sites(bench_path='.'): backup_site(site, bench_path=bench_path) -def drop_privileges(uid_name='nobody', gid_name='nogroup'): - # from http://stackoverflow.com/a/2699996 - if os.getuid() != 0: - # We're not root so, like, whatever dude - return - - # Get the uid/gid from the name - running_uid = pwd.getpwnam(uid_name).pw_uid - running_gid = grp.getgrnam(gid_name).gr_gid - - # Remove group privileges - os.setgroups([]) - - # Try setting the new uid/gid - os.setgid(running_gid) - os.setuid(running_uid) - - # Ensure a very conservative umask - os.umask(0o22) - - def fix_prod_setup_perms(bench_path='.', frappe_user=None): from glob import glob from bench.bench import Bench From c4b839171662092dc0754586e5e14304e55be613 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 13 Nov 2021 00:20:05 +0530 Subject: [PATCH 41/91] fix: Raise and handle Exception class * Instead of sys.exit, raise ValidationError * Let's not handle BaseException, just Exception * Use ValidationError instead of invalid syntax raise str --- bench/config/nginx.py | 2 +- bench/config/production_setup.py | 2 +- bench/release.py | 4 ++-- bench/utils/__init__.py | 7 ++++--- bench/utils/bench.py | 4 ++-- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/bench/config/nginx.py b/bench/config/nginx.py index 53685e60..a3117f9c 100644 --- a/bench/config/nginx.py +++ b/bench/config/nginx.py @@ -201,7 +201,7 @@ def get_sites_with_config(bench_path): "If you want this command to pass, instead of just throwing an error,", "You may remove the 'strict_nginx' flag from common_site_config.json or set it to 0", "\n\n") - raise (e) + raise e else: print(f"\n\nWARNING: The site config for the site {site} is broken.", "If you want this command to fail, instead of just showing a warning,", diff --git a/bench/config/production_setup.py b/bench/config/production_setup.py index 7632f830..ad233e31 100755 --- a/bench/config/production_setup.py +++ b/bench/config/production_setup.py @@ -182,7 +182,7 @@ def reload_supervisor(): def reload_nginx(): try: exec_cmd(f"sudo {which('nginx')} -t") - except: + except Exception: raise service('nginx', 'reload') diff --git a/bench/release.py b/bench/release.py index d236e1c4..303795d8 100755 --- a/bench/release.py +++ b/bench/release.py @@ -191,10 +191,10 @@ def get_bumped_version(version, bump_type): v.prerelease = ('beta', str(int(v.prerelease[1]) + 1)) else: - raise ("Something wen't wrong while doing a prerelease") + raise ValidationError("Something wen't wrong while doing a prerelease") else: - raise ("bump_type not amongst [major, minor, patch, prerelease]") + raise ValidationError("bump_type not amongst [major, minor, patch, prerelease]") return str(v) diff --git a/bench/utils/__init__.py b/bench/utils/__init__.py index dc51e446..181908f4 100644 --- a/bench/utils/__init__.py +++ b/bench/utils/__init__.py @@ -239,15 +239,16 @@ def get_bench_name(bench_path): def set_git_remote_url(git_url, bench_path='.'): "Set app remote git url" + from bench.app import get_repo_dir from bench.bench import Bench app = git_url.rsplit('/', 1)[1].rsplit('.', 1)[0] if app not in Bench(bench_path).apps: - print(f"No app named {app}") - sys.exit(1) + raise ValidationError(f"No app named {app}") + + app_dir = get_repo_dir(app, bench_path=bench_path) - app_dir = bench.app.get_repo_dir(app, bench_path=bench_path) if os.path.exists(os.path.join(app_dir, '.git')): exec_cmd(f"git remote set-url upstream {git_url}", cwd=app_dir) diff --git a/bench/utils/bench.py b/bench/utils/bench.py index 0062ce99..5bb4e649 100644 --- a/bench/utils/bench.py +++ b/bench/utils/bench.py @@ -182,7 +182,7 @@ def migrate_env(python, backup=False): exec_cmd(f'{redis} FLUSHALL') logger.log('Clearing Redis DataBase...') exec_cmd(f'{redis} FLUSHDB') - except: + except Exception: logger.warning('Please ensure Redis Connections are running or Daemonized.') # Backup venv: restore using `virtualenv --relocatable` if needed @@ -213,7 +213,7 @@ def migrate_env(python, backup=False): packages_setup = exec_cmd(f'{pvenv} -m pip install -q -U {apps}') logger.log(f'Migration Successful to {python}') - except: + except Exception: if venv_creation or packages_setup: logger.warning('Migration Error') From fafb29477c67ed470ab4f042eff1a354ae709fee Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 13 Nov 2021 00:24:32 +0530 Subject: [PATCH 42/91] test: Use frappe/chat as the test install app --- bench/tests/test_init.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/bench/tests/test_init.py b/bench/tests/test_init.py index 56b05946..b0b6e090 100755 --- a/bench/tests/test_init.py +++ b/bench/tests/test_init.py @@ -15,6 +15,8 @@ from bench.release import get_bumped_version from bench.tests.test_base import FRAPPE_BRANCH, TestBenchBase +TEST_FRAPPE_APP = "chat" + class TestBenchInit(TestBenchBase): def test_semantic_version(self): self.assertEqual( get_bumped_version('11.0.4', 'major'), '12.0.0' ) @@ -97,9 +99,9 @@ class TestBenchInit(TestBenchBase): def test_get_app(self): self.init_bench("test-bench") bench_path = os.path.join(self.benches_path, "test-bench") - bench.utils.exec_cmd("bench get-app frappe_theme", cwd=bench_path) - self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "frappe_theme"))) - app_installed_in_env = "frappe_theme" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8') + bench.utils.exec_cmd(f"bench get-app {TEST_FRAPPE_APP}", cwd=bench_path) + self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", TEST_FRAPPE_APP))) + app_installed_in_env = TEST_FRAPPE_APP in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8') self.assertTrue(app_installed_in_env) @@ -111,22 +113,22 @@ class TestBenchInit(TestBenchBase): self.init_bench(bench_name) bench.utils.exec_cmd("bench setup requirements --node", cwd=bench_path) bench.utils.exec_cmd("bench build", cwd=bench_path) - bench.utils.exec_cmd("bench get-app frappe_theme --branch master", cwd=bench_path) + bench.utils.exec_cmd(f"bench get-app {TEST_FRAPPE_APP} --branch master", cwd=bench_path) - self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "frappe_theme"))) + self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", TEST_FRAPPE_APP))) # check if app is installed - app_installed_in_env = "frappe_theme" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8') + app_installed_in_env = TEST_FRAPPE_APP in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8') self.assertTrue(app_installed_in_env) # create and install app on site self.new_site(site_name, bench_name) - installed_app = not bench.utils.exec_cmd(f"bench --site {site_name} install-app frappe_theme", cwd=bench_path) + installed_app = not bench.utils.exec_cmd(f"bench --site {site_name} install-app {TEST_FRAPPE_APP}", cwd=bench_path) app_installed_on_site = subprocess.check_output(["bench", "--site", site_name, "list-apps"], cwd=bench_path).decode('utf8') if installed_app: - self.assertTrue("frappe_theme" in app_installed_on_site) + self.assertTrue(TEST_FRAPPE_APP in app_installed_on_site) def test_remove_app(self): @@ -134,13 +136,13 @@ class TestBenchInit(TestBenchBase): bench_path = os.path.join(self.benches_path, "test-bench") bench.utils.exec_cmd("bench setup requirements --node", cwd=bench_path) - bench.utils.exec_cmd("bench get-app frappe_theme --branch master --overwrite", cwd=bench_path) - bench.utils.exec_cmd("bench remove-app frappe_theme", cwd=bench_path) + bench.utils.exec_cmd(f"bench get-app {TEST_FRAPPE_APP} --branch master --overwrite", cwd=bench_path) + bench.utils.exec_cmd(f"bench remove-app {TEST_FRAPPE_APP}", cwd=bench_path) with open(os.path.join(bench_path, "sites", "apps.txt")) as f: - self.assertFalse("frappe_theme" in f.read()) - self.assertFalse("frappe_theme" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8')) - self.assertFalse(os.path.exists(os.path.join(bench_path, "apps", "frappe_theme"))) + self.assertFalse(TEST_FRAPPE_APP in f.read()) + self.assertFalse(TEST_FRAPPE_APP in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8')) + self.assertFalse(os.path.exists(os.path.join(bench_path, "apps", TEST_FRAPPE_APP))) def test_switch_to_branch(self): From b2e0fd15c6162d0942708755916eada942b05d61 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 13 Nov 2021 00:25:01 +0530 Subject: [PATCH 43/91] ci: Don't install bench in sudo --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f5f6d6e4..e2b718ae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -90,7 +90,6 @@ install: mkdir -p ~/.bench; cp -r $TRAVIS_BUILD_DIR/* ~/.bench; pip3 install -q -U -e ~/.bench; - sudo pip install -q -U -e ~/.bench; mysql -u root -e "SET GLOBAL character_set_server = 'utf8mb4'"; mysql -u root -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'"; From e2fd9de921b5b5f26a98dd7fd38745c1eac7b9d8 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 13 Nov 2021 01:48:30 +0530 Subject: [PATCH 44/91] fix: Cleanup import hell There was this bug that said `bench.utils.bench` doesn't have attribute VERSION or whatever bench/__init__ was supposed to have. Now bench/utils/__init__ had an `import bench` statement that was supposed to ask for the top level module...but conflicted with the utils.bench namespace. Changed the line in utils to `from bench import PROJECT_NAME, VERSION` and it just works now...oh well Other changes made were an attempt to cleanup and simplify the multi level dotted path calls --- bench/cli.py | 2 +- bench/commands/setup.py | 55 +++++++++++++++++++++-------------------- bench/utils/__init__.py | 15 +++++++---- 3 files changed, 39 insertions(+), 33 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index 66b4d21d..491b54f9 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -20,13 +20,13 @@ from bench.utils import ( find_parent_bench, generate_command_cache, get_cmd_output, - get_env_cmd, is_bench_directory, is_dist_editable, is_root, log, setup_logging, ) +from bench.utils.bench import get_env_cmd from_command_line = False change_uid_msg = "You should not run this command as root" diff --git a/bench/commands/setup.py b/bench/commands/setup.py index d049b953..654f0240 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -6,11 +6,6 @@ import sys import click # imports - module imports -import bench.config.procfile -import bench.config.redis -import bench.config.site_config -import bench.config.supervisor -import bench.utils from bench.utils import exec_cmd, run_playbook @@ -22,22 +17,23 @@ def setup(): @click.command("sudoers", help="Add commands to sudoers list for execution without password") @click.argument("user") def setup_sudoers(user): - bench.utils.setup_sudoers(user) + from bench.utils.system import setup_sudoers + setup_sudoers(user) @click.command("nginx", help="Generate configuration files for NGINX") @click.option("--yes", help="Yes to regeneration of nginx config file", default=False, is_flag=True) def setup_nginx(yes=False): - import bench.config.nginx + from bench.config.nginx import make_nginx_conf - bench.config.nginx.make_nginx_conf(bench_path=".", yes=yes) + make_nginx_conf(bench_path=".", yes=yes) @click.command("reload-nginx", help="Checks NGINX config file and reloads service") def reload_nginx(): - import bench.config.production_setup + from bench.config.production_setup import reload_nginx - bench.config.production_setup.reload_nginx() + reload_nginx() @click.command("supervisor", help="Generate configuration for supervisor") @@ -45,27 +41,30 @@ def reload_nginx(): @click.option("--yes", help="Yes to regeneration of supervisor config", is_flag=True, default=False) @click.option("--skip-redis", help="Skip redis configuration", is_flag=True, default=False) def setup_supervisor(user=None, yes=False, skip_redis=False): - bench.config.supervisor.update_supervisord_config(user=user, yes=yes) - bench.config.supervisor.generate_supervisor_config(bench_path=".", user=user, yes=yes, skip_redis=skip_redis) + from bench.config.supervisor import update_supervisord_config, generate_supervisor_config + + update_supervisord_config(user=user, yes=yes) + generate_supervisor_config(bench_path=".", user=user, yes=yes, skip_redis=skip_redis) @click.command("redis", help="Generates configuration for Redis") def setup_redis(): - bench.config.redis.generate_config(".") + from bench.config.redis import generate_config + generate_config(".") @click.command("fonts", help="Add Frappe fonts to system") def setup_fonts(): - bench.utils.setup_fonts() + from bench.utils.system import setup_fonts + setup_fonts() @click.command("production", help="Setup Frappe production environment for specific user") @click.argument("user") @click.option("--yes", help="Yes to regeneration config", is_flag=True, default=False) def setup_production(user, yes=False): - import bench.config.production_setup - - bench.config.production_setup.setup_production(user=user, yes=yes) + from bench.config.production_setup import setup_production + setup_production(user=user, yes=yes) @click.command("backups", help="Add cronjob for bench backups") @@ -109,9 +108,8 @@ def set_ssh_port(port, force=False): @click.option("--custom-domain") @click.option('-n', '--non-interactive', default=False, is_flag=True, help="Run command non-interactively. This flag restarts nginx and runs certbot non interactively. Shouldn't be used on 1'st attempt") def setup_letsencrypt(site, custom_domain, non_interactive): - import bench.config.lets_encrypt - - bench.config.lets_encrypt.setup_letsencrypt(site, custom_domain, bench_path=".", interactive=not non_interactive) + from bench.config.lets_encrypt import setup_letsencrypt + setup_letsencrypt(site, custom_domain, bench_path=".", interactive=not non_interactive) @click.command("wildcard-ssl", help="Setup wildcard SSL certificate for multi-tenant bench") @@ -119,14 +117,14 @@ def setup_letsencrypt(site, custom_domain, non_interactive): @click.option("--email") @click.option("--exclude-base-domain", default=False, is_flag=True, help="SSL Certificate not applicable for base domain") def setup_wildcard_ssl(domain, email, exclude_base_domain): - import bench.config.lets_encrypt - - bench.config.lets_encrypt.setup_wildcard_ssl(domain, email, bench_path=".", exclude_base_domain=exclude_base_domain) + from bench.config.lets_encrypt import setup_wildcard_ssl + setup_wildcard_ssl(domain, email, bench_path=".", exclude_base_domain=exclude_base_domain) @click.command("procfile", help="Generate Procfile for bench start") def setup_procfile(): - bench.config.procfile.setup_procfile(".") + from bench.config.procfile import setup_procfile + setup_procfile(".") @click.command("socketio", help="[DEPRECATED] Setup node dependencies for socketio server") @@ -213,7 +211,8 @@ def add_domain(domain, site=None, ssl_certificate=None, ssl_certificate_key=None print("Please specify site") sys.exit(1) - bench.config.site_config.add_domain(site, domain, ssl_certificate, ssl_certificate_key, bench_path=".") + from bench.config.site_config import add_domain + add_domain(site, domain, ssl_certificate, ssl_certificate_key, bench_path=".") @click.command("remove-domain", help="Remove custom domain from a site") @@ -224,7 +223,8 @@ def remove_domain(domain, site=None): print("Please specify site") sys.exit(1) - bench.config.site_config.remove_domain(site, domain, bench_path=".") + from bench.config.site_config import remove_domain + remove_domain(site, domain, bench_path=".") @click.command("sync-domains", help="Check if there is a change in domains. If yes, updates the domains list.") @@ -241,7 +241,8 @@ def sync_domains(domain=None, site=None): print("Domains should be a json list of strings or dictionaries") sys.exit(1) - changed = bench.config.site_config.sync_domains(site, domains, bench_path=".") + from bench.config.site_config import sync_domains + changed = sync_domains(site, domains, bench_path=".") # if changed, success, else failure sys.exit(0 if changed else 1) diff --git a/bench/utils/__init__.py b/bench/utils/__init__.py index 181908f4..40a40c95 100644 --- a/bench/utils/__init__.py +++ b/bench/utils/__init__.py @@ -12,11 +12,12 @@ from shlex import split import click # imports - module imports -import bench +from bench import PROJECT_NAME, VERSION + from bench.exceptions import InvalidRemoteException, ValidationError -logger = logging.getLogger(bench.PROJECT_NAME) +logger = logging.getLogger(PROJECT_NAME) bench_cache_file = '.bench.cmd' paths_in_app = ('hooks.py', 'modules.txt', 'patches.txt', 'public') paths_in_bench = ('apps', 'sites', 'config', 'logs', 'config/pids') @@ -65,7 +66,7 @@ def log(message, level=0): def check_latest_version(): - if bench.VERSION.endswith("dev"): + if VERSION.endswith("dev"): return import requests @@ -81,7 +82,7 @@ def check_latest_version(): if pypi_request.status_code == 200: pypi_version_str = pypi_request.json().get('info').get('version') pypi_version = Version(pypi_version_str) - local_version = Version(bench.VERSION) + local_version = Version(VERSION) if pypi_version > local_version: log(f"A newer version of bench is available: {local_version} → {pypi_version}") @@ -137,7 +138,7 @@ def setup_logging(bench_path='.'): else: hdlr = logging.NullHandler() - logger = logging.getLogger(bench.PROJECT_NAME) + logger = logging.getLogger(PROJECT_NAME) formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') hdlr.setFormatter(formatter) logger.addHandler(hdlr) @@ -180,6 +181,7 @@ def is_root(): def run_frappe_cmd(*args, **kwargs): from bench.cli import from_command_line + from bench.utils.bench import get_env_cmd bench_path = kwargs.get('bench_path', '.') f = get_env_cmd('python', bench_path=bench_path) @@ -395,6 +397,9 @@ def is_git_url(url): def drop_privileges(uid_name='nobody', gid_name='nogroup'): + import grp + import pwd + # from http://stackoverflow.com/a/2699996 if os.getuid() != 0: # We're not root so, like, whatever dude From d2fba5fe52529c2c59b01038561cab71dc8be49d Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sun, 14 Nov 2021 09:16:19 +0530 Subject: [PATCH 45/91] fix: Cleanup import hell (contd) --- bench/tests/test_base.py | 11 ++++++----- bench/tests/test_init.py | 26 ++++++++++++-------------- bench/tests/test_setup_production.py | 18 +++++++++--------- bench/utils/__init__.py | 3 +++ bench/utils/app.py | 1 + 5 files changed, 31 insertions(+), 28 deletions(-) diff --git a/bench/tests/test_base.py b/bench/tests/test_base.py index ad884ba5..75b1646c 100644 --- a/bench/tests/test_base.py +++ b/bench/tests/test_base.py @@ -10,7 +10,8 @@ import unittest # imports - module imports import bench -import bench.utils +from bench.utils import paths_in_bench, exec_cmd +from bench.utils.system import init from bench.bench import Bench if sys.version_info.major == 2: @@ -35,7 +36,7 @@ class TestBenchBase(unittest.TestCase): shutil.rmtree(bench_path, ignore_errors=True) def assert_folders(self, bench_name): - for folder in bench.utils.paths_in_bench: + for folder in paths_in_bench: self.assert_exists(bench_name, folder) self.assert_exists(bench_name, "apps", "frappe") @@ -83,7 +84,7 @@ class TestBenchBase(unittest.TestCase): frappe_tmp_path = "/tmp/frappe" if not os.path.exists(frappe_tmp_path): - bench.utils.exec_cmd(f"git clone https://github.com/frappe/frappe -b {FRAPPE_BRANCH} --depth 1 --origin upstream {frappe_tmp_path}") + exec_cmd(f"git clone https://github.com/frappe/frappe -b {FRAPPE_BRANCH} --depth 1 --origin upstream {frappe_tmp_path}") kwargs.update(dict( python=sys.executable, @@ -93,8 +94,8 @@ class TestBenchBase(unittest.TestCase): )) if not os.path.exists(os.path.join(self.benches_path, bench_name)): - bench.utils.init(bench_name, **kwargs) - bench.utils.exec_cmd("git remote set-url upstream https://github.com/frappe/frappe", cwd=os.path.join(self.benches_path, bench_name, "apps", "frappe")) + init(bench_name, **kwargs) + exec_cmd("git remote set-url upstream https://github.com/frappe/frappe", cwd=os.path.join(self.benches_path, bench_name, "apps", "frappe")) def file_exists(self, path): if os.environ.get("CI"): diff --git a/bench/tests/test_init.py b/bench/tests/test_init.py index b0b6e090..6b13d838 100755 --- a/bench/tests/test_init.py +++ b/bench/tests/test_init.py @@ -8,9 +8,7 @@ import unittest import git # imports - module imports -import bench -import bench.cli -import bench.utils +from bench.utils import exec_cmd from bench.release import get_bumped_version from bench.tests.test_base import FRAPPE_BRANCH, TestBenchBase @@ -80,7 +78,7 @@ class TestBenchInit(TestBenchBase): site_config_path = os.path.join(site_path, "site_config.json") self.init_bench(bench_name) - bench.utils.exec_cmd("bench setup requirements --node", cwd=bench_path) + exec_cmd("bench setup requirements --node", cwd=bench_path) self.new_site(site_name, bench_name) self.assertTrue(os.path.exists(site_path)) @@ -99,7 +97,7 @@ class TestBenchInit(TestBenchBase): def test_get_app(self): self.init_bench("test-bench") bench_path = os.path.join(self.benches_path, "test-bench") - bench.utils.exec_cmd(f"bench get-app {TEST_FRAPPE_APP}", cwd=bench_path) + exec_cmd(f"bench get-app {TEST_FRAPPE_APP}", cwd=bench_path) self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", TEST_FRAPPE_APP))) app_installed_in_env = TEST_FRAPPE_APP in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8') self.assertTrue(app_installed_in_env) @@ -111,9 +109,9 @@ class TestBenchInit(TestBenchBase): bench_path = os.path.join(self.benches_path, "test-bench") self.init_bench(bench_name) - bench.utils.exec_cmd("bench setup requirements --node", cwd=bench_path) - bench.utils.exec_cmd("bench build", cwd=bench_path) - bench.utils.exec_cmd(f"bench get-app {TEST_FRAPPE_APP} --branch master", cwd=bench_path) + exec_cmd("bench setup requirements --node", cwd=bench_path) + exec_cmd("bench build", cwd=bench_path) + exec_cmd(f"bench get-app {TEST_FRAPPE_APP} --branch master", cwd=bench_path) self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", TEST_FRAPPE_APP))) @@ -123,7 +121,7 @@ class TestBenchInit(TestBenchBase): # create and install app on site self.new_site(site_name, bench_name) - installed_app = not bench.utils.exec_cmd(f"bench --site {site_name} install-app {TEST_FRAPPE_APP}", cwd=bench_path) + installed_app = not exec_cmd(f"bench --site {site_name} install-app {TEST_FRAPPE_APP}", cwd=bench_path) app_installed_on_site = subprocess.check_output(["bench", "--site", site_name, "list-apps"], cwd=bench_path).decode('utf8') @@ -135,9 +133,9 @@ class TestBenchInit(TestBenchBase): self.init_bench("test-bench") bench_path = os.path.join(self.benches_path, "test-bench") - bench.utils.exec_cmd("bench setup requirements --node", cwd=bench_path) - bench.utils.exec_cmd(f"bench get-app {TEST_FRAPPE_APP} --branch master --overwrite", cwd=bench_path) - bench.utils.exec_cmd(f"bench remove-app {TEST_FRAPPE_APP}", cwd=bench_path) + exec_cmd("bench setup requirements --node", cwd=bench_path) + exec_cmd(f"bench get-app {TEST_FRAPPE_APP} --branch master --overwrite", cwd=bench_path) + exec_cmd(f"bench remove-app {TEST_FRAPPE_APP}", cwd=bench_path) with open(os.path.join(bench_path, "sites", "apps.txt")) as f: self.assertFalse(TEST_FRAPPE_APP in f.read()) @@ -150,12 +148,12 @@ class TestBenchInit(TestBenchBase): bench_path = os.path.join(self.benches_path, "test-bench") app_path = os.path.join(bench_path, "apps", "frappe") - successful_switch = not bench.utils.exec_cmd("bench switch-to-branch version-13 frappe --upgrade", cwd=bench_path) + successful_switch = not exec_cmd("bench switch-to-branch version-13 frappe --upgrade", cwd=bench_path) app_branch_after_switch = str(git.Repo(path=app_path).active_branch) if successful_switch: self.assertEqual("version-13", app_branch_after_switch) - successful_switch = not bench.utils.exec_cmd("bench switch-to-branch develop frappe --upgrade", cwd=bench_path) + successful_switch = not exec_cmd("bench switch-to-branch develop frappe --upgrade", cwd=bench_path) app_branch_after_second_switch = str(git.Repo(path=app_path).active_branch) if successful_switch: self.assertEqual("develop", app_branch_after_second_switch) diff --git a/bench/tests/test_setup_production.py b/bench/tests/test_setup_production.py index 47358dc5..af9c0b5b 100644 --- a/bench/tests/test_setup_production.py +++ b/bench/tests/test_setup_production.py @@ -7,7 +7,7 @@ import time import unittest # imports - module imports -import bench.utils +from bench.utils import exec_cmd, get_cmd_output, which from bench.config.production_setup import get_supervisor_confdir from bench.tests.test_base import TestBenchBase @@ -19,18 +19,18 @@ class TestSetupProduction(TestBenchBase): for bench_name in ("test-bench-1", "test-bench-2"): bench_path = os.path.join(os.path.abspath(self.benches_path), bench_name) self.init_bench(bench_name) - bench.utils.exec_cmd(f"sudo bench setup production {user} --yes", cwd=bench_path) + exec_cmd(f"sudo bench setup production {user} --yes", cwd=bench_path) self.assert_nginx_config(bench_name) self.assert_supervisor_config(bench_name) self.assert_supervisor_process(bench_name) self.assert_nginx_process() - bench.utils.exec_cmd(f"sudo bench setup sudoers {user}") + exec_cmd(f"sudo bench setup sudoers {user}") self.assert_sudoers(user) for bench_name in self.benches: bench_path = os.path.join(os.path.abspath(self.benches_path), bench_name) - bench.utils.exec_cmd("sudo bench disable-production", cwd=bench_path) + exec_cmd("sudo bench disable-production", cwd=bench_path) def production(self): @@ -62,14 +62,14 @@ class TestSetupProduction(TestBenchBase): def assert_nginx_process(self): - out = bench.utils.get_cmd_output("sudo nginx -t 2>&1") + out = get_cmd_output("sudo nginx -t 2>&1") self.assertTrue("nginx: configuration file /etc/nginx/nginx.conf test is successful" in out) def assert_sudoers(self, user): sudoers_file = '/etc/sudoers.d/frappe' - service = bench.utils.which("service") - nginx = bench.utils.which("nginx") + service = which("service") + nginx = which("nginx") self.assertTrue(self.file_exists(sudoers_file)) @@ -133,12 +133,12 @@ class TestSetupProduction(TestBenchBase): def assert_supervisor_process(self, bench_name, use_rq=True, disable_production=False): - out = bench.utils.get_cmd_output("supervisorctl status") + out = get_cmd_output("supervisorctl status") while "STARTING" in out: print ("Waiting for all processes to start...") time.sleep(10) - out = bench.utils.get_cmd_output("supervisorctl status") + out = get_cmd_output("supervisorctl status") tests = [ "{bench_name}-web:{bench_name}-frappe-web[\s]+RUNNING", diff --git a/bench/utils/__init__.py b/bench/utils/__init__.py index 40a40c95..8bcb9ee0 100644 --- a/bench/utils/__init__.py +++ b/bench/utils/__init__.py @@ -256,6 +256,8 @@ def set_git_remote_url(git_url, bench_path='.'): def run_playbook(playbook_name, extra_vars=None, tag=None): + import bench + if not which('ansible'): print("Ansible is needed to run this command, please install it using 'pip install ansible'") sys.exit(1) @@ -326,6 +328,7 @@ def generate_command_cache(bench_path='.'): """Caches all available commands (even custom apps) via Frappe Default caching behaviour: generated the first time any command (for a specific bench directory) """ + from bench.utils.bench import get_env_cmd python = get_env_cmd('python', bench_path=bench_path) sites_path = os.path.join(bench_path, 'sites') diff --git a/bench/utils/app.py b/bench/utils/app.py index 64d6bf9b..ab2f0b50 100644 --- a/bench/utils/app.py +++ b/bench/utils/app.py @@ -130,6 +130,7 @@ def get_current_frappe_version(bench_path='.'): return 0 def get_current_branch(app, bench_path='.'): + from bench.utils import get_cmd_output repo_dir = get_repo_dir(app, bench_path=bench_path) return get_cmd_output("basename $(git symbolic-ref -q HEAD)", cwd=repo_dir) From 1330e66d073887b6aa9e95cc097e0c25857f8f31 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 13 Nov 2021 02:59:01 +0530 Subject: [PATCH 46/91] style: Flake8 + Black-ish Tried styling and following standards over teh following modules: * bench.app * bench.bench * bench.exceptions * bench.utils --- bench/__init__.py | 5 +- bench/app.py | 168 +++++++++++-------- bench/bench.py | 78 +++++---- bench/exceptions.py | 5 + bench/utils/__init__.py | 116 +++++++------ bench/utils/app.py | 139 ++++++++++------ bench/utils/bench.py | 325 +++++++++++++++++++++---------------- bench/utils/system.py | 109 ++++++++----- bench/utils/translation.py | 18 +- 9 files changed, 577 insertions(+), 386 deletions(-) diff --git a/bench/__init__.py b/bench/__init__.py index e2f6fd6d..77c6e130 100644 --- a/bench/__init__.py +++ b/bench/__init__.py @@ -5,8 +5,9 @@ current_path = None updated_path = None -def set_frappe_version(bench_path='.'): +def set_frappe_version(bench_path="."): from .utils.app import get_current_frappe_version + global FRAPPE_VERSION if not FRAPPE_VERSION: - FRAPPE_VERSION = get_current_frappe_version(bench_path=bench_path) \ No newline at end of file + FRAPPE_VERSION = get_current_frappe_version(bench_path=bench_path) diff --git a/bench/app.py b/bench/app.py index d3aee384..e36cc73f 100755 --- a/bench/app.py +++ b/bench/app.py @@ -13,8 +13,19 @@ import click # imports - module imports import bench -from bench.utils import exec_cmd, is_bench_directory, run_frappe_cmd, is_git_url, fetch_details_from_tag -from bench.utils.bench import get_env_cmd, build_assets, restart_supervisor_processes, restart_systemd_processes +from bench.utils import ( + exec_cmd, + is_bench_directory, + run_frappe_cmd, + is_git_url, + fetch_details_from_tag, +) +from bench.utils.bench import ( + get_env_cmd, + build_assets, + restart_supervisor_processes, + restart_systemd_processes, +) logger = logging.getLogger(bench.PROJECT_NAME) @@ -22,8 +33,9 @@ logger = logging.getLogger(bench.PROJECT_NAME) if typing.TYPE_CHECKING: from bench.bench import Bench + class AppMeta: - def __init__(self, name: str, branch : str = None): + def __init__(self, name: str, branch: str = None): """ name (str): This could look something like 1. https://github.com/frappe/healthcare.git @@ -96,23 +108,22 @@ class AppMeta: class App(AppMeta): - def __init__(self, name: str, branch : str = None, bench : "Bench" = None): + def __init__(self, name: str, branch: str = None, bench: "Bench" = None): super().__init__(name, branch) self.bench = bench def get(self): - branch = f'--branch {self.tag}' if self.tag else '' - shallow = '--depth 1' if self.bench.shallow_clone else '' + branch = f"--branch {self.tag}" if self.tag else "" + shallow = "--depth 1" if self.bench.shallow_clone else "" self.bench.run( f"git clone {self.url} {branch} {shallow} --origin upstream", - cwd=os.path.join(self.bench.name, 'apps') + cwd=os.path.join(self.bench.name, "apps"), ) def remove(self): shutil.move( - os.path.join("apps", self.repo), - os.path.join("archived", "apps", self.repo), + os.path.join("apps", self.repo), os.path.join("archived", "apps", self.repo), ) def install(self, skip_assets=False, verbose=False): @@ -131,12 +142,10 @@ class App(AppMeta): def uninstall(self): env_python = get_env_cmd("python", bench_path=self.bench.name) - self.bench.run( - f"{env_python} -m pip uninstall -y {self.repo}" - ) + self.bench.run(f"{env_python} -m pip uninstall -y {self.repo}") -def add_to_appstxt(app, bench_path='.'): +def add_to_appstxt(app, bench_path="."): from bench.bench import Bench apps = Bench(bench_path).apps @@ -145,7 +154,8 @@ def add_to_appstxt(app, bench_path='.'): apps.append(app) return write_appstxt(apps, bench_path=bench_path) -def remove_from_appstxt(app, bench_path='.'): + +def remove_from_appstxt(app, bench_path="."): from bench.bench import Bench apps = Bench(bench_path).apps @@ -154,37 +164,43 @@ def remove_from_appstxt(app, bench_path='.'): apps.remove(app) return write_appstxt(apps, bench_path=bench_path) -def write_appstxt(apps, bench_path='.'): - with open(os.path.join(bench_path, 'sites', 'apps.txt'), 'w') as f: - return f.write('\n'.join(apps)) -def get_excluded_apps(bench_path='.'): +def write_appstxt(apps, bench_path="."): + with open(os.path.join(bench_path, "sites", "apps.txt"), "w") as f: + return f.write("\n".join(apps)) + + +def get_excluded_apps(bench_path="."): try: - with open(os.path.join(bench_path, 'sites', 'excluded_apps.txt')) as f: - return f.read().strip().split('\n') + with open(os.path.join(bench_path, "sites", "excluded_apps.txt")) as f: + return f.read().strip().split("\n") except IOError: return [] -def add_to_excluded_apps_txt(app, bench_path='.'): - if app == 'frappe': - raise ValueError('Frappe app cannot be excludeed from update') - if app not in os.listdir('apps'): - raise ValueError(f'The app {app} does not exist') + +def add_to_excluded_apps_txt(app, bench_path="."): + if app == "frappe": + raise ValueError("Frappe app cannot be excludeed from update") + if app not in os.listdir("apps"): + raise ValueError(f"The app {app} does not exist") apps = get_excluded_apps(bench_path=bench_path) if app not in apps: apps.append(app) return write_excluded_apps_txt(apps, bench_path=bench_path) -def write_excluded_apps_txt(apps, bench_path='.'): - with open(os.path.join(bench_path, 'sites', 'excluded_apps.txt'), 'w') as f: - return f.write('\n'.join(apps)) -def remove_from_excluded_apps_txt(app, bench_path='.'): +def write_excluded_apps_txt(apps, bench_path="."): + with open(os.path.join(bench_path, "sites", "excluded_apps.txt"), "w") as f: + return f.write("\n".join(apps)) + + +def remove_from_excluded_apps_txt(app, bench_path="."): apps = get_excluded_apps(bench_path=bench_path) if app in apps: apps.remove(app) return write_excluded_apps_txt(apps, bench_path=bench_path) + def generate_bench_name(git_url, bench_path): if os.path.exists(git_url): guessed_app_name = os.path.basename(git_url) @@ -194,25 +210,29 @@ def generate_bench_name(git_url, bench_path): return os.path.join(bench_path, f"{guessed_app_name}-bench") -def setup_app_dependencies(repo_name, bench_path='.', branch=None): + +def setup_app_dependencies(repo_name, bench_path=".", branch=None): # branch kwarg is somewhat of a hack here; since we're assuming the same branches for all apps # for eg: if you're installing erpnext@develop, you'll want frappe@develop and healthcare@develop too import glob - apps_path = os.path.join(os.path.abspath(bench_path), 'apps') - files = glob.glob(os.path.join(apps_path, repo_name, '**', 'hooks.py')) + apps_path = os.path.join(os.path.abspath(bench_path), "apps") + files = glob.glob(os.path.join(apps_path, repo_name, "**", "hooks.py")) if files: with open(files[0]) as f: - lines = [x for x in f.read().split('\n') if x.strip().startswith('required_apps')] + lines = [x for x in f.read().split("\n") if x.strip().startswith("required_apps")] if lines: - required_apps = eval(lines[0].strip('required_apps').strip().lstrip('=').strip()) + required_apps = eval(lines[0].strip("required_apps").strip().lstrip("=").strip()) # TODO: when the time comes, add version check here for app in required_apps: if app not in Bench(bench_path).apps: get_app(app, bench_path=bench_path, branch=branch) -def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=False, overwrite=False): + +def get_app( + git_url, branch=None, bench_path=".", skip_assets=False, verbose=False, overwrite=False +): """bench get-app clones a Frappe App from remote (GitHub or any other git server), and installs it on the current bench. This also resolves dependencies based on the apps' required_apps defined in the hooks.py file. @@ -231,9 +251,10 @@ def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=Fal if not is_bench_directory(bench_path): bench_path = generate_bench_name(git_url, bench_path) from bench.commands.make import init + click.get_current_context().invoke(init, path=bench_path, frappe_branch=branch) - cloned_path = os.path.join(bench_path, 'apps', repo_name) + cloned_path = os.path.join(bench_path, "apps", repo_name) dir_already_exists = os.path.isdir(cloned_path) if dir_already_exists: @@ -244,6 +265,7 @@ def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=Fal "Do you want to continue and overwrite it?" ): import shutil + shutil.rmtree(cloned_path) elif click.confirm("Do you want to reinstall the existing application?", abort=True): pass @@ -256,20 +278,27 @@ def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=Fal app.install(verbose=verbose, skip_assets=skip_assets) -def new_app(app, bench_path='.'): +def new_app(app, bench_path="."): # For backwards compatibility app = app.lower().replace(" ", "_").replace("-", "_") - logger.log(f'creating new app {app}') - apps = os.path.abspath(os.path.join(bench_path, 'apps')) - run_frappe_cmd('make-app', apps, app, bench_path=bench_path) + logger.log(f"creating new app {app}") + apps = os.path.abspath(os.path.join(bench_path, "apps")) + run_frappe_cmd("make-app", apps, app, bench_path=bench_path) install_app(app, bench_path=bench_path) -def install_app(app, bench_path=".", verbose=False, no_cache=False, restart_bench=True, skip_assets=False): +def install_app( + app, + bench_path=".", + verbose=False, + no_cache=False, + restart_bench=True, + skip_assets=False, +): from bench.bench import Bench from bench.utils.bench import get_env_cmd - install_text = f'Installing {app}' + install_text = f"Installing {app}" click.secho(install_text, fg="yellow") logger.log(install_text) @@ -280,7 +309,7 @@ def install_app(app, bench_path=".", verbose=False, no_cache=False, restart_benc exec_cmd(f"{python_path} -m pip install {quiet_flag} -U -e {app_path} {cache_flag}") - if os.path.exists(os.path.join(app_path, 'package.json')): + if os.path.exists(os.path.join(app_path, "package.json")): exec_cmd("yarn install", cwd=app_path) add_to_appstxt(app, bench_path=bench_path) @@ -289,25 +318,26 @@ def install_app(app, bench_path=".", verbose=False, no_cache=False, restart_benc if conf.get("developer_mode"): from bench.utils.bench import install_python_dev_dependencies + install_python_dev_dependencies(apps=app) if not skip_assets: build_assets(bench_path=bench_path, app=app) if restart_bench: - if conf.get('restart_supervisor_on_update'): + if conf.get("restart_supervisor_on_update"): restart_supervisor_processes(bench_path=bench_path) - if conf.get('restart_systemd_on_update'): + if conf.get("restart_systemd_on_update"): restart_systemd_processes(bench_path=bench_path) -def pull_apps(apps=None, bench_path='.', reset=False): - '''Check all apps if there no local changes, pull''' +def pull_apps(apps=None, bench_path=".", reset=False): + """Check all apps if there no local changes, pull""" from bench.bench import Bench from bench.utils.app import get_remote, get_current_branch bench = Bench(bench_path) - rebase = '--rebase' if bench.conf.get('rebase_on_pull') else '' + rebase = "--rebase" if bench.conf.get("rebase_on_pull") else "" apps = apps or bench.apps excluded_apps = bench.excluded_apps @@ -318,11 +348,12 @@ def pull_apps(apps=None, bench_path='.', reset=False): print(f"Skipping reset for app {app}") continue app_dir = get_repo_dir(app, bench_path=bench_path) - if os.path.exists(os.path.join(app_dir, '.git')): - out = subprocess.check_output('git status', shell=True, cwd=app_dir) - out = out.decode('utf-8') - if not re.search(r'nothing to commit, working (directory|tree) clean', out): - print(f''' + if os.path.exists(os.path.join(app_dir, ".git")): + out = subprocess.check_output("git status", shell=True, cwd=app_dir) + out = out.decode("utf-8") + if not re.search(r"nothing to commit, working (directory|tree) clean", out): + print( + f""" Cannot proceed with update: You have local changes in app "{app}" that are not committed. @@ -332,7 +363,8 @@ Here are your choices: 1. Temporarily remove your changes with "git stash" or discard them completely with "bench update --reset" or for individual repositries "git reset --hard" 2. If your changes are helpful for others, send in a pull request via GitHub and - wait for them to be merged in the core.''') + wait for them to be merged in the core.""" + ) sys.exit(1) for app in apps: @@ -340,15 +372,18 @@ Here are your choices: print(f"Skipping pull for app {app}") continue app_dir = get_repo_dir(app, bench_path=bench_path) - if os.path.exists(os.path.join(app_dir, '.git')): + if os.path.exists(os.path.join(app_dir, ".git")): remote = get_remote(app) if not remote: # remote is False, i.e. remote doesn't exist, add the app to excluded_apps.txt add_to_excluded_apps_txt(app, bench_path=bench_path) - print(f"Skipping pull for app {app}, since remote doesn't exist, and adding it to excluded apps") + print( + f"Skipping pull for app {app}, since remote doesn't exist, and" + " adding it to excluded apps" + ) continue - if not bench.conf.get('shallow_clone') or not reset: + if not bench.conf.get("shallow_clone") or not reset: is_shallow = os.path.exists(os.path.join(app_dir, ".git", "shallow")) if is_shallow: s = " to safely pull remote changes." if not reset else "" @@ -356,10 +391,10 @@ Here are your choices: bench.run(f"git fetch {remote} --unshallow", cwd=app_dir) branch = get_current_branch(app, bench_path=bench_path) - logger.log(f'pulling {app}') + logger.log(f"pulling {app}") if reset: reset_cmd = f"git reset --hard {remote}/{branch}" - if bench.conf.get('shallow_clone'): + if bench.conf.get("shallow_clone"): bench.run(f"git fetch --depth=1 --no-tags {remote} {branch}", cwd=app_dir) bench.run(reset_cmd, cwd=app_dir) bench.run("git reflog expire --all", cwd=app_dir) @@ -374,21 +409,24 @@ Here are your choices: def use_rq(bench_path): bench_path = os.path.abspath(bench_path) - celery_app = os.path.join(bench_path, 'apps', 'frappe', 'frappe', 'celery_app.py') + celery_app = os.path.join(bench_path, "apps", "frappe", "frappe", "celery_app.py") return not os.path.exists(celery_app) -def get_repo_dir(app, bench_path='.'): - return os.path.join(bench_path, 'apps', app) -def install_apps_from_path(path, bench_path='.'): +def get_repo_dir(app, bench_path="."): + return os.path.join(bench_path, "apps", app) + + +def install_apps_from_path(path, bench_path="."): apps = get_apps_json(path) for app in apps: - get_app(app['url'], branch=app.get('branch'), bench_path=bench_path, skip_assets=True) + get_app(app["url"], branch=app.get("branch"), bench_path=bench_path, skip_assets=True) + def get_apps_json(path): import requests - if path.startswith('http'): + if path.startswith("http"): r = requests.get(path) return r.json() diff --git a/bench/bench.py b/bench/bench.py index e1db12b6..8add1744 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -7,8 +7,21 @@ from typing import MutableSequence, TYPE_CHECKING import bench from bench.exceptions import ValidationError from bench.config.common_site_config import setup_config -from bench.utils import paths_in_bench, exec_cmd, is_frappe_app, get_git_version, run_frappe_cmd -from bench.utils.bench import validate_app_installed_on_sites, restart_supervisor_processes, restart_systemd_processes, remove_backups_crontab, get_venv_path, get_env_cmd +from bench.utils import ( + paths_in_bench, + exec_cmd, + is_frappe_app, + get_git_version, + run_frappe_cmd, +) +from bench.utils.bench import ( + validate_app_installed_on_sites, + restart_supervisor_processes, + restart_systemd_processes, + remove_backups_crontab, + get_venv_path, + get_env_cmd, +) if TYPE_CHECKING: @@ -39,15 +52,15 @@ class Bench(Base, Validator): self.teardown = BenchTearDown(self) self.apps = BenchApps(self) - self.apps_txt = os.path.join(self.name, 'sites', 'apps.txt') - self.excluded_apps_txt = os.path.join(self.name, 'sites', 'excluded_apps.txt') + self.apps_txt = os.path.join(self.name, "sites", "apps.txt") + self.excluded_apps_txt = os.path.join(self.name, "sites", "excluded_apps.txt") @property def shallow_clone(self): config = self.conf if config: - if config.get('release_bench') or not config.get('shallow_clone'): + if config.get("release_bench") or not config.get("shallow_clone"): return False if get_git_version() > 1.9: @@ -57,22 +70,22 @@ class Bench(Base, Validator): def excluded_apps(self): try: with open(self.excluded_apps_txt) as f: - return f.read().strip().split('\n') + return f.read().strip().split("\n") except Exception: return [] @property def sites(self): return [ - path for path in os.listdir(os.path.join(self.name, 'sites')) - if os.path.exists( - os.path.join("sites", path, "site_config.json") - ) + path + for path in os.listdir(os.path.join(self.name, "sites")) + if os.path.exists(os.path.join("sites", path, "site_config.json")) ] @property def conf(self): from bench.config.common_site_config import get_config + return get_config(self.name) def init(self): @@ -112,14 +125,14 @@ class Bench(Base, Validator): def reload(self): conf = self.conf - if conf.get('restart_supervisor_on_update'): + if conf.get("restart_supervisor_on_update"): restart_supervisor_processes(bench_path=self.name) - if conf.get('restart_systemd_on_update'): + if conf.get("restart_systemd_on_update"): restart_systemd_processes(bench_path=self.name) class BenchApps(MutableSequence): - def __init__(self, bench : Bench): + def __init__(self, bench: Bench): self.bench = bench self.initialize_apps() @@ -130,25 +143,27 @@ class BenchApps(MutableSequence): def initialize_apps(self): try: - self.apps = [x for x in os.listdir( - os.path.join(self.bench.name, "apps") - ) if is_frappe_app(os.path.join(self.bench.name, "apps", x))] + self.apps = [ + x + for x in os.listdir(os.path.join(self.bench.name, "apps")) + if is_frappe_app(os.path.join(self.bench.name, "apps", x)) + ] self.apps.sort() except FileNotFoundError: self.apps = [] def __getitem__(self, key): - ''' retrieves an item by its index, key''' + """ retrieves an item by its index, key""" return self.apps[key] def __setitem__(self, key, value): - ''' set the item at index, key, to value ''' + """ set the item at index, key, to value """ # should probably not be allowed # self.apps[key] = value raise NotImplementedError def __delitem__(self, key): - ''' removes the item at index, key ''' + """ removes the item at index, key """ # TODO: uninstall and delete app from bench del self.apps[key] @@ -156,7 +171,7 @@ class BenchApps(MutableSequence): return len(self.apps) def insert(self, key, value): - ''' add an item, value, at index, key. ''' + """ add an item, value, at index, key. """ # TODO: fetch and install app to bench self.apps.insert(key, value) @@ -171,7 +186,7 @@ class BenchApps(MutableSequence): app.remove() super().remove(app.repo) - def append(self, app : "App"): + def append(self, app: "App"): return self.add(app) def __repr__(self): @@ -182,7 +197,7 @@ class BenchApps(MutableSequence): class BenchSetup(Base): - def __init__(self, bench : Bench): + def __init__(self, bench: Bench): self.bench = bench self.cwd = self.bench.cwd @@ -219,41 +234,46 @@ class BenchSetup(Base): if redis: from bench.config.redis import generate_config + generate_config(self.bench.name) if procfile: from bench.config.procfile import setup_procfile + setup_procfile(self.bench.name, skip_redis=not redis) def logging(self): from bench.utils import setup_logging + return setup_logging(bench_path=self.bench.name) def patches(self): shutil.copy( - os.path.join(os.path.dirname(os.path.abspath(__file__)), 'patches', 'patches.txt'), - os.path.join(self.bench.name, 'patches.txt') + os.path.join(os.path.dirname(os.path.abspath(__file__)), "patches", "patches.txt"), + os.path.join(self.bench.name, "patches.txt"), ) def backups(self): # TODO: to something better for logging data? - maybe a wrapper that auto-logs with more context - logger.log('setting up backups') + logger.log("setting up backups") from crontab import CronTab bench_dir = os.path.abspath(self.bench.name) - user = self.bench.conf.get('frappe_user') - logfile = os.path.join(bench_dir, 'logs', 'backup.log') + user = self.bench.conf.get("frappe_user") + logfile = os.path.join(bench_dir, "logs", "backup.log") system_crontab = CronTab(user=user) backup_command = f"cd {bench_dir} && {sys.argv[0]} --verbose --site all backup" job_command = f"{backup_command} >> {logfile} 2>&1" if job_command not in str(system_crontab): - job = system_crontab.new(command=job_command, comment="bench auto backups set for every 6 hours") + job = system_crontab.new( + command=job_command, comment="bench auto backups set for every 6 hours" + ) job.every(6).hours() system_crontab.write() - logger.log('backups were set up') + logger.log("backups were set up") class BenchTearDown: diff --git a/bench/exceptions.py b/bench/exceptions.py index b3f206c7..3002314b 100644 --- a/bench/exceptions.py +++ b/bench/exceptions.py @@ -1,17 +1,22 @@ class InvalidBranchException(Exception): pass + class InvalidRemoteException(Exception): pass + class PatchError(Exception): pass + class CommandFailedError(Exception): pass + class BenchNotFoundError(Exception): pass + class ValidationError(Exception): pass diff --git a/bench/utils/__init__.py b/bench/utils/__init__.py index 8bcb9ee0..525c6079 100644 --- a/bench/utils/__init__.py +++ b/bench/utils/__init__.py @@ -18,10 +18,10 @@ from bench.exceptions import InvalidRemoteException, ValidationError logger = logging.getLogger(PROJECT_NAME) -bench_cache_file = '.bench.cmd' -paths_in_app = ('hooks.py', 'modules.txt', 'patches.txt', 'public') -paths_in_bench = ('apps', 'sites', 'config', 'logs', 'config/pids') -sudoers_file = '/etc/sudoers.d/frappe' +bench_cache_file = ".bench.cmd" +paths_in_app = ("hooks.py", "modules.txt", "patches.txt", "public") +paths_in_bench = ("apps", "sites", "config", "logs", "config/pids") +sudoers_file = "/etc/sudoers.d/frappe" def is_bench_directory(directory=os.path.curdir): @@ -49,15 +49,12 @@ def is_frappe_app(directory): def log(message, level=0): levels = { - 0: ("blue", "INFO"), # normal - 1: ("green", "SUCCESS"), # success - 2: ("red", "ERROR"), # fail - 3: ("yellow", "WARN") # warn/suggest - } - loggers = { - 2: logger.error, - 3: logger.warning + 0: ("blue", "INFO"), # normal + 1: ("green", "SUCCESS"), # success + 2: ("red", "ERROR"), # fail + 3: ("yellow", "WARN"), # warn/suggest } + loggers = {2: logger.error, 3: logger.warning} color, prefix = levels.get(level, levels[0]) level_logger = loggers.get(level, logger.info) @@ -80,7 +77,7 @@ def check_latest_version(): return if pypi_request.status_code == 200: - pypi_version_str = pypi_request.json().get('info').get('version') + pypi_version_str = pypi_request.json().get("info").get("version") pypi_version = Version(pypi_version_str) local_version = Version(VERSION) @@ -98,11 +95,11 @@ def pause_exec(seconds=10): print(" " * 40, end="\r") -def exec_cmd(cmd, cwd='.', env=None): +def exec_cmd(cmd, cwd=".", env=None): if env: env.update(os.environ.copy()) - click.secho(f"$ {cmd}", fg='bright_black') + click.secho(f"$ {cmd}", fg="bright_black") cwd_info = f"cd {cwd} && " if cwd != "." else "" cmd_log = f"{cwd_info}{cmd}" @@ -119,27 +116,29 @@ def which(executable, raise_err=False): exec_ = which(executable) if not exec_ and raise_err: - raise ValueError(f'{executable} not found.') + raise ValueError(f"{executable} not found.") return exec_ -def setup_logging(bench_path='.'): +def setup_logging(bench_path="."): LOG_LEVEL = 15 logging.addLevelName(LOG_LEVEL, "LOG") + def logv(self, message, *args, **kws): if self.isEnabledFor(LOG_LEVEL): self._log(LOG_LEVEL, message, args, **kws) + logging.Logger.log = logv - if os.path.exists(os.path.join(bench_path, 'logs')): - log_file = os.path.join(bench_path, 'logs', 'bench.log') + if os.path.exists(os.path.join(bench_path, "logs")): + log_file = os.path.join(bench_path, "logs", "bench.log") hdlr = logging.FileHandler(log_file) else: hdlr = logging.NullHandler() logger = logging.getLogger(PROJECT_NAME) - formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') + formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s") hdlr.setFormatter(formatter) logger.addHandler(hdlr) logger.setLevel(logging.DEBUG) @@ -148,25 +147,27 @@ def setup_logging(bench_path='.'): def get_process_manager(): - for proc_man in ['honcho', 'foreman', 'forego']: + for proc_man in ["honcho", "foreman", "forego"]: proc_man_path = which(proc_man) if proc_man_path: return proc_man_path def get_git_version(): - '''returns git version from `git --version` - extracts version number from string `get version 1.9.1` etc''' + """returns git version from `git --version` + extracts version number from string `get version 1.9.1` etc""" version = get_cmd_output("git --version") version = version.strip().split()[2] - version = '.'.join(version.split('.')[0:2]) + version = ".".join(version.split(".")[0:2]) return float(version) -def get_cmd_output(cmd, cwd='.', _raise=True): +def get_cmd_output(cmd, cwd=".", _raise=True): output = "" try: - output = subprocess.check_output(cmd, cwd=cwd, shell=True, stderr=subprocess.PIPE, encoding="utf-8").strip() + output = subprocess.check_output( + cmd, cwd=cwd, shell=True, stderr=subprocess.PIPE, encoding="utf-8" + ).strip() except subprocess.CalledProcessError as e: if e.output: output = e.output @@ -183,9 +184,9 @@ def run_frappe_cmd(*args, **kwargs): from bench.cli import from_command_line from bench.utils.bench import get_env_cmd - bench_path = kwargs.get('bench_path', '.') - f = get_env_cmd('python', bench_path=bench_path) - sites_dir = os.path.join(bench_path, 'sites') + bench_path = kwargs.get("bench_path", ".") + f = get_env_cmd("python", bench_path=bench_path) + sites_dir = os.path.join(bench_path, "sites") is_async = False if from_command_line else True if is_async: @@ -193,8 +194,12 @@ def run_frappe_cmd(*args, **kwargs): else: stderr = stdout = None - p = subprocess.Popen((f, '-m', 'frappe.utils.bench_helper', 'frappe') + args, - cwd=sites_dir, stdout=stdout, stderr=stderr) + p = subprocess.Popen( + (f, "-m", "frappe.utils.bench_helper", "frappe") + args, + cwd=sites_dir, + stdout=stdout, + stderr=stderr, + ) if is_async: return_code = print_output(p) @@ -217,20 +222,20 @@ def print_output(p): buf = p.stdout.read(1) if not len(buf): break - if buf == '\r' or buf == '\n': + if buf == "\r" or buf == "\n": send_buffer.append(buf) - log_line(''.join(send_buffer), 'stdout') + log_line("".join(send_buffer), "stdout") send_buffer = [] else: send_buffer.append(buf) if fd == p.stderr.fileno(): - log_line(p.stderr.readline(), 'stderr') + log_line(p.stderr.readline(), "stderr") return p.poll() def log_line(data, stream): - if stream == 'stderr': + if stream == "stderr": return sys.stderr.write(data) return sys.stdout.write(data) @@ -239,37 +244,40 @@ def get_bench_name(bench_path): return os.path.basename(os.path.abspath(bench_path)) -def set_git_remote_url(git_url, bench_path='.'): +def set_git_remote_url(git_url, bench_path="."): "Set app remote git url" from bench.app import get_repo_dir from bench.bench import Bench - app = git_url.rsplit('/', 1)[1].rsplit('.', 1)[0] + app = git_url.rsplit("/", 1)[1].rsplit(".", 1)[0] if app not in Bench(bench_path).apps: raise ValidationError(f"No app named {app}") app_dir = get_repo_dir(app, bench_path=bench_path) - if os.path.exists(os.path.join(app_dir, '.git')): + if os.path.exists(os.path.join(app_dir, ".git")): exec_cmd(f"git remote set-url upstream {git_url}", cwd=app_dir) def run_playbook(playbook_name, extra_vars=None, tag=None): import bench - if not which('ansible'): - print("Ansible is needed to run this command, please install it using 'pip install ansible'") + if not which("ansible"): + print( + "Ansible is needed to run this command, please install it using 'pip" + " install ansible'" + ) sys.exit(1) - args = ['ansible-playbook', '-c', 'local', playbook_name, '-vvvv'] + args = ["ansible-playbook", "-c", "local", playbook_name, "-vvvv"] if extra_vars: - args.extend(['-e', json.dumps(extra_vars)]) + args.extend(["-e", json.dumps(extra_vars)]) if tag: - args.extend(['-t', tag]) + args.extend(["-t", tag]) - subprocess.check_call(args, cwd=os.path.join(bench.__path__[0], 'playbooks')) + subprocess.check_call(args, cwd=os.path.join(bench.__path__[0], "playbooks")) def find_benches(directory=None): @@ -304,7 +312,7 @@ def find_benches(directory=None): def is_dist_editable(dist): """Is distribution an editable install?""" for path_item in sys.path: - egg_link = os.path.join(path_item, dist + '.egg-link') + egg_link = os.path.join(path_item, dist + ".egg-link") if os.path.isfile(egg_link): return True return False @@ -324,21 +332,23 @@ def find_parent_bench(path): return find_parent_bench(parent_dir) -def generate_command_cache(bench_path='.'): +def generate_command_cache(bench_path="."): """Caches all available commands (even custom apps) via Frappe Default caching behaviour: generated the first time any command (for a specific bench directory) """ from bench.utils.bench import get_env_cmd - python = get_env_cmd('python', bench_path=bench_path) - sites_path = os.path.join(bench_path, 'sites') + python = get_env_cmd("python", bench_path=bench_path) + sites_path = os.path.join(bench_path, "sites") if os.path.exists(bench_cache_file): os.remove(bench_cache_file) try: - output = get_cmd_output(f"{python} -m frappe.utils.bench_helper get-frappe-commands", cwd=sites_path) - with open(bench_cache_file, 'w') as f: + output = get_cmd_output( + f"{python} -m frappe.utils.bench_helper get-frappe-commands", cwd=sites_path + ) + with open(bench_cache_file, "w") as f: json.dump(eval(output), f) return json.loads(output) @@ -349,7 +359,7 @@ def generate_command_cache(bench_path='.'): return [] -def clear_command_cache(bench_path='.'): +def clear_command_cache(bench_path="."): """Clears commands cached Default invalidation behaviour: destroyed on each run of `bench update` """ @@ -366,7 +376,7 @@ def find_org(org_repo): org_repo = org_repo[0] for org in ["frappe", "erpnext"]: - res = requests.head(f'https://api.github.com/repos/{org}/{org_repo}') + res = requests.head(f"https://api.github.com/repos/{org}/{org_repo}") if res.ok: return org, org_repo @@ -399,7 +409,7 @@ def is_git_url(url): return bool(re.match(pattern, url)) -def drop_privileges(uid_name='nobody', gid_name='nogroup'): +def drop_privileges(uid_name="nobody", gid_name="nogroup"): import grp import pwd diff --git a/bench/utils/app.py b/bench/utils/app.py index ab2f0b50..61a22462 100644 --- a/bench/utils/app.py +++ b/bench/utils/app.py @@ -4,15 +4,21 @@ from setuptools.config import read_configuration import bench import sys import subprocess -from bench.exceptions import InvalidRemoteException, InvalidBranchException, CommandFailedError +from bench.exceptions import ( + InvalidRemoteException, + InvalidBranchException, + CommandFailedError, +) from bench.app import get_repo_dir -def is_version_upgrade(app='frappe', bench_path='.', branch=None): +def is_version_upgrade(app="frappe", bench_path=".", branch=None): upstream_version = get_upstream_version(app=app, branch=branch, bench_path=bench_path) if not upstream_version: - raise InvalidBranchException(f'Specified branch of app {app} is not in upstream remote') + raise InvalidBranchException( + f"Specified branch of app {app} is not in upstream remote" + ) local_version = get_major_version(get_current_version(app, bench_path=bench_path)) upstream_version = get_major_version(upstream_version) @@ -23,21 +29,28 @@ def is_version_upgrade(app='frappe', bench_path='.', branch=None): return (False, local_version, upstream_version) -def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrade=True): +def switch_branch(branch, apps=None, bench_path=".", upgrade=False, check_upgrade=True): import git import importlib - from bench.utils import update_requirements, update_node_packages, backup_all_sites, patch_sites, post_upgrade + from bench.utils import ( + update_requirements, + update_node_packages, + backup_all_sites, + patch_sites, + post_upgrade, + ) from bench.utils.bench import build_assets - apps_dir = os.path.join(bench_path, 'apps') + apps_dir = os.path.join(bench_path, "apps") version_upgrade = (False,) switched_apps = [] if not apps: - apps = [name for name in os.listdir(apps_dir) - if os.path.isdir(os.path.join(apps_dir, name))] - if branch=="v4.x.x": - apps.append('shopping_cart') + apps = [ + name for name in os.listdir(apps_dir) if os.path.isdir(os.path.join(apps_dir, name)) + ] + if branch == "v4.x.x": + apps.append("shopping_cart") for app in apps: app_dir = os.path.join(apps_dir, app) @@ -48,18 +61,27 @@ def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrad repo = git.Repo(app_dir) unshallow_flag = os.path.exists(os.path.join(app_dir, ".git", "shallow")) - bench.utils.log(f"Fetching upstream {'unshallow ' if unshallow_flag else ''}for {app}") + bench.utils.log( + f"Fetching upstream {'unshallow ' if unshallow_flag else ''}for {app}" + ) bench.utils.exec_cmd("git remote set-branches upstream '*'", cwd=app_dir) - bench.utils.exec_cmd(f"git fetch --all{' --unshallow' if unshallow_flag else ''} --quiet", cwd=app_dir) + bench.utils.exec_cmd( + f"git fetch --all{' --unshallow' if unshallow_flag else ''} --quiet", cwd=app_dir + ) if check_upgrade: version_upgrade = is_version_upgrade(app=app, bench_path=bench_path, branch=branch) if version_upgrade[0] and not upgrade: - bench.utils.log(f"Switching to {branch} will cause upgrade from {version_upgrade[1]} to {version_upgrade[2]}. Pass --upgrade to confirm", level=2) + bench.utils.log( + f"Switching to {branch} will cause upgrade from" + f" {version_upgrade[1]} to {version_upgrade[2]}. Pass --upgrade to" + " confirm", + level=2, + ) sys.exit(1) - print("Switching for "+app) + print("Switching for " + app) bench.utils.exec_cmd(f"git checkout -f {branch}", cwd=app_dir) if str(repo.active_branch) == branch: @@ -68,8 +90,13 @@ def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrad bench.utils.log(f"Switching branches failed for: {app}", level=2) if switched_apps: - bench.utils.log("Successfully switched branches for: " + ", ".join(switched_apps), level=1) - print('Please run `bench update --patch` to be safe from any differences in database schema') + bench.utils.log( + "Successfully switched branches for: " + ", ".join(switched_apps), level=1 + ) + print( + "Please run `bench update --patch` to be safe from any differences in" + " database schema" + ) if version_upgrade[0] and upgrade: update_requirements() @@ -81,40 +108,51 @@ def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrad post_upgrade(version_upgrade[1], version_upgrade[2]) -def switch_to_branch(branch=None, apps=None, bench_path='.', upgrade=False): +def switch_to_branch(branch=None, apps=None, bench_path=".", upgrade=False): switch_branch(branch, apps=apps, bench_path=bench_path, upgrade=upgrade) -def switch_to_develop(apps=None, bench_path='.', upgrade=True): - switch_branch('develop', apps=apps, bench_path=bench_path, upgrade=upgrade) -def get_version_from_string(contents, field='__version__'): +def switch_to_develop(apps=None, bench_path=".", upgrade=True): + switch_branch("develop", apps=apps, bench_path=bench_path, upgrade=upgrade) + + +def get_version_from_string(contents, field="__version__"): match = re.search(r"^(\s*%s\s*=\s*['\\\"])(.+?)(['\"])(?sm)" % field, contents) return match.group(2) + def get_major_version(version): import semantic_version return semantic_version.Version(version).major -def get_develop_version(app, bench_path='.'): - repo_dir = get_repo_dir(app, bench_path=bench_path) - with open(os.path.join(repo_dir, os.path.basename(repo_dir), 'hooks.py')) as f: - return get_version_from_string(f.read(), field='develop_version') -def get_upstream_version(app, branch=None, bench_path='.'): +def get_develop_version(app, bench_path="."): + repo_dir = get_repo_dir(app, bench_path=bench_path) + with open(os.path.join(repo_dir, os.path.basename(repo_dir), "hooks.py")) as f: + return get_version_from_string(f.read(), field="develop_version") + + +def get_upstream_version(app, branch=None, bench_path="."): repo_dir = get_repo_dir(app, bench_path=bench_path) if not branch: branch = get_current_branch(app, bench_path=bench_path) try: - subprocess.call(f'git fetch --depth=1 --no-tags upstream {branch}', shell=True, cwd=repo_dir) + subprocess.call( + f"git fetch --depth=1 --no-tags upstream {branch}", shell=True, cwd=repo_dir + ) except CommandFailedError: - raise InvalidRemoteException(f'Failed to fetch from remote named upstream for {app}') + raise InvalidRemoteException(f"Failed to fetch from remote named upstream for {app}") try: - contents = subprocess.check_output(f'git show upstream/{branch}:{app}/__init__.py', - shell=True, cwd=repo_dir, stderr=subprocess.STDOUT) - contents = contents.decode('utf-8') + contents = subprocess.check_output( + f"git show upstream/{branch}:{app}/__init__.py", + shell=True, + cwd=repo_dir, + stderr=subprocess.STDOUT, + ) + contents = contents.decode("utf-8") except subprocess.CalledProcessError as e: if b"Invalid object" in e.output: return None @@ -123,23 +161,28 @@ def get_upstream_version(app, branch=None, bench_path='.'): return get_version_from_string(contents) -def get_current_frappe_version(bench_path='.'): +def get_current_frappe_version(bench_path="."): try: - return get_major_version(get_current_version('frappe', bench_path=bench_path)) + return get_major_version(get_current_version("frappe", bench_path=bench_path)) except IOError: return 0 -def get_current_branch(app, bench_path='.'): + +def get_current_branch(app, bench_path="."): from bench.utils import get_cmd_output + repo_dir = get_repo_dir(app, bench_path=bench_path) return get_cmd_output("basename $(git symbolic-ref -q HEAD)", cwd=repo_dir) -def get_remote(app, bench_path='.'): + +def get_remote(app, bench_path="."): repo_dir = get_repo_dir(app, bench_path=bench_path) - contents = subprocess.check_output(['git', 'remote', '-v'], cwd=repo_dir, stderr=subprocess.STDOUT) - contents = contents.decode('utf-8') - if re.findall('upstream[\s]+', contents): - return 'upstream' + contents = subprocess.check_output( + ["git", "remote", "-v"], cwd=repo_dir, stderr=subprocess.STDOUT + ) + contents = contents.decode("utf-8") + if re.findall(r"upstream[\s]+", contents): + return "upstream" elif not contents: # if contents is an empty string => remote doesn't exist return False @@ -150,17 +193,17 @@ def get_remote(app, bench_path='.'): def get_app_name(bench_path, repo_name): app_name = None - apps_path = os.path.join(os.path.abspath(bench_path), 'apps') - config_path = os.path.join(apps_path, repo_name, 'setup.cfg') + apps_path = os.path.join(os.path.abspath(bench_path), "apps") + config_path = os.path.join(apps_path, repo_name, "setup.cfg") if os.path.exists(config_path): config = read_configuration(config_path) - app_name = config.get('metadata', {}).get('name') + app_name = config.get("metadata", {}).get("name") if not app_name: # retrieve app name from setup.py as fallback - app_path = os.path.join(apps_path, repo_name, 'setup.py') - with open(app_path, 'rb') as f: - app_name = re.search(r'name\s*=\s*[\'"](.*)[\'"]', f.read().decode('utf-8')).group(1) + app_path = os.path.join(apps_path, repo_name, "setup.py") + with open(app_path, "rb") as f: + app_name = re.search(r'name\s*=\s*[\'"](.*)[\'"]', f.read().decode("utf-8")).group(1) if app_name and repo_name != app_name: os.rename(os.path.join(apps_path, repo_name), os.path.join(apps_path, app_name)) @@ -169,12 +212,12 @@ def get_app_name(bench_path, repo_name): return repo_name -def get_current_version(app, bench_path='.'): +def get_current_version(app, bench_path="."): current_version = None repo_dir = get_repo_dir(app, bench_path=bench_path) config_path = os.path.join(repo_dir, "setup.cfg") - init_path = os.path.join(repo_dir, os.path.basename(repo_dir), '__init__.py') - setup_path = os.path.join(repo_dir, 'setup.py') + init_path = os.path.join(repo_dir, os.path.basename(repo_dir), "__init__.py") + setup_path = os.path.join(repo_dir, "setup.py") try: if os.path.exists(config_path): @@ -187,6 +230,6 @@ def get_current_version(app, bench_path='.'): except AttributeError: # backward compatibility with open(setup_path) as f: - current_version = get_version_from_string(f.read(), field='version') + current_version = get_version_from_string(f.read(), field="version") return current_version diff --git a/bench/utils/bench.py b/bench/utils/bench.py index 5bb4e649..cc8e0561 100644 --- a/bench/utils/bench.py +++ b/bench/utils/bench.py @@ -18,17 +18,19 @@ from bench.exceptions import PatchError, ValidationError logger = logging.getLogger(bench.PROJECT_NAME) -def get_env_cmd(cmd, bench_path='.'): - return os.path.abspath(os.path.join(bench_path, 'env', 'bin', cmd)) +def get_env_cmd(cmd, bench_path="."): + return os.path.abspath(os.path.join(bench_path, "env", "bin", cmd)) def get_venv_path(): - venv = which('virtualenv') + venv = which("virtualenv") if not venv: current_python = sys.executable with open(os.devnull, "wb") as devnull: - is_venv_installed = not subprocess.call([current_python, "-m", "venv", "--help"], stdout=devnull) + is_venv_installed = not subprocess.call( + [current_python, "-m", "venv", "--help"], stdout=devnull + ) if is_venv_installed: venv = f"{current_python} -m venv" @@ -40,14 +42,14 @@ def update_env_pip(bench_path): exec_cmd(f"{env_py} -m pip install -q -U pip") -def update_requirements(bench_path='.'): +def update_requirements(bench_path="."): from bench.app import install_app from bench.bench import Bench bench = Bench(bench_path) apps = [app for app in bench.apps if app not in bench.excluded_apps] - print(f"Updating env pip...") + print("Updating env pip...") update_env_pip(bench_path) @@ -57,14 +59,14 @@ def update_requirements(bench_path='.'): install_app(app, bench_path=bench_path, skip_assets=True, restart_bench=False) -def update_python_packages(bench_path='.'): +def update_python_packages(bench_path="."): from bench.bench import Bench bench = Bench(bench_path) env_py = get_env_cmd("python") apps = [app for app in bench.apps if app not in bench.excluded_apps] - print('Updating Python libraries...') + print("Updating Python libraries...") update_env_pip(bench_path) @@ -74,21 +76,22 @@ def update_python_packages(bench_path='.'): bench.run(f"{env_py} -m pip install -q -U -e {app_path}") -def update_node_packages(bench_path='.'): - print('Updating node packages...') +def update_node_packages(bench_path="."): + print("Updating node packages...") from bench.utils.app import get_develop_version from distutils.version import LooseVersion - v = LooseVersion(get_develop_version('frappe', bench_path = bench_path)) + + v = LooseVersion(get_develop_version("frappe", bench_path=bench_path)) # After rollup was merged, frappe_version = 10.1 # if develop_verion is 11 and up, only then install yarn - if v < LooseVersion('11.x.x-develop'): + if v < LooseVersion("11.x.x-develop"): update_npm_packages(bench_path) else: update_yarn_packages(bench_path) -def install_python_dev_dependencies(bench_path='.', apps=None): +def install_python_dev_dependencies(bench_path=".", apps=None): from bench.bench import Bench bench = Bench(bench_path) @@ -104,43 +107,43 @@ def install_python_dev_dependencies(bench_path='.', apps=None): dev_requirements_path = os.path.join(app_path, "dev-requirements.txt") if os.path.exists(dev_requirements_path): - log(f'Installing python development dependencies for {app}') + log(f"Installing python development dependencies for {app}") bench.run(f"{env_py} -m pip install -q -r {dev_requirements_path}") -def update_yarn_packages(bench_path='.'): +def update_yarn_packages(bench_path="."): from bench.bench import Bench bench = Bench(bench_path) apps = [app for app in bench.apps if app not in bench.excluded_apps] - apps_dir = os.path.join(bench.name, 'apps') + apps_dir = os.path.join(bench.name, "apps") # TODO: Check for stuff like this early on only?? - if not which('yarn'): + if not which("yarn"): print("Please install yarn using below command and try again.") print("`npm install -g yarn`") return for app in apps: app_path = os.path.join(apps_dir, app) - if os.path.exists(os.path.join(app_path, 'package.json')): + if os.path.exists(os.path.join(app_path, "package.json")): click.secho(f"\nInstalling node dependencies for {app}", fg="yellow") - bench.run('yarn install', cwd=app_path) + bench.run("yarn install", cwd=app_path) -def update_npm_packages(bench_path='.'): - apps_dir = os.path.join(bench_path, 'apps') +def update_npm_packages(bench_path="."): + apps_dir = os.path.join(bench_path, "apps") package_json = {} for app in os.listdir(apps_dir): - package_json_path = os.path.join(apps_dir, app, 'package.json') + package_json_path = os.path.join(apps_dir, app, "package.json") if os.path.exists(package_json_path): with open(package_json_path, "r") as f: app_package_json = json.loads(f.read()) # package.json is usually a dict in a dict for key, value in app_package_json.items(): - if not key in package_json: + if key not in package_json: package_json[key] = value else: if isinstance(value, dict): @@ -151,13 +154,13 @@ def update_npm_packages(bench_path='.'): package_json[key] = value if package_json is {}: - with open(os.path.join(os.path.dirname(__file__), 'package.json'), 'r') as f: + with open(os.path.join(os.path.dirname(__file__), "package.json"), "r") as f: package_json = json.loads(f.read()) - with open(os.path.join(bench_path, 'package.json'), 'w') as f: + with open(os.path.join(bench_path, "package.json"), "w") as f: f.write(json.dumps(package_json, indent=1, sort_keys=True)) - exec_cmd('npm install', cwd=bench_path) + exec_cmd("npm install", cwd=bench_path) def migrate_env(python, backup=False): @@ -166,38 +169,38 @@ def migrate_env(python, backup=False): from bench.bench import Bench bench = Bench(".") - nvenv = 'env' + nvenv = "env" path = os.getcwd() python = which(python) - virtualenv = which('virtualenv') + virtualenv = which("virtualenv") pvenv = os.path.join(path, nvenv) # Clear Cache before Bench Dies. try: config = bench.conf - rredis = urlparse(config['redis_cache']) - redis = f"{which('redis-cli')} -p {rredis.port}" + rredis = urlparse(config["redis_cache"]) + redis = f"{which('redis-cli')} -p {rredis.port}" - logger.log('Clearing Redis Cache...') - exec_cmd(f'{redis} FLUSHALL') - logger.log('Clearing Redis DataBase...') - exec_cmd(f'{redis} FLUSHDB') + logger.log("Clearing Redis Cache...") + exec_cmd(f"{redis} FLUSHALL") + logger.log("Clearing Redis DataBase...") + exec_cmd(f"{redis} FLUSHDB") except Exception: - logger.warning('Please ensure Redis Connections are running or Daemonized.') + logger.warning("Please ensure Redis Connections are running or Daemonized.") # Backup venv: restore using `virtualenv --relocatable` if needed if backup: from datetime import datetime - parch = os.path.join(path, 'archived', 'envs') + parch = os.path.join(path, "archived", "envs") if not os.path.exists(parch): os.mkdir(parch) - source = os.path.join(path, 'env') + source = os.path.join(path, "env") target = parch - logger.log('Backing up Virtual Environment') - stamp = datetime.now().strftime('%Y%m%d_%H%M%S') + logger.log("Backing up Virtual Environment") + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") dest = os.path.join(path, str(stamp)) os.rename(source, dest) @@ -206,24 +209,25 @@ def migrate_env(python, backup=False): # Create virtualenv using specified python venv_creation, packages_setup = 1, 1 try: - logger.log(f'Setting up a New Virtual {python} Environment') - venv_creation = exec_cmd(f'{virtualenv} --python {python} {pvenv}') + logger.log(f"Setting up a New Virtual {python} Environment") + venv_creation = exec_cmd(f"{virtualenv} --python {python} {pvenv}") - apps = ' '.join([f"-e {os.path.join('apps', app)}" for app in bench.apps]) - packages_setup = exec_cmd(f'{pvenv} -m pip install -q -U {apps}') + apps = " ".join([f"-e {os.path.join('apps', app)}" for app in bench.apps]) + packages_setup = exec_cmd(f"{pvenv} -m pip install -q -U {apps}") - logger.log(f'Migration Successful to {python}') + logger.log(f"Migration Successful to {python}") except Exception: if venv_creation or packages_setup: - logger.warning('Migration Error') + logger.warning("Migration Error") -def validate_upgrade(from_ver, to_ver, bench_path='.'): + +def validate_upgrade(from_ver, to_ver, bench_path="."): if to_ver >= 6: - if not which('npm') and not (which('node') or which('nodejs')): + if not which("npm") and not (which("node") or which("nodejs")): raise Exception("Please install nodejs and npm") -def post_upgrade(from_ver, to_ver, bench_path='.'): +def post_upgrade(from_ver, to_ver, bench_path="."): from bench.config import redis from bench.config.supervisor import generate_supervisor_config from bench.config.nginx import make_nginx_conf @@ -232,18 +236,19 @@ def post_upgrade(from_ver, to_ver, bench_path='.'): conf = Bench(bench_path).conf print("-" * 80 + f"Your bench was upgraded to version {to_ver}") - if conf.get('restart_supervisor_on_update'): + if conf.get("restart_supervisor_on_update"): redis.generate_config(bench_path=bench_path) generate_supervisor_config(bench_path=bench_path) make_nginx_conf(bench_path=bench_path) print( - "As you have setup your bench for production, you will have to reload configuration for " - "nginx and supervisor. To complete the migration, please run the following commands:" - "\nsudo service nginx restart" - "\nsudo supervisorctl reload" + "As you have setup your bench for production, you will have to reload" + " configuration for nginx and supervisor. To complete the migration, please" + " run the following commands:\nsudo service nginx restart\nsudo" + " supervisorctl reload" ) -def patch_sites(bench_path='.'): + +def patch_sites(bench_path="."): from bench.bench import Bench from bench.utils.system import migrate_site @@ -255,56 +260,79 @@ def patch_sites(bench_path='.'): except subprocess.CalledProcessError: raise PatchError -def restart_supervisor_processes(bench_path='.', web_workers=False): + +def restart_supervisor_processes(bench_path=".", web_workers=False): from bench.bench import Bench bench = Bench(bench_path) conf = bench.conf - cmd = conf.get('supervisor_restart_cmd') + cmd = conf.get("supervisor_restart_cmd") bench_name = get_bench_name(bench_path) if cmd: bench.run(cmd) else: - supervisor_status = get_cmd_output('supervisorctl status', cwd=bench_path) + supervisor_status = get_cmd_output("supervisorctl status", cwd=bench_path) - if web_workers and f'{bench_name}-web:' in supervisor_status: - group = f'{bench_name}-web:\t' + if web_workers and f"{bench_name}-web:" in supervisor_status: + group = f"{bench_name}-web:\t" - elif f'{bench_name}-workers:' in supervisor_status: - group = f'{bench_name}-workers: {bench_name}-web:' + elif f"{bench_name}-workers:" in supervisor_status: + group = f"{bench_name}-workers: {bench_name}-web:" # backward compatibility - elif f'{bench_name}-processes:' in supervisor_status: - group = f'{bench_name}-processes:' + elif f"{bench_name}-processes:" in supervisor_status: + group = f"{bench_name}-processes:" # backward compatibility else: - group = 'frappe:' + group = "frappe:" bench.run(f"supervisorctl restart {group}") -def restart_systemd_processes(bench_path='.', web_workers=False): +def restart_systemd_processes(bench_path=".", web_workers=False): bench_name = get_bench_name(bench_path) - exec_cmd(f'sudo systemctl stop -- $(systemctl show -p Requires {bench_name}.target | cut -d= -f2)') - exec_cmd(f'sudo systemctl start -- $(systemctl show -p Requires {bench_name}.target | cut -d= -f2)') + exec_cmd( + f"sudo systemctl stop -- $(systemctl show -p Requires {bench_name}.target | cut" + " -d= -f2)" + ) + exec_cmd( + f"sudo systemctl start -- $(systemctl show -p Requires {bench_name}.target |" + " cut -d= -f2)" + ) -def build_assets(bench_path='.', app=None): - command = 'bench build' +def build_assets(bench_path=".", app=None): + command = "bench build" if app: - command += f' --app {app}' + command += f" --app {app}" exec_cmd(command, cwd=bench_path, env={"BENCH_DEVELOPER": "1"}) -def update(pull=False, apps=None, patch=False, build=False, requirements=False, backup=True, compile=True, - force=False, reset=False, restart_supervisor=False, restart_systemd=False): + +def update( + pull=False, + apps=None, + patch=False, + build=False, + requirements=False, + backup=True, + compile=True, + force=False, + reset=False, + restart_supervisor=False, + restart_systemd=False, +): """command: bench update""" import re from bench import patches from bench.utils import clear_command_cache, pause_exec, log - from bench.utils.bench import restart_supervisor_processes, restart_systemd_processes, backup_all_sites + from bench.utils.bench import ( + restart_supervisor_processes, + restart_systemd_processes, + backup_all_sites, + ) from bench.app import pull_apps from bench.utils.app import is_version_upgrade from bench.config.common_site_config import update_config @@ -318,10 +346,10 @@ def update(pull=False, apps=None, patch=False, build=False, requirements=False, if apps and not pull: apps = [] - clear_command_cache(bench_path='.') + clear_command_cache(bench_path=".") - if conf.get('release_bench'): - print('Release bench detected, cannot update!') + if conf.get("release_bench"): + print("Release bench detected, cannot update!") sys.exit(1) if not (pull or patch or build or requirements): @@ -332,50 +360,58 @@ def update(pull=False, apps=None, patch=False, build=False, requirements=False, if version_upgrade[0]: if force: - log("""Force flag has been used for a major version change in Frappe and it's apps. -This will take significant time to migrate and might break custom apps.""", level=3) + log( + """Force flag has been used for a major version change in Frappe and it's apps. +This will take significant time to migrate and might break custom apps.""", + level=3, + ) else: - print(f"""This update will cause a major version change in Frappe/ERPNext from {version_upgrade[1]} to {version_upgrade[2]}. -This would take significant time to migrate and might break custom apps.""") - click.confirm('Do you want to continue?', abort=True) + print( + f"""This update will cause a major version change in Frappe/ERPNext from {version_upgrade[1]} to {version_upgrade[2]}. +This would take significant time to migrate and might break custom apps.""" + ) + click.confirm("Do you want to continue?", abort=True) - if not reset and conf.get('shallow_clone'): - log("""shallow_clone is set in your bench config. + if not reset and conf.get("shallow_clone"): + log( + """shallow_clone is set in your bench config. However without passing the --reset flag, your repositories will be unshallowed. To avoid this, cancel this operation and run `bench update --reset`. Consider the consequences of `git reset --hard` on your apps before you run that. To avoid seeing this warning, set shallow_clone to false in your common_site_config.json - """, level=3) + """, + level=3, + ) pause_exec(seconds=10) if version_upgrade[0] or (not version_upgrade[0] and force): validate_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) - conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 }) + conf.update({"maintenance_mode": 1, "pause_scheduler": 1}) update_config(conf, bench_path=bench_path) if backup: - print('Backing up sites...') + print("Backing up sites...") backup_all_sites(bench_path=bench_path) if apps: apps = [app.strip() for app in re.split(",| ", apps) if app] if pull: - print('Updating apps source...') + print("Updating apps source...") pull_apps(apps=apps, bench_path=bench_path, reset=reset) if requirements: - print('Setting up requirements...') + print("Setting up requirements...") update_requirements(bench_path=bench_path) update_node_packages(bench_path=bench_path) if patch: - print('Patching sites...') + print("Patching sites...") patch_sites(bench_path=bench_path) if build: - print('Building assets...') + print("Building assets...") build_assets(bench_path=bench_path) if version_upgrade[0] or (not version_upgrade[0] and force): @@ -384,102 +420,112 @@ To avoid seeing this warning, set shallow_clone to false in your common_site_con if pull and compile: from compileall import compile_dir - print('Compiling Python files...') - apps_dir = os.path.join(bench_path, 'apps') - compile_dir(apps_dir, quiet=1, rx=re.compile('.*node_modules.*')) + print("Compiling Python files...") + apps_dir = os.path.join(bench_path, "apps") + compile_dir(apps_dir, quiet=1, rx=re.compile(".*node_modules.*")) - if restart_supervisor or conf.get('restart_supervisor_on_update'): + if restart_supervisor or conf.get("restart_supervisor_on_update"): restart_supervisor_processes(bench_path=bench_path) - if restart_systemd or conf.get('restart_systemd_on_update'): + if restart_systemd or conf.get("restart_systemd_on_update"): restart_systemd_processes(bench_path=bench_path) - conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 }) + conf.update({"maintenance_mode": 0, "pause_scheduler": 0}) update_config(conf, bench_path=bench_path) - print("_" * 80 + "\nBench: Deployment tool for Frappe and Frappe Applications (https://frappe.io/bench).\nOpen source depends on your contributions, so do give back by submitting bug reports, patches and fixes and be a part of the community :)") + print( + "_" * 80 + + "\nBench: Deployment tool for Frappe and Frappe Applications" + " (https://frappe.io/bench).\nOpen source depends on your contributions, so do" + " give back by submitting bug reports, patches and fixes and be a part of the" + " community :)" + ) def clone_apps_from(bench_path, clone_from, update_app=True): from bench.app import install_app - print(f'Copying apps from {clone_from}...') - subprocess.check_output(['cp', '-R', os.path.join(clone_from, 'apps'), bench_path]) - node_modules_path = os.path.join(clone_from, 'node_modules') + print(f"Copying apps from {clone_from}...") + subprocess.check_output(["cp", "-R", os.path.join(clone_from, "apps"), bench_path]) + + node_modules_path = os.path.join(clone_from, "node_modules") if os.path.exists(node_modules_path): - print(f'Copying node_modules from {clone_from}...') - subprocess.check_output(['cp', '-R', node_modules_path, bench_path]) + print(f"Copying node_modules from {clone_from}...") + subprocess.check_output(["cp", "-R", node_modules_path, bench_path]) def setup_app(app): # run git reset --hard in each branch, pull latest updates and install_app - app_path = os.path.join(bench_path, 'apps', app) + app_path = os.path.join(bench_path, "apps", app) # remove .egg-ino - subprocess.check_output(['rm', '-rf', app + '.egg-info'], cwd=app_path) + subprocess.check_output(["rm", "-rf", app + ".egg-info"], cwd=app_path) - if update_app and os.path.exists(os.path.join(app_path, '.git')): - remotes = subprocess.check_output(['git', 'remote'], cwd=app_path).strip().split() - if 'upstream' in remotes: - remote = 'upstream' + if update_app and os.path.exists(os.path.join(app_path, ".git")): + remotes = subprocess.check_output(["git", "remote"], cwd=app_path).strip().split() + if "upstream" in remotes: + remote = "upstream" else: remote = remotes[0] - print(f'Cleaning up {app}') - branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], cwd=app_path).strip() - subprocess.check_output(['git', 'reset', '--hard'], cwd=app_path) - subprocess.check_output(['git', 'pull', '--rebase', remote, branch], cwd=app_path) + print(f"Cleaning up {app}") + branch = subprocess.check_output( + ["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=app_path + ).strip() + subprocess.check_output(["git", "reset", "--hard"], cwd=app_path) + subprocess.check_output(["git", "pull", "--rebase", remote, branch], cwd=app_path) install_app(app, bench_path, restart_bench=False) - with open(os.path.join(clone_from, 'sites', 'apps.txt'), 'r') as f: + with open(os.path.join(clone_from, "sites", "apps.txt"), "r") as f: apps = f.read().splitlines() for app in apps: setup_app(app) -def remove_backups_crontab(bench_path='.'): +def remove_backups_crontab(bench_path="."): from crontab import CronTab from bench.bench import Bench - logger.log('removing backup cronjob') + logger.log("removing backup cronjob") bench_dir = os.path.abspath(bench_path) - user = Bench(bench_dir).conf.get('frappe_user') - logfile = os.path.join(bench_dir, 'logs', 'backup.log') + user = Bench(bench_dir).conf.get("frappe_user") + logfile = os.path.join(bench_dir, "logs", "backup.log") system_crontab = CronTab(user=user) backup_command = f"cd {bench_dir} && {sys.argv[0]} --verbose --site all backup" job_command = f"{backup_command} >> {logfile} 2>&1" system_crontab.remove_all(command=job_command) -def set_mariadb_host(host, bench_path='.'): - update_common_site_config({'db_host': host}, bench_path=bench_path) + +def set_mariadb_host(host, bench_path="."): + update_common_site_config({"db_host": host}, bench_path=bench_path) -def set_redis_cache_host(host, bench_path='.'): - update_common_site_config({'redis_cache': f"redis://{host}"}, bench_path=bench_path) +def set_redis_cache_host(host, bench_path="."): + update_common_site_config({"redis_cache": f"redis://{host}"}, bench_path=bench_path) -def set_redis_queue_host(host, bench_path='.'): - update_common_site_config({'redis_queue': f"redis://{host}"}, bench_path=bench_path) +def set_redis_queue_host(host, bench_path="."): + update_common_site_config({"redis_queue": f"redis://{host}"}, bench_path=bench_path) -def set_redis_socketio_host(host, bench_path='.'): - update_common_site_config({'redis_socketio': f"redis://{host}"}, bench_path=bench_path) +def set_redis_socketio_host(host, bench_path="."): + update_common_site_config({"redis_socketio": f"redis://{host}"}, bench_path=bench_path) -def update_common_site_config(ddict, bench_path='.'): - filename = os.path.join(bench_path, 'sites', 'common_site_config.json') +def update_common_site_config(ddict, bench_path="."): + filename = os.path.join(bench_path, "sites", "common_site_config.json") if os.path.exists(filename): - with open(filename, 'r') as f: + with open(filename, "r") as f: content = json.load(f) else: content = {} content.update(ddict) - with open(filename, 'w') as f: + with open(filename, "w") as f: json.dump(content, f, indent=1, sort_keys=True) @@ -499,7 +545,7 @@ def check_app_installed(app, bench_path="."): ["bench", "--site", "all", "list-apps", "--format", "json"], stderr=open(os.devnull, "wb"), cwd=bench_path, - ).decode('utf-8') + ).decode("utf-8") except subprocess.CalledProcessError: return None @@ -514,16 +560,19 @@ def check_app_installed(app, bench_path="."): def check_app_installed_legacy(app, bench_path="."): - site_path = os.path.join(bench_path, 'sites') + site_path = os.path.join(bench_path, "sites") for site in os.listdir(site_path): - req_file = os.path.join(site_path, site, 'site_config.json') + req_file = os.path.join(site_path, site, "site_config.json") if os.path.exists(req_file): - out = subprocess.check_output(["bench", "--site", site, "list-apps"], cwd=bench_path).decode('utf-8') - if re.search(r'\b' + app + r'\b', out): + out = subprocess.check_output( + ["bench", "--site", site, "list-apps"], cwd=bench_path + ).decode("utf-8") + if re.search(r"\b" + app + r"\b", out): print(f"Cannot remove, app is installed on site: {site}") sys.exit(1) + def validate_branch(): from bench.bench import Bench from bench.utils.app import get_current_branch @@ -531,14 +580,15 @@ def validate_branch(): apps = Bench(".").apps installed_apps = set(apps) - check_apps = set(['frappe', 'erpnext']) + check_apps = set(["frappe", "erpnext"]) intersection_apps = installed_apps.intersection(check_apps) for app in intersection_apps: branch = get_current_branch(app) if branch == "master": - print("""'master' branch is renamed to 'version-11' since 'version-12' release. + print( + """'master' branch is renamed to 'version-11' since 'version-12' release. As of January 2020, the following branches are version Frappe ERPNext 11 version-11 version-11 @@ -547,6 +597,7 @@ version Frappe ERPNext 14 develop develop Please switch to new branches to get future updates. -To switch to your required branch, run the following commands: bench switch-to-branch [branch-name]""") +To switch to your required branch, run the following commands: bench switch-to-branch [branch-name]""" + ) sys.exit(1) diff --git a/bench/utils/system.py b/bench/utils/system.py index d0a2c428..297c3fc4 100644 --- a/bench/utils/system.py +++ b/bench/utils/system.py @@ -7,15 +7,31 @@ import sys # imports - module imports import bench -from bench.utils import exec_cmd, get_process_manager, log, run_frappe_cmd, sudoers_file, which +from bench.utils import ( + exec_cmd, + get_process_manager, + log, + run_frappe_cmd, + sudoers_file, + which, +) from bench.utils.bench import build_assets, clone_apps_from - -def init(path, apps_path=None, no_procfile=False, no_backups=False, - frappe_path=None, frappe_branch=None, verbose=False, clone_from=None, - skip_redis_config_generation=False, clone_without_update=False, skip_assets=False, - python='python3'): +def init( + path, + apps_path=None, + no_procfile=False, + no_backups=False, + frappe_path=None, + frappe_branch=None, + verbose=False, + clone_from=None, + skip_redis_config_generation=False, + clone_without_update=False, + skip_assets=False, + python="python3", +): """Initialize a new bench directory * create a bench directory in the given path @@ -45,13 +61,17 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, # local apps if clone_from: - clone_apps_from(bench_path=path, clone_from=clone_from, update_app=not clone_without_update) + clone_apps_from( + bench_path=path, clone_from=clone_from, update_app=not clone_without_update + ) # remote apps else: - frappe_path = frappe_path or 'https://github.com/frappe/frappe.git' + frappe_path = frappe_path or "https://github.com/frappe/frappe.git" - get_app(frappe_path, branch=frappe_branch, bench_path=path, skip_assets=True, verbose=verbose) + get_app( + frappe_path, branch=frappe_branch, bench_path=path, skip_assets=True, verbose=verbose + ) # fetch remote apps using config file - deprecate this! if apps_path: @@ -65,29 +85,32 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, if not no_backups: bench.setup.backups() + def setup_sudoers(user): - if not os.path.exists('/etc/sudoers.d'): - os.makedirs('/etc/sudoers.d') + if not os.path.exists("/etc/sudoers.d"): + os.makedirs("/etc/sudoers.d") set_permissions = False - if not os.path.exists('/etc/sudoers'): + if not os.path.exists("/etc/sudoers"): set_permissions = True - with open('/etc/sudoers', 'a') as f: - f.write('\n#includedir /etc/sudoers.d\n') + with open("/etc/sudoers", "a") as f: + f.write("\n#includedir /etc/sudoers.d\n") if set_permissions: - os.chmod('/etc/sudoers', 0o440) + os.chmod("/etc/sudoers", 0o440) - template = bench.config.env().get_template('frappe_sudoers') - frappe_sudoers = template.render(**{ - 'user': user, - 'service': which('service'), - 'systemctl': which('systemctl'), - 'nginx': which('nginx'), - }) + template = bench.config.env().get_template("frappe_sudoers") + frappe_sudoers = template.render( + **{ + "user": user, + "service": which("service"), + "systemctl": which("systemctl"), + "nginx": which("nginx"), + } + ) - with open(sudoers_file, 'w') as f: + with open(sudoers_file, "w") as f: f.write(frappe_sudoers) os.chmod(sudoers_file, 0o440) @@ -103,43 +126,43 @@ def start(no_dev=False, concurrency=None, procfile=None, no_prefix=False, procma if not program: raise Exception("No process manager found") - os.environ['PYTHONUNBUFFERED'] = "true" + os.environ["PYTHONUNBUFFERED"] = "true" if not no_dev: - os.environ['DEV_SERVER'] = "true" + os.environ["DEV_SERVER"] = "true" - command = [program, 'start'] + command = [program, "start"] if concurrency: - command.extend(['-c', concurrency]) + command.extend(["-c", concurrency]) if procfile: - command.extend(['-f', procfile]) + command.extend(["-f", procfile]) if no_prefix: - command.extend(['--no-prefix']) + command.extend(["--no-prefix"]) os.execv(program, command) -def migrate_site(site, bench_path='.'): - run_frappe_cmd('--site', site, 'migrate', bench_path=bench_path) +def migrate_site(site, bench_path="."): + run_frappe_cmd("--site", site, "migrate", bench_path=bench_path) -def backup_site(site, bench_path='.'): - run_frappe_cmd('--site', site, 'backup', bench_path=bench_path) +def backup_site(site, bench_path="."): + run_frappe_cmd("--site", site, "backup", bench_path=bench_path) -def backup_all_sites(bench_path='.'): +def backup_all_sites(bench_path="."): from bench.bench import Bench for site in Bench(bench_path).sites: backup_site(site, bench_path=bench_path) -def fix_prod_setup_perms(bench_path='.', frappe_user=None): +def fix_prod_setup_perms(bench_path=".", frappe_user=None): from glob import glob from bench.bench import Bench - frappe_user = frappe_user or Bench(bench_path).conf.get('frappe_user') + frappe_user = frappe_user or Bench(bench_path).conf.get("frappe_user") if not frappe_user: print("frappe user not set") @@ -154,15 +177,15 @@ def fix_prod_setup_perms(bench_path='.', frappe_user=None): def setup_fonts(): - fonts_path = os.path.join('/tmp', 'fonts') + fonts_path = os.path.join("/tmp", "fonts") - if os.path.exists('/etc/fonts_backup'): + if os.path.exists("/etc/fonts_backup"): return - exec_cmd("git clone https://github.com/frappe/fonts.git", cwd='/tmp') - os.rename('/etc/fonts', '/etc/fonts_backup') - os.rename('/usr/share/fonts', '/usr/share/fonts_backup') - os.rename(os.path.join(fonts_path, 'etc_fonts'), '/etc/fonts') - os.rename(os.path.join(fonts_path, 'usr_share_fonts'), '/usr/share/fonts') + exec_cmd("git clone https://github.com/frappe/fonts.git", cwd="/tmp") + os.rename("/etc/fonts", "/etc/fonts_backup") + os.rename("/usr/share/fonts", "/usr/share/fonts_backup") + os.rename(os.path.join(fonts_path, "etc_fonts"), "/etc/fonts") + os.rename(os.path.join(fonts_path, "usr_share_fonts"), "/usr/share/fonts") shutil.rmtree(fonts_path) exec_cmd("fc-cache -fv") diff --git a/bench/utils/translation.py b/bench/utils/translation.py index d4420eac..ad3a202d 100644 --- a/bench/utils/translation.py +++ b/bench/utils/translation.py @@ -10,7 +10,7 @@ def update_translations_p(args): try: update_translations(*args) except requests.exceptions.HTTPError: - print('Download failed for', args[0], args[1]) + print("Download failed for", args[0], args[1]) def download_translations_p(): @@ -19,7 +19,7 @@ def download_translations_p(): pool = multiprocessing.Pool(multiprocessing.cpu_count()) langs = get_langs() - apps = ('frappe', 'erpnext') + apps = ("frappe", "erpnext") args = list(itertools.product(apps, langs)) pool.map(update_translations_p, args) @@ -27,32 +27,32 @@ def download_translations_p(): def download_translations(): langs = get_langs() - apps = ('frappe', 'erpnext') + apps = ("frappe", "erpnext") for app, lang in itertools.product(apps, langs): update_translations(app, lang) def get_langs(): - lang_file = 'apps/frappe/frappe/geo/languages.json' + lang_file = "apps/frappe/frappe/geo/languages.json" with open(lang_file) as f: langs = json.loads(f.read()) - return [d['code'] for d in langs] + return [d["code"] for d in langs] def update_translations(app, lang): import requests - translations_dir = os.path.join('apps', app, app, 'translations') - csv_file = os.path.join(translations_dir, lang + '.csv') + translations_dir = os.path.join("apps", app, app, "translations") + csv_file = os.path.join(translations_dir, lang + ".csv") url = f"https://translate.erpnext.com/files/{app}-{lang}.csv" r = requests.get(url, stream=True) r.raise_for_status() - with open(csv_file, 'wb') as f: + with open(csv_file, "wb") as f: for chunk in r.iter_content(chunk_size=1024): # filter out keep-alive new chunks if chunk: f.write(chunk) f.flush() - print('downloaded for', app, lang) + print("downloaded for", app, lang) From 17edb4797baf70f7397e44ccaef7dbaeac403b0e Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 13 Nov 2021 14:23:08 +0530 Subject: [PATCH 47/91] fix: Don't fetch meta for rm app --- bench/app.py | 27 +++++++++++++++++++++------ bench/bench.py | 4 ++-- bench/utils/__init__.py | 2 +- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/bench/app.py b/bench/app.py index e36cc73f..4aaed5bf 100755 --- a/bench/app.py +++ b/bench/app.py @@ -35,7 +35,7 @@ if typing.TYPE_CHECKING: class AppMeta: - def __init__(self, name: str, branch: str = None): + def __init__(self, name: str, branch: str = None, to_clone: bool = True): """ name (str): This could look something like 1. https://github.com/frappe/healthcare.git @@ -53,14 +53,21 @@ class AppMeta: """ self.name = name self.remote_server = "github.com" + self.to_clone = to_clone self.on_disk = False self.use_ssh = False + self.from_apps = False self.branch = branch self.setup_details() def setup_details(self): + # fetch meta from installed apps + if not self.to_clone and hasattr(self, "bench") and os.path.exists(os.path.join(self.bench.name, "apps", self.name)): + self.from_apps = True + self._setup_details_from_installed_apps() + # fetch meta for repo on mounted disk - if os.path.exists(self.name): + elif os.path.exists(self.name): self.on_disk = True self._setup_details_from_mounted_disk() @@ -72,11 +79,16 @@ class AppMeta: else: self._setup_details_from_name_tag() + def _setup_details_from_mounted_disk(self): + self.org, self.repo, self.tag = os.path.split(self.name)[-2:] + (self.branch,) + def _setup_details_from_name_tag(self): self.org, self.repo, self.tag = fetch_details_from_tag(self.name) - def _setup_details_from_mounted_disk(self): - self.org, self.repo, self.tag = os.path.split(self.name)[-2:] + (self.branch,) + def _setup_details_from_installed_apps(self): + self.org, self.repo, self.tag = os.path.split( + os.path.join(self.bench.name, "apps", self.name) + )[-2:] + (self.branch,) def _setup_details_from_git_url(self): return self.__setup_details_from_git() @@ -92,6 +104,9 @@ class AppMeta: @property def url(self): + if self.from_apps: + return os.path.abspath(os.path.join("apps", self.name)) + if self.on_disk: return os.path.abspath(self.name) @@ -108,9 +123,9 @@ class AppMeta: class App(AppMeta): - def __init__(self, name: str, branch: str = None, bench: "Bench" = None): - super().__init__(name, branch) + def __init__(self, name: str, branch: str = None, bench: "Bench" = None, *args, **kwargs): self.bench = bench + super().__init__(name, branch, *args, **kwargs) def get(self): branch = f"--branch {self.tag}" if self.tag else "" diff --git a/bench/bench.py b/bench/bench.py index 8add1744..35861bac 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -114,9 +114,9 @@ class Bench(Base, Validator): from bench.app import App self.validate_app_uninstall(app) - self.apps.remove(App(app, bench=self)) + self.apps.remove(App(app, bench=self, to_clone=False)) self.apps.sync() - self.build() + # self.build() - removed because it seems unnecessary self.reload() def build(self): diff --git a/bench/utils/__init__.py b/bench/utils/__init__.py index 525c6079..6ea4cc1a 100644 --- a/bench/utils/__init__.py +++ b/bench/utils/__init__.py @@ -397,7 +397,7 @@ def fetch_details_from_tag(_tag): try: org, repo = org_repo - except ValueError: + except Exception: org, repo = find_org(org_repo) return org, repo, tag From 37096822319983302df1a9daf72d41035ad43f5b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 13 Nov 2021 14:23:49 +0530 Subject: [PATCH 48/91] fix: Handle collisions while moving apps --- bench/app.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/bench/app.py b/bench/app.py index 4aaed5bf..32a6b892 100755 --- a/bench/app.py +++ b/bench/app.py @@ -137,9 +137,22 @@ class App(AppMeta): ) def remove(self): - shutil.move( - os.path.join("apps", self.repo), os.path.join("archived", "apps", self.repo), - ) + import datetime + from bench.utils import log + + def get_archived_app_name(): + dt = f"{self.repo}-{datetime.date.today()}" + if os.path.exists(os.path.join("archived", "apps", dt)): + for num in range(1, 100): + dt = f"{dt}_{num}" + if not os.path.exists(os.path.join("archived", "apps", dt)): + return dt + return dt + + src = os.path.join("apps", self.repo) + dst = os.path.join("archived", "apps", get_archived_app_name()) + log(f"App moved from {src} to {dst}") + shutil.move(src, dst) def install(self, skip_assets=False, verbose=False): from bench.utils.app import get_app_name @@ -291,6 +304,7 @@ def get_app( app.get() app.install(verbose=verbose, skip_assets=skip_assets) + bench.apps.sync() def new_app(app, bench_path="."): From 6123cd5b49c27e857a4b53f50d4564c2c5647ba1 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 13 Nov 2021 14:24:39 +0530 Subject: [PATCH 49/91] feat: Restart process manager for develop mode --- bench/bench.py | 4 +++- bench/utils/app.py | 6 +++--- bench/utils/bench.py | 12 ++++++++++-- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/bench/bench.py b/bench/bench.py index 35861bac..bf3f2fe7 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -18,6 +18,7 @@ from bench.utils.bench import ( validate_app_installed_on_sites, restart_supervisor_processes, restart_systemd_processes, + restart_process_manager, remove_backups_crontab, get_venv_path, get_env_cmd, @@ -129,7 +130,8 @@ class Bench(Base, Validator): restart_supervisor_processes(bench_path=self.name) if conf.get("restart_systemd_on_update"): restart_systemd_processes(bench_path=self.name) - + if conf.get("developer_mode"): + restart_process_manager(bench_path=self.name) class BenchApps(MutableSequence): def __init__(self, bench: Bench): diff --git a/bench/utils/app.py b/bench/utils/app.py index 61a22462..fa86e081 100644 --- a/bench/utils/app.py +++ b/bench/utils/app.py @@ -32,14 +32,14 @@ def is_version_upgrade(app="frappe", bench_path=".", branch=None): def switch_branch(branch, apps=None, bench_path=".", upgrade=False, check_upgrade=True): import git import importlib - from bench.utils import ( + from bench.utils.bench import ( + build_assets, update_requirements, update_node_packages, - backup_all_sites, patch_sites, post_upgrade, ) - from bench.utils.bench import build_assets + from bench.utils.system import backup_all_sites apps_dir = os.path.join(bench_path, "apps") version_upgrade = (False,) diff --git a/bench/utils/bench.py b/bench/utils/bench.py index cc8e0561..5ebec530 100644 --- a/bench/utils/bench.py +++ b/bench/utils/bench.py @@ -12,7 +12,7 @@ import click import bench # imports - module imports -from bench.utils import which, log, exec_cmd, get_bench_name, get_cmd_output +from bench.utils import get_process_manager, which, log, exec_cmd, get_bench_name, get_cmd_output from bench.exceptions import PatchError, ValidationError logger = logging.getLogger(bench.PROJECT_NAME) @@ -304,6 +304,14 @@ def restart_systemd_processes(bench_path=".", web_workers=False): ) +def restart_process_manager(bench_path="."): + # only overmind has the restart feature, not sure other supported procmans do + if which("overmind") and os.path.exists( + os.path.join(bench_path, ".overmind.sock") + ): + exec_cmd("overmind restart", cwd=bench_path) + + def build_assets(bench_path=".", app=None): command = "bench build" if app: @@ -331,8 +339,8 @@ def update( from bench.utils.bench import ( restart_supervisor_processes, restart_systemd_processes, - backup_all_sites, ) + from bench.utils.system import backup_all_sites from bench.app import pull_apps from bench.utils.app import is_version_upgrade from bench.config.common_site_config import update_config From e074a1d886b1cb00f792a8cc781059b7ea2c2a4a Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 13 Nov 2021 14:26:37 +0530 Subject: [PATCH 50/91] fix: Show traceback in dev mode and set exit code on errors --- bench/cli.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index 491b54f9..118bff00 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -4,6 +4,7 @@ import json import os import pwd import sys +from traceback import format_exception # imports - third party imports import click @@ -34,7 +35,7 @@ src = os.path.dirname(__file__) def cli(): - global from_command_line + global from_command_line, bench_config from_command_line = True command = " ".join(sys.argv) is_envvar_warn_set = not (os.environ.get("BENCH_DEVELOPER") or os.environ.get("CI")) @@ -43,6 +44,8 @@ def cli(): logger = setup_logging() logger.info(command) + bench_config = get_config(".") + if len(sys.argv) > 1 and sys.argv[1] not in ("src",): check_uid() change_uid() @@ -52,7 +55,7 @@ def cli(): is_dist_editable(bench.PROJECT_NAME) and len(sys.argv) > 1 and sys.argv[1] != "src" - and not get_config(".").get("developer_mode") + and not bench_config.get("developer_mode") ): log( "bench is installed in editable mode!\n\nThis is not the recommended mode" @@ -98,8 +101,12 @@ def cli(): return_code = getattr(e, "code", 0) if return_code: logger.warning(f"{command} executed with exit code {return_code}") + if isinstance(e, Exception): + if os.environ.get("BENCH_DEVELOPER") or bench_config.get("developer_mode"): + click.secho("".join(format_exception(*sys.exc_info()))[:-1]) click.secho(f"ERROR: {e}", fg="red") + return_code = 1 raise e finally: try: @@ -152,7 +159,7 @@ def change_dir(): def change_uid(): if is_root() and not cmd_requires_root(): - frappe_user = get_config(".").get("frappe_user") + frappe_user = bench_config.get("frappe_user") if frappe_user: drop_privileges(uid_name=frappe_user, gid_name=frappe_user) os.environ["HOME"] = pwd.getpwnam(frappe_user).pw_dir From 6890d98847bd308d7e0be88369475ed4db062e34 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 13 Nov 2021 14:27:25 +0530 Subject: [PATCH 51/91] feat: Add alias for remove-app: rm, remove --- bench/commands/make.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/commands/make.py b/bench/commands/make.py index 618af9ad..5717d22e 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -94,7 +94,7 @@ def new_app(app_name): new_app(app_name) -@click.command('remove-app', help='Completely remove app from bench and re-build assets if not installed on any site') +@click.command(['remove', 'rm', 'remove-app'], help='Completely remove app from bench and re-build assets if not installed on any site') @click.argument('app-name') def remove_app(app_name): from bench.bench import Bench From c636c7c133f6559d0b18c0c855b88e4419c412bb Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 13 Nov 2021 14:27:45 +0530 Subject: [PATCH 52/91] fix: Use test frappe app - frappe_docs instead of wiki or chat --- bench/tests/test_init.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bench/tests/test_init.py b/bench/tests/test_init.py index 6b13d838..331de422 100755 --- a/bench/tests/test_init.py +++ b/bench/tests/test_init.py @@ -13,7 +13,10 @@ from bench.release import get_bumped_version from bench.tests.test_base import FRAPPE_BRANCH, TestBenchBase -TEST_FRAPPE_APP = "chat" +# changed from frappe_theme because it wasn't maintained and incompatible, +# chat app & wiki was breaking too. hopefully frappe_docs will be maintained +# for longer since docs.erpnext.com is powered by it ;) +TEST_FRAPPE_APP = "frappe_docs" class TestBenchInit(TestBenchBase): def test_semantic_version(self): From 79980c3c55229ca0a4342618d4c2c60e78810d3d Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 13 Nov 2021 14:54:08 +0530 Subject: [PATCH 53/91] fix: Archived app naming, apps.txt sync * apps.txt wasn't being synced in time so the build was failing, this changes that * Apps archived on the same day weren't being numbered properly...it went from _1 to _1_2 and so forth. This is fixed to show _1, _2 and so on --- bench/app.py | 13 ++++++------- bench/utils/system.py | 2 -- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/bench/app.py b/bench/app.py index 32a6b892..fde06db3 100755 --- a/bench/app.py +++ b/bench/app.py @@ -144,9 +144,9 @@ class App(AppMeta): dt = f"{self.repo}-{datetime.date.today()}" if os.path.exists(os.path.join("archived", "apps", dt)): for num in range(1, 100): - dt = f"{dt}_{num}" - if not os.path.exists(os.path.join("archived", "apps", dt)): - return dt + _dt = f"{dt}_{num}" + if not os.path.exists(os.path.join("archived", "apps", _dt)): + return _dt return dt src = os.path.join("apps", self.repo) @@ -304,7 +304,6 @@ def get_app( app.get() app.install(verbose=verbose, skip_assets=skip_assets) - bench.apps.sync() def new_app(app, bench_path="."): @@ -341,10 +340,10 @@ def install_app( if os.path.exists(os.path.join(app_path, "package.json")): exec_cmd("yarn install", cwd=app_path) - add_to_appstxt(app, bench_path=bench_path) - - conf = Bench(bench_path).conf + bench = Bench(bench_path) + bench.apps.sync() + conf = bench.conf if conf.get("developer_mode"): from bench.utils.bench import install_python_dev_dependencies diff --git a/bench/utils/system.py b/bench/utils/system.py index 297c3fc4..6d741b3d 100644 --- a/bench/utils/system.py +++ b/bench/utils/system.py @@ -77,8 +77,6 @@ def init( if apps_path: install_apps_from_path(apps_path, bench_path=path) - bench.apps.sync() - if not skip_assets: build_assets(bench_path=path) From 68c0549c18d09e6c255f78b17246179b973b8df5 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 15 Nov 2021 12:37:03 +0530 Subject: [PATCH 54/91] refactor(get_app): Workflow for clones/install --- bench/app.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/bench/app.py b/bench/app.py index fde06db3..c0dd3961 100755 --- a/bench/app.py +++ b/bench/app.py @@ -284,26 +284,29 @@ def get_app( cloned_path = os.path.join(bench_path, "apps", repo_name) dir_already_exists = os.path.isdir(cloned_path) + to_clone = not dir_already_exists - if dir_already_exists: - # application directory already exists - # prompt user to overwrite it - if overwrite or click.confirm( - f"A directory for the application '{repo_name}' already exists." + # application directory already exists + # prompt user to overwrite it + if dir_already_exists and ( + overwrite + or click.confirm( + f"A directory for the application '{repo_name}' already exists. " "Do you want to continue and overwrite it?" - ): - import shutil + ) + ): + shutil.rmtree(cloned_path) + to_clone = True - shutil.rmtree(cloned_path) - elif click.confirm("Do you want to reinstall the existing application?", abort=True): - pass + if to_clone: + app.get() - fetch_txt = f"Getting {repo_name}" - click.secho(fetch_txt, fg="yellow") - logger.log(fetch_txt) - - app.get() - app.install(verbose=verbose, skip_assets=skip_assets) + if ( + to_clone + or overwrite + or click.confirm("Do you want to reinstall the existing application?") + ): + app.install(verbose=verbose, skip_assets=skip_assets) def new_app(app, bench_path="."): From 5170048d6d6d2e15c47ad7a6e2330688738649b2 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 15 Nov 2021 12:37:43 +0530 Subject: [PATCH 55/91] style: Black-ish app.py for readability --- bench/app.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/bench/app.py b/bench/app.py index c0dd3961..8b1db57f 100755 --- a/bench/app.py +++ b/bench/app.py @@ -62,7 +62,11 @@ class AppMeta: def setup_details(self): # fetch meta from installed apps - if not self.to_clone and hasattr(self, "bench") and os.path.exists(os.path.join(self.bench.name, "apps", self.name)): + if ( + not self.to_clone + and hasattr(self, "bench") + and os.path.exists(os.path.join(self.bench.name, "apps", self.name)) + ): self.from_apps = True self._setup_details_from_installed_apps() @@ -123,7 +127,9 @@ class AppMeta: class App(AppMeta): - def __init__(self, name: str, branch: str = None, bench: "Bench" = None, *args, **kwargs): + def __init__( + self, name: str, branch: str = None, bench: "Bench" = None, *args, **kwargs + ): self.bench = bench super().__init__(name, branch, *args, **kwargs) @@ -165,7 +171,7 @@ class App(AppMeta): ) install_app( - app=app_name, bench_path=self.bench.name, verbose=verbose, skip_assets=skip_assets + app=app_name, bench_path=self.bench.name, verbose=verbose, skip_assets=skip_assets, ) def uninstall(self): @@ -259,7 +265,12 @@ def setup_app_dependencies(repo_name, bench_path=".", branch=None): def get_app( - git_url, branch=None, bench_path=".", skip_assets=False, verbose=False, overwrite=False + git_url, + branch=None, + bench_path=".", + skip_assets=False, + verbose=False, + overwrite=False, ): """bench get-app clones a Frappe App from remote (GitHub or any other git server), and installs it on the current bench. This also resolves dependencies based on the @@ -451,7 +462,9 @@ def get_repo_dir(app, bench_path="."): def install_apps_from_path(path, bench_path="."): apps = get_apps_json(path) for app in apps: - get_app(app["url"], branch=app.get("branch"), bench_path=bench_path, skip_assets=True) + get_app( + app["url"], branch=app.get("branch"), bench_path=bench_path, skip_assets=True, + ) def get_apps_json(path): From 13ed9beca5e42a139e05836713ade6bb1dfc2513 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 15 Nov 2021 12:38:18 +0530 Subject: [PATCH 56/91] fix: frappe app can exist without public folder other change: * sync apps.txt manually after dep apps are also setup; right before build step --- bench/app.py | 3 ++- bench/utils/__init__.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bench/app.py b/bench/app.py index 8b1db57f..5be70aa8 100755 --- a/bench/app.py +++ b/bench/app.py @@ -355,7 +355,6 @@ def install_app( exec_cmd("yarn install", cwd=app_path) bench = Bench(bench_path) - bench.apps.sync() conf = bench.conf if conf.get("developer_mode"): @@ -363,6 +362,8 @@ def install_app( install_python_dev_dependencies(apps=app) + bench.apps.sync() + if not skip_assets: build_assets(bench_path=bench_path, app=app) diff --git a/bench/utils/__init__.py b/bench/utils/__init__.py index 6ea4cc1a..d80cfca5 100644 --- a/bench/utils/__init__.py +++ b/bench/utils/__init__.py @@ -19,7 +19,7 @@ from bench.exceptions import InvalidRemoteException, ValidationError logger = logging.getLogger(PROJECT_NAME) bench_cache_file = ".bench.cmd" -paths_in_app = ("hooks.py", "modules.txt", "patches.txt", "public") +paths_in_app = ("hooks.py", "modules.txt", "patches.txt") paths_in_bench = ("apps", "sites", "config", "logs", "config/pids") sudoers_file = "/etc/sudoers.d/frappe" From 4ba5422b65a3d29b462bdec1bf8dc4126483e20a Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 15 Nov 2021 12:40:20 +0530 Subject: [PATCH 57/91] fix: Move logging to lower level App.get API --- bench/app.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bench/app.py b/bench/app.py index 5be70aa8..591b6715 100755 --- a/bench/app.py +++ b/bench/app.py @@ -137,6 +137,10 @@ class App(AppMeta): branch = f"--branch {self.tag}" if self.tag else "" shallow = "--depth 1" if self.bench.shallow_clone else "" + fetch_txt = f"Getting {self.repo}" + click.secho(fetch_txt, fg="yellow") + logger.log(fetch_txt) + self.bench.run( f"git clone {self.url} {branch} {shallow} --origin upstream", cwd=os.path.join(self.bench.name, "apps"), From b6fc562b48c2f8acdd379c10050f0fb1358c1b12 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 17 Nov 2021 23:45:30 +0530 Subject: [PATCH 58/91] refactor: get_app flow * Changes in get_app fix path/cwd issues and order of resolving deps: py -> dev py -> yarn * Pass verbosity in setup_app_dependencies via get_app * Say hi to new utils: get_available_folder_name, get_traceback --- bench/app.py | 74 +++++++++++++++-------------------------- bench/bench.py | 6 ---- bench/utils/__init__.py | 25 ++++++++++++++ 3 files changed, 52 insertions(+), 53 deletions(-) diff --git a/bench/app.py b/bench/app.py index 591b6715..2ff0c252 100755 --- a/bench/app.py +++ b/bench/app.py @@ -14,11 +14,12 @@ import click # imports - module imports import bench from bench.utils import ( - exec_cmd, - is_bench_directory, - run_frappe_cmd, - is_git_url, fetch_details_from_tag, + get_available_folder_name, + is_bench_directory, + is_git_url, + log, + run_frappe_cmd, ) from bench.utils.bench import ( get_env_cmd, @@ -147,31 +148,23 @@ class App(AppMeta): ) def remove(self): - import datetime - from bench.utils import log + active_app_path = os.path.join("apps", self.repo) + archived_path = os.path.join("archived", "apps") + archived_name = get_available_folder_name( + f"{self.repo}-{date.today()}", archived_path + ) + archived_app_path = os.path.join(archived_path, archived_name) + log(f"App moved from {active_app_path} to {archived_app_path}") + shutil.move(active_app_path, archived_app_path) - def get_archived_app_name(): - dt = f"{self.repo}-{datetime.date.today()}" - if os.path.exists(os.path.join("archived", "apps", dt)): - for num in range(1, 100): - _dt = f"{dt}_{num}" - if not os.path.exists(os.path.join("archived", "apps", _dt)): - return _dt - return dt - - src = os.path.join("apps", self.repo) - dst = os.path.join("archived", "apps", get_archived_app_name()) - log(f"App moved from {src} to {dst}") - shutil.move(src, dst) - - def install(self, skip_assets=False, verbose=False): + def install(self, skip_assets=False, verbose=True): from bench.utils.app import get_app_name app_name = get_app_name(self.bench.name, self.repo) # TODO: this should go inside install_app only tho - issue: default/resolved branch setup_app_dependencies( - repo_name=self.repo, bench_path=self.bench.name, branch=self.tag + repo_name=self.repo, bench_path=self.bench.name, branch=self.tag, verbose=verbose, ) install_app( @@ -239,20 +232,11 @@ def remove_from_excluded_apps_txt(app, bench_path="."): return write_excluded_apps_txt(apps, bench_path=bench_path) -def generate_bench_name(git_url, bench_path): - if os.path.exists(git_url): - guessed_app_name = os.path.basename(git_url) - else: - app = App(git_url) - guessed_app_name = f"{app.org}_{app.repo}" - - return os.path.join(bench_path, f"{guessed_app_name}-bench") - - -def setup_app_dependencies(repo_name, bench_path=".", branch=None): +def setup_app_dependencies(repo_name, bench_path=".", branch=None, verbose=True): # branch kwarg is somewhat of a hack here; since we're assuming the same branches for all apps # for eg: if you're installing erpnext@develop, you'll want frappe@develop and healthcare@develop too import glob + from bench.bench import Bench apps_path = os.path.join(os.path.abspath(bench_path), "apps") files = glob.glob(os.path.join(apps_path, repo_name, "**", "hooks.py")) @@ -265,7 +249,7 @@ def setup_app_dependencies(repo_name, bench_path=".", branch=None): # TODO: when the time comes, add version check here for app in required_apps: if app not in Bench(bench_path).apps: - get_app(app, bench_path=bench_path, branch=branch) + get_app(app, bench_path=bench_path, branch=branch, verbose=verbose) def get_app( @@ -342,29 +326,25 @@ def install_app( skip_assets=False, ): from bench.bench import Bench - from bench.utils.bench import get_env_cmd install_text = f"Installing {app}" click.secho(install_text, fg="yellow") logger.log(install_text) + bench = Bench(bench_path) + conf = bench.conf python_path = get_env_cmd("python", bench_path=bench_path) quiet_flag = "-q" if not verbose else "" - app_path = os.path.join(bench_path, "apps", app) + app_path = os.path.realpath(os.path.join(bench_path, "apps", app)) cache_flag = "--no-cache-dir" if no_cache else "" - exec_cmd(f"{python_path} -m pip install {quiet_flag} -U -e {app_path} {cache_flag}") + bench.run(f"{python_path} -m pip install {quiet_flag} -U -e {app_path} {cache_flag}") + + if conf.get("developer_mode"): + install_python_dev_dependencies(apps=app) if os.path.exists(os.path.join(app_path, "package.json")): - exec_cmd("yarn install", cwd=app_path) - - bench = Bench(bench_path) - - conf = bench.conf - if conf.get("developer_mode"): - from bench.utils.bench import install_python_dev_dependencies - - install_python_dev_dependencies(apps=app) + bench.run("yarn install", cwd=app_path) bench.apps.sync() @@ -381,7 +361,7 @@ def install_app( def pull_apps(apps=None, bench_path=".", reset=False): """Check all apps if there no local changes, pull""" from bench.bench import Bench - from bench.utils.app import get_remote, get_current_branch + from bench.utils.app import get_current_branch, get_remote bench = Bench(bench_path) rebase = "--rebase" if bench.conf.get("rebase_on_pull") else "" diff --git a/bench/bench.py b/bench/bench.py index bf3f2fe7..308435dd 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -98,12 +98,6 @@ class Bench(Base, Validator): self.teardown.backups() self.teardown.dirs() - def get_app(self, app, version=None): - pass - - def drop_app(self, app, version=None): - pass - def install(self, app, branch=None): from bench.app import App diff --git a/bench/utils/__init__.py b/bench/utils/__init__.py index d80cfca5..002ac912 100644 --- a/bench/utils/__init__.py +++ b/bench/utils/__init__.py @@ -431,3 +431,28 @@ def drop_privileges(uid_name="nobody", gid_name="nogroup"): # Ensure a very conservative umask os.umask(0o22) + + +def get_available_folder_name(name: str, path: str) -> str: + """Subfixes the passed name with -1 uptil -100 whatever's available + """ + if os.path.exists(os.path.join(path, name)): + for num in range(1, 100): + _dt = f"{name}_{num}" + if not os.path.exists(os.path.join(path, _dt)): + return _dt + return name + + +def get_traceback() -> str: + """ + Returns the traceback of the Exception + """ + from traceback import format_exception + exc_type, exc_value, exc_tb = sys.exc_info() + + if not any([exc_type, exc_value, exc_tb]): + return "" + + trace_list = format_exception(exc_type, exc_value, exc_tb) + return "".join(trace_list) From 67e5db69797d44cd263634c2d11e05ae4b35769b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 17 Nov 2021 23:49:11 +0530 Subject: [PATCH 59/91] chore: Drop dead code bench.prepare_staging module wasn't being utilized from any internal APIs. Dropped it ;) --- bench/prepare_staging.py | 71 ---------------------------------------- 1 file changed, 71 deletions(-) delete mode 100755 bench/prepare_staging.py diff --git a/bench/prepare_staging.py b/bench/prepare_staging.py deleted file mode 100755 index 9d004684..00000000 --- a/bench/prepare_staging.py +++ /dev/null @@ -1,71 +0,0 @@ -#! env python -import os -import git -import click -from .config.common_site_config import get_config - -github_username = None -github_password = None - -def prepare_staging(bench_path, app, remote='upstream'): - from .release import get_release_message - validate(bench_path) - - repo_path = os.path.join(bench_path, 'apps', app) - update_branches(repo_path, remote) - message = get_release_message(repo_path, from_branch='develop', to_branch='staging', remote=remote) - - if not message: - print('No commits to release') - return - - print(message) - - click.confirm('Do you want to continue?', abort=True) - - create_staging(repo_path) - push_commits(repo_path) - -def validate(bench_path): - from .release import validate - - config = get_config(bench_path) - validate(bench_path, config) - -def update_branches(repo_path, remote): - from .release import update_branch - update_branch(repo_path, 'staging', remote) - update_branch(repo_path, 'develop', remote) - - git.Repo(repo_path).git.checkout('develop') - -def create_staging(repo_path, from_branch='develop'): - from .release import handle_merge_error - - print('creating staging from', from_branch) - repo = git.Repo(repo_path) - g = repo.git - g.checkout('staging') - try: - g.merge(from_branch, '--no-ff') - except git.exc.GitCommandError as e: - handle_merge_error(e, source=from_branch, target='staging') - - g.checkout(from_branch) - try: - g.merge('staging') - except git.exc.GitCommandError as e: - handle_merge_error(e, source='staging', target=from_branch) - -def push_commits(repo_path, remote='upstream'): - print('pushing staging branch of', repo_path) - - repo = git.Repo(repo_path) - g = repo.git - - args = [ - 'develop:develop', - 'staging:staging' - ] - - print(g.push(remote, *args)) From a192240cec9156c83804a35e1aeeadff885ea185 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 17 Nov 2021 23:51:28 +0530 Subject: [PATCH 60/91] fix: Check if app is installed in env via initialize_apps Consequence: Bench.apps only will have valid frappe apps that are installed in env --- bench/bench.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/bench/bench.py b/bench/bench.py index 308435dd..fc5f79ea 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -4,6 +4,7 @@ import sys import logging from typing import MutableSequence, TYPE_CHECKING +# imports - module imports import bench from bench.exceptions import ValidationError from bench.config.common_site_config import setup_config @@ -11,6 +12,7 @@ from bench.utils import ( paths_in_bench, exec_cmd, is_frappe_app, + get_cmd_output, get_git_version, run_frappe_cmd, ) @@ -138,11 +140,23 @@ class BenchApps(MutableSequence): return f.write("\n".join(self.apps)) def initialize_apps(self): + cmd = f"{get_env_cmd('python', bench_path=self.bench.name)} -m pip freeze" + is_installed = lambda app: app in installed_packages + + try: + installed_packages = get_cmd_output(cmd=cmd, cwd=self.bench.name) + except Exception: + self.apps = [] + return + try: self.apps = [ x for x in os.listdir(os.path.join(self.bench.name, "apps")) - if is_frappe_app(os.path.join(self.bench.name, "apps", x)) + if ( + is_frappe_app(os.path.join(self.bench.name, "apps", x)) + and is_installed(x) + ) ] self.apps.sort() except FileNotFoundError: From 30e3e725a88b081c0fdeb41c960dd3067a45dfe7 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 17 Nov 2021 23:53:58 +0530 Subject: [PATCH 61/91] fix: Show traceback on bench init failure Other changes: * Invoke internal init function instead of invoking click command. This was done to avoid handling random Exceptions that are done in the command definition * os.chdir after initializing new bench in get_app --- bench/app.py | 7 ++++--- bench/commands/make.py | 5 +++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/bench/app.py b/bench/app.py index 2ff0c252..b30a958d 100755 --- a/bench/app.py +++ b/bench/app.py @@ -276,10 +276,11 @@ def get_app( branch = app.tag if not is_bench_directory(bench_path): - bench_path = generate_bench_name(git_url, bench_path) - from bench.commands.make import init + from bench.utils.system import init - click.get_current_context().invoke(init, path=bench_path, frappe_branch=branch) + bench_path = get_available_folder_name(f"{app.repo}-bench", bench_path) + init(path=bench_path, frappe_branch=branch) + os.chdir(bench_path) cloned_path = os.path.join(bench_path, "apps", repo_name) dir_already_exists = os.path.isdir(cloned_path) diff --git a/bench/commands/make.py b/bench/commands/make.py index 5717d22e..458ca606 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -44,10 +44,11 @@ def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, c except SystemExit: raise except Exception as e: - import os, shutil, time + import shutil, time + from bench.utils import get_traceback # add a sleep here so that the traceback of other processes doesnt overlap with the prompts time.sleep(1) - print(e) + print(get_traceback()) log(f"There was a problem while creating {path}", level=2) if click.confirm("Do you want to rollback these changes?"): print(f'Rolling back Bench "{path}"') From cd252d3476723a2fdec307ec27f405d613c867f7 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 17 Nov 2021 23:56:55 +0530 Subject: [PATCH 62/91] refactor: setup dev requirements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Remove unwanted log message 🤢. It was inconsistent...this had to be done...rip * Fix missing import - install_python_dev_dependencies * Don't setup node + python requirements when --dev flag is passed --- bench/app.py | 3 ++- bench/commands/setup.py | 8 ++++---- bench/utils/bench.py | 1 - 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bench/app.py b/bench/app.py index b30a958d..d0d583ca 100755 --- a/bench/app.py +++ b/bench/app.py @@ -22,8 +22,9 @@ from bench.utils import ( run_frappe_cmd, ) from bench.utils.bench import ( - get_env_cmd, build_assets, + get_env_cmd, + install_python_dev_dependencies, restart_supervisor_processes, restart_systemd_processes, ) diff --git a/bench/commands/setup.py b/bench/commands/setup.py index 654f0240..a02e39a3 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -136,19 +136,19 @@ def setup_socketio(): @click.option("--python", help="Update only Python packages", default=False, is_flag=True) @click.option("--dev", help="Install optional python development dependencies", default=False, is_flag=True) def setup_requirements(node=False, python=False, dev=False): - if not (node or python): + if not (node or python or dev): from bench.utils.bench import update_requirements update_requirements() - elif not node: + elif not node and not dev: from bench.utils.bench import update_python_packages update_python_packages() - elif not python: + elif not python and not dev: from bench.utils.bench import update_node_packages update_node_packages() - if dev: + else: from bench.utils.bench import install_python_dev_dependencies install_python_dev_dependencies() diff --git a/bench/utils/bench.py b/bench/utils/bench.py index 5ebec530..3c0eeee4 100644 --- a/bench/utils/bench.py +++ b/bench/utils/bench.py @@ -107,7 +107,6 @@ def install_python_dev_dependencies(bench_path=".", apps=None): dev_requirements_path = os.path.join(app_path, "dev-requirements.txt") if os.path.exists(dev_requirements_path): - log(f"Installing python development dependencies for {app}") bench.run(f"{env_py} -m pip install -q -r {dev_requirements_path}") From 84d1b209baba2ce9104071034455c73474eaddd9 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 18 Nov 2021 00:03:42 +0530 Subject: [PATCH 63/91] chore: Added Typing + styling --- bench/utils/__init__.py | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/bench/utils/__init__.py b/bench/utils/__init__.py index 002ac912..db455a68 100644 --- a/bench/utils/__init__.py +++ b/bench/utils/__init__.py @@ -7,6 +7,7 @@ import re import sys from glob import glob from shlex import split +from typing import List, Tuple # imports - third party imports import click @@ -34,7 +35,7 @@ def is_bench_directory(directory=os.path.curdir): return is_bench -def is_frappe_app(directory): +def is_frappe_app(directory: str) -> bool: is_frappe_app = True for folder in paths_in_app: @@ -110,7 +111,7 @@ def exec_cmd(cmd, cwd=".", env=None): logger.warning(f"{cmd_log} executed with exit code {return_code}") -def which(executable, raise_err=False): +def which(executable: str, raise_err: bool = False) -> str: from shutil import which exec_ = which(executable) @@ -121,7 +122,7 @@ def which(executable, raise_err=False): return exec_ -def setup_logging(bench_path="."): +def setup_logging(bench_path=".") -> "logger": LOG_LEVEL = 15 logging.addLevelName(LOG_LEVEL, "LOG") @@ -146,14 +147,14 @@ def setup_logging(bench_path="."): return logger -def get_process_manager(): +def get_process_manager() -> str: for proc_man in ["honcho", "foreman", "forego"]: proc_man_path = which(proc_man) if proc_man_path: return proc_man_path -def get_git_version(): +def get_git_version() -> float: """returns git version from `git --version` extracts version number from string `get version 1.9.1` etc""" version = get_cmd_output("git --version") @@ -280,7 +281,7 @@ def run_playbook(playbook_name, extra_vars=None, tag=None): subprocess.check_call(args, cwd=os.path.join(bench.__path__[0], "playbooks")) -def find_benches(directory=None): +def find_benches(directory: str = None) -> List: if not directory: directory = os.path.expanduser("~") elif os.path.exists(directory): @@ -309,16 +310,16 @@ def find_benches(directory=None): return benches -def is_dist_editable(dist): +def is_dist_editable(dist: str) -> bool: """Is distribution an editable install?""" for path_item in sys.path: - egg_link = os.path.join(path_item, dist + ".egg-link") + egg_link = os.path.join(path_item, f"{dist}.egg-link") if os.path.isfile(egg_link): return True return False -def find_parent_bench(path): +def find_parent_bench(path: str) -> str: """Checks if parent directories are benches""" if is_bench_directory(directory=path): return path @@ -332,7 +333,7 @@ def find_parent_bench(path): return find_parent_bench(parent_dir) -def generate_command_cache(bench_path="."): +def generate_command_cache(bench_path=".") -> List: """Caches all available commands (even custom apps) via Frappe Default caching behaviour: generated the first time any command (for a specific bench directory) """ @@ -383,7 +384,7 @@ def find_org(org_repo): raise InvalidRemoteException -def fetch_details_from_tag(_tag): +def fetch_details_from_tag(_tag: str) -> Tuple[str, str, str]: if not _tag: raise Exception("Tag is not provided") @@ -403,7 +404,7 @@ def fetch_details_from_tag(_tag): return org, repo, tag -def is_git_url(url): +def is_git_url(url: str) -> bool: # modified to allow without the tailing .git from https://github.com/jonschlinkert/is-git-url.git pattern = r"(?:git|ssh|https?|\w*@[-\w.]+):(\/\/)?(.*?)(\.git)?(\/?|\#[-\d\w._]+?)$" return bool(re.match(pattern, url)) @@ -434,8 +435,7 @@ def drop_privileges(uid_name="nobody", gid_name="nogroup"): def get_available_folder_name(name: str, path: str) -> str: - """Subfixes the passed name with -1 uptil -100 whatever's available - """ + """Subfixes the passed name with -1 uptil -100 whatever's available""" if os.path.exists(os.path.join(path, name)): for num in range(1, 100): _dt = f"{name}_{num}" @@ -445,10 +445,9 @@ def get_available_folder_name(name: str, path: str) -> str: def get_traceback() -> str: - """ - Returns the traceback of the Exception - """ + """Returns the traceback of the Exception""" from traceback import format_exception + exc_type, exc_value, exc_tb = sys.exc_info() if not any([exc_type, exc_value, exc_tb]): From 4655f6e7857df3109d70a5c52bd42776255f80ea Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 18 Nov 2021 00:10:25 +0530 Subject: [PATCH 64/91] refactor(minor): Dependency resolving * Added a requirements method under BenchSetup * Resolve Frappe dependencies prior to other apps * Fixed issues with missing path in update_env_pip --- bench/bench.py | 4 ++++ bench/utils/bench.py | 35 ++++++++++++++++++++++------------- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/bench/bench.py b/bench/bench.py index fc5f79ea..9064805f 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -285,6 +285,10 @@ class BenchSetup(Base): logger.log("backups were set up") + def requirements(self): + from bench.utils.bench import update_requirements + update_requirements(bench=self.bench) + class BenchTearDown: def __init__(self, bench): diff --git a/bench/utils/bench.py b/bench/utils/bench.py index 3c0eeee4..690d9fea 100644 --- a/bench/utils/bench.py +++ b/bench/utils/bench.py @@ -6,6 +6,7 @@ import re import subprocess import sys from json.decoder import JSONDecodeError +import typing # imports - third party imports import click @@ -15,6 +16,10 @@ import bench from bench.utils import get_process_manager, which, log, exec_cmd, get_bench_name, get_cmd_output from bench.exceptions import PatchError, ValidationError + +if typing.TYPE_CHECKING: + from bench.bench import Bench + logger = logging.getLogger(bench.PROJECT_NAME) @@ -38,41 +43,46 @@ def get_venv_path(): def update_env_pip(bench_path): - env_py = get_env_cmd("python") + env_py = get_env_cmd("python", bench_path=bench_path) exec_cmd(f"{env_py} -m pip install -q -U pip") -def update_requirements(bench_path="."): +def update_requirements(bench: "Bench" = None, bench_path="."): from bench.app import install_app - from bench.bench import Bench - bench = Bench(bench_path) + if not bench: + from bench.bench import Bench + + bench = Bench(bench_path) + apps = [app for app in bench.apps if app not in bench.excluded_apps] + apps.remove("frappe") + apps.insert(0, "frappe") print("Updating env pip...") - - update_env_pip(bench_path) + update_env_pip(bench.name) print(f"Installing {len(apps)} applications...") - for app in apps: - install_app(app, bench_path=bench_path, skip_assets=True, restart_bench=False) + install_app(app, bench_path=bench.name, skip_assets=True, restart_bench=False) def update_python_packages(bench_path="."): from bench.bench import Bench bench = Bench(bench_path) - env_py = get_env_cmd("python") + env_py = get_env_cmd("python", bench_path=bench.name) + apps = [app for app in bench.apps if app not in bench.excluded_apps] + apps.remove("frappe") + apps.insert(0, "frappe") print("Updating Python libraries...") - update_env_pip(bench_path) for app in apps: - click.secho(f"\nInstalling python dependencies for {app}", fg="yellow") app_path = os.path.join(bench_path, "apps", app) + click.secho(f"\nInstalling python dependencies for {app}", fg="yellow") bench.run(f"{env_py} -m pip install -q -U -e {app_path}") @@ -410,8 +420,7 @@ To avoid seeing this warning, set shallow_clone to false in your common_site_con if requirements: print("Setting up requirements...") - update_requirements(bench_path=bench_path) - update_node_packages(bench_path=bench_path) + bench.setup.requirements() if patch: print("Patching sites...") From 58e3d4931b22587bcdb0623e9bdfc70bbb31addb Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 18 Nov 2021 00:18:10 +0530 Subject: [PATCH 65/91] refactor: bench update workflow * Separate out branch validation logic * Add type hints * Raise named Exception "CannotUpdateReleaseBench" instead of SystemExit * Use "newer" APIs wherever possible --- bench/utils/bench.py | 116 ++++++++++++++++++++++++------------------- 1 file changed, 65 insertions(+), 51 deletions(-) diff --git a/bench/utils/bench.py b/bench/utils/bench.py index 690d9fea..d31f78e6 100644 --- a/bench/utils/bench.py +++ b/bench/utils/bench.py @@ -13,7 +13,13 @@ import click import bench # imports - module imports -from bench.utils import get_process_manager, which, log, exec_cmd, get_bench_name, get_cmd_output +from bench.utils import ( + which, + log, + exec_cmd, + get_bench_name, + get_cmd_output, +) from bench.exceptions import PatchError, ValidationError @@ -328,52 +334,8 @@ def build_assets(bench_path=".", app=None): exec_cmd(command, cwd=bench_path, env={"BENCH_DEVELOPER": "1"}) -def update( - pull=False, - apps=None, - patch=False, - build=False, - requirements=False, - backup=True, - compile=True, - force=False, - reset=False, - restart_supervisor=False, - restart_systemd=False, -): - """command: bench update""" - import re - from bench import patches - from bench.utils import clear_command_cache, pause_exec, log - from bench.utils.bench import ( - restart_supervisor_processes, - restart_systemd_processes, - ) - from bench.utils.system import backup_all_sites - from bench.app import pull_apps - from bench.utils.app import is_version_upgrade - from bench.config.common_site_config import update_config - from bench.bench import Bench - - bench_path = os.path.abspath(".") - bench = Bench(bench_path) - patches.run(bench_path=bench_path) - conf = bench.conf - - if apps and not pull: - apps = [] - - clear_command_cache(bench_path=".") - - if conf.get("release_bench"): - print("Release bench detected, cannot update!") - sys.exit(1) - - if not (pull or patch or build or requirements): - pull, patch, build, requirements = True, True, True, True - - validate_branch() - version_upgrade = is_version_upgrade() +def handle_version_upgrade(version_upgrade, bench_path, force, reset, conf): + from bench.utils import pause_exec, log if version_upgrade[0]: if force: @@ -404,6 +366,61 @@ To avoid seeing this warning, set shallow_clone to false in your common_site_con if version_upgrade[0] or (not version_upgrade[0] and force): validate_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) + + +def update( + pull: bool = False, + apps: str = None, + patch: bool = False, + build: bool = False, + requirements: bool = False, + backup: bool = True, + compile: bool = True, + force: bool = False, + reset: bool = False, + restart_supervisor: bool = False, + restart_systemd: bool = False, +): + """command: bench update""" + import re + from bench import patches + + from bench.app import pull_apps + from bench.bench import Bench + from bench.config.common_site_config import update_config + from bench.exceptions import CannotUpdateReleaseBench + + from bench.utils import clear_command_cache + from bench.utils.app import is_version_upgrade + from bench.utils.bench import ( + restart_supervisor_processes, + restart_systemd_processes, + ) + from bench.utils.system import backup_all_sites + + bench_path = os.path.abspath(".") + bench = Bench(bench_path) + patches.run(bench_path=bench_path) + conf = bench.conf + + clear_command_cache(bench_path=".") + + if conf.get("release_bench"): + raise CannotUpdateReleaseBench("Release bench detected, cannot update!") + + if not (pull or patch or build or requirements): + pull, patch, build, requirements = True, True, True, True + + if apps and pull: + apps = [app.strip() for app in re.split(",| ", apps) if app] + else: + apps = [] + + validate_branch() + + version_upgrade = is_version_upgrade() + handle_version_upgrade(version_upgrade, bench_path, force, reset, conf) + conf.update({"maintenance_mode": 1, "pause_scheduler": 1}) update_config(conf, bench_path=bench_path) @@ -411,9 +428,6 @@ To avoid seeing this warning, set shallow_clone to false in your common_site_con print("Backing up sites...") backup_all_sites(bench_path=bench_path) - if apps: - apps = [app.strip() for app in re.split(",| ", apps) if app] - if pull: print("Updating apps source...") pull_apps(apps=apps, bench_path=bench_path, reset=reset) @@ -428,7 +442,7 @@ To avoid seeing this warning, set shallow_clone to false in your common_site_con if build: print("Building assets...") - build_assets(bench_path=bench_path) + bench.build() if version_upgrade[0] or (not version_upgrade[0] and force): post_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) From 747be65f92be0da945e6f0b01992c845f6e8c83a Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 18 Nov 2021 00:31:51 +0530 Subject: [PATCH 66/91] refactor(minor): switch branch * Don't update node packages twice (already managed via update_requirements) * Don't reload bench.utils - this made sense back when bench was updated along with the apps * Fix utils namespaces. Import instead of .... --- bench/utils/app.py | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/bench/utils/app.py b/bench/utils/app.py index fa86e081..9a059d98 100644 --- a/bench/utils/app.py +++ b/bench/utils/app.py @@ -1,7 +1,6 @@ import os import re from setuptools.config import read_configuration -import bench import sys import subprocess from bench.exceptions import ( @@ -31,11 +30,10 @@ def is_version_upgrade(app="frappe", bench_path=".", branch=None): def switch_branch(branch, apps=None, bench_path=".", upgrade=False, check_upgrade=True): import git - import importlib + from bench.utils import log, exec_cmd from bench.utils.bench import ( build_assets, update_requirements, - update_node_packages, patch_sites, post_upgrade, ) @@ -56,24 +54,22 @@ def switch_branch(branch, apps=None, bench_path=".", upgrade=False, check_upgrad app_dir = os.path.join(apps_dir, app) if not os.path.exists(app_dir): - bench.utils.log(f"{app} does not exist!", level=2) + log(f"{app} does not exist!", level=2) continue repo = git.Repo(app_dir) unshallow_flag = os.path.exists(os.path.join(app_dir, ".git", "shallow")) - bench.utils.log( - f"Fetching upstream {'unshallow ' if unshallow_flag else ''}for {app}" - ) + log(f"Fetching upstream {'unshallow ' if unshallow_flag else ''}for {app}") - bench.utils.exec_cmd("git remote set-branches upstream '*'", cwd=app_dir) - bench.utils.exec_cmd( + exec_cmd("git remote set-branches upstream '*'", cwd=app_dir) + exec_cmd( f"git fetch --all{' --unshallow' if unshallow_flag else ''} --quiet", cwd=app_dir ) if check_upgrade: version_upgrade = is_version_upgrade(app=app, bench_path=bench_path, branch=branch) if version_upgrade[0] and not upgrade: - bench.utils.log( + log( f"Switching to {branch} will cause upgrade from" f" {version_upgrade[1]} to {version_upgrade[2]}. Pass --upgrade to" " confirm", @@ -82,17 +78,15 @@ def switch_branch(branch, apps=None, bench_path=".", upgrade=False, check_upgrad sys.exit(1) print("Switching for " + app) - bench.utils.exec_cmd(f"git checkout -f {branch}", cwd=app_dir) + exec_cmd(f"git checkout -f {branch}", cwd=app_dir) if str(repo.active_branch) == branch: switched_apps.append(app) else: - bench.utils.log(f"Switching branches failed for: {app}", level=2) + log(f"Switching branches failed for: {app}", level=2) if switched_apps: - bench.utils.log( - "Successfully switched branches for: " + ", ".join(switched_apps), level=1 - ) + log(f"Successfully switched branches for: {', '.join(switched_apps)}", level=1) print( "Please run `bench update --patch` to be safe from any differences in" " database schema" @@ -100,8 +94,6 @@ def switch_branch(branch, apps=None, bench_path=".", upgrade=False, check_upgrad if version_upgrade[0] and upgrade: update_requirements() - update_node_packages() - importlib.reload(bench.utils) backup_all_sites() patch_sites() build_assets() From ac07d8dc522f9688a4f6056b988991d1a368dff8 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 18 Nov 2021 00:36:13 +0530 Subject: [PATCH 67/91] refactor: bench.utils.log Maintain `bench.LOG_BUFFER` for entire process when `bench.cli.fancy` is set --- bench/cli.py | 1 + bench/utils/__init__.py | 25 +++++++++++++++++++------ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index 118bff00..a257b630 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -30,6 +30,7 @@ from bench.utils import ( from bench.utils.bench import get_env_cmd from_command_line = False +bench.LOG_BUFFER = [] change_uid_msg = "You should not run this command as root" src = os.path.dirname(__file__) diff --git a/bench/utils/__init__.py b/bench/utils/__init__.py index db455a68..1a52291b 100644 --- a/bench/utils/__init__.py +++ b/bench/utils/__init__.py @@ -48,19 +48,32 @@ def is_frappe_app(directory: str) -> bool: return bool(is_frappe_app) -def log(message, level=0): +def log(message, level=0, no_log=False): + import bench + import bench.cli + levels = { 0: ("blue", "INFO"), # normal 1: ("green", "SUCCESS"), # success 2: ("red", "ERROR"), # fail 3: ("yellow", "WARN"), # warn/suggest } - loggers = {2: logger.error, 3: logger.warning} - color, prefix = levels.get(level, levels[0]) - level_logger = loggers.get(level, logger.info) - level_logger(message) - click.secho(f"{prefix}: {message}", fg=color) + color, prefix = levels.get(level, levels[0]) + + if bench.cli.from_command_line and bench.cli.fancy: + bench.LOG_BUFFER.append( + {"prefix": prefix, "message": message, "color": color,} + ) + + if no_log: + click.secho(message, fg=color) + else: + loggers = {2: logger.error, 3: logger.warning} + level_logger = loggers.get(level, logger.info) + + level_logger(message) + click.secho(f"{prefix}: {message}", fg=color) def check_latest_version(): From cd1f526d0907a6fbe8e912920ce9abc39d4adc27 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 18 Nov 2021 00:39:48 +0530 Subject: [PATCH 68/91] perf: Cache Bench and App instantiation --- bench/app.py | 4 +++- bench/bench.py | 3 +++ bench/cli.py | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/bench/app.py b/bench/app.py index d0d583ca..f9ab0b82 100755 --- a/bench/app.py +++ b/bench/app.py @@ -1,4 +1,5 @@ # imports - standard imports +import functools import json import logging import os @@ -7,6 +8,7 @@ import shutil import subprocess import sys import typing +from datetime import date # imports - third party imports import click @@ -127,7 +129,7 @@ class AppMeta: def get_ssh_url(self): return f"git@{self.remote_server}:{self.org}/{self.repo}.git" - +@functools.lru_cache(maxsize=None) class App(AppMeta): def __init__( self, name: str, branch: str = None, bench: "Bench" = None, *args, **kwargs diff --git a/bench/bench.py b/bench/bench.py index 9064805f..a3b2cb55 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -1,3 +1,5 @@ +# imports - standard imports +import functools import os import shutil import sys @@ -45,6 +47,7 @@ class Validator: validate_app_installed_on_sites(app, bench_path=self.name) +@functools.lru_cache(maxsize=None) class Bench(Base, Validator): def __init__(self, path): self.name = path diff --git a/bench/cli.py b/bench/cli.py index a257b630..c628d9dd 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -29,6 +29,7 @@ from bench.utils import ( ) from bench.utils.bench import get_env_cmd +fancy = True from_command_line = False bench.LOG_BUFFER = [] change_uid_msg = "You should not run this command as root" From 3995b9237b07ef5bef2fbd6a35000b7faa88ab2c Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 19 Nov 2021 09:16:19 +0530 Subject: [PATCH 69/91] fix: raise (by default) when exec_cmd fails --- bench/exceptions.py | 3 +++ bench/utils/__init__.py | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/bench/exceptions.py b/bench/exceptions.py index 3002314b..011f0400 100644 --- a/bench/exceptions.py +++ b/bench/exceptions.py @@ -20,3 +20,6 @@ class BenchNotFoundError(Exception): class ValidationError(Exception): pass + +class CannotUpdateReleaseBench(ValidationError): + pass diff --git a/bench/utils/__init__.py b/bench/utils/__init__.py index 1a52291b..9cd363ed 100644 --- a/bench/utils/__init__.py +++ b/bench/utils/__init__.py @@ -15,7 +15,7 @@ import click # imports - module imports from bench import PROJECT_NAME, VERSION -from bench.exceptions import InvalidRemoteException, ValidationError +from bench.exceptions import CommandFailedError, InvalidRemoteException, ValidationError logger = logging.getLogger(PROJECT_NAME) @@ -109,7 +109,7 @@ def pause_exec(seconds=10): print(" " * 40, end="\r") -def exec_cmd(cmd, cwd=".", env=None): +def exec_cmd(cmd, cwd=".", env=None, _raise=True): if env: env.update(os.environ.copy()) @@ -122,6 +122,9 @@ def exec_cmd(cmd, cwd=".", env=None): return_code = subprocess.call(cmd, cwd=cwd, universal_newlines=True, env=env) if return_code: logger.warning(f"{cmd_log} executed with exit code {return_code}") + if _raise: + raise CommandFailedError + return return_code def which(executable: str, raise_err: bool = False) -> str: From f117959801ea615495c733b689027d53d0302dba Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 19 Nov 2021 10:16:19 +0530 Subject: [PATCH 70/91] feat: Dynamic Output rendering Each operation can be broken down to multiple jobs. For instance, the update operation can be broken down into: setting up bench dirs, env, backups, requirements, etc. Each step has a lot of output since Frappe requires a lot of complex stuff to be done as a pre-requisite. Bench tries to simplify a lot in a single command. Once, a step is completed, it's output is not really required. So, we can ignore it to reduce the noise. Here's where the `bench.cli.fancy` variable kicks in. Along with the refactored log and new step wrapper, it makes the above thing possible. At this point, there's no way to set this var from the end user...given I'm still developing this. In the later commits, the idea is to pass a flag similar to pip's --use-feature flag. --- bench/app.py | 6 ++++ bench/bench.py | 8 ++++++ bench/cli.py | 8 ++++-- bench/utils/render.py | 65 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 bench/utils/render.py diff --git a/bench/app.py b/bench/app.py index f9ab0b82..baf68115 100755 --- a/bench/app.py +++ b/bench/app.py @@ -30,6 +30,8 @@ from bench.utils.bench import ( restart_supervisor_processes, restart_systemd_processes, ) +from bench.utils.render import step + logger = logging.getLogger(bench.PROJECT_NAME) @@ -137,6 +139,7 @@ class App(AppMeta): self.bench = bench super().__init__(name, branch, *args, **kwargs) + @step(title="Fetching App {repo}", success="App {repo} Fetched") def get(self): branch = f"--branch {self.tag}" if self.tag else "" shallow = "--depth 1" if self.bench.shallow_clone else "" @@ -150,6 +153,7 @@ class App(AppMeta): cwd=os.path.join(self.bench.name, "apps"), ) + @step(title="Archiving App {repo}", success="App {repo} Archived") def remove(self): active_app_path = os.path.join("apps", self.repo) archived_path = os.path.join("archived", "apps") @@ -160,6 +164,7 @@ class App(AppMeta): log(f"App moved from {active_app_path} to {archived_app_path}") shutil.move(active_app_path, archived_app_path) + @step(title="Installing App {repo}", success="App {repo} Installed") def install(self, skip_assets=False, verbose=True): from bench.utils.app import get_app_name @@ -174,6 +179,7 @@ class App(AppMeta): app=app_name, bench_path=self.bench.name, verbose=verbose, skip_assets=skip_assets, ) + @step(title="Uninstalling App {repo}", success="App {repo} Uninstalled") def uninstall(self): env_python = get_env_cmd("python", bench_path=self.bench.name) self.bench.run(f"{env_python} -m pip uninstall -y {self.repo}") diff --git a/bench/bench.py b/bench/bench.py index a3b2cb55..c9e16086 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -27,6 +27,7 @@ from bench.utils.bench import ( get_venv_path, get_env_cmd, ) +from bench.utils.render import step if TYPE_CHECKING: @@ -119,6 +120,7 @@ class Bench(Base, Validator): # self.build() - removed because it seems unnecessary self.reload() + @step(title="Building Bench Assets", success="Bench Assets Built") def build(self): # build assets & stuff run_frappe_cmd("build", bench_path=self.name) @@ -214,12 +216,14 @@ class BenchSetup(Base): self.bench = bench self.cwd = self.bench.cwd + @step(title="Setting Up Directories", success="Directories Set Up") def dirs(self): os.makedirs(self.bench.name, exist_ok=True) for dirname in paths_in_bench: os.makedirs(os.path.join(self.bench.name, dirname), exist_ok=True) + @step(title="Setting Up Environment", success="Environment Set Up") def env(self, python="python3"): """Setup env folder - create env if not exists @@ -238,6 +242,7 @@ class BenchSetup(Base): if os.path.exists(frappe): self.run(f"{env_python} -m pip install -q -U -e {frappe}") + @step(title="Setting Up Bench Config", success="Bench Config Set Up") def config(self, redis=True, procfile=True): """Setup config folder - create pids folder @@ -260,12 +265,14 @@ class BenchSetup(Base): return setup_logging(bench_path=self.bench.name) + @step(title="Setting Up Bench Patches", success="Bench Patches Set Up") def patches(self): shutil.copy( os.path.join(os.path.dirname(os.path.abspath(__file__)), "patches", "patches.txt"), os.path.join(self.bench.name, "patches.txt"), ) + @step(title="Setting Up Backups Cronjob", success="Backups Cronjob Set Up") def backups(self): # TODO: to something better for logging data? - maybe a wrapper that auto-logs with more context logger.log("setting up backups") @@ -288,6 +295,7 @@ class BenchSetup(Base): logger.log("backups were set up") + @step(title="Setting Up Bench Dependencies", success="Bench Dependencies Set Up") def requirements(self): from bench.utils.bench import update_requirements update_requirements(bench=self.bench) diff --git a/bench/cli.py b/bench/cli.py index c628d9dd..4df81ec6 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -29,9 +29,13 @@ from bench.utils import ( ) from bench.utils.bench import get_env_cmd -fancy = True -from_command_line = False +# these variables are used to show dynamic outputs on the terminal +fancy = False bench.LOG_BUFFER = [] + +# set when commands are executed via the CLI +from_command_line = False + change_uid_msg = "You should not run this command as root" src = os.path.dirname(__file__) diff --git a/bench/utils/render.py b/bench/utils/render.py new file mode 100644 index 00000000..48417155 --- /dev/null +++ b/bench/utils/render.py @@ -0,0 +1,65 @@ +# imports - standard imports +import sys +from io import StringIO + +# imports - third party imports +import click + + +class Capturing(list): + """ + Util to consume the stdout encompassed in it and push it to a list + + with Capturing() as output: + subprocess.check_output("ls", shell=True) + + print(output) + # ["b'Applications\\nDesktop\\nDocuments\\nDownloads\\n'"] + """ + + def __enter__(self): + self._stdout = sys.stdout + sys.stdout = self._stringio = StringIO() + return self + + def __exit__(self, *args): + self.extend(self._stringio.getvalue().splitlines()) + del self._stringio # free up some memory + sys.stdout = self._stdout + + +def step(title: str = None, success: str = None): + """Supposed to be wrapped around the smallest possible atomic step in a given operation. + For instance, `building assets` is a step in the update operation. + """ + + def innfn(fn): + def wrapper_fn(*args, **kwargs): + import bench.cli + + if bench.cli.from_command_line and bench.cli.fancy: + kw = args[0].__dict__ + + _title = f"{click.style('⏼', fg='bright_yellow')} {title.format(**kw)}" + click.secho(_title) + + retval = fn(*args) + + if bench.cli.from_command_line and bench.cli.fancy: + click.clear() + + for l in bench.LOG_BUFFER: + click.secho(l["message"], fg=l["color"]) + + _success = f"{click.style('✔', fg='green')} {success.format(**kw)}" + click.echo(_success) + + bench.LOG_BUFFER.append( + {"message": _success, "color": None,} + ) + + return retval + + return wrapper_fn + + return innfn From 806d543fac93fafb844bd19e5b44d4b6196faa42 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 19 Nov 2021 11:16:19 +0530 Subject: [PATCH 71/91] fix: Show traceback if error occurs during init, get outside bench dir --- bench/cli.py | 11 +++++++++-- bench/commands/make.py | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index 4df81ec6..410b3057 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -109,8 +109,15 @@ def cli(): logger.warning(f"{command} executed with exit code {return_code}") if isinstance(e, Exception): - if os.environ.get("BENCH_DEVELOPER") or bench_config.get("developer_mode"): - click.secho("".join(format_exception(*sys.exc_info()))[:-1]) + if ( + os.environ.get("BENCH_DEVELOPER") + or bench_config.get("developer_mode") + or ( + sys.argv[1] in ("init", "get", "get-app") and not in_bench + ) + ): + from bench.utils import get_traceback + click.echo(get_traceback()) click.secho(f"ERROR: {e}", fg="red") return_code = 1 raise e diff --git a/bench/commands/make.py b/bench/commands/make.py index 458ca606..cba4a2cf 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -50,7 +50,7 @@ def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, c time.sleep(1) print(get_traceback()) log(f"There was a problem while creating {path}", level=2) - if click.confirm("Do you want to rollback these changes?"): + if click.confirm("Do you want to rollback these changes?", abort=True): print(f'Rolling back Bench "{path}"') if os.path.exists(path): shutil.rmtree(path) From b92fb1401bd9bb0272241e99668bd14bdd32e7ab Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 20 Nov 2021 11:16:19 +0530 Subject: [PATCH 72/91] feat: Pass --use-feature='dynamic-feed' for new CLI --- bench/cli.py | 2 +- bench/commands/__init__.py | 50 +++++++++++++++----------------------- bench/exceptions.py | 3 +++ bench/utils/__init__.py | 2 +- bench/utils/cli.py | 42 ++++++++++++++++++++++++++++++++ bench/utils/render.py | 4 +-- 6 files changed, 68 insertions(+), 35 deletions(-) create mode 100644 bench/utils/cli.py diff --git a/bench/cli.py b/bench/cli.py index 410b3057..041bc94f 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -30,7 +30,7 @@ from bench.utils import ( from bench.utils.bench import get_env_cmd # these variables are used to show dynamic outputs on the terminal -fancy = False +dynamic_feed = False bench.LOG_BUFFER = [] # set when commands are executed via the CLI diff --git a/bench/commands/__init__.py b/bench/commands/__init__.py index 547bf08c..35de746c 100755 --- a/bench/commands/__init__.py +++ b/bench/commands/__init__.py @@ -1,40 +1,28 @@ +# imports - third party imports import click -from click.core import _check_multicommand -def print_bench_version(ctx, param, value): - """Prints current bench version""" - if not value or ctx.resilient_parsing: - return - - import bench - click.echo(bench.VERSION) - ctx.exit() - -class MultiCommandGroup(click.Group): - def add_command(self, cmd, name=None): - """Registers another :class:`Command` with this group. If the name - is not provided, the name of the command is used. - - Note: This is a custom Group that allows passing a list of names for - the command name. - """ - name = name or cmd.name - if name is None: - raise TypeError('Command has no name.') - _check_multicommand(self, name, cmd, register=True) - - try: - self.commands[name] = cmd - except TypeError: - if isinstance(name, list): - for _name in name: - self.commands[_name] = cmd +# imports - module imports +from bench.utils.cli import ( + MultiCommandGroup, + print_bench_version, + use_experimental_feature, +) @click.group(cls=MultiCommandGroup) -@click.option('--version', is_flag=True, is_eager=True, callback=print_bench_version, expose_value=False) -def bench_command(bench_path='.'): +@click.option( + "--version", + is_flag=True, + is_eager=True, + callback=print_bench_version, + expose_value=False, +) +@click.option( + "--use-feature", is_eager=True, callback=use_experimental_feature, expose_value=False +) +def bench_command(bench_path="."): import bench + bench.set_frappe_version(bench_path=bench_path) diff --git a/bench/exceptions.py b/bench/exceptions.py index 011f0400..d4ecb119 100644 --- a/bench/exceptions.py +++ b/bench/exceptions.py @@ -23,3 +23,6 @@ class ValidationError(Exception): class CannotUpdateReleaseBench(ValidationError): pass + +class FeatureDoesNotExistError(CommandFailedError): + pass diff --git a/bench/utils/__init__.py b/bench/utils/__init__.py index 9cd363ed..81a00612 100644 --- a/bench/utils/__init__.py +++ b/bench/utils/__init__.py @@ -61,7 +61,7 @@ def log(message, level=0, no_log=False): color, prefix = levels.get(level, levels[0]) - if bench.cli.from_command_line and bench.cli.fancy: + if bench.cli.from_command_line and bench.cli.dynamic_feed: bench.LOG_BUFFER.append( {"prefix": prefix, "message": message, "color": color,} ) diff --git a/bench/utils/cli.py b/bench/utils/cli.py new file mode 100644 index 00000000..ee81d220 --- /dev/null +++ b/bench/utils/cli.py @@ -0,0 +1,42 @@ +import click +from click.core import _check_multicommand + + +def print_bench_version(ctx, param, value): + """Prints current bench version""" + if not value or ctx.resilient_parsing: + return + + import bench + click.echo(bench.VERSION) + ctx.exit() + + +class MultiCommandGroup(click.Group): + def add_command(self, cmd, name=None): + """Registers another :class:`Command` with this group. If the name + is not provided, the name of the command is used. + + Note: This is a custom Group that allows passing a list of names for + the command name. + """ + name = name or cmd.name + if name is None: + raise TypeError('Command has no name.') + _check_multicommand(self, name, cmd, register=True) + + try: + self.commands[name] = cmd + except TypeError: + if isinstance(name, list): + for _name in name: + self.commands[_name] = cmd + + +def use_experimental_feature(ctx, param, value): + if value == "dynamic-feed": + import bench.cli + bench.cli.dynamic_feed = True + else: + from bench.exceptions import FeatureDoesNotExistError + raise FeatureDoesNotExistError(f"Feature {value} does not exist") diff --git a/bench/utils/render.py b/bench/utils/render.py index 48417155..11a6e0f6 100644 --- a/bench/utils/render.py +++ b/bench/utils/render.py @@ -37,7 +37,7 @@ def step(title: str = None, success: str = None): def wrapper_fn(*args, **kwargs): import bench.cli - if bench.cli.from_command_line and bench.cli.fancy: + if bench.cli.from_command_line and bench.cli.dynamic_feed: kw = args[0].__dict__ _title = f"{click.style('⏼', fg='bright_yellow')} {title.format(**kw)}" @@ -45,7 +45,7 @@ def step(title: str = None, success: str = None): retval = fn(*args) - if bench.cli.from_command_line and bench.cli.fancy: + if bench.cli.from_command_line and bench.cli.dynamic_feed: click.clear() for l in bench.LOG_BUFFER: From 7c4b6cea2bbb7589db31ce167442b9198cb8e364 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 20 Nov 2021 12:16:19 +0530 Subject: [PATCH 73/91] style: Black-ish + Flake8 --- bench/commands/__init__.py | 55 +++++++++++++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 7 deletions(-) diff --git a/bench/commands/__init__.py b/bench/commands/__init__.py index 35de746c..a4d5a9f1 100755 --- a/bench/commands/__init__.py +++ b/bench/commands/__init__.py @@ -26,7 +26,17 @@ def bench_command(bench_path="."): bench.set_frappe_version(bench_path=bench_path) -from bench.commands.make import init, drop, get_app, new_app, remove_app, exclude_app_for_update, include_app_for_update, pip +from bench.commands.make import ( + drop, + exclude_app_for_update, + get_app, + include_app_for_update, + init, + new_app, + pip, + remove_app, +) + bench_command.add_command(init) bench_command.add_command(drop) bench_command.add_command(get_app) @@ -37,17 +47,44 @@ bench_command.add_command(include_app_for_update) bench_command.add_command(pip) -from bench.commands.update import update, retry_upgrade, switch_to_branch, switch_to_develop +from bench.commands.update import ( + retry_upgrade, + switch_to_branch, + switch_to_develop, + update, +) + bench_command.add_command(update) bench_command.add_command(retry_upgrade) bench_command.add_command(switch_to_branch) bench_command.add_command(switch_to_develop) -from bench.commands.utils import (start, restart, set_nginx_port, set_ssl_certificate, set_ssl_certificate_key, set_url_root, - set_mariadb_host, download_translations, backup_site, backup_all_sites, release, renew_lets_encrypt, - disable_production, bench_src, prepare_beta_release, set_redis_cache_host, set_redis_queue_host, set_redis_socketio_host, find_benches, migrate_env, - generate_command_cache, clear_command_cache) +from bench.commands.utils import ( + backup_all_sites, + backup_site, + bench_src, + clear_command_cache, + disable_production, + download_translations, + find_benches, + generate_command_cache, + migrate_env, + prepare_beta_release, + release, + renew_lets_encrypt, + restart, + set_mariadb_host, + set_nginx_port, + set_redis_cache_host, + set_redis_queue_host, + set_redis_socketio_host, + set_ssl_certificate, + set_ssl_certificate_key, + set_url_root, + start, +) + bench_command.add_command(start) bench_command.add_command(restart) bench_command.add_command(set_nginx_port) @@ -72,16 +109,20 @@ bench_command.add_command(generate_command_cache) bench_command.add_command(clear_command_cache) from bench.commands.setup import setup + bench_command.add_command(setup) from bench.commands.config import config + bench_command.add_command(config) -from bench.commands.git import remote_set_url, remote_reset_url, remote_urls +from bench.commands.git import remote_reset_url, remote_set_url, remote_urls + bench_command.add_command(remote_set_url) bench_command.add_command(remote_reset_url) bench_command.add_command(remote_urls) from bench.commands.install import install + bench_command.add_command(install) From 6d10d75bb169b3ed3ad91c9c297cefc93f494448 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 20 Nov 2021 02:16:19 +0530 Subject: [PATCH 74/91] fix: Show pip's help menu when --help is passed in pip command --- bench/commands/make.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/commands/make.py b/bench/commands/make.py index cba4a2cf..0a7a02ae 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -118,7 +118,7 @@ def include_app_for_update(app_name): remove_from_excluded_apps_txt(app_name) -@click.command('pip', context_settings={"ignore_unknown_options": True}, help="For pip help use `bench pip help [COMMAND]` or `bench pip [COMMAND] -h`") +@click.command('pip', context_settings={"ignore_unknown_options": True, "help_option_names": []}, help="For pip help use `bench pip help [COMMAND]` or `bench pip [COMMAND] -h`") @click.argument('args', nargs=-1) @click.pass_context def pip(ctx, args): From f2307789722710415d4e8ee66214ba0c5c8312de Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 18 Nov 2021 18:10:01 +0530 Subject: [PATCH 75/91] refactor: CLI Rendering Engine * Hanlde job-step renders * Use context manager for managing renders and state * Job and Step remain process agnostic - neither are dependant on the other --- bench/utils/render.py | 98 +++++++++++++++++++++++++++++++------------ 1 file changed, 72 insertions(+), 26 deletions(-) diff --git a/bench/utils/render.py b/bench/utils/render.py index 11a6e0f6..9c862b4b 100644 --- a/bench/utils/render.py +++ b/bench/utils/render.py @@ -5,6 +5,9 @@ from io import StringIO # imports - third party imports import click +# imports - module imports +import bench + class Capturing(list): """ @@ -28,38 +31,81 @@ class Capturing(list): sys.stdout = self._stdout +class Rendering: + def __init__(self, success, title, is_parent, args, kwargs): + self.dynamic_feed = bench.cli.from_command_line and bench.cli.dynamic_feed + + if not self.dynamic_feed: + return + + try: + self.kw = args[0].__dict__ + except Exception: + self.kw = kwargs + + self.is_parent = is_parent + self.title = title + self.success = success + + def __enter__(self, *args, **kwargs): + if not self.dynamic_feed: + return + + _prefix = click.style('⏼', fg='bright_yellow') + _hierarchy = " " if not self.is_parent else "" + self._title = self.title.format(**self.kw) + click.secho(f"{_hierarchy}{_prefix} {self._title}") + + bench.LOG_BUFFER.append( + {"message": self._title, "prefix": _prefix, "color": None, "is_parent": self.is_parent} + ) + + def __exit__(self, *args, **kwargs): + if not self.dynamic_feed: + return + + self._prefix = click.style('✔', fg='green') + self._success = self.success.format(**self.kw) + + self.render_screen() + + def render_screen(self): + click.clear() + + for l in bench.LOG_BUFFER: + if l["message"] == self._title: + l["prefix"] = self._prefix + l["message"] = self._success + _hierarchy = " " if not l["is_parent"] else "" + click.secho(f'{_hierarchy}{l["prefix"]} {l["message"]}', fg=l["color"]) + + +def job(title: str = None, success: str = None): + """Supposed to be wrapped around an atomic job in a given process. + For instance, the `get-app` command consists of two jobs: `initializing bench` + and `fetching and installing app`. + """ + def innfn(fn): + def wrapper_fn(*args, **kwargs): + with Rendering( + success=success, title=title, is_parent=True, args=args, kwargs=kwargs, + ): + return fn(*args, **kwargs) + + return wrapper_fn + return innfn + + def step(title: str = None, success: str = None): """Supposed to be wrapped around the smallest possible atomic step in a given operation. For instance, `building assets` is a step in the update operation. """ - def innfn(fn): def wrapper_fn(*args, **kwargs): - import bench.cli - - if bench.cli.from_command_line and bench.cli.dynamic_feed: - kw = args[0].__dict__ - - _title = f"{click.style('⏼', fg='bright_yellow')} {title.format(**kw)}" - click.secho(_title) - - retval = fn(*args) - - if bench.cli.from_command_line and bench.cli.dynamic_feed: - click.clear() - - for l in bench.LOG_BUFFER: - click.secho(l["message"], fg=l["color"]) - - _success = f"{click.style('✔', fg='green')} {success.format(**kw)}" - click.echo(_success) - - bench.LOG_BUFFER.append( - {"message": _success, "color": None,} - ) - - return retval + with Rendering( + success=success, title=title, is_parent=False, args=args, kwargs=kwargs, + ): + return fn(*args, **kwargs) return wrapper_fn - return innfn From 225fce5be68bf889e3b1cb8e7d52b212a2167a32 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 20 Nov 2021 03:16:19 +0530 Subject: [PATCH 76/91] fix: Show bench init job via bench get-app (conditional) This happens when dynamic-feed is enabled --- bench/app.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/bench/app.py b/bench/app.py index baf68115..f4554146 100755 --- a/bench/app.py +++ b/bench/app.py @@ -277,12 +277,14 @@ def get_app( git_url parameter. """ from bench.bench import Bench + import bench as bench_cli bench = Bench(bench_path) app = App(git_url, branch=branch, bench=bench) git_url = app.url repo_name = app.repo branch = app.tag + bench_setup = False if not is_bench_directory(bench_path): from bench.utils.system import init @@ -290,6 +292,16 @@ def get_app( bench_path = get_available_folder_name(f"{app.repo}-bench", bench_path) init(path=bench_path, frappe_branch=branch) os.chdir(bench_path) + bench_setup = True + + if bench_setup and bench_cli.cli.from_command_line and bench_cli.cli.dynamic_feed: + bench_cli.LOG_BUFFER.append({ + "message": f"Fetching App {repo_name}", + "prefix": click.style('⏼', fg='bright_yellow'), + "is_parent": True, + "color": None, + }) + cloned_path = os.path.join(bench_path, "apps", repo_name) dir_already_exists = os.path.isdir(cloned_path) From 7c653db70322604471fad120b779212119e3d010 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 20 Nov 2021 04:16:19 +0530 Subject: [PATCH 77/91] refactor: CLI entry point * Optimized condiitonal checks * Removed warnings for src, --version * Handled positional shifts due to usages of --use-feature (by set.intersection) * Raise Exception always instead of hiding the traceback --- bench/cli.py | 46 +++++++++++++++++----------------------------- 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index 041bc94f..02103396 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -4,7 +4,6 @@ import json import os import pwd import sys -from traceback import format_exception # imports - third party imports import click @@ -42,9 +41,12 @@ src = os.path.dirname(__file__) def cli(): global from_command_line, bench_config + from_command_line = True command = " ".join(sys.argv) + argv = set(sys.argv) is_envvar_warn_set = not (os.environ.get("BENCH_DEVELOPER") or os.environ.get("CI")) + is_cli_command = len(sys.argv) > 1 and not argv.intersection({"src", "--version"}) change_working_directory() logger = setup_logging() @@ -52,15 +54,15 @@ def cli(): bench_config = get_config(".") - if len(sys.argv) > 1 and sys.argv[1] not in ("src",): + if is_cli_command: check_uid() change_uid() change_dir() - if is_envvar_warn_set and ( - is_dist_editable(bench.PROJECT_NAME) - and len(sys.argv) > 1 - and sys.argv[1] != "src" + if ( + is_envvar_warn_set + and is_cli_command + and is_dist_editable(bench.PROJECT_NAME) and not bench_config.get("developer_mode") ): log( @@ -74,9 +76,9 @@ def cli(): if ( not in_bench - and not cmd_requires_root() and len(sys.argv) > 1 - and sys.argv[1] not in ("init", "find", "src", "drop", "get", "get-app") + and not argv.intersection({"init", "find", "src", "drop", "get", "get-app", "--version"}) + and not cmd_requires_root() ): log("Command not being executed in bench directory", level=3) @@ -98,35 +100,21 @@ def cli(): if sys.argv[1] in Bench(".").apps: app_cmd() - if not (len(sys.argv) > 1 and sys.argv[1] == "src"): + if not is_cli_command: atexit.register(check_latest_version) try: bench_command() except BaseException as e: - return_code = getattr(e, "code", 0) + return_code = getattr(e, "code", 1) + + if isinstance(e, Exception): + click.secho(f"ERROR: {e}", fg="red") + if return_code: logger.warning(f"{command} executed with exit code {return_code}") - if isinstance(e, Exception): - if ( - os.environ.get("BENCH_DEVELOPER") - or bench_config.get("developer_mode") - or ( - sys.argv[1] in ("init", "get", "get-app") and not in_bench - ) - ): - from bench.utils import get_traceback - click.echo(get_traceback()) - click.secho(f"ERROR: {e}", fg="red") - return_code = 1 - raise e - finally: - try: - return_code - except NameError: - return_code = 0 - sys.exit(return_code) + raise e def check_uid(): From a8564f39918cf57dfd12f8c5a6969c865a791431 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 18 Nov 2021 18:22:32 +0530 Subject: [PATCH 78/91] chore: Mark methods under job/step --- bench/bench.py | 1 + bench/utils/cli.py | 2 ++ bench/utils/system.py | 2 ++ 3 files changed, 5 insertions(+) diff --git a/bench/bench.py b/bench/bench.py index c9e16086..f845906e 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -125,6 +125,7 @@ class Bench(Base, Validator): # build assets & stuff run_frappe_cmd("build", bench_path=self.name) + @step(title="Reloading Bench Processes", success="Bench Processes Reloaded") def reload(self): conf = self.conf if conf.get("restart_supervisor_on_update"): diff --git a/bench/utils/cli.py b/bench/utils/cli.py index ee81d220..66cbb766 100644 --- a/bench/utils/cli.py +++ b/bench/utils/cli.py @@ -34,6 +34,8 @@ class MultiCommandGroup(click.Group): def use_experimental_feature(ctx, param, value): + if not value: + return if value == "dynamic-feed": import bench.cli bench.cli.dynamic_feed = True diff --git a/bench/utils/system.py b/bench/utils/system.py index 6d741b3d..c232744c 100644 --- a/bench/utils/system.py +++ b/bench/utils/system.py @@ -16,8 +16,10 @@ from bench.utils import ( which, ) from bench.utils.bench import build_assets, clone_apps_from +from bench.utils.render import job +@job(title="Initializing Bench {path}", success="Bench {path} initialized") def init( path, apps_path=None, From 27128c7c18f7df694344305460eacb358df6ef35 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 20 Nov 2021 06:16:19 +0530 Subject: [PATCH 79/91] fix: Import bench.cli For whatever reason, Travis complained saying "AttributeError: module 'bench' has no attribute 'cli'" but works fine locally :') --- bench/utils/render.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bench/utils/render.py b/bench/utils/render.py index 9c862b4b..b0ca6a19 100644 --- a/bench/utils/render.py +++ b/bench/utils/render.py @@ -33,6 +33,8 @@ class Capturing(list): class Rendering: def __init__(self, success, title, is_parent, args, kwargs): + import bench.cli + self.dynamic_feed = bench.cli.from_command_line and bench.cli.dynamic_feed if not self.dynamic_feed: From d63a9883e0fa5aed1a2faaf00d2aa5f797d2b77f Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 26 Nov 2021 11:54:32 +0530 Subject: [PATCH 80/91] feat(minor): Bench.python property that points to env python --- bench/app.py | 8 +++----- bench/bench.py | 25 +++++++++++++------------ bench/utils/bench.py | 6 ++---- 3 files changed, 18 insertions(+), 21 deletions(-) diff --git a/bench/app.py b/bench/app.py index f4554146..5e1eab13 100755 --- a/bench/app.py +++ b/bench/app.py @@ -181,8 +181,7 @@ class App(AppMeta): @step(title="Uninstalling App {repo}", success="App {repo} Uninstalled") def uninstall(self): - env_python = get_env_cmd("python", bench_path=self.bench.name) - self.bench.run(f"{env_python} -m pip uninstall -y {self.repo}") + self.bench.run(f"{self.bench.python} -m pip uninstall -y {self.repo}") def add_to_appstxt(app, bench_path="."): @@ -355,12 +354,11 @@ def install_app( bench = Bench(bench_path) conf = bench.conf - python_path = get_env_cmd("python", bench_path=bench_path) - quiet_flag = "-q" if not verbose else "" + quiet_flag = "" if verbose else "-q" app_path = os.path.realpath(os.path.join(bench_path, "apps", app)) cache_flag = "--no-cache-dir" if no_cache else "" - bench.run(f"{python_path} -m pip install {quiet_flag} -U -e {app_path} {cache_flag}") + bench.run(f"{bench.python} -m pip install {quiet_flag} -U -e {app_path} {cache_flag}") if conf.get("developer_mode"): install_python_dev_dependencies(apps=app) diff --git a/bench/bench.py b/bench/bench.py index f845906e..248e9bd5 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -4,7 +4,7 @@ import os import shutil import sys import logging -from typing import MutableSequence, TYPE_CHECKING +from typing import List, MutableSequence, TYPE_CHECKING # imports - module imports import bench @@ -63,18 +63,21 @@ class Bench(Base, Validator): self.excluded_apps_txt = os.path.join(self.name, "sites", "excluded_apps.txt") @property - def shallow_clone(self): + def python(self) -> str: + return get_env_cmd("python", bench_path=self.name) + + @property + def shallow_clone(self) -> bool: config = self.conf if config: if config.get("release_bench") or not config.get("shallow_clone"): return False - if get_git_version() > 1.9: - return True + return get_git_version() > 1.9 @property - def excluded_apps(self): + def excluded_apps(self) -> List: try: with open(self.excluded_apps_txt) as f: return f.read().strip().split("\n") @@ -82,7 +85,7 @@ class Bench(Base, Validator): return [] @property - def sites(self): + def sites(self) -> List: return [ path for path in os.listdir(os.path.join(self.name, "sites")) @@ -146,11 +149,10 @@ class BenchApps(MutableSequence): return f.write("\n".join(self.apps)) def initialize_apps(self): - cmd = f"{get_env_cmd('python', bench_path=self.bench.name)} -m pip freeze" is_installed = lambda app: app in installed_packages try: - installed_packages = get_cmd_output(cmd=cmd, cwd=self.bench.name) + installed_packages = get_cmd_output(f"{self.bench.python} -m pip freeze", cwd=self.bench.name) except Exception: self.apps = [] return @@ -232,16 +234,15 @@ class BenchSetup(Base): - install frappe python dependencies """ frappe = os.path.join(self.bench.name, "apps", "frappe") - env_python = get_env_cmd("python", bench_path=self.bench.name) virtualenv = get_venv_path() - if not os.path.exists(env_python): + if not os.path.exists(self.bench.python): self.run(f"{virtualenv} -q env -p {python}") - self.run(f"{env_python} -m pip install -q -U pip") + self.run(f"{self.bench.python} -m pip install -U pip") if os.path.exists(frappe): - self.run(f"{env_python} -m pip install -q -U -e {frappe}") + self.run(f"{self.bench.python} -m pip install -U -e {frappe}") @step(title="Setting Up Bench Config", success="Bench Config Set Up") def config(self, redis=True, procfile=True): diff --git a/bench/utils/bench.py b/bench/utils/bench.py index d31f78e6..a9621b48 100644 --- a/bench/utils/bench.py +++ b/bench/utils/bench.py @@ -77,7 +77,6 @@ def update_python_packages(bench_path="."): from bench.bench import Bench bench = Bench(bench_path) - env_py = get_env_cmd("python", bench_path=bench.name) apps = [app for app in bench.apps if app not in bench.excluded_apps] apps.remove("frappe") @@ -89,7 +88,7 @@ def update_python_packages(bench_path="."): for app in apps: app_path = os.path.join(bench_path, "apps", app) click.secho(f"\nInstalling python dependencies for {app}", fg="yellow") - bench.run(f"{env_py} -m pip install -q -U -e {app_path}") + bench.run(f"{bench.python} -m pip install -U -e {app_path}") def update_node_packages(bench_path="."): @@ -117,13 +116,12 @@ def install_python_dev_dependencies(bench_path=".", apps=None): elif apps is None: apps = [app for app in bench.apps if app not in bench.excluded_apps] - env_py = get_env_cmd("python") for app in apps: app_path = os.path.join(bench_path, "apps", app) dev_requirements_path = os.path.join(app_path, "dev-requirements.txt") if os.path.exists(dev_requirements_path): - bench.run(f"{env_py} -m pip install -q -r {dev_requirements_path}") + bench.run(f"{bench.python} -m pip install -r {dev_requirements_path}") def update_yarn_packages(bench_path="."): From 4798992abf55e3fb9786ddd303cfdfd0e25b2529 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 26 Nov 2021 11:55:21 +0530 Subject: [PATCH 81/91] chore: Update setup.py * Add long description using README.md * Add project URLs. This will brighten up https://pypi.org/project/frappe-bench/ a bit * Update contact, license info, etc * Add project classifiers --- setup.py | 41 +++++++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/setup.py b/setup.py index 36835aa6..bb48a876 100644 --- a/setup.py +++ b/setup.py @@ -1,22 +1,43 @@ from setuptools import find_packages, setup from bench import PROJECT_NAME, VERSION -with open('requirements.txt') as f: - install_requires = f.read().strip().split('\n') +with open("requirements.txt") as f: + install_requires = f.read().strip().split("\n") + +with open("README.md") as f: + long_description = f.read() setup( name=PROJECT_NAME, - description='CLI to manage Multi-tenant deployments for Frappe apps', - author='Frappe Technologies', - author_email='info@frappe.io', + description="CLI to manage Multi-tenant deployments for Frappe apps", + long_description=long_description, + long_description_content_type="text/markdown", version=VERSION, + license="GPLv3", + author="Frappe Technologies Pvt Ltd", + author_email="developers@frappe.io", + url="https://frappe.io/bench", + project_urls={ + "Documentation": "https://frappeframework.com/docs/user/en/bench", + "Source": "https://github.com/frappe/bench", + "Changelog": "https://github.com/frappe/bench/releases", + }, + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Framework :: Frappe Framework", + "License :: OSI Approved :: GNU Affero General Public License v3", + "Natural Language :: English", + "Operating System :: MacOS", + "Operating System :: OS Independent", + "Topic :: Software Development :: Build Tools", + "Topic :: Software Development :: User Interfaces", + "Topic :: System :: Installation/Setup", + ], packages=find_packages(), - python_requires='~=3.6', + python_requires="~=3.6", zip_safe=False, include_package_data=True, install_requires=install_requires, - entry_points=''' -[console_scripts] -bench=bench.cli:cli -''', + entry_points={"console_scripts": ["bench=bench.cli:cli"]}, ) From 6081690d7b4b72286a608baf7e551f997f4b0264 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 26 Nov 2021 12:47:29 +0530 Subject: [PATCH 82/91] refactor(minor): python packages setup --- bench/app.py | 4 ++-- bench/bench.py | 11 ++++++++--- bench/utils/bench.py | 6 +++--- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/bench/app.py b/bench/app.py index 5e1eab13..cd02390a 100755 --- a/bench/app.py +++ b/bench/app.py @@ -354,11 +354,11 @@ def install_app( bench = Bench(bench_path) conf = bench.conf - quiet_flag = "" if verbose else "-q" + quiet_flag = "" if verbose else "--quiet" app_path = os.path.realpath(os.path.join(bench_path, "apps", app)) cache_flag = "--no-cache-dir" if no_cache else "" - bench.run(f"{bench.python} -m pip install {quiet_flag} -U -e {app_path} {cache_flag}") + bench.run(f"{bench.python} -m pip install {quiet_flag} --upgrade -e {app_path} {cache_flag}") if conf.get("developer_mode"): install_python_dev_dependencies(apps=app) diff --git a/bench/bench.py b/bench/bench.py index 248e9bd5..f3002396 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -237,12 +237,12 @@ class BenchSetup(Base): virtualenv = get_venv_path() if not os.path.exists(self.bench.python): - self.run(f"{virtualenv} -q env -p {python}") + self.run(f"{virtualenv} env -p {python}") - self.run(f"{self.bench.python} -m pip install -U pip") + self.pip() if os.path.exists(frappe): - self.run(f"{self.bench.python} -m pip install -U -e {frappe}") + self.run(f"{self.bench.python} -m pip install --upgrade -e {frappe}") @step(title="Setting Up Bench Config", success="Bench Config Set Up") def config(self, redis=True, procfile=True): @@ -262,6 +262,11 @@ class BenchSetup(Base): setup_procfile(self.bench.name, skip_redis=not redis) + def pip(self): + """Updates env pip; assumes that env is setup + """ + return self.run(f"{self.bench.python} -m pip install --upgrade pip") + def logging(self): from bench.utils import setup_logging diff --git a/bench/utils/bench.py b/bench/utils/bench.py index a9621b48..e791b9b5 100644 --- a/bench/utils/bench.py +++ b/bench/utils/bench.py @@ -83,12 +83,12 @@ def update_python_packages(bench_path="."): apps.insert(0, "frappe") print("Updating Python libraries...") - update_env_pip(bench_path) + bench.setup.pip() for app in apps: app_path = os.path.join(bench_path, "apps", app) click.secho(f"\nInstalling python dependencies for {app}", fg="yellow") - bench.run(f"{bench.python} -m pip install -U -e {app_path}") + bench.run(f"{bench.python} -m pip install --upgrade -e {app_path}") def update_node_packages(bench_path="."): @@ -226,7 +226,7 @@ def migrate_env(python, backup=False): venv_creation = exec_cmd(f"{virtualenv} --python {python} {pvenv}") apps = " ".join([f"-e {os.path.join('apps', app)}" for app in bench.apps]) - packages_setup = exec_cmd(f"{pvenv} -m pip install -q -U {apps}") + packages_setup = exec_cmd(f"{pvenv} -m pip install --upgrade {apps}") logger.log(f"Migration Successful to {python}") except Exception: From 1b2bb87ab73a268cb7a7fdffb92587dd47744c51 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 26 Nov 2021 16:24:56 +0530 Subject: [PATCH 83/91] refactor: Bench verbosity It's off by default. If you want it enabled, pass --verbose or -v following `bench`. The verbosity only changes how pip installs work. Other stuff is immaterial at this point. Eg: bench -v setup requirements --- bench/app.py | 16 +++++++--- bench/bench.py | 62 ++++++++++++++++++++++++++++++++++---- bench/cli.py | 3 +- bench/commands/__init__.py | 6 +++- bench/commands/setup.py | 13 ++++---- bench/utils/__init__.py | 2 +- bench/utils/app.py | 6 ++-- bench/utils/bench.py | 52 +++++--------------------------- bench/utils/cli.py | 9 ++++++ bench/utils/system.py | 3 ++ 10 files changed, 104 insertions(+), 68 deletions(-) diff --git a/bench/app.py b/bench/app.py index cd02390a..396cbc40 100755 --- a/bench/app.py +++ b/bench/app.py @@ -165,9 +165,11 @@ class App(AppMeta): shutil.move(active_app_path, archived_app_path) @step(title="Installing App {repo}", success="App {repo} Installed") - def install(self, skip_assets=False, verbose=True): + def install(self, skip_assets=False, verbose=False): + import bench.cli from bench.utils.app import get_app_name + verbose = bench.cli.verbose or verbose app_name = get_app_name(self.bench.name, self.repo) # TODO: this should go inside install_app only tho - issue: default/resolved branch @@ -240,12 +242,14 @@ def remove_from_excluded_apps_txt(app, bench_path="."): return write_excluded_apps_txt(apps, bench_path=bench_path) -def setup_app_dependencies(repo_name, bench_path=".", branch=None, verbose=True): +def setup_app_dependencies(repo_name, bench_path=".", branch=None, verbose=False): # branch kwarg is somewhat of a hack here; since we're assuming the same branches for all apps # for eg: if you're installing erpnext@develop, you'll want frappe@develop and healthcare@develop too import glob + import bench.cli from bench.bench import Bench + verbose = bench.cli.verbose or verbose apps_path = os.path.join(os.path.abspath(bench_path), "apps") files = glob.glob(os.path.join(apps_path, repo_name, "**", "hooks.py")) @@ -346,6 +350,7 @@ def install_app( restart_bench=True, skip_assets=False, ): + import bench.cli as bench_cli from bench.bench import Bench install_text = f"Installing {app}" @@ -354,14 +359,17 @@ def install_app( bench = Bench(bench_path) conf = bench.conf + + verbose = bench_cli.verbose or verbose quiet_flag = "" if verbose else "--quiet" - app_path = os.path.realpath(os.path.join(bench_path, "apps", app)) cache_flag = "--no-cache-dir" if no_cache else "" + app_path = os.path.realpath(os.path.join(bench_path, "apps", app)) + bench.run(f"{bench.python} -m pip install {quiet_flag} --upgrade -e {app_path} {cache_flag}") if conf.get("developer_mode"): - install_python_dev_dependencies(apps=app) + install_python_dev_dependencies(apps=app, bench_path=bench_path, verbose=verbose) if os.path.exists(os.path.join(app_path, "package.json")): bench.run("yarn install", cwd=app_path) diff --git a/bench/bench.py b/bench/bench.py index f3002396..70934efd 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -16,6 +16,7 @@ from bench.utils import ( is_frappe_app, get_cmd_output, get_git_version, + log, run_frappe_cmd, ) from bench.utils.bench import ( @@ -233,16 +234,19 @@ class BenchSetup(Base): - upgrade env pip - install frappe python dependencies """ + import bench.cli + frappe = os.path.join(self.bench.name, "apps", "frappe") virtualenv = get_venv_path() + quiet_flag = "" if bench.cli.verbose else "--quiet" if not os.path.exists(self.bench.python): - self.run(f"{virtualenv} env -p {python}") + self.run(f"{virtualenv} {quiet_flag} env -p {python}") self.pip() if os.path.exists(frappe): - self.run(f"{self.bench.python} -m pip install --upgrade -e {frappe}") + self.run(f"{self.bench.python} -m pip install {quiet_flag} --upgrade -e {frappe}") @step(title="Setting Up Bench Config", success="Bench Config Set Up") def config(self, redis=True, procfile=True): @@ -262,10 +266,16 @@ class BenchSetup(Base): setup_procfile(self.bench.name, skip_redis=not redis) - def pip(self): + @step(title="Updating pip", success="Updated pip") + def pip(self, verbose=False): """Updates env pip; assumes that env is setup """ - return self.run(f"{self.bench.python} -m pip install --upgrade pip") + import bench.cli + + verbose = bench.cli.verbose or verbose + quiet_flag = "" if verbose else "--quiet" + + return self.run(f"{self.bench.python} -m pip install {quiet_flag} --upgrade pip") def logging(self): from bench.utils import setup_logging @@ -302,10 +312,50 @@ class BenchSetup(Base): logger.log("backups were set up") + def __get_installed_apps(self) -> List: + """Returns list of installed apps on bench, not in excluded_apps.txt + """ + apps = [app for app in self.bench.apps if app not in self.bench.excluded_apps] + apps.remove("frappe") + apps.insert(0, "frappe") + return apps + @step(title="Setting Up Bench Dependencies", success="Bench Dependencies Set Up") def requirements(self): - from bench.utils.bench import update_requirements - update_requirements(bench=self.bench) + """Install and upgrade all installed apps on given Bench + """ + from bench.app import App + + apps = self.__get_installed_apps() + + self.pip() + + print(f"Installing {len(apps)} applications...") + for app in apps: + App(app, bench=self.bench, to_clone=False).install() + + def python(self): + """Install and upgrade Python dependencies for installed apps on given Bench + """ + import bench.cli + + apps = self.__get_installed_apps() + + quiet_flag = "" if bench.cli.verbose else "--quiet" + + self.pip() + + for app in apps: + app_path = os.path.join(self.bench.name, "apps", app) + log(f"\nInstalling python dependencies for {app}", level=3, no_log=True) + self.run(f"{self.bench.python} -m pip install {quiet_flag} --upgrade -e {app_path}") + + def node(self): + """Install and upgrade Node dependencies for all apps on given Bench + """ + from bench.utils.bench import update_node_packages + + return update_node_packages(bench_path=self.bench.name) class BenchTearDown: diff --git a/bench/cli.py b/bench/cli.py index 02103396..844d64b9 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -30,6 +30,7 @@ from bench.utils.bench import get_env_cmd # these variables are used to show dynamic outputs on the terminal dynamic_feed = False +verbose = False bench.LOG_BUFFER = [] # set when commands are executed via the CLI @@ -88,7 +89,7 @@ def cli(): print(get_frappe_help()) return - if sys.argv[1] in ["--site", "--verbose", "--force", "--profile"]: + if sys.argv[1] in ["--site", "--force", "--profile"]: frappe_cmd() if sys.argv[1] in get_cached_frappe_commands(): diff --git a/bench/commands/__init__.py b/bench/commands/__init__.py index a4d5a9f1..08a493b9 100755 --- a/bench/commands/__init__.py +++ b/bench/commands/__init__.py @@ -6,6 +6,7 @@ from bench.utils.cli import ( MultiCommandGroup, print_bench_version, use_experimental_feature, + setup_verbosity, ) @@ -18,7 +19,10 @@ from bench.utils.cli import ( expose_value=False, ) @click.option( - "--use-feature", is_eager=True, callback=use_experimental_feature, expose_value=False + "--use-feature", is_eager=True, callback=use_experimental_feature, expose_value=False, +) +@click.option( + "-v", "--verbose", is_flag=True, callback=setup_verbosity, expose_value=False, ) def bench_command(bench_path="."): import bench diff --git a/bench/commands/setup.py b/bench/commands/setup.py index a02e39a3..006eba74 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -136,17 +136,18 @@ def setup_socketio(): @click.option("--python", help="Update only Python packages", default=False, is_flag=True) @click.option("--dev", help="Install optional python development dependencies", default=False, is_flag=True) def setup_requirements(node=False, python=False, dev=False): + from bench.bench import Bench + + bench = Bench(".") + if not (node or python or dev): - from bench.utils.bench import update_requirements - update_requirements() + bench.setup.requirements() elif not node and not dev: - from bench.utils.bench import update_python_packages - update_python_packages() + bench.setup.python() elif not python and not dev: - from bench.utils.bench import update_node_packages - update_node_packages() + bench.setup.node() else: from bench.utils.bench import install_python_dev_dependencies diff --git a/bench/utils/__init__.py b/bench/utils/__init__.py index 81a00612..adfbf6c0 100644 --- a/bench/utils/__init__.py +++ b/bench/utils/__init__.py @@ -63,7 +63,7 @@ def log(message, level=0, no_log=False): if bench.cli.from_command_line and bench.cli.dynamic_feed: bench.LOG_BUFFER.append( - {"prefix": prefix, "message": message, "color": color,} + {"prefix": prefix, "message": message, "color": color} ) if no_log: diff --git a/bench/utils/app.py b/bench/utils/app.py index 9a059d98..ae980743 100644 --- a/bench/utils/app.py +++ b/bench/utils/app.py @@ -30,10 +30,10 @@ def is_version_upgrade(app="frappe", bench_path=".", branch=None): def switch_branch(branch, apps=None, bench_path=".", upgrade=False, check_upgrade=True): import git + from bench.bench import Bench from bench.utils import log, exec_cmd from bench.utils.bench import ( build_assets, - update_requirements, patch_sites, post_upgrade, ) @@ -47,8 +47,6 @@ def switch_branch(branch, apps=None, bench_path=".", upgrade=False, check_upgrad apps = [ name for name in os.listdir(apps_dir) if os.path.isdir(os.path.join(apps_dir, name)) ] - if branch == "v4.x.x": - apps.append("shopping_cart") for app in apps: app_dir = os.path.join(apps_dir, app) @@ -93,7 +91,7 @@ def switch_branch(branch, apps=None, bench_path=".", upgrade=False, check_upgrad ) if version_upgrade[0] and upgrade: - update_requirements() + Bench(bench_path).setup.requirements() backup_all_sites() patch_sites() build_assets() diff --git a/bench/utils/bench.py b/bench/utils/bench.py index e791b9b5..d2f67127 100644 --- a/bench/utils/bench.py +++ b/bench/utils/bench.py @@ -48,49 +48,6 @@ def get_venv_path(): return venv or log("virtualenv cannot be found", level=2) -def update_env_pip(bench_path): - env_py = get_env_cmd("python", bench_path=bench_path) - exec_cmd(f"{env_py} -m pip install -q -U pip") - - -def update_requirements(bench: "Bench" = None, bench_path="."): - from bench.app import install_app - - if not bench: - from bench.bench import Bench - - bench = Bench(bench_path) - - apps = [app for app in bench.apps if app not in bench.excluded_apps] - apps.remove("frappe") - apps.insert(0, "frappe") - - print("Updating env pip...") - update_env_pip(bench.name) - - print(f"Installing {len(apps)} applications...") - for app in apps: - install_app(app, bench_path=bench.name, skip_assets=True, restart_bench=False) - - -def update_python_packages(bench_path="."): - from bench.bench import Bench - - bench = Bench(bench_path) - - apps = [app for app in bench.apps if app not in bench.excluded_apps] - apps.remove("frappe") - apps.insert(0, "frappe") - - print("Updating Python libraries...") - bench.setup.pip() - - for app in apps: - app_path = os.path.join(bench_path, "apps", app) - click.secho(f"\nInstalling python dependencies for {app}", fg="yellow") - bench.run(f"{bench.python} -m pip install --upgrade -e {app_path}") - - def update_node_packages(bench_path="."): print("Updating node packages...") from bench.utils.app import get_develop_version @@ -106,9 +63,13 @@ def update_node_packages(bench_path="."): update_yarn_packages(bench_path) -def install_python_dev_dependencies(bench_path=".", apps=None): +def install_python_dev_dependencies(bench_path=".", apps=None, verbose=False): + import bench.cli from bench.bench import Bench + verbose = bench.cli.verbose or verbose + quiet_flag = "" if verbose else "--quiet" + bench = Bench(bench_path) if isinstance(apps, str): @@ -118,10 +79,11 @@ def install_python_dev_dependencies(bench_path=".", apps=None): for app in apps: app_path = os.path.join(bench_path, "apps", app) + dev_requirements_path = os.path.join(app_path, "dev-requirements.txt") if os.path.exists(dev_requirements_path): - bench.run(f"{bench.python} -m pip install -r {dev_requirements_path}") + bench.run(f"{bench.python} -m pip install {quiet_flag} --upgrade -r {dev_requirements_path}") def update_yarn_packages(bench_path="."): diff --git a/bench/utils/cli.py b/bench/utils/cli.py index 66cbb766..1c664b25 100644 --- a/bench/utils/cli.py +++ b/bench/utils/cli.py @@ -39,6 +39,15 @@ def use_experimental_feature(ctx, param, value): if value == "dynamic-feed": import bench.cli bench.cli.dynamic_feed = True + bench.cli.verbose = True else: from bench.exceptions import FeatureDoesNotExistError raise FeatureDoesNotExistError(f"Feature {value} does not exist") + + +def setup_verbosity(ctx, param, value): + if not value: + return + + import bench.cli + bench.cli.verbose = True diff --git a/bench/utils/system.py b/bench/utils/system.py index c232744c..b6bd9dd6 100644 --- a/bench/utils/system.py +++ b/bench/utils/system.py @@ -50,9 +50,12 @@ def init( # Use print("\033c", end="") to clear entire screen after each step and re-render each list # another way => https://stackoverflow.com/a/44591228/10309266 + import bench.cli from bench.app import get_app, install_apps_from_path from bench.bench import Bench + verbose = bench.cli.verbose or verbose + bench = Bench(path) bench.setup.dirs() From b7994e21b18b5d970e46f55e8fc2db89a2a4407e Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 26 Nov 2021 16:28:18 +0530 Subject: [PATCH 84/91] fix: Show warning while using experimental features This shows a "WARNING:..." whenever any --use-feature flag is used --- bench/bench.py | 5 +++-- bench/cli.py | 7 +++---- bench/utils/cli.py | 19 ++++++++++++++++++- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/bench/bench.py b/bench/bench.py index 70934efd..ede4bedd 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -28,7 +28,7 @@ from bench.utils.bench import ( get_venv_path, get_env_cmd, ) -from bench.utils.render import step +from bench.utils.render import job, step if TYPE_CHECKING: @@ -320,7 +320,7 @@ class BenchSetup(Base): apps.insert(0, "frappe") return apps - @step(title="Setting Up Bench Dependencies", success="Bench Dependencies Set Up") + @job(title="Setting Up Bench Dependencies", success="Bench Dependencies Set Up") def requirements(self): """Install and upgrade all installed apps on given Bench """ @@ -331,6 +331,7 @@ class BenchSetup(Base): self.pip() print(f"Installing {len(apps)} applications...") + for app in apps: App(app, bench=self.bench, to_clone=False).install() diff --git a/bench/cli.py b/bench/cli.py index 844d64b9..b2eeac13 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -31,17 +31,16 @@ from bench.utils.bench import get_env_cmd # these variables are used to show dynamic outputs on the terminal dynamic_feed = False verbose = False +is_envvar_warn_set = None +from_command_line = False # set when commands are executed via the CLI bench.LOG_BUFFER = [] -# set when commands are executed via the CLI -from_command_line = False - change_uid_msg = "You should not run this command as root" src = os.path.dirname(__file__) def cli(): - global from_command_line, bench_config + global from_command_line, bench_config, is_envvar_warn_set from_command_line = True command = " ".join(sys.argv) diff --git a/bench/utils/cli.py b/bench/utils/cli.py index 1c664b25..7749e00f 100644 --- a/bench/utils/cli.py +++ b/bench/utils/cli.py @@ -8,6 +8,7 @@ def print_bench_version(ctx, param, value): return import bench + click.echo(bench.VERSION) ctx.exit() @@ -22,7 +23,7 @@ class MultiCommandGroup(click.Group): """ name = name or cmd.name if name is None: - raise TypeError('Command has no name.') + raise TypeError("Command has no name.") _check_multicommand(self, name, cmd, register=True) try: @@ -36,18 +37,34 @@ class MultiCommandGroup(click.Group): def use_experimental_feature(ctx, param, value): if not value: return + if value == "dynamic-feed": import bench.cli + bench.cli.dynamic_feed = True bench.cli.verbose = True else: from bench.exceptions import FeatureDoesNotExistError + raise FeatureDoesNotExistError(f"Feature {value} does not exist") + from bench.cli import is_envvar_warn_set + + if is_envvar_warn_set: + return + + click.secho( + "WARNING: bench is using it's new CLI rendering engine. This behaviour has" + f" been enabled by passing --{value} in the command. This feature is" + " experimental and may not be implemented for all commands yet.", + fg="yellow", + ) + def setup_verbosity(ctx, param, value): if not value: return import bench.cli + bench.cli.verbose = True From 877e812fbbee2fcb65e3759175883096f5ea55a5 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 26 Nov 2021 18:53:13 +0530 Subject: [PATCH 85/91] fix: Patch to move archived_sites to archived/sites This patch runs only if Frappe >= v14. Doesn't do anything else Frappe PR: https://github.com/frappe/frappe/pull/15060 --- bench/patches/patches.txt | 3 +- bench/patches/v5/update_archived_sites.py | 51 +++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 bench/patches/v5/update_archived_sites.py diff --git a/bench/patches/patches.txt b/bench/patches/patches.txt index da51be2e..4be1468c 100644 --- a/bench/patches/patches.txt +++ b/bench/patches/patches.txt @@ -6,4 +6,5 @@ bench.patches.v4.update_socketio bench.patches.v4.install_yarn #2 bench.patches.v5.fix_user_permissions bench.patches.v5.fix_backup_cronjob -bench.patches.v5.set_live_reload_config \ No newline at end of file +bench.patches.v5.set_live_reload_config +bench.patches.v5.update_archived_sites \ No newline at end of file diff --git a/bench/patches/v5/update_archived_sites.py b/bench/patches/v5/update_archived_sites.py new file mode 100644 index 00000000..76507daf --- /dev/null +++ b/bench/patches/v5/update_archived_sites.py @@ -0,0 +1,51 @@ +""" +Deprecate archived_sites folder for consistency. This change is +only for Frappe v14 benches. If not a v14 bench yet, skip this +patch and try again later. + +1. Rename folder `./archived_sites` to `./archived/sites` +2. Create a symlink `./archived_sites` => `./archived/sites` + +Corresponding changes in frappe/frappe via https://github.com/frappe/frappe/pull/15060 +""" +import os +import shutil +from pathlib import Path + +import click +from bench.utils.app import get_current_version +from semantic_version import Version + + +def execute(bench_path): + frappe_version = Version(get_current_version('frappe')) + + if frappe_version.major < 14 or os.name != "posix": + # Returning False means patch has been skipped + return False + + pre_patch_dir = os.getcwd() + old_directory = Path(bench_path, "archived_sites") + new_directory = Path(bench_path, "archived", "sites") + + if old_directory.is_symlink(): + return True + + os.chdir(bench_path) + + if not os.path.exists(new_directory): + os.makedirs(new_directory) + + for archived_site_path in old_directory.glob("*"): + shutil.move(archived_site_path, new_directory) + + click.secho(f"Archived sites are now stored under {new_directory}") + + if not os.listdir(old_directory): + os.rmdir(old_directory) + + os.symlink(new_directory, old_directory) + + click.secho(f"Symlink {old_directory} that points to {new_directory}") + + os.chdir(pre_patch_dir) From a2ccc30bc365fb88a320559828082706944e0072 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 29 Nov 2021 18:33:32 +0530 Subject: [PATCH 86/91] fix: Initialize bench in get-app with --init-bench --- bench/__init__.py | 1 + bench/app.py | 16 +++++++++++++--- bench/bench.py | 3 ++- bench/commands/make.py | 31 +++++++++++++++++++++++-------- bench/exceptions.py | 4 ++++ 5 files changed, 43 insertions(+), 12 deletions(-) diff --git a/bench/__init__.py b/bench/__init__.py index 77c6e130..6c99d660 100644 --- a/bench/__init__.py +++ b/bench/__init__.py @@ -3,6 +3,7 @@ PROJECT_NAME = "frappe-bench" FRAPPE_VERSION = None current_path = None updated_path = None +LOG_BUFFER = [] def set_frappe_version(bench_path="."): diff --git a/bench/app.py b/bench/app.py index 396cbc40..af800540 100755 --- a/bench/app.py +++ b/bench/app.py @@ -15,6 +15,7 @@ import click # imports - module imports import bench +from bench.exceptions import NotInBenchDirectoryError from bench.utils import ( fetch_details_from_tag, get_available_folder_name, @@ -271,6 +272,7 @@ def get_app( skip_assets=False, verbose=False, overwrite=False, + init_bench=False, ): """bench get-app clones a Frappe App from remote (GitHub or any other git server), and installs it on the current bench. This also resolves dependencies based on the @@ -280,7 +282,8 @@ def get_app( git_url parameter. """ from bench.bench import Bench - import bench as bench_cli + import bench as _bench + import bench.cli as bench_cli bench = Bench(bench_path) app = App(git_url, branch=branch, bench=bench) @@ -290,6 +293,12 @@ def get_app( bench_setup = False if not is_bench_directory(bench_path): + if not init_bench: + raise NotInBenchDirectoryError( + f"{os.path.realpath(bench_path)} is not a valid bench directory. " + "Run with --init-bench if you'd like to create a Bench too." + ) + from bench.utils.system import init bench_path = get_available_folder_name(f"{app.repo}-bench", bench_path) @@ -297,8 +306,9 @@ def get_app( os.chdir(bench_path) bench_setup = True - if bench_setup and bench_cli.cli.from_command_line and bench_cli.cli.dynamic_feed: - bench_cli.LOG_BUFFER.append({ + + if bench_setup and bench_cli.from_command_line and bench_cli.dynamic_feed: + _bench.LOG_BUFFER.append({ "message": f"Fetching App {repo_name}", "prefix": click.style('⏼', fg='bright_yellow'), "is_parent": True, diff --git a/bench/bench.py b/bench/bench.py index ede4bedd..157cfef8 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -13,6 +13,7 @@ from bench.config.common_site_config import setup_config from bench.utils import ( paths_in_bench, exec_cmd, + is_bench_directory, is_frappe_app, get_cmd_output, get_git_version, @@ -54,7 +55,7 @@ class Bench(Base, Validator): def __init__(self, path): self.name = path self.cwd = os.path.abspath(path) - self.exists = os.path.exists(self.name) + self.exists = is_bench_directory(self.name) self.setup = BenchSetup(self) self.teardown = BenchTearDown(self) diff --git a/bench/commands/make.py b/bench/commands/make.py index 0a7a02ae..a393017d 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -76,16 +76,31 @@ def drop(path): -@click.command(['get', 'get-app'], help='Clone an app from the internet or filesystem and set it up in your bench') -@click.argument('name', nargs=-1) # Dummy argument for backward compatibility -@click.argument('git-url') -@click.option('--branch', default=None, help="branch to checkout") -@click.option('--overwrite', is_flag=True, default=False) -@click.option('--skip-assets', is_flag=True, default=False, help="Do not build assets") -def get_app(git_url, branch, name=None, overwrite=False, skip_assets=False): +@click.command( + ["get", "get-app"], + help="Clone an app from the internet or filesystem and set it up in your bench", +) +@click.argument("name", nargs=-1) # Dummy argument for backward compatibility +@click.argument("git-url") +@click.option("--branch", default=None, help="branch to checkout") +@click.option("--overwrite", is_flag=True, default=False) +@click.option("--skip-assets", is_flag=True, default=False, help="Do not build assets") +@click.option( + "--init-bench", is_flag=True, default=False, help="Initialize Bench if not in one" +) +def get_app( + git_url, branch, name=None, overwrite=False, skip_assets=False, init_bench=False +): "clone an app from the internet and set it up in your bench" from bench.app import get_app - get_app(git_url, branch=branch, skip_assets=skip_assets, overwrite=overwrite) + + get_app( + git_url, + branch=branch, + skip_assets=skip_assets, + overwrite=overwrite, + init_bench=init_bench, + ) @click.command('new-app', help='Create a new Frappe application under apps folder') diff --git a/bench/exceptions.py b/bench/exceptions.py index d4ecb119..0d517fe4 100644 --- a/bench/exceptions.py +++ b/bench/exceptions.py @@ -26,3 +26,7 @@ class CannotUpdateReleaseBench(ValidationError): class FeatureDoesNotExistError(CommandFailedError): pass + + +class NotInBenchDirectoryError(Exception): + pass From 8480eb5df749b1c2a73c9ed79bb5c7d8ace48397 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 29 Nov 2021 18:41:45 +0530 Subject: [PATCH 87/91] feat(minor): --install-app option in init --- bench/commands/make.py | 82 ++++++++++++++++++++++++++++++++---------- bench/utils/system.py | 7 ++++ 2 files changed, 71 insertions(+), 18 deletions(-) diff --git a/bench/commands/make.py b/bench/commands/make.py index a393017d..dd6f5d1d 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -2,22 +2,63 @@ import click -@click.command('init', help='Initialize a new bench instance in the specified path') -@click.argument('path') -@click.option('--version', '--frappe-branch', 'frappe_branch', default=None, help="Clone a particular branch of frappe") -@click.option('--ignore-exist', is_flag = True, default = False, help = "Ignore if Bench instance exists.") -@click.option('--python', type = str, default = 'python3', help = 'Path to Python Executable.') -@click.option('--apps_path', default=None, help="path to json files with apps to install after init") -@click.option('--frappe-path', default=None, help="path to frappe repo") -@click.option('--clone-from', default=None, help="copy repos from path") -@click.option('--clone-without-update', is_flag=True, help="copy repos from path without update") -@click.option('--no-procfile', is_flag=True, help="Do not create a Procfile") -@click.option('--no-backups',is_flag=True, help="Do not set up automatic periodic backups for all sites on this bench") -@click.option('--skip-redis-config-generation', is_flag=True, help="Skip redis config generation if already specifying the common-site-config file") -@click.option('--skip-assets',is_flag=True, default=False, help="Do not build assets") -@click.option('--verbose',is_flag=True, help="Verbose output during install") -def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, clone_from, verbose, skip_redis_config_generation, clone_without_update, ignore_exist=False, skip_assets=False, python='python3'): +@click.command("init", help="Initialize a new bench instance in the specified path") +@click.argument("path") +@click.option( + "--version", + "--frappe-branch", + "frappe_branch", + default=None, + help="Clone a particular branch of frappe", +) +@click.option( + "--ignore-exist", is_flag=True, default=False, help="Ignore if Bench instance exists." +) +@click.option( + "--python", type=str, default="python3", help="Path to Python Executable." +) +@click.option( + "--apps_path", default=None, help="path to json files with apps to install after init" +) +@click.option("--frappe-path", default=None, help="path to frappe repo") +@click.option("--clone-from", default=None, help="copy repos from path") +@click.option( + "--clone-without-update", is_flag=True, help="copy repos from path without update" +) +@click.option("--no-procfile", is_flag=True, help="Do not create a Procfile") +@click.option( + "--no-backups", + is_flag=True, + help="Do not set up automatic periodic backups for all sites on this bench", +) +@click.option( + "--skip-redis-config-generation", + is_flag=True, + help="Skip redis config generation if already specifying the common-site-config file", +) +@click.option("--skip-assets", is_flag=True, default=False, help="Do not build assets") +@click.option( + "--install-app", help="Install particular app after initialization" +) +@click.option("--verbose", is_flag=True, help="Verbose output during install") +def init( + path, + apps_path, + frappe_path, + frappe_branch, + no_procfile, + no_backups, + clone_from, + verbose, + skip_redis_config_generation, + clone_without_update, + ignore_exist=False, + skip_assets=False, + python="python3", + install_app=None, +): import os + from bench.utils import log from bench.utils.system import init @@ -33,6 +74,7 @@ def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, c no_backups=no_backups, frappe_path=frappe_path, frappe_branch=frappe_branch, + install_app=install_app, clone_from=clone_from, skip_redis_config_generation=skip_redis_config_generation, clone_without_update=clone_without_update, @@ -43,15 +85,19 @@ def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, c log(f'Bench {path} initialized', level=1) except SystemExit: raise - except Exception as e: - import shutil, time + except Exception: + import shutil + import time + from bench.utils import get_traceback + # add a sleep here so that the traceback of other processes doesnt overlap with the prompts time.sleep(1) print(get_traceback()) + log(f"There was a problem while creating {path}", level=2) if click.confirm("Do you want to rollback these changes?", abort=True): - print(f'Rolling back Bench "{path}"') + log(f'Rolling back Bench "{path}"') if os.path.exists(path): shutil.rmtree(path) diff --git a/bench/utils/system.py b/bench/utils/system.py index b6bd9dd6..50c8b1d8 100644 --- a/bench/utils/system.py +++ b/bench/utils/system.py @@ -33,6 +33,7 @@ def init( clone_without_update=False, skip_assets=False, python="python3", + install_app=None, ): """Initialize a new bench directory @@ -82,6 +83,12 @@ def init( if apps_path: install_apps_from_path(apps_path, bench_path=path) + # getting app on bench init using --install-app + if install_app: + get_app( + install_app, branch=frappe_branch, bench_path=path, skip_assets=True, verbose=verbose + ) + if not skip_assets: build_assets(bench_path=path) From 8275d678f4dc0ad1902123f4f1c5f6bf4bce76b8 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 29 Nov 2021 18:55:47 +0530 Subject: [PATCH 88/91] fix: Pass `skip_assets` while resolving dependant apps --- bench/app.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/bench/app.py b/bench/app.py index af800540..857236c9 100755 --- a/bench/app.py +++ b/bench/app.py @@ -132,6 +132,7 @@ class AppMeta: def get_ssh_url(self): return f"git@{self.remote_server}:{self.org}/{self.repo}.git" + @functools.lru_cache(maxsize=None) class App(AppMeta): def __init__( @@ -175,7 +176,11 @@ class App(AppMeta): # TODO: this should go inside install_app only tho - issue: default/resolved branch setup_app_dependencies( - repo_name=self.repo, bench_path=self.bench.name, branch=self.tag, verbose=verbose, + repo_name=self.repo, + bench_path=self.bench.name, + branch=self.tag, + verbose=verbose, + skip_assets=skip_assets, ) install_app( @@ -243,7 +248,9 @@ def remove_from_excluded_apps_txt(app, bench_path="."): return write_excluded_apps_txt(apps, bench_path=bench_path) -def setup_app_dependencies(repo_name, bench_path=".", branch=None, verbose=False): +def setup_app_dependencies( + repo_name, bench_path=".", branch=None, skip_assets=False, verbose=False +): # branch kwarg is somewhat of a hack here; since we're assuming the same branches for all apps # for eg: if you're installing erpnext@develop, you'll want frappe@develop and healthcare@develop too import glob @@ -262,7 +269,13 @@ def setup_app_dependencies(repo_name, bench_path=".", branch=None, verbose=False # TODO: when the time comes, add version check here for app in required_apps: if app not in Bench(bench_path).apps: - get_app(app, bench_path=bench_path, branch=branch, verbose=verbose) + get_app( + app, + bench_path=bench_path, + branch=branch, + skip_assets=skip_assets, + verbose=verbose, + ) def get_app( From a17af670df4dd1eccffe6eadf2a0f22d71dc074a Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 29 Nov 2021 18:56:33 +0530 Subject: [PATCH 89/91] style: Black-ish make & app --- bench/app.py | 1 - bench/commands/make.py | 48 +++++++++++++++++++++++++++--------------- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/bench/app.py b/bench/app.py index 857236c9..1b14947c 100755 --- a/bench/app.py +++ b/bench/app.py @@ -319,7 +319,6 @@ def get_app( os.chdir(bench_path) bench_setup = True - if bench_setup and bench_cli.from_command_line and bench_cli.dynamic_feed: _bench.LOG_BUFFER.append({ "message": f"Fetching App {repo_name}", diff --git a/bench/commands/make.py b/bench/commands/make.py index dd6f5d1d..7c3c2dda 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -82,7 +82,7 @@ def init( python=python, verbose=verbose, ) - log(f'Bench {path} initialized', level=1) + log(f"Bench {path} initialized", level=1) except SystemExit: raise except Exception: @@ -102,8 +102,8 @@ def init( shutil.rmtree(path) -@click.command('drop') -@click.argument('path') +@click.command("drop") +@click.argument("path") def drop(path): from bench.bench import Bench from bench.exceptions import BenchNotFoundError, ValidationError @@ -118,8 +118,7 @@ def drop(path): bench.drop() - print('Bench dropped') - + print("Bench dropped") @click.command( @@ -149,42 +148,57 @@ def get_app( ) -@click.command('new-app', help='Create a new Frappe application under apps folder') -@click.argument('app-name') +@click.command("new-app", help="Create a new Frappe application under apps folder") +@click.argument("app-name") def new_app(app_name): from bench.app import new_app + new_app(app_name) -@click.command(['remove', 'rm', 'remove-app'], help='Completely remove app from bench and re-build assets if not installed on any site') -@click.argument('app-name') +@click.command( + ["remove", "rm", "remove-app"], + help=( + "Completely remove app from bench and re-build assets if not installed on any site" + ), +) +@click.argument("app-name") def remove_app(app_name): from bench.bench import Bench + bench = Bench(".") bench.uninstall(app_name) -@click.command('exclude-app', help='Exclude app from updating') -@click.argument('app_name') +@click.command("exclude-app", help="Exclude app from updating") +@click.argument("app_name") def exclude_app_for_update(app_name): from bench.app import add_to_excluded_apps_txt + add_to_excluded_apps_txt(app_name) -@click.command('include-app', help='Include app for updating') -@click.argument('app_name') +@click.command("include-app", help="Include app for updating") +@click.argument("app_name") def include_app_for_update(app_name): "Include app from updating" from bench.app import remove_from_excluded_apps_txt + remove_from_excluded_apps_txt(app_name) -@click.command('pip', context_settings={"ignore_unknown_options": True, "help_option_names": []}, help="For pip help use `bench pip help [COMMAND]` or `bench pip [COMMAND] -h`") -@click.argument('args', nargs=-1) +@click.command( + "pip", + context_settings={"ignore_unknown_options": True, "help_option_names": []}, + help="For pip help use `bench pip help [COMMAND]` or `bench pip [COMMAND] -h`", +) +@click.argument("args", nargs=-1) @click.pass_context def pip(ctx, args): "Run pip commands in bench env" import os + from bench.utils.bench import get_env_cmd - env_py = get_env_cmd('python') - os.execv(env_py, (env_py, '-m', 'pip') + args) + + env_py = get_env_cmd("python") + os.execv(env_py, (env_py, "-m", "pip") + args) From 357fcc0bbd7e0ea6f7546d61c5223f1288b89517 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 30 Nov 2021 11:33:02 +0530 Subject: [PATCH 90/91] fix: Trailing slash not setting app name properly Re-do of https://github.com/frappe/bench/pull/1021, somewhat --- bench/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/app.py b/bench/app.py index 1b14947c..9881b9af 100755 --- a/bench/app.py +++ b/bench/app.py @@ -58,7 +58,7 @@ class AppMeta: class Healthcare(AppConfig): dependencies = [{"frappe/erpnext": "~13.17.0"}] """ - self.name = name + self.name = name.rstrip('/') self.remote_server = "github.com" self.to_clone = to_clone self.on_disk = False From dc197cf9741b0ed15d13199cbee03a672924eab9 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 30 Nov 2021 15:43:16 +0530 Subject: [PATCH 91/91] docs: Update releases information in README --- README.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/README.md b/README.md index 43f209c4..faf2f726 100755 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ Bench is a command-line utility that helps you to install, update, and manage mu - [Guides](#guides) - [Resources](#resources) - [Development](#development) + - [Releases](#releases) - [License](#license) @@ -254,6 +255,27 @@ $ pip3 install -U frappe-bench To confirm the switch, check the output of `bench src`. It should change from something like `$HOME/bench-repo` to `/usr/local/lib/python3.6/dist-packages` and stop the editable install warnings from getting triggered at every command. + +## Releases + +Bench's version information can be accessed via `bench.VERSION` in the package's __init__.py file. Eversince the v5.0 release, we've started publishing releases on GitHub, and PyPI. + +GitHub: https://github.com/frappe/bench/releases + +PyPI: https://pypi.org/project/frappe-bench + + +From v5.3.0, we partially automated the release process using [@semantic-release](.github/workflows/release.yml). Under this new pipeline, we do the following steps to make a release: + +1. Merge `develop` into the `staging` branch +1. Merge `staging` into the latest stable branch, which is `v5.x` at this point. + +This triggers a GitHub Action job that generates a bump commit, drafts and generates a GitHub release, builds a Python package and publishes it to PyPI. + +The intermediate `staging` branch exists to mediate the `bench.VERSION` conflict that would arise while merging `develop` and stable. On develop, the version has to be manually updated (for major release changes). The version tag plays a role in deciding when checks have to be made for new Bench releases. + +> Note: We may want to kill the convention of separate branches for different version releases of Bench. We don't need to maintain this the way we do for Frappe & ERPNext. A single branch named `stable` would sustain. + ## License This repository has been released under the [GNU GPLv3 License](LICENSE). \ No newline at end of file