From a4916d7b895f8d9840a0dbff647a66de06bcf6d5 Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Tue, 26 May 2020 10:51:34 +0530 Subject: [PATCH 01/14] fix: set referrer-policy header in nginx config sets the "Referrer-Policy" header to "strict-origin-when-cross-origin", with "same-origin" as a fallback setting the referrer policy prevents sharing site context to external links, preventing cross-site hijacking or tab nagging. Signed-off-by: Chinmay D. Pai --- bench/config/templates/nginx.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/bench/config/templates/nginx.conf b/bench/config/templates/nginx.conf index 9df365f3..d5e5109e 100644 --- a/bench/config/templates/nginx.conf +++ b/bench/config/templates/nginx.conf @@ -49,6 +49,7 @@ server { add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload"; add_header X-Content-Type-Options nosniff; add_header X-XSS-Protection "1; mode=block"; + add_header Referrer-Policy "same-origin, strict-origin-when-cross-origin"; location /assets { try_files $uri =404; From 6137086e036997f123447bf1c8d0b0a743e88dee Mon Sep 17 00:00:00 2001 From: Abhishek Balam Date: Mon, 30 Aug 2021 12:29:09 +0530 Subject: [PATCH 02/14] feat: Set default version for easy install to v13 (#1189) * feat: set default version for easy install to v13 * fix: python version based version picking fixed and reverted changes to node version in installation script * fix: revert test version Co-authored-by: gavin --- bench/app.py | 3 ++- bench/tests/test_init.py | 4 ++-- install.py | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/bench/app.py b/bench/app.py index bcfc22b5..352f858d 100755 --- a/bench/app.py +++ b/bench/app.py @@ -528,7 +528,8 @@ As of January 2020, the following branches are version Frappe ERPNext 11 version-11 version-11 12 version-12 version-12 -13 develop develop +13 version-13 version-13 +14 develop develop Please switch to new branches to get future updates. To switch to your required branch, run the following commands: bench switch-to-branch [branch-name]""") diff --git a/bench/tests/test_init.py b/bench/tests/test_init.py index cbc6a0ea..56b05946 100755 --- a/bench/tests/test_init.py +++ b/bench/tests/test_init.py @@ -148,10 +148,10 @@ class TestBenchInit(TestBenchBase): bench_path = os.path.join(self.benches_path, "test-bench") app_path = os.path.join(bench_path, "apps", "frappe") - successful_switch = not bench.utils.exec_cmd("bench switch-to-branch version-12 frappe --upgrade", cwd=bench_path) + successful_switch = not bench.utils.exec_cmd("bench switch-to-branch version-13 frappe --upgrade", cwd=bench_path) app_branch_after_switch = str(git.Repo(path=app_path).active_branch) if successful_switch: - self.assertEqual("version-12", app_branch_after_switch) + self.assertEqual("version-13", app_branch_after_switch) successful_switch = not bench.utils.exec_cmd("bench switch-to-branch develop frappe --upgrade", cwd=bench_path) app_branch_after_second_switch = str(git.Repo(path=app_path).active_branch) diff --git a/install.py b/install.py index d42da5b8..16b99be1 100644 --- a/install.py +++ b/install.py @@ -251,8 +251,8 @@ def install_bench(args): if args.production: extra_vars.update(max_worker_connections=multiprocessing.cpu_count() * 1024) - frappe_branch = 'version-12' - erpnext_branch = 'version-12' + frappe_branch = 'version-13' + erpnext_branch = 'version-13' if args.version: if args.version <= 10: From e12f53d1a56d598d90307bdfa755db02cc37a30c Mon Sep 17 00:00:00 2001 From: Francisco Roldan Date: Mon, 30 Aug 2021 12:35:53 -0300 Subject: [PATCH 03/14] feat: added option to provide custom queues --- bench/config/supervisor.py | 1 + bench/config/templates/supervisor.conf | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 8c036624..9aa1a2be 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -42,6 +42,7 @@ def generate_supervisor_config(bench_path, user=None, yes=False, skip_redis=Fals "background_workers": config.get('background_workers') or 1, "bench_cmd": which('bench'), "skip_redis": skip_redis, + "custom_queues": config.get("custom_queues", {}), }) conf_path = os.path.join(bench_path, 'config', 'supervisor.conf') diff --git a/bench/config/templates/supervisor.conf b/bench/config/templates/supervisor.conf index d24b1cb6..06872b00 100644 --- a/bench/config/templates/supervisor.conf +++ b/bench/config/templates/supervisor.conf @@ -65,6 +65,22 @@ killasgroup=true numprocs={{ background_workers }} process_name=%(program_name)s-%(process_num)d +{% for queue_name, queue_details in custom_queues.items() %} +[program:{{ bench_name }}-frappe-{{ queue_name }}-worker] +command={{ bench_cmd }} worker --queue {{ queue_name }} +priority=4 +autostart=true +autorestart=true +stdout_logfile={{ bench_dir }}/logs/worker.log +stderr_logfile={{ bench_dir }}/logs/worker.error.log +user={{ user }} +stopwaitsecs={{ queue_details["timeout"] }} +directory={{ bench_dir }} +killasgroup=true +numprocs={{ queue_details["workers"] or background_workers }} +process_name=%(program_name)s-%(process_num)d +{% endfor %} + {% else %} [program:{{ bench_name }}-frappe-workerbeat] command={{ bench_dir }}/env/bin/python -m frappe.celery_app beat -s beat.schedule From a94ea19bf43d0359c5f59d2aa886742eb8832c65 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 9 Sep 2021 13:19:44 +0530 Subject: [PATCH 04/14] fix: Hit command cache before fetching all commands --- bench/cli.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index 9fb29966..765ae758 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -45,7 +45,11 @@ def cli(): return old_frappe_cli() elif len(sys.argv) > 1: - if sys.argv[1] in get_frappe_commands() + ["--site", "--verbose", "--force", "--profile"]: + if sys.argv[1] in ["--site", "--verbose", "--force", "--profile"]: + return frappe_cmd() + if sys.argv[1] in get_cached_frappe_commands(): + return frappe_cmd() + if sys.argv[1] in get_frappe_commands(): return frappe_cmd() elif sys.argv[1] == "--help": @@ -124,16 +128,16 @@ def frappe_cmd(bench_path='.'): os.execv(f, [f] + ['-m', 'frappe.utils.bench_helper', 'frappe'] + sys.argv[1:]) -def get_frappe_commands(): - if not is_bench_directory(): - return [] - +def get_cached_frappe_commands(): if os.path.exists(bench_cache_file): command_dump = open(bench_cache_file, 'r').read() or '[]' return json.loads(command_dump) - else: - return generate_command_cache() +def get_frappe_commands(): + if not is_bench_directory(): + return [] + + return generate_command_cache() def get_frappe_help(bench_path='.'): From 552b935f6b7d35933907c05421171827b30a967c Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 9 Sep 2021 13:20:26 +0530 Subject: [PATCH 05/14] fix: Always set return code via cli --- bench/cli.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bench/cli.py b/bench/cli.py index 765ae758..3d3d2454 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -69,6 +69,13 @@ def cli(): return_code = getattr(e, "code", 0) if return_code: logger.warning(f"{command} executed with exit code {return_code}") + if isinstance(e, Exception): + raise e + finally: + try: + return_code + except NameError: + return_code = 0 sys.exit(return_code) From 6160ff48ab1c04ebcf4436e40a51efd564647aee Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 9 Sep 2021 13:54:19 +0530 Subject: [PATCH 06/14] style: Black --- bench/cli.py | 107 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 76 insertions(+), 31 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index 3d3d2454..b2a79e2c 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -1,7 +1,6 @@ # imports - standard imports import atexit import json -import logging import os import pwd import sys @@ -14,7 +13,21 @@ import bench from bench.app import get_apps from bench.commands import bench_command from bench.config.common_site_config import get_config -from bench.utils import PatchError, bench_cache_file, check_latest_version, drop_privileges, find_parent_bench, generate_command_cache, get_cmd_output, get_env_cmd, get_frappe, is_bench_directory, is_dist_editable, is_root, log, setup_logging +from bench.utils import ( + bench_cache_file, + check_latest_version, + drop_privileges, + find_parent_bench, + generate_command_cache, + get_cmd_output, + get_env_cmd, + get_frappe, + is_bench_directory, + is_dist_editable, + is_root, + log, + setup_logging, +) from_command_line = False change_uid_msg = "You should not run this command as root" @@ -30,15 +43,30 @@ def cli(): logger = setup_logging() logger.info(command) - if len(sys.argv) > 1 and sys.argv[1] not in ("src", ): + if len(sys.argv) > 1 and sys.argv[1] not in ("src",): check_uid() change_uid() change_dir() - if is_dist_editable(bench.PROJECT_NAME) and len(sys.argv) > 1 and sys.argv[1] != "src" and not get_config(".").get("developer_mode"): - log("bench is installed in editable mode!\n\nThis is not the recommended mode of installation for production. Instead, install the package from PyPI with: `pip install frappe-bench`\n", level=3) + if ( + is_dist_editable(bench.PROJECT_NAME) + and len(sys.argv) > 1 + and sys.argv[1] != "src" + and not get_config(".").get("developer_mode") + ): + log( + "bench is installed in editable mode!\n\nThis is not the recommended mode" + " of installation for production. Instead, install the package from PyPI" + " with: `pip install frappe-bench`\n", + level=3, + ) - if not is_bench_directory() and not cmd_requires_root() and len(sys.argv) > 1 and sys.argv[1] not in ("init", "find", "src"): + if ( + not is_bench_directory() + and not cmd_requires_root() + and len(sys.argv) > 1 + and sys.argv[1] not in ("init", "find", "src") + ): log("Command not being executed in bench directory", level=3) if len(sys.argv) > 2 and sys.argv[1] == "frappe": @@ -81,24 +109,38 @@ def cli(): def check_uid(): if cmd_requires_root() and not is_root(): - log('superuser privileges required for this command', level=3) + log("superuser privileges required for this command", level=3) sys.exit(1) def cmd_requires_root(): - if len(sys.argv) > 2 and sys.argv[2] in ('production', 'sudoers', 'lets-encrypt', 'fonts', - 'print', 'firewall', 'ssh-port', 'role', 'fail2ban', 'wildcard-ssl'): + if len(sys.argv) > 2 and sys.argv[2] in ( + "production", + "sudoers", + "lets-encrypt", + "fonts", + "print", + "firewall", + "ssh-port", + "role", + "fail2ban", + "wildcard-ssl", + ): return True - if len(sys.argv) >= 2 and sys.argv[1] in ('patch', 'renew-lets-encrypt', 'disable-production'): + if len(sys.argv) >= 2 and sys.argv[1] in ( + "patch", + "renew-lets-encrypt", + "disable-production", + ): return True - if len(sys.argv) > 2 and sys.argv[1] in ('install'): + if len(sys.argv) > 2 and sys.argv[1] in ("install"): return True def change_dir(): - if os.path.exists('config.json') or "init" in sys.argv: + if os.path.exists("config.json") or "init" in sys.argv: return - dir_path_file = '/etc/frappe_bench_dir' + dir_path_file = "/etc/frappe_bench_dir" if os.path.exists(dir_path_file): with open(dir_path_file) as f: dir_path = f.read().strip() @@ -108,38 +150,39 @@ def change_dir(): def change_uid(): if is_root() and not cmd_requires_root(): - frappe_user = get_config(".").get('frappe_user') + frappe_user = get_config(".").get("frappe_user") if frappe_user: drop_privileges(uid_name=frappe_user, gid_name=frappe_user) - os.environ['HOME'] = pwd.getpwnam(frappe_user).pw_dir + os.environ["HOME"] = pwd.getpwnam(frappe_user).pw_dir else: log(change_uid_msg, level=3) sys.exit(1) -def old_frappe_cli(bench_path='.'): +def old_frappe_cli(bench_path="."): f = get_frappe(bench_path=bench_path) - os.chdir(os.path.join(bench_path, 'sites')) + os.chdir(os.path.join(bench_path, "sites")) os.execv(f, [f] + sys.argv[2:]) -def app_cmd(bench_path='.'): - f = get_env_cmd('python', bench_path=bench_path) - os.chdir(os.path.join(bench_path, 'sites')) - os.execv(f, [f] + ['-m', 'frappe.utils.bench_helper'] + sys.argv[1:]) +def app_cmd(bench_path="."): + f = get_env_cmd("python", bench_path=bench_path) + os.chdir(os.path.join(bench_path, "sites")) + os.execv(f, [f] + ["-m", "frappe.utils.bench_helper"] + sys.argv[1:]) -def frappe_cmd(bench_path='.'): - f = get_env_cmd('python', bench_path=bench_path) - os.chdir(os.path.join(bench_path, 'sites')) - os.execv(f, [f] + ['-m', 'frappe.utils.bench_helper', 'frappe'] + sys.argv[1:]) +def frappe_cmd(bench_path="."): + f = get_env_cmd("python", bench_path=bench_path) + os.chdir(os.path.join(bench_path, "sites")) + os.execv(f, [f] + ["-m", "frappe.utils.bench_helper", "frappe"] + sys.argv[1:]) def get_cached_frappe_commands(): if os.path.exists(bench_cache_file): - command_dump = open(bench_cache_file, 'r').read() or '[]' + command_dump = open(bench_cache_file, "r").read() or "[]" return json.loads(command_dump) + def get_frappe_commands(): if not is_bench_directory(): return [] @@ -147,12 +190,14 @@ def get_frappe_commands(): return generate_command_cache() -def get_frappe_help(bench_path='.'): - python = get_env_cmd('python', bench_path=bench_path) - sites_path = os.path.join(bench_path, 'sites') +def get_frappe_help(bench_path="."): + python = get_env_cmd("python", bench_path=bench_path) + sites_path = os.path.join(bench_path, "sites") try: - out = get_cmd_output(f"{python} -m frappe.utils.bench_helper get-frappe-help", cwd=sites_path) - return "\n\nFramework commands:\n" + out.split('Commands:')[1] + out = get_cmd_output( + f"{python} -m frappe.utils.bench_helper get-frappe-help", cwd=sites_path + ) + return "\n\nFramework commands:\n" + out.split("Commands:")[1] except: return "" From a01550169905a20b8322931338331cc80bc57766 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 9 Sep 2021 13:54:35 +0530 Subject: [PATCH 07/14] fix: Return fallback list in cached_frappe_commands --- bench/cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bench/cli.py b/bench/cli.py index b2a79e2c..9d49dbbe 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -181,6 +181,7 @@ def get_cached_frappe_commands(): if os.path.exists(bench_cache_file): command_dump = open(bench_cache_file, "r").read() or "[]" return json.loads(command_dump) + return [] def get_frappe_commands(): From b35af193d34d45ed068888bba8bf4d5490154e9f Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 9 Sep 2021 14:56:32 +0530 Subject: [PATCH 08/14] chore: Remove unnecessary returns, re-order cli resolution Changes suggested by LGTM https://lgtm.com/projects/g/frappe/bench/rev/pr-560978725461013796d605b409ffc5c85c774829 --- bench/cli.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index 9d49dbbe..f194349c 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -70,23 +70,25 @@ def cli(): log("Command not being executed in bench directory", level=3) if len(sys.argv) > 2 and sys.argv[1] == "frappe": - return old_frappe_cli() + old_frappe_cli() elif len(sys.argv) > 1: - if sys.argv[1] in ["--site", "--verbose", "--force", "--profile"]: - return frappe_cmd() - if sys.argv[1] in get_cached_frappe_commands(): - return frappe_cmd() - if sys.argv[1] in get_frappe_commands(): - return frappe_cmd() - - elif sys.argv[1] == "--help": + if sys.argv[1] == "--help": print(click.Context(bench_command).get_help()) print(get_frappe_help()) return - elif sys.argv[1] in get_apps(): - return app_cmd() + if sys.argv[1] in ["--site", "--verbose", "--force", "--profile"]: + frappe_cmd() + + if sys.argv[1] in get_cached_frappe_commands(): + frappe_cmd() + + if sys.argv[1] in get_frappe_commands(): + frappe_cmd() + + if sys.argv[1] in get_apps(): + app_cmd() if not (len(sys.argv) > 1 and sys.argv[1] == "src"): atexit.register(check_latest_version) @@ -199,7 +201,7 @@ def get_frappe_help(bench_path="."): f"{python} -m frappe.utils.bench_helper get-frappe-help", cwd=sites_path ) return "\n\nFramework commands:\n" + out.split("Commands:")[1] - except: + except Exception: return "" From 6fd3ad738b75a974763aec26ff8124c4fcf4147e Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 9 Sep 2021 16:05:42 +0530 Subject: [PATCH 09/14] fix(command_cache): Return iterable as fallback --- bench/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bench/utils.py b/bench/utils.py index 39d0ca56..bfe6469c 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -1076,6 +1076,8 @@ def generate_command_cache(bench_path='.'): if hasattr(e, "stderr"): print(e.stderr.decode('utf-8')) + return [] + def clear_command_cache(bench_path='.'): """Clears commands cached From b51f0ed2b2c3c26b29e8d038bec134275dbff808 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 13 Sep 2021 20:55:21 +0530 Subject: [PATCH 10/14] feat: `bench setup requirements --dev` (#1196) * chore: typo * chore: update project description * feat: command to install dev-requirements.txt Often applications have development or test specific requirements which are not required in production. - Add new command `bench setup dev-requirements` - installs all `dev-requirements.txt` in app's root folder. * refactor: remove duplicate function * refactor: use `log` instead of print * refactor: merge dev-requirement command * feat: install dev-dependencies in get-app if dev When developer mode is enabled install all dev dependencies too while doing `get-app` * fix: warn about --dev not supporting node --- bench/app.py | 8 ++++++-- bench/commands/setup.py | 10 +++++++++- bench/utils.py | 24 ++++++++++++++++++++++-- docs/bench_custom_cmd.md | 2 +- setup.py | 2 +- 5 files changed, 39 insertions(+), 7 deletions(-) diff --git a/bench/app.py b/bench/app.py index 352f858d..97cbe6a8 100755 --- a/bench/app.py +++ b/bench/app.py @@ -181,12 +181,16 @@ def install_app(app, bench_path=".", verbose=False, no_cache=False, restart_benc add_to_appstxt(app, bench_path=bench_path) + conf = get_config(bench_path=bench_path) + + if conf.get("developer_mode"): + from bench.utils import install_python_dev_dependencies + install_python_dev_dependencies(apps=app) + if not skip_assets: build_assets(bench_path=bench_path, app=app) if restart_bench: - conf = get_config(bench_path=bench_path) - if conf.get('restart_supervisor_on_update'): restart_supervisor_processes(bench_path=bench_path) if conf.get('restart_systemd_on_update'): diff --git a/bench/commands/setup.py b/bench/commands/setup.py index 51f87030..73b3eb3a 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -135,7 +135,8 @@ def setup_socketio(): @click.command("requirements", help="Setup Python and Node dependencies") @click.option("--node", help="Update only Node packages", default=False, is_flag=True) @click.option("--python", help="Update only Python packages", default=False, is_flag=True) -def setup_requirements(node=False, python=False): +@click.option("--dev", help="Install optional python development dependencies", default=False, is_flag=True) +def setup_requirements(node=False, python=False, dev=False): if not (node or python): from bench.utils import update_requirements update_requirements() @@ -148,6 +149,13 @@ def setup_requirements(node=False, python=False): from bench.utils import update_node_packages update_node_packages() + if dev: + from bench.utils import install_python_dev_dependencies + install_python_dev_dependencies() + + if node: + click.secho("--dev flag only supports python dependencies. All node development dependencies are installed by default.", fg="yellow") + @click.command("manager", help="Setup bench-manager.local site with the bench_manager app installed on it") @click.option("--yes", help="Yes to regeneration of nginx config file", default=False, is_flag=True) diff --git a/bench/utils.py b/bench/utils.py index bfe6469c..99f7870b 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -577,7 +577,7 @@ def set_default_site(site, bench_path='.'): def update_env_pip(bench_path): - env_py = os.path.join(bench_path, 'env', 'bin', 'python') + env_py = get_env_cmd("python") exec_cmd(f"{env_py} -m pip install -q -U pip") @@ -593,7 +593,7 @@ def update_requirements(bench_path='.'): def update_python_packages(bench_path='.'): from bench.app import get_apps - env_py = os.path.join(bench_path, "env", "bin", "python") + env_py = get_env_cmd("python") print('Updating Python libraries...') update_env_pip(bench_path) @@ -617,6 +617,26 @@ def update_node_packages(bench_path='.'): update_yarn_packages(bench_path) +def install_python_dev_dependencies(bench_path='.', apps=None): + from bench.app import get_apps + + if isinstance(apps, str): + apps = [apps] + elif apps is None: + apps = get_apps() + + env_py = get_env_cmd("python") + for app in apps: + app_path = os.path.join(bench_path, "apps", app) + dev_requirements_path = os.path.join(app_path, "dev-requirements.txt") + + if os.path.exists(dev_requirements_path): + log(f'Installing python development dependencies for {app}') + exec_cmd(f"{env_py} -m pip install -q -r {dev_requirements_path}", cwd=bench_path) + else: + log(f'dev-requirements.txt not found in {app}', level=3) + + def update_yarn_packages(bench_path='.'): apps_dir = os.path.join(bench_path, 'apps') diff --git a/docs/bench_custom_cmd.md b/docs/bench_custom_cmd.md index 693739ba..72ef1559 100644 --- a/docs/bench_custom_cmd.md +++ b/docs/bench_custom_cmd.md @@ -7,7 +7,7 @@ bench utilizes `frappe.utils.bench_manager` to get the framework's as well as th Along with the framework commands, Frappe's `bench_manager` module also searches for any commands in your custom applications. Thereby, bench communicates with the respective bench's Frappe which in turn checks for available commands in all of the applications. -To make your custom command available to bench, just create a `commands` module under your parent module and write the command with a click wrapper and a variable commands which contains a list of click functions, which are your own commands. The directory strcuture may be visualized as: +To make your custom command available to bench, just create a `commands` module under your parent module and write the command with a click wrapper and a variable commands which contains a list of click functions, which are your own commands. The directory structure may be visualized as: ``` frappe-bench diff --git a/setup.py b/setup.py index be99d6c2..36835aa6 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open('requirements.txt') as f: setup( name=PROJECT_NAME, - description='Metadata driven, full-stack web framework', + description='CLI to manage Multi-tenant deployments for Frappe apps', author='Frappe Technologies', author_email='info@frappe.io', version=VERSION, From 75f84625ad4b19f74bffa989c878ecda07b5c68b Mon Sep 17 00:00:00 2001 From: Francisco Roldan Date: Tue, 14 Sep 2021 08:42:06 -0300 Subject: [PATCH 11/14] fix: rename key --- bench/config/supervisor.py | 2 +- bench/config/templates/supervisor.conf | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 9aa1a2be..cfbaebbd 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -42,7 +42,7 @@ def generate_supervisor_config(bench_path, user=None, yes=False, skip_redis=Fals "background_workers": config.get('background_workers') or 1, "bench_cmd": which('bench'), "skip_redis": skip_redis, - "custom_queues": config.get("custom_queues", {}), + "workers": config.get("workers", {}), }) conf_path = os.path.join(bench_path, 'config', 'supervisor.conf') diff --git a/bench/config/templates/supervisor.conf b/bench/config/templates/supervisor.conf index 06872b00..085cc2cf 100644 --- a/bench/config/templates/supervisor.conf +++ b/bench/config/templates/supervisor.conf @@ -65,19 +65,19 @@ killasgroup=true numprocs={{ background_workers }} process_name=%(program_name)s-%(process_num)d -{% for queue_name, queue_details in custom_queues.items() %} -[program:{{ bench_name }}-frappe-{{ queue_name }}-worker] -command={{ bench_cmd }} worker --queue {{ queue_name }} +{% for worker_name, worker_details in workers.items() %} +[program:{{ bench_name }}-frappe-{{ worker_name }}-worker] +command={{ bench_cmd }} worker --queue {{ worker_name }} priority=4 autostart=true autorestart=true stdout_logfile={{ bench_dir }}/logs/worker.log stderr_logfile={{ bench_dir }}/logs/worker.error.log user={{ user }} -stopwaitsecs={{ queue_details["timeout"] }} +stopwaitsecs={{ worker_details["timeout"] }} directory={{ bench_dir }} killasgroup=true -numprocs={{ queue_details["workers"] or background_workers }} +numprocs={{ worker_details["background_workers"] or background_workers }} process_name=%(program_name)s-%(process_num)d {% endfor %} From 641181260e698073c9df6a68ae8ef75c40769eaa Mon Sep 17 00:00:00 2001 From: kevgeni <90869237+kevgeni@users.noreply.github.com> Date: Fri, 17 Sep 2021 05:59:14 -0400 Subject: [PATCH 12/14] fix: Allow git url with custom username (#1200) Allow git url with custom user, and not just the git user. --- bench/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/app.py b/bench/app.py index 97cbe6a8..9fe122ab 100755 --- a/bench/app.py +++ b/bench/app.py @@ -53,7 +53,7 @@ def write_appstxt(apps, bench_path='.'): def is_git_url(url): # modified to allow without the tailing .git from https://github.com/jonschlinkert/is-git-url.git - pattern = r"(?:git|ssh|https?|git@[-\w.]+):(\/\/)?(.*?)(\.git)?(\/?|\#[-\d\w._]+?)$" + pattern = r"(?:git|ssh|https?|\w*@[-\w.]+):(\/\/)?(.*?)(\.git)?(\/?|\#[-\d\w._]+?)$" return bool(re.match(pattern, url)) def get_excluded_apps(bench_path='.'): From 593b80eec96502901068bf5a7c734f48b71e5d31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Rold=C3=A1n?= Date: Thu, 7 Oct 2021 03:50:36 -0300 Subject: [PATCH 13/14] feat: Add custom workers in Procfile (#1201) * feat: updated procfile * fix: add missing context * fix: add missing context --- bench/config/procfile.py | 3 ++- bench/config/templates/Procfile | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/bench/config/procfile.py b/bench/config/procfile.py index 50a45560..8a479b2e 100755 --- a/bench/config/procfile.py +++ b/bench/config/procfile.py @@ -23,7 +23,8 @@ def setup_procfile(bench_path, yes=False, skip_redis=False): use_rq=use_rq(bench_path), webserver_port=config.get('webserver_port'), CI=os.environ.get('CI'), - skip_redis=skip_redis) + skip_redis=skip_redis, + workers=config.get("workers", {})) with open(procfile_path, 'w') as f: f.write(procfile) diff --git a/bench/config/templates/Procfile b/bench/config/templates/Procfile index f810506e..d9391087 100644 --- a/bench/config/templates/Procfile +++ b/bench/config/templates/Procfile @@ -14,6 +14,9 @@ schedule: bench schedule worker_short: bench worker --queue short 1>> logs/worker.log 2>> logs/worker.error.log worker_long: bench worker --queue long 1>> logs/worker.log 2>> logs/worker.error.log worker_default: bench worker --queue default 1>> logs/worker.log 2>> logs/worker.error.log +{% for worker_name, worker_details in workers.items() %} +worker_{{ worker_name }}: bench worker --queue {{ worker_name }} 1>> logs/worker.log 2>> logs/worker.error.log +{% endfor %} {% else %} workerbeat: sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app beat -s scheduler.schedule' worker: sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app worker -n jobs@%h -Ofair --soft-time-limit 360 --time-limit 390' From 7ad4813cc3a308bd231fb872f032261ca1383ee5 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 12 Oct 2021 21:47:20 +0530 Subject: [PATCH 14/14] build: Loose-er dependencies * Replaced one of the = with ~ to allow any patch releases to be compatible * Removed pinning from requests, honcho & Click since we're using the more basic APIs * Made these changes so that it's easier to install bench globally :') --- requirements.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/requirements.txt b/requirements.txt index 66c2c947..e84b98c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ -Click==7.0 -GitPython==2.1.15 -honcho==1.0.1 -Jinja2==2.11.3 -python-crontab==2.4.0 -requests==2.22.0 -semantic-version==2.8.2 +Click +GitPython~=2.1.15 +honcho +Jinja2~=2.11.3 +python-crontab~=2.4.0 +requests +semantic-version~=2.8.2 setuptools virtualenv