diff --git a/bench/app.py b/bench/app.py index bcfc22b5..9fe122ab 100755 --- a/bench/app.py +++ b/bench/app.py @@ -53,7 +53,7 @@ def write_appstxt(apps, bench_path='.'): def is_git_url(url): # modified to allow without the tailing .git from https://github.com/jonschlinkert/is-git-url.git - pattern = r"(?:git|ssh|https?|git@[-\w.]+):(\/\/)?(.*?)(\.git)?(\/?|\#[-\d\w._]+?)$" + pattern = r"(?:git|ssh|https?|\w*@[-\w.]+):(\/\/)?(.*?)(\.git)?(\/?|\#[-\d\w._]+?)$" return bool(re.match(pattern, url)) def get_excluded_apps(bench_path='.'): @@ -181,12 +181,16 @@ def install_app(app, bench_path=".", verbose=False, no_cache=False, restart_benc add_to_appstxt(app, bench_path=bench_path) + conf = get_config(bench_path=bench_path) + + if conf.get("developer_mode"): + from bench.utils import install_python_dev_dependencies + install_python_dev_dependencies(apps=app) + if not skip_assets: build_assets(bench_path=bench_path, app=app) if restart_bench: - conf = get_config(bench_path=bench_path) - if conf.get('restart_supervisor_on_update'): restart_supervisor_processes(bench_path=bench_path) if conf.get('restart_systemd_on_update'): @@ -528,7 +532,8 @@ As of January 2020, the following branches are version Frappe ERPNext 11 version-11 version-11 12 version-12 version-12 -13 develop develop +13 version-13 version-13 +14 develop develop Please switch to new branches to get future updates. To switch to your required branch, run the following commands: bench switch-to-branch [branch-name]""") diff --git a/bench/cli.py b/bench/cli.py index 9fb29966..f194349c 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -1,7 +1,6 @@ # imports - standard imports import atexit import json -import logging import os import pwd import sys @@ -14,7 +13,21 @@ import bench from bench.app import get_apps from bench.commands import bench_command from bench.config.common_site_config import get_config -from bench.utils import PatchError, bench_cache_file, check_latest_version, drop_privileges, find_parent_bench, generate_command_cache, get_cmd_output, get_env_cmd, get_frappe, is_bench_directory, is_dist_editable, is_root, log, setup_logging +from bench.utils import ( + bench_cache_file, + check_latest_version, + drop_privileges, + find_parent_bench, + generate_command_cache, + get_cmd_output, + get_env_cmd, + get_frappe, + is_bench_directory, + is_dist_editable, + is_root, + log, + setup_logging, +) from_command_line = False change_uid_msg = "You should not run this command as root" @@ -30,31 +43,52 @@ def cli(): logger = setup_logging() logger.info(command) - if len(sys.argv) > 1 and sys.argv[1] not in ("src", ): + if len(sys.argv) > 1 and sys.argv[1] not in ("src",): check_uid() change_uid() change_dir() - if is_dist_editable(bench.PROJECT_NAME) and len(sys.argv) > 1 and sys.argv[1] != "src" and not get_config(".").get("developer_mode"): - log("bench is installed in editable mode!\n\nThis is not the recommended mode of installation for production. Instead, install the package from PyPI with: `pip install frappe-bench`\n", level=3) + if ( + is_dist_editable(bench.PROJECT_NAME) + and len(sys.argv) > 1 + and sys.argv[1] != "src" + and not get_config(".").get("developer_mode") + ): + log( + "bench is installed in editable mode!\n\nThis is not the recommended mode" + " of installation for production. Instead, install the package from PyPI" + " with: `pip install frappe-bench`\n", + level=3, + ) - if not is_bench_directory() and not cmd_requires_root() and len(sys.argv) > 1 and sys.argv[1] not in ("init", "find", "src"): + if ( + not is_bench_directory() + and not cmd_requires_root() + and len(sys.argv) > 1 + and sys.argv[1] not in ("init", "find", "src") + ): log("Command not being executed in bench directory", level=3) if len(sys.argv) > 2 and sys.argv[1] == "frappe": - return old_frappe_cli() + old_frappe_cli() elif len(sys.argv) > 1: - if sys.argv[1] in get_frappe_commands() + ["--site", "--verbose", "--force", "--profile"]: - return frappe_cmd() - - elif sys.argv[1] == "--help": + if sys.argv[1] == "--help": print(click.Context(bench_command).get_help()) print(get_frappe_help()) return - elif sys.argv[1] in get_apps(): - return app_cmd() + if sys.argv[1] in ["--site", "--verbose", "--force", "--profile"]: + frappe_cmd() + + if sys.argv[1] in get_cached_frappe_commands(): + frappe_cmd() + + if sys.argv[1] in get_frappe_commands(): + frappe_cmd() + + if sys.argv[1] in get_apps(): + app_cmd() if not (len(sys.argv) > 1 and sys.argv[1] == "src"): atexit.register(check_latest_version) @@ -65,29 +99,50 @@ def cli(): return_code = getattr(e, "code", 0) if return_code: logger.warning(f"{command} executed with exit code {return_code}") + if isinstance(e, Exception): + raise e + finally: + try: + return_code + except NameError: + return_code = 0 sys.exit(return_code) def check_uid(): if cmd_requires_root() and not is_root(): - log('superuser privileges required for this command', level=3) + log("superuser privileges required for this command", level=3) sys.exit(1) def cmd_requires_root(): - if len(sys.argv) > 2 and sys.argv[2] in ('production', 'sudoers', 'lets-encrypt', 'fonts', - 'print', 'firewall', 'ssh-port', 'role', 'fail2ban', 'wildcard-ssl'): + if len(sys.argv) > 2 and sys.argv[2] in ( + "production", + "sudoers", + "lets-encrypt", + "fonts", + "print", + "firewall", + "ssh-port", + "role", + "fail2ban", + "wildcard-ssl", + ): return True - if len(sys.argv) >= 2 and sys.argv[1] in ('patch', 'renew-lets-encrypt', 'disable-production'): + if len(sys.argv) >= 2 and sys.argv[1] in ( + "patch", + "renew-lets-encrypt", + "disable-production", + ): return True - if len(sys.argv) > 2 and sys.argv[1] in ('install'): + if len(sys.argv) > 2 and sys.argv[1] in ("install"): return True def change_dir(): - if os.path.exists('config.json') or "init" in sys.argv: + if os.path.exists("config.json") or "init" in sys.argv: return - dir_path_file = '/etc/frappe_bench_dir' + dir_path_file = "/etc/frappe_bench_dir" if os.path.exists(dir_path_file): with open(dir_path_file) as f: dir_path = f.read().strip() @@ -97,52 +152,56 @@ def change_dir(): def change_uid(): if is_root() and not cmd_requires_root(): - frappe_user = get_config(".").get('frappe_user') + frappe_user = get_config(".").get("frappe_user") if frappe_user: drop_privileges(uid_name=frappe_user, gid_name=frappe_user) - os.environ['HOME'] = pwd.getpwnam(frappe_user).pw_dir + os.environ["HOME"] = pwd.getpwnam(frappe_user).pw_dir else: log(change_uid_msg, level=3) sys.exit(1) -def old_frappe_cli(bench_path='.'): +def old_frappe_cli(bench_path="."): f = get_frappe(bench_path=bench_path) - os.chdir(os.path.join(bench_path, 'sites')) + os.chdir(os.path.join(bench_path, "sites")) os.execv(f, [f] + sys.argv[2:]) -def app_cmd(bench_path='.'): - f = get_env_cmd('python', bench_path=bench_path) - os.chdir(os.path.join(bench_path, 'sites')) - os.execv(f, [f] + ['-m', 'frappe.utils.bench_helper'] + sys.argv[1:]) +def app_cmd(bench_path="."): + f = get_env_cmd("python", bench_path=bench_path) + os.chdir(os.path.join(bench_path, "sites")) + os.execv(f, [f] + ["-m", "frappe.utils.bench_helper"] + sys.argv[1:]) -def frappe_cmd(bench_path='.'): - f = get_env_cmd('python', bench_path=bench_path) - os.chdir(os.path.join(bench_path, 'sites')) - os.execv(f, [f] + ['-m', 'frappe.utils.bench_helper', 'frappe'] + sys.argv[1:]) +def frappe_cmd(bench_path="."): + f = get_env_cmd("python", bench_path=bench_path) + os.chdir(os.path.join(bench_path, "sites")) + os.execv(f, [f] + ["-m", "frappe.utils.bench_helper", "frappe"] + sys.argv[1:]) + + +def get_cached_frappe_commands(): + if os.path.exists(bench_cache_file): + command_dump = open(bench_cache_file, "r").read() or "[]" + return json.loads(command_dump) + return [] def get_frappe_commands(): if not is_bench_directory(): return [] - if os.path.exists(bench_cache_file): - command_dump = open(bench_cache_file, 'r').read() or '[]' - return json.loads(command_dump) - - else: - return generate_command_cache() + return generate_command_cache() -def get_frappe_help(bench_path='.'): - python = get_env_cmd('python', bench_path=bench_path) - sites_path = os.path.join(bench_path, 'sites') +def get_frappe_help(bench_path="."): + python = get_env_cmd("python", bench_path=bench_path) + sites_path = os.path.join(bench_path, "sites") try: - out = get_cmd_output(f"{python} -m frappe.utils.bench_helper get-frappe-help", cwd=sites_path) - return "\n\nFramework commands:\n" + out.split('Commands:')[1] - except: + out = get_cmd_output( + f"{python} -m frappe.utils.bench_helper get-frappe-help", cwd=sites_path + ) + return "\n\nFramework commands:\n" + out.split("Commands:")[1] + except Exception: return "" diff --git a/bench/commands/setup.py b/bench/commands/setup.py index 51f87030..73b3eb3a 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -135,7 +135,8 @@ def setup_socketio(): @click.command("requirements", help="Setup Python and Node dependencies") @click.option("--node", help="Update only Node packages", default=False, is_flag=True) @click.option("--python", help="Update only Python packages", default=False, is_flag=True) -def setup_requirements(node=False, python=False): +@click.option("--dev", help="Install optional python development dependencies", default=False, is_flag=True) +def setup_requirements(node=False, python=False, dev=False): if not (node or python): from bench.utils import update_requirements update_requirements() @@ -148,6 +149,13 @@ def setup_requirements(node=False, python=False): from bench.utils import update_node_packages update_node_packages() + if dev: + from bench.utils import install_python_dev_dependencies + install_python_dev_dependencies() + + if node: + click.secho("--dev flag only supports python dependencies. All node development dependencies are installed by default.", fg="yellow") + @click.command("manager", help="Setup bench-manager.local site with the bench_manager app installed on it") @click.option("--yes", help="Yes to regeneration of nginx config file", default=False, is_flag=True) diff --git a/bench/config/procfile.py b/bench/config/procfile.py index 50a45560..8a479b2e 100755 --- a/bench/config/procfile.py +++ b/bench/config/procfile.py @@ -23,7 +23,8 @@ def setup_procfile(bench_path, yes=False, skip_redis=False): use_rq=use_rq(bench_path), webserver_port=config.get('webserver_port'), CI=os.environ.get('CI'), - skip_redis=skip_redis) + skip_redis=skip_redis, + workers=config.get("workers", {})) with open(procfile_path, 'w') as f: f.write(procfile) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 8c036624..cfbaebbd 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -42,6 +42,7 @@ def generate_supervisor_config(bench_path, user=None, yes=False, skip_redis=Fals "background_workers": config.get('background_workers') or 1, "bench_cmd": which('bench'), "skip_redis": skip_redis, + "workers": config.get("workers", {}), }) conf_path = os.path.join(bench_path, 'config', 'supervisor.conf') diff --git a/bench/config/templates/Procfile b/bench/config/templates/Procfile index f810506e..d9391087 100644 --- a/bench/config/templates/Procfile +++ b/bench/config/templates/Procfile @@ -14,6 +14,9 @@ schedule: bench schedule worker_short: bench worker --queue short 1>> logs/worker.log 2>> logs/worker.error.log worker_long: bench worker --queue long 1>> logs/worker.log 2>> logs/worker.error.log worker_default: bench worker --queue default 1>> logs/worker.log 2>> logs/worker.error.log +{% for worker_name, worker_details in workers.items() %} +worker_{{ worker_name }}: bench worker --queue {{ worker_name }} 1>> logs/worker.log 2>> logs/worker.error.log +{% endfor %} {% else %} workerbeat: sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app beat -s scheduler.schedule' worker: sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app worker -n jobs@%h -Ofair --soft-time-limit 360 --time-limit 390' diff --git a/bench/config/templates/nginx.conf b/bench/config/templates/nginx.conf index 1879be63..77d865a2 100644 --- a/bench/config/templates/nginx.conf +++ b/bench/config/templates/nginx.conf @@ -52,6 +52,7 @@ server { add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload"; add_header X-Content-Type-Options nosniff; add_header X-XSS-Protection "1; mode=block"; + add_header Referrer-Policy "same-origin, strict-origin-when-cross-origin"; location /assets { try_files $uri =404; diff --git a/bench/config/templates/supervisor.conf b/bench/config/templates/supervisor.conf index d24b1cb6..085cc2cf 100644 --- a/bench/config/templates/supervisor.conf +++ b/bench/config/templates/supervisor.conf @@ -65,6 +65,22 @@ killasgroup=true numprocs={{ background_workers }} process_name=%(program_name)s-%(process_num)d +{% for worker_name, worker_details in workers.items() %} +[program:{{ bench_name }}-frappe-{{ worker_name }}-worker] +command={{ bench_cmd }} worker --queue {{ worker_name }} +priority=4 +autostart=true +autorestart=true +stdout_logfile={{ bench_dir }}/logs/worker.log +stderr_logfile={{ bench_dir }}/logs/worker.error.log +user={{ user }} +stopwaitsecs={{ worker_details["timeout"] }} +directory={{ bench_dir }} +killasgroup=true +numprocs={{ worker_details["background_workers"] or background_workers }} +process_name=%(program_name)s-%(process_num)d +{% endfor %} + {% else %} [program:{{ bench_name }}-frappe-workerbeat] command={{ bench_dir }}/env/bin/python -m frappe.celery_app beat -s beat.schedule diff --git a/bench/tests/test_init.py b/bench/tests/test_init.py index cbc6a0ea..56b05946 100755 --- a/bench/tests/test_init.py +++ b/bench/tests/test_init.py @@ -148,10 +148,10 @@ class TestBenchInit(TestBenchBase): bench_path = os.path.join(self.benches_path, "test-bench") app_path = os.path.join(bench_path, "apps", "frappe") - successful_switch = not bench.utils.exec_cmd("bench switch-to-branch version-12 frappe --upgrade", cwd=bench_path) + successful_switch = not bench.utils.exec_cmd("bench switch-to-branch version-13 frappe --upgrade", cwd=bench_path) app_branch_after_switch = str(git.Repo(path=app_path).active_branch) if successful_switch: - self.assertEqual("version-12", app_branch_after_switch) + self.assertEqual("version-13", app_branch_after_switch) successful_switch = not bench.utils.exec_cmd("bench switch-to-branch develop frappe --upgrade", cwd=bench_path) app_branch_after_second_switch = str(git.Repo(path=app_path).active_branch) diff --git a/bench/utils.py b/bench/utils.py index 39d0ca56..99f7870b 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -577,7 +577,7 @@ def set_default_site(site, bench_path='.'): def update_env_pip(bench_path): - env_py = os.path.join(bench_path, 'env', 'bin', 'python') + env_py = get_env_cmd("python") exec_cmd(f"{env_py} -m pip install -q -U pip") @@ -593,7 +593,7 @@ def update_requirements(bench_path='.'): def update_python_packages(bench_path='.'): from bench.app import get_apps - env_py = os.path.join(bench_path, "env", "bin", "python") + env_py = get_env_cmd("python") print('Updating Python libraries...') update_env_pip(bench_path) @@ -617,6 +617,26 @@ def update_node_packages(bench_path='.'): update_yarn_packages(bench_path) +def install_python_dev_dependencies(bench_path='.', apps=None): + from bench.app import get_apps + + if isinstance(apps, str): + apps = [apps] + elif apps is None: + apps = get_apps() + + env_py = get_env_cmd("python") + for app in apps: + app_path = os.path.join(bench_path, "apps", app) + dev_requirements_path = os.path.join(app_path, "dev-requirements.txt") + + if os.path.exists(dev_requirements_path): + log(f'Installing python development dependencies for {app}') + exec_cmd(f"{env_py} -m pip install -q -r {dev_requirements_path}", cwd=bench_path) + else: + log(f'dev-requirements.txt not found in {app}', level=3) + + def update_yarn_packages(bench_path='.'): apps_dir = os.path.join(bench_path, 'apps') @@ -1076,6 +1096,8 @@ def generate_command_cache(bench_path='.'): if hasattr(e, "stderr"): print(e.stderr.decode('utf-8')) + return [] + def clear_command_cache(bench_path='.'): """Clears commands cached diff --git a/docs/bench_custom_cmd.md b/docs/bench_custom_cmd.md index 693739ba..72ef1559 100644 --- a/docs/bench_custom_cmd.md +++ b/docs/bench_custom_cmd.md @@ -7,7 +7,7 @@ bench utilizes `frappe.utils.bench_manager` to get the framework's as well as th Along with the framework commands, Frappe's `bench_manager` module also searches for any commands in your custom applications. Thereby, bench communicates with the respective bench's Frappe which in turn checks for available commands in all of the applications. -To make your custom command available to bench, just create a `commands` module under your parent module and write the command with a click wrapper and a variable commands which contains a list of click functions, which are your own commands. The directory strcuture may be visualized as: +To make your custom command available to bench, just create a `commands` module under your parent module and write the command with a click wrapper and a variable commands which contains a list of click functions, which are your own commands. The directory structure may be visualized as: ``` frappe-bench diff --git a/install.py b/install.py index d42da5b8..16b99be1 100644 --- a/install.py +++ b/install.py @@ -251,8 +251,8 @@ def install_bench(args): if args.production: extra_vars.update(max_worker_connections=multiprocessing.cpu_count() * 1024) - frappe_branch = 'version-12' - erpnext_branch = 'version-12' + frappe_branch = 'version-13' + erpnext_branch = 'version-13' if args.version: if args.version <= 10: diff --git a/requirements.txt b/requirements.txt index 66c2c947..e84b98c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ -Click==7.0 -GitPython==2.1.15 -honcho==1.0.1 -Jinja2==2.11.3 -python-crontab==2.4.0 -requests==2.22.0 -semantic-version==2.8.2 +Click +GitPython~=2.1.15 +honcho +Jinja2~=2.11.3 +python-crontab~=2.4.0 +requests +semantic-version~=2.8.2 setuptools virtualenv diff --git a/setup.py b/setup.py index be99d6c2..36835aa6 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open('requirements.txt') as f: setup( name=PROJECT_NAME, - description='Metadata driven, full-stack web framework', + description='CLI to manage Multi-tenant deployments for Frappe apps', author='Frappe Technologies', author_email='info@frappe.io', version=VERSION,