2
0
mirror of https://github.com/frappe/bench.git synced 2024-11-12 08:16:28 +00:00

Merge branch 'develop' into add-license-scan-badge

This commit is contained in:
gavin 2021-10-13 20:26:33 +05:30 committed by GitHub
commit b9cc6eeecb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 181 additions and 65 deletions

View File

@ -53,7 +53,7 @@ def write_appstxt(apps, bench_path='.'):
def is_git_url(url): def is_git_url(url):
# modified to allow without the tailing .git from https://github.com/jonschlinkert/is-git-url.git # modified to allow without the tailing .git from https://github.com/jonschlinkert/is-git-url.git
pattern = r"(?:git|ssh|https?|git@[-\w.]+):(\/\/)?(.*?)(\.git)?(\/?|\#[-\d\w._]+?)$" pattern = r"(?:git|ssh|https?|\w*@[-\w.]+):(\/\/)?(.*?)(\.git)?(\/?|\#[-\d\w._]+?)$"
return bool(re.match(pattern, url)) return bool(re.match(pattern, url))
def get_excluded_apps(bench_path='.'): def get_excluded_apps(bench_path='.'):
@ -181,12 +181,16 @@ def install_app(app, bench_path=".", verbose=False, no_cache=False, restart_benc
add_to_appstxt(app, bench_path=bench_path) add_to_appstxt(app, bench_path=bench_path)
conf = get_config(bench_path=bench_path)
if conf.get("developer_mode"):
from bench.utils import install_python_dev_dependencies
install_python_dev_dependencies(apps=app)
if not skip_assets: if not skip_assets:
build_assets(bench_path=bench_path, app=app) build_assets(bench_path=bench_path, app=app)
if restart_bench: if restart_bench:
conf = get_config(bench_path=bench_path)
if conf.get('restart_supervisor_on_update'): if conf.get('restart_supervisor_on_update'):
restart_supervisor_processes(bench_path=bench_path) restart_supervisor_processes(bench_path=bench_path)
if conf.get('restart_systemd_on_update'): if conf.get('restart_systemd_on_update'):
@ -528,7 +532,8 @@ As of January 2020, the following branches are
version Frappe ERPNext version Frappe ERPNext
11 version-11 version-11 11 version-11 version-11
12 version-12 version-12 12 version-12 version-12
13 develop develop 13 version-13 version-13
14 develop develop
Please switch to new branches to get future updates. Please switch to new branches to get future updates.
To switch to your required branch, run the following commands: bench switch-to-branch [branch-name]""") To switch to your required branch, run the following commands: bench switch-to-branch [branch-name]""")

View File

@ -1,7 +1,6 @@
# imports - standard imports # imports - standard imports
import atexit import atexit
import json import json
import logging
import os import os
import pwd import pwd
import sys import sys
@ -14,7 +13,21 @@ import bench
from bench.app import get_apps from bench.app import get_apps
from bench.commands import bench_command from bench.commands import bench_command
from bench.config.common_site_config import get_config from bench.config.common_site_config import get_config
from bench.utils import PatchError, bench_cache_file, check_latest_version, drop_privileges, find_parent_bench, generate_command_cache, get_cmd_output, get_env_cmd, get_frappe, is_bench_directory, is_dist_editable, is_root, log, setup_logging from bench.utils import (
bench_cache_file,
check_latest_version,
drop_privileges,
find_parent_bench,
generate_command_cache,
get_cmd_output,
get_env_cmd,
get_frappe,
is_bench_directory,
is_dist_editable,
is_root,
log,
setup_logging,
)
from_command_line = False from_command_line = False
change_uid_msg = "You should not run this command as root" change_uid_msg = "You should not run this command as root"
@ -35,26 +48,47 @@ def cli():
change_uid() change_uid()
change_dir() change_dir()
if is_dist_editable(bench.PROJECT_NAME) and len(sys.argv) > 1 and sys.argv[1] != "src" and not get_config(".").get("developer_mode"): if (
log("bench is installed in editable mode!\n\nThis is not the recommended mode of installation for production. Instead, install the package from PyPI with: `pip install frappe-bench`\n", level=3) is_dist_editable(bench.PROJECT_NAME)
and len(sys.argv) > 1
and sys.argv[1] != "src"
and not get_config(".").get("developer_mode")
):
log(
"bench is installed in editable mode!\n\nThis is not the recommended mode"
" of installation for production. Instead, install the package from PyPI"
" with: `pip install frappe-bench`\n",
level=3,
)
if not is_bench_directory() and not cmd_requires_root() and len(sys.argv) > 1 and sys.argv[1] not in ("init", "find", "src"): if (
not is_bench_directory()
and not cmd_requires_root()
and len(sys.argv) > 1
and sys.argv[1] not in ("init", "find", "src")
):
log("Command not being executed in bench directory", level=3) log("Command not being executed in bench directory", level=3)
if len(sys.argv) > 2 and sys.argv[1] == "frappe": if len(sys.argv) > 2 and sys.argv[1] == "frappe":
return old_frappe_cli() old_frappe_cli()
elif len(sys.argv) > 1: elif len(sys.argv) > 1:
if sys.argv[1] in get_frappe_commands() + ["--site", "--verbose", "--force", "--profile"]: if sys.argv[1] == "--help":
return frappe_cmd()
elif sys.argv[1] == "--help":
print(click.Context(bench_command).get_help()) print(click.Context(bench_command).get_help())
print(get_frappe_help()) print(get_frappe_help())
return return
elif sys.argv[1] in get_apps(): if sys.argv[1] in ["--site", "--verbose", "--force", "--profile"]:
return app_cmd() frappe_cmd()
if sys.argv[1] in get_cached_frappe_commands():
frappe_cmd()
if sys.argv[1] in get_frappe_commands():
frappe_cmd()
if sys.argv[1] in get_apps():
app_cmd()
if not (len(sys.argv) > 1 and sys.argv[1] == "src"): if not (len(sys.argv) > 1 and sys.argv[1] == "src"):
atexit.register(check_latest_version) atexit.register(check_latest_version)
@ -65,29 +99,50 @@ def cli():
return_code = getattr(e, "code", 0) return_code = getattr(e, "code", 0)
if return_code: if return_code:
logger.warning(f"{command} executed with exit code {return_code}") logger.warning(f"{command} executed with exit code {return_code}")
if isinstance(e, Exception):
raise e
finally:
try:
return_code
except NameError:
return_code = 0
sys.exit(return_code) sys.exit(return_code)
def check_uid(): def check_uid():
if cmd_requires_root() and not is_root(): if cmd_requires_root() and not is_root():
log('superuser privileges required for this command', level=3) log("superuser privileges required for this command", level=3)
sys.exit(1) sys.exit(1)
def cmd_requires_root(): def cmd_requires_root():
if len(sys.argv) > 2 and sys.argv[2] in ('production', 'sudoers', 'lets-encrypt', 'fonts', if len(sys.argv) > 2 and sys.argv[2] in (
'print', 'firewall', 'ssh-port', 'role', 'fail2ban', 'wildcard-ssl'): "production",
"sudoers",
"lets-encrypt",
"fonts",
"print",
"firewall",
"ssh-port",
"role",
"fail2ban",
"wildcard-ssl",
):
return True return True
if len(sys.argv) >= 2 and sys.argv[1] in ('patch', 'renew-lets-encrypt', 'disable-production'): if len(sys.argv) >= 2 and sys.argv[1] in (
"patch",
"renew-lets-encrypt",
"disable-production",
):
return True return True
if len(sys.argv) > 2 and sys.argv[1] in ('install'): if len(sys.argv) > 2 and sys.argv[1] in ("install"):
return True return True
def change_dir(): def change_dir():
if os.path.exists('config.json') or "init" in sys.argv: if os.path.exists("config.json") or "init" in sys.argv:
return return
dir_path_file = '/etc/frappe_bench_dir' dir_path_file = "/etc/frappe_bench_dir"
if os.path.exists(dir_path_file): if os.path.exists(dir_path_file):
with open(dir_path_file) as f: with open(dir_path_file) as f:
dir_path = f.read().strip() dir_path = f.read().strip()
@ -97,52 +152,56 @@ def change_dir():
def change_uid(): def change_uid():
if is_root() and not cmd_requires_root(): if is_root() and not cmd_requires_root():
frappe_user = get_config(".").get('frappe_user') frappe_user = get_config(".").get("frappe_user")
if frappe_user: if frappe_user:
drop_privileges(uid_name=frappe_user, gid_name=frappe_user) drop_privileges(uid_name=frappe_user, gid_name=frappe_user)
os.environ['HOME'] = pwd.getpwnam(frappe_user).pw_dir os.environ["HOME"] = pwd.getpwnam(frappe_user).pw_dir
else: else:
log(change_uid_msg, level=3) log(change_uid_msg, level=3)
sys.exit(1) sys.exit(1)
def old_frappe_cli(bench_path='.'): def old_frappe_cli(bench_path="."):
f = get_frappe(bench_path=bench_path) f = get_frappe(bench_path=bench_path)
os.chdir(os.path.join(bench_path, 'sites')) os.chdir(os.path.join(bench_path, "sites"))
os.execv(f, [f] + sys.argv[2:]) os.execv(f, [f] + sys.argv[2:])
def app_cmd(bench_path='.'): def app_cmd(bench_path="."):
f = get_env_cmd('python', bench_path=bench_path) f = get_env_cmd("python", bench_path=bench_path)
os.chdir(os.path.join(bench_path, 'sites')) os.chdir(os.path.join(bench_path, "sites"))
os.execv(f, [f] + ['-m', 'frappe.utils.bench_helper'] + sys.argv[1:]) os.execv(f, [f] + ["-m", "frappe.utils.bench_helper"] + sys.argv[1:])
def frappe_cmd(bench_path='.'): def frappe_cmd(bench_path="."):
f = get_env_cmd('python', bench_path=bench_path) f = get_env_cmd("python", bench_path=bench_path)
os.chdir(os.path.join(bench_path, 'sites')) os.chdir(os.path.join(bench_path, "sites"))
os.execv(f, [f] + ['-m', 'frappe.utils.bench_helper', 'frappe'] + sys.argv[1:]) os.execv(f, [f] + ["-m", "frappe.utils.bench_helper", "frappe"] + sys.argv[1:])
def get_cached_frappe_commands():
if os.path.exists(bench_cache_file):
command_dump = open(bench_cache_file, "r").read() or "[]"
return json.loads(command_dump)
return []
def get_frappe_commands(): def get_frappe_commands():
if not is_bench_directory(): if not is_bench_directory():
return [] return []
if os.path.exists(bench_cache_file):
command_dump = open(bench_cache_file, 'r').read() or '[]'
return json.loads(command_dump)
else:
return generate_command_cache() return generate_command_cache()
def get_frappe_help(bench_path='.'): def get_frappe_help(bench_path="."):
python = get_env_cmd('python', bench_path=bench_path) python = get_env_cmd("python", bench_path=bench_path)
sites_path = os.path.join(bench_path, 'sites') sites_path = os.path.join(bench_path, "sites")
try: try:
out = get_cmd_output(f"{python} -m frappe.utils.bench_helper get-frappe-help", cwd=sites_path) out = get_cmd_output(
return "\n\nFramework commands:\n" + out.split('Commands:')[1] f"{python} -m frappe.utils.bench_helper get-frappe-help", cwd=sites_path
except: )
return "\n\nFramework commands:\n" + out.split("Commands:")[1]
except Exception:
return "" return ""

View File

@ -135,7 +135,8 @@ def setup_socketio():
@click.command("requirements", help="Setup Python and Node dependencies") @click.command("requirements", help="Setup Python and Node dependencies")
@click.option("--node", help="Update only Node packages", default=False, is_flag=True) @click.option("--node", help="Update only Node packages", default=False, is_flag=True)
@click.option("--python", help="Update only Python packages", default=False, is_flag=True) @click.option("--python", help="Update only Python packages", default=False, is_flag=True)
def setup_requirements(node=False, python=False): @click.option("--dev", help="Install optional python development dependencies", default=False, is_flag=True)
def setup_requirements(node=False, python=False, dev=False):
if not (node or python): if not (node or python):
from bench.utils import update_requirements from bench.utils import update_requirements
update_requirements() update_requirements()
@ -148,6 +149,13 @@ def setup_requirements(node=False, python=False):
from bench.utils import update_node_packages from bench.utils import update_node_packages
update_node_packages() update_node_packages()
if dev:
from bench.utils import install_python_dev_dependencies
install_python_dev_dependencies()
if node:
click.secho("--dev flag only supports python dependencies. All node development dependencies are installed by default.", fg="yellow")
@click.command("manager", help="Setup bench-manager.local site with the bench_manager app installed on it") @click.command("manager", help="Setup bench-manager.local site with the bench_manager app installed on it")
@click.option("--yes", help="Yes to regeneration of nginx config file", default=False, is_flag=True) @click.option("--yes", help="Yes to regeneration of nginx config file", default=False, is_flag=True)

View File

@ -23,7 +23,8 @@ def setup_procfile(bench_path, yes=False, skip_redis=False):
use_rq=use_rq(bench_path), use_rq=use_rq(bench_path),
webserver_port=config.get('webserver_port'), webserver_port=config.get('webserver_port'),
CI=os.environ.get('CI'), CI=os.environ.get('CI'),
skip_redis=skip_redis) skip_redis=skip_redis,
workers=config.get("workers", {}))
with open(procfile_path, 'w') as f: with open(procfile_path, 'w') as f:
f.write(procfile) f.write(procfile)

View File

@ -42,6 +42,7 @@ def generate_supervisor_config(bench_path, user=None, yes=False, skip_redis=Fals
"background_workers": config.get('background_workers') or 1, "background_workers": config.get('background_workers') or 1,
"bench_cmd": which('bench'), "bench_cmd": which('bench'),
"skip_redis": skip_redis, "skip_redis": skip_redis,
"workers": config.get("workers", {}),
}) })
conf_path = os.path.join(bench_path, 'config', 'supervisor.conf') conf_path = os.path.join(bench_path, 'config', 'supervisor.conf')

View File

@ -14,6 +14,9 @@ schedule: bench schedule
worker_short: bench worker --queue short 1>> logs/worker.log 2>> logs/worker.error.log worker_short: bench worker --queue short 1>> logs/worker.log 2>> logs/worker.error.log
worker_long: bench worker --queue long 1>> logs/worker.log 2>> logs/worker.error.log worker_long: bench worker --queue long 1>> logs/worker.log 2>> logs/worker.error.log
worker_default: bench worker --queue default 1>> logs/worker.log 2>> logs/worker.error.log worker_default: bench worker --queue default 1>> logs/worker.log 2>> logs/worker.error.log
{% for worker_name, worker_details in workers.items() %}
worker_{{ worker_name }}: bench worker --queue {{ worker_name }} 1>> logs/worker.log 2>> logs/worker.error.log
{% endfor %}
{% else %} {% else %}
workerbeat: sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app beat -s scheduler.schedule' workerbeat: sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app beat -s scheduler.schedule'
worker: sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app worker -n jobs@%h -Ofair --soft-time-limit 360 --time-limit 390' worker: sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app worker -n jobs@%h -Ofair --soft-time-limit 360 --time-limit 390'

View File

@ -52,6 +52,7 @@ server {
add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload"; add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload";
add_header X-Content-Type-Options nosniff; add_header X-Content-Type-Options nosniff;
add_header X-XSS-Protection "1; mode=block"; add_header X-XSS-Protection "1; mode=block";
add_header Referrer-Policy "same-origin, strict-origin-when-cross-origin";
location /assets { location /assets {
try_files $uri =404; try_files $uri =404;

View File

@ -65,6 +65,22 @@ killasgroup=true
numprocs={{ background_workers }} numprocs={{ background_workers }}
process_name=%(program_name)s-%(process_num)d process_name=%(program_name)s-%(process_num)d
{% for worker_name, worker_details in workers.items() %}
[program:{{ bench_name }}-frappe-{{ worker_name }}-worker]
command={{ bench_cmd }} worker --queue {{ worker_name }}
priority=4
autostart=true
autorestart=true
stdout_logfile={{ bench_dir }}/logs/worker.log
stderr_logfile={{ bench_dir }}/logs/worker.error.log
user={{ user }}
stopwaitsecs={{ worker_details["timeout"] }}
directory={{ bench_dir }}
killasgroup=true
numprocs={{ worker_details["background_workers"] or background_workers }}
process_name=%(program_name)s-%(process_num)d
{% endfor %}
{% else %} {% else %}
[program:{{ bench_name }}-frappe-workerbeat] [program:{{ bench_name }}-frappe-workerbeat]
command={{ bench_dir }}/env/bin/python -m frappe.celery_app beat -s beat.schedule command={{ bench_dir }}/env/bin/python -m frappe.celery_app beat -s beat.schedule

View File

@ -148,10 +148,10 @@ class TestBenchInit(TestBenchBase):
bench_path = os.path.join(self.benches_path, "test-bench") bench_path = os.path.join(self.benches_path, "test-bench")
app_path = os.path.join(bench_path, "apps", "frappe") app_path = os.path.join(bench_path, "apps", "frappe")
successful_switch = not bench.utils.exec_cmd("bench switch-to-branch version-12 frappe --upgrade", cwd=bench_path) successful_switch = not bench.utils.exec_cmd("bench switch-to-branch version-13 frappe --upgrade", cwd=bench_path)
app_branch_after_switch = str(git.Repo(path=app_path).active_branch) app_branch_after_switch = str(git.Repo(path=app_path).active_branch)
if successful_switch: if successful_switch:
self.assertEqual("version-12", app_branch_after_switch) self.assertEqual("version-13", app_branch_after_switch)
successful_switch = not bench.utils.exec_cmd("bench switch-to-branch develop frappe --upgrade", cwd=bench_path) successful_switch = not bench.utils.exec_cmd("bench switch-to-branch develop frappe --upgrade", cwd=bench_path)
app_branch_after_second_switch = str(git.Repo(path=app_path).active_branch) app_branch_after_second_switch = str(git.Repo(path=app_path).active_branch)

View File

@ -577,7 +577,7 @@ def set_default_site(site, bench_path='.'):
def update_env_pip(bench_path): def update_env_pip(bench_path):
env_py = os.path.join(bench_path, 'env', 'bin', 'python') env_py = get_env_cmd("python")
exec_cmd(f"{env_py} -m pip install -q -U pip") exec_cmd(f"{env_py} -m pip install -q -U pip")
@ -593,7 +593,7 @@ def update_requirements(bench_path='.'):
def update_python_packages(bench_path='.'): def update_python_packages(bench_path='.'):
from bench.app import get_apps from bench.app import get_apps
env_py = os.path.join(bench_path, "env", "bin", "python") env_py = get_env_cmd("python")
print('Updating Python libraries...') print('Updating Python libraries...')
update_env_pip(bench_path) update_env_pip(bench_path)
@ -617,6 +617,26 @@ def update_node_packages(bench_path='.'):
update_yarn_packages(bench_path) update_yarn_packages(bench_path)
def install_python_dev_dependencies(bench_path='.', apps=None):
from bench.app import get_apps
if isinstance(apps, str):
apps = [apps]
elif apps is None:
apps = get_apps()
env_py = get_env_cmd("python")
for app in apps:
app_path = os.path.join(bench_path, "apps", app)
dev_requirements_path = os.path.join(app_path, "dev-requirements.txt")
if os.path.exists(dev_requirements_path):
log(f'Installing python development dependencies for {app}')
exec_cmd(f"{env_py} -m pip install -q -r {dev_requirements_path}", cwd=bench_path)
else:
log(f'dev-requirements.txt not found in {app}', level=3)
def update_yarn_packages(bench_path='.'): def update_yarn_packages(bench_path='.'):
apps_dir = os.path.join(bench_path, 'apps') apps_dir = os.path.join(bench_path, 'apps')
@ -1076,6 +1096,8 @@ def generate_command_cache(bench_path='.'):
if hasattr(e, "stderr"): if hasattr(e, "stderr"):
print(e.stderr.decode('utf-8')) print(e.stderr.decode('utf-8'))
return []
def clear_command_cache(bench_path='.'): def clear_command_cache(bench_path='.'):
"""Clears commands cached """Clears commands cached

View File

@ -7,7 +7,7 @@ bench utilizes `frappe.utils.bench_manager` to get the framework's as well as th
Along with the framework commands, Frappe's `bench_manager` module also searches for any commands in your custom applications. Thereby, bench communicates with the respective bench's Frappe which in turn checks for available commands in all of the applications. Along with the framework commands, Frappe's `bench_manager` module also searches for any commands in your custom applications. Thereby, bench communicates with the respective bench's Frappe which in turn checks for available commands in all of the applications.
To make your custom command available to bench, just create a `commands` module under your parent module and write the command with a click wrapper and a variable commands which contains a list of click functions, which are your own commands. The directory strcuture may be visualized as: To make your custom command available to bench, just create a `commands` module under your parent module and write the command with a click wrapper and a variable commands which contains a list of click functions, which are your own commands. The directory structure may be visualized as:
``` ```
frappe-bench frappe-bench

View File

@ -251,8 +251,8 @@ def install_bench(args):
if args.production: if args.production:
extra_vars.update(max_worker_connections=multiprocessing.cpu_count() * 1024) extra_vars.update(max_worker_connections=multiprocessing.cpu_count() * 1024)
frappe_branch = 'version-12' frappe_branch = 'version-13'
erpnext_branch = 'version-12' erpnext_branch = 'version-13'
if args.version: if args.version:
if args.version <= 10: if args.version <= 10:

View File

@ -1,9 +1,9 @@
Click==7.0 Click
GitPython==2.1.15 GitPython~=2.1.15
honcho==1.0.1 honcho
Jinja2==2.11.3 Jinja2~=2.11.3
python-crontab==2.4.0 python-crontab~=2.4.0
requests==2.22.0 requests
semantic-version==2.8.2 semantic-version~=2.8.2
setuptools setuptools
virtualenv virtualenv

View File

@ -6,7 +6,7 @@ with open('requirements.txt') as f:
setup( setup(
name=PROJECT_NAME, name=PROJECT_NAME,
description='Metadata driven, full-stack web framework', description='CLI to manage Multi-tenant deployments for Frappe apps',
author='Frappe Technologies', author='Frappe Technologies',
author_email='info@frappe.io', author_email='info@frappe.io',
version=VERSION, version=VERSION,