From 51b81c399760581a57184e368f4a5e2bc4743b5f Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 13 Mar 2020 16:41:00 +0530 Subject: [PATCH 01/47] fix: remove bench and supervisor from sudoers chore: move production prerequisites into setup_production --- bench/commands/setup.py | 10 ---------- bench/config/production_setup.py | 21 ++++++++++++++++++--- bench/config/templates/frappe_sudoers | 5 ----- bench/tests/test_setup_production.py | 4 ++-- bench/utils.py | 9 +++------ 5 files changed, 23 insertions(+), 26 deletions(-) diff --git a/bench/commands/setup.py b/bench/commands/setup.py index 352a781b..84e267c7 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -60,16 +60,6 @@ def setup_fonts(): @click.option("--yes", help="Yes to regeneration config", is_flag=True, default=False) def setup_production(user, yes=False): from bench.config.production_setup import setup_production - # Install prereqs for production - from distutils.spawn import find_executable - if not find_executable("ansible"): - exec_cmd("sudo -H {0} -m pip install ansible".format(sys.executable)) - if not find_executable("fail2ban-client"): - exec_cmd("bench setup role fail2ban") - if not find_executable("nginx"): - exec_cmd("bench setup role nginx") - if not find_executable("supervisord"): - exec_cmd("bench setup role supervisor") setup_production(user=user, yes=yes) diff --git a/bench/config/production_setup.py b/bench/config/production_setup.py index 8b2e7801..e6d09098 100755 --- a/bench/config/production_setup.py +++ b/bench/config/production_setup.py @@ -4,8 +4,23 @@ from bench.config.systemd import generate_systemd_config from bench.config.nginx import make_nginx_conf from bench.config.common_site_config import get_config import os, subprocess +import sys +from distutils.spawn import find_executable + + +def setup_production_prerequisites(): + if not find_executable("ansible"): + exec_cmd("sudo {0} -m pip install ansible".format(sys.executable)) + if not find_executable("fail2ban-client"): + exec_cmd("bench setup role fail2ban") + if not find_executable("nginx"): + exec_cmd("bench setup role nginx") + if not find_executable("supervisord"): + exec_cmd("bench setup role supervisor") + def setup_production(user, bench_path='.', yes=False): + setup_production_prerequisites() if get_config(bench_path).get('restart_supervisor_on_update') and get_config(bench_path).get('restart_systemd_on_update'): raise Exception("You cannot use supervisor and systemd at the same time. Modify your common_site_config accordingly." ) @@ -109,15 +124,15 @@ def reload_supervisor(): try: # first try reread/update - exec_cmd('sudo {0} reread'.format(supervisorctl)) - exec_cmd('sudo {0} update'.format(supervisorctl)) + exec_cmd('{0} reread'.format(supervisorctl)) + exec_cmd('{0} update'.format(supervisorctl)) return except CommandFailedError: pass try: # something is wrong, so try reloading - exec_cmd('sudo {0} reload'.format(supervisorctl)) + exec_cmd('{0} reload'.format(supervisorctl)) return except CommandFailedError: pass diff --git a/bench/config/templates/frappe_sudoers b/bench/config/templates/frappe_sudoers index 5394444f..51595c8f 100644 --- a/bench/config/templates/frappe_sudoers +++ b/bench/config/templates/frappe_sudoers @@ -8,13 +8,8 @@ {{ user }} ALL = (root) NOPASSWD: {{ systemctl }} * nginx {{ user }} ALL = (root) NOPASSWD: {{ systemctl }} * supervisord {% endif %} -{% if supervisorctl %} -{{ user }} ALL = (root) NOPASSWD: {{ supervisorctl }} -{% endif %} {% if nginx %} {{ user }} ALL = (root) NOPASSWD: {{ nginx }} {% endif %} {{ user }} ALL = (root) NOPASSWD: /opt/certbot-auto -{{ user }} ALL = (root) NOPASSWD: {{ bench }} Defaults:{{ user }} !requiretty - diff --git a/bench/tests/test_setup_production.py b/bench/tests/test_setup_production.py index ca5b4abc..8d0bbc7a 100644 --- a/bench/tests/test_setup_production.py +++ b/bench/tests/test_setup_production.py @@ -126,12 +126,12 @@ class TestSetupProduction(TestBenchBase): def assert_supervisor_process(self, bench_name, use_rq=True, disable_production=False): - out = bench.utils.get_cmd_output("sudo supervisorctl status") + out = bench.utils.get_cmd_output("supervisorctl status") while "STARTING" in out: print ("Waiting for all processes to start...") time.sleep(10) - out = bench.utils.get_cmd_output("sudo supervisorctl status") + out = bench.utils.get_cmd_output("supervisorctl status") tests = [ "{bench_name}-web:{bench_name}-frappe-web[\s]+RUNNING", diff --git a/bench/utils.py b/bench/utils.py index bd42556f..0f534c7a 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -37,6 +37,7 @@ class CommandFailedError(Exception): logger = logging.getLogger(__name__) folders_in_bench = ('apps', 'sites', 'config', 'logs', 'config/pids') +sudoers_file = '/etc/sudoers.d/frappe' class color: @@ -423,16 +424,12 @@ def setup_sudoers(user): if set_permissions: os.chmod('/etc/sudoers', 0o440) - sudoers_file = '/etc/sudoers.d/frappe' - template = env.get_template('frappe_sudoers') frappe_sudoers = template.render(**{ 'user': user, 'service': find_executable('service'), 'systemctl': find_executable('systemctl'), - 'supervisorctl': find_executable('supervisorctl'), 'nginx': find_executable('nginx'), - 'bench': find_executable('bench') }) frappe_sudoers = safe_decode(frappe_sudoers) @@ -548,7 +545,7 @@ def restart_supervisor_processes(bench_path='.', web_workers=False): exec_cmd(cmd, cwd=bench_path) else: - supervisor_status = subprocess.check_output(['sudo', 'supervisorctl', 'status'], cwd=bench_path) + supervisor_status = subprocess.check_output(['supervisorctl', 'status'], cwd=bench_path) supervisor_status = safe_decode(supervisor_status) if web_workers and '{bench_name}-web:'.format(bench_name=bench_name) in supervisor_status: @@ -565,7 +562,7 @@ def restart_supervisor_processes(bench_path='.', web_workers=False): else: group = 'frappe:' - exec_cmd('sudo supervisorctl restart {group}'.format(group=group), cwd=bench_path) + exec_cmd('supervisorctl restart {group}'.format(group=group), cwd=bench_path) def restart_systemd_processes(bench_path='.', web_workers=False): From a03252881a0b0392a149eeb4963d651921783f22 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 13 Mar 2020 17:57:43 +0530 Subject: [PATCH 02/47] fix: get_cmd_output handles exit codes if no output eg: running "supervisorctl status" returns non zero codes in case certain processes its running are in failed state. get_cmd_output will now handle such situations chore: dropped bench.utils.get_program --- bench/utils.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/bench/utils.py b/bench/utils.py index 0f534c7a..464d03a9 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -450,17 +450,11 @@ def setup_logging(bench_path='.'): logger.setLevel(logging.DEBUG) -def get_program(programs): - program = None - for p in programs: - program = find_executable(p) - if program: - break - return program - - def get_process_manager(): - return get_program(['foreman', 'forego', 'honcho']) + for proc_man in ['honcho', 'foreman', 'forego']: + proc_man_path = find_executable(proc_man) + if proc_man_path: + return proc_man_path def start(no_dev=False, concurrency=None, procfile=None): @@ -516,14 +510,15 @@ def check_git_for_shallow_clone(): def get_cmd_output(cmd, cwd='.'): + print("{0}$ {1}{2}".format(color.silver, cmd, color.nc)) try: output = subprocess.check_output(cmd, cwd=cwd, shell=True, stderr=subprocess.PIPE).strip() - output = output.decode('utf-8') - return output except subprocess.CalledProcessError as e: if e.output: - print(e.output) - raise + output = e.output + else: + raise + return safe_decode(output) def safe_encode(what, encoding = 'utf-8'): @@ -545,7 +540,7 @@ def restart_supervisor_processes(bench_path='.', web_workers=False): exec_cmd(cmd, cwd=bench_path) else: - supervisor_status = subprocess.check_output(['supervisorctl', 'status'], cwd=bench_path) + supervisor_status = get_cmd_output('supervisorctl status', cwd=bench_path) supervisor_status = safe_decode(supervisor_status) if web_workers and '{bench_name}-web:'.format(bench_name=bench_name) in supervisor_status: From 39f8223d6d0b82c7b867789a5593f11e4d353bef Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 13 Mar 2020 18:47:05 +0530 Subject: [PATCH 03/47] fix: remove supervisord under service and systemctl in sudoers --- bench/config/templates/frappe_sudoers | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bench/config/templates/frappe_sudoers b/bench/config/templates/frappe_sudoers index 51595c8f..f87e01d5 100644 --- a/bench/config/templates/frappe_sudoers +++ b/bench/config/templates/frappe_sudoers @@ -1,15 +1,16 @@ {% if service %} {{ user }} ALL = (root) {{ service }} {{ user }} ALL = (root) NOPASSWD: {{ service }} nginx * -{{ user }} ALL = (root) NOPASSWD: {{ service }} supervisord * {% endif %} + {% if systemctl %} {{ user }} ALL = (root) {{ systemctl }} {{ user }} ALL = (root) NOPASSWD: {{ systemctl }} * nginx -{{ user }} ALL = (root) NOPASSWD: {{ systemctl }} * supervisord {% endif %} + {% if nginx %} {{ user }} ALL = (root) NOPASSWD: {{ nginx }} {% endif %} + {{ user }} ALL = (root) NOPASSWD: /opt/certbot-auto Defaults:{{ user }} !requiretty From d2a70badd3c61bd50b1fdc5dfd456ed77601f446 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 13 Mar 2020 18:48:22 +0530 Subject: [PATCH 04/47] chore: optimized and removed deprecated api usage and imports --- bench/config/production_setup.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/bench/config/production_setup.py b/bench/config/production_setup.py index e6d09098..42729a22 100755 --- a/bench/config/production_setup.py +++ b/bench/config/production_setup.py @@ -1,12 +1,15 @@ -from bench.utils import get_program, exec_cmd, get_cmd_output, fix_prod_setup_perms, get_bench_name, find_executable, CommandFailedError -from bench.config.supervisor import generate_supervisor_config -from bench.config.systemd import generate_systemd_config -from bench.config.nginx import make_nginx_conf -from bench.config.common_site_config import get_config -import os, subprocess +# imports - standard imports +import os import sys from distutils.spawn import find_executable +# imports - module imports +from bench.config.common_site_config import get_config +from bench.config.nginx import make_nginx_conf +from bench.config.supervisor import generate_supervisor_config +from bench.config.systemd import generate_systemd_config +from bench.utils import CommandFailedError, exec_cmd, fix_prod_setup_perms, get_bench_name, get_cmd_output + def setup_production_prerequisites(): if not find_executable("ansible"): @@ -55,6 +58,7 @@ def setup_production(user, bench_path='.', yes=False): reload_nginx() + def disable_production(bench_path='.'): bench_name = get_bench_name(bench_path) @@ -77,10 +81,11 @@ def disable_production(bench_path='.'): reload_nginx() + def service(service, option): - if os.path.basename(get_program(['systemctl']) or '') == 'systemctl' and is_running_systemd(): + if os.path.basename(find_executable('systemctl') or '') == 'systemctl' and is_running_systemd(): exec_cmd("sudo {service_manager} {option} {service}".format(service_manager='systemctl', option=option, service=service)) - elif os.path.basename(get_program(['service']) or '') == 'service': + elif os.path.basename(find_executable('service') or '') == 'service': exec_cmd("sudo {service_manager} {service} {option} ".format(service_manager='service', service=service, option=option)) else: # look for 'service_manager' and 'service_manager_command' in environment @@ -93,12 +98,14 @@ def service(service, option): else: raise Exception('No service manager found') + def get_supervisor_confdir(): possiblities = ('/etc/supervisor/conf.d', '/etc/supervisor.d/', '/etc/supervisord/conf.d', '/etc/supervisord.d') for possiblity in possiblities: if os.path.exists(possiblity): return possiblity + def remove_default_nginx_configs(): default_nginx_configs = ['/etc/nginx/conf.d/default.conf', '/etc/nginx/sites-enabled/default'] @@ -110,6 +117,7 @@ def remove_default_nginx_configs(): def is_centos7(): return os.path.exists('/etc/redhat-release') and get_cmd_output("cat /etc/redhat-release | sed 's/Linux\ //g' | cut -d' ' -f3 | cut -d. -f1").strip() == '7' + def is_running_systemd(): with open('/proc/1/comm') as f: comm = f.read().strip() @@ -119,6 +127,7 @@ def is_running_systemd(): return True return False + def reload_supervisor(): supervisorctl = find_executable('supervisorctl') @@ -153,7 +162,7 @@ def reload_supervisor(): def reload_nginx(): try: - subprocess.check_output(['sudo', find_executable('nginx'), '-t']) + exec_cmd('sudo {0} -t'.format(find_executable('nginx'))) except: raise From 1f0a5ce5569ae2846294c73565f82bd356ae2b1d Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 13 Mar 2020 19:38:13 +0530 Subject: [PATCH 05/47] fix: update supervisord.conf while generating supervisor.conf --- bench/config/supervisor.py | 44 ++++++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 60db0018..10e88916 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -1,17 +1,26 @@ -import os, getpass, click +# imports - standard imports +import configparser +import getpass +import os + +# imports - module imports import bench +from bench.app import get_current_frappe_version, use_rq +from bench.utils import get_bench_name, find_executable +from bench.config.common_site_config import get_config, update_config, get_gunicorn_workers + +# imports - third party imports +import click + def generate_supervisor_config(bench_path, user=None, yes=False): - from bench.app import get_current_frappe_version, use_rq - from bench.utils import get_bench_name, find_executable - from bench.config.common_site_config import get_config, update_config, get_gunicorn_workers - - template = bench.env.get_template('supervisor.conf') if not user: user = getpass.getuser() - config = get_config(bench_path=bench_path) + update_supervisord_conf(user=user) + template = bench.env.get_template('supervisor.conf') + config = get_config(bench_path=bench_path) bench_dir = os.path.abspath(bench_path) config = template.render(**{ @@ -44,3 +53,24 @@ def generate_supervisor_config(bench_path, user=None, yes=False): update_config({'restart_supervisor_on_update': True}, bench_path=bench_path) update_config({'restart_systemd_on_update': False}, bench_path=bench_path) + +def get_supervisord_conf(): + possibilities = ("supervisord.conf", "etc/supervisord.conf", "/etc/supervisord.conf", "/etc/supervisor/supervisord.conf", "/etc/supervisord.conf") + + for possibility in possibilities: + if os.path.exists(possibility): + return possibility + + +def update_supervisord_conf(user): + """From bench v5.0, we're moving to supervisor running as user""" + supervisord_conf = get_supervisord_conf() or "supervisord.conf" + section = "unix_http_server" + + config = configparser.ConfigParser() + config.read(supervisord_conf) + config[section]["chmod"] = "0760" + config[section]["chown"] = user + + with open(supervisord_conf, "w") as f: + config.write(f) From 48f70aca65b8627dce6ff124ad1509c9641f1895 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 13 Mar 2020 20:01:37 +0530 Subject: [PATCH 06/47] fix: restart supervisor service after conf update --- bench/config/supervisor.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 10e88916..141e97a6 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -8,6 +8,7 @@ import bench from bench.app import get_current_frappe_version, use_rq from bench.utils import get_bench_name, find_executable from bench.config.common_site_config import get_config, update_config, get_gunicorn_workers +from bench.config.production_setup import service # imports - third party imports import click @@ -74,3 +75,6 @@ def update_supervisord_conf(user): with open(supervisord_conf, "w") as f: config.write(f) + + # restart supervisor to take new changes into effect + service('supervisor', 'restart') From e12208dc25a3c034fc0a39dc17f52ce3e2e77d76 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sun, 15 Mar 2020 19:31:27 +0530 Subject: [PATCH 07/47] fix: update supervisord and sudoers for old benches --- bench/cli.py | 3 ++- bench/config/templates/frappe_sudoers | 4 ++++ bench/patches/patches.txt | 1 + bench/patches/v5/__init__.py | 0 bench/patches/v5/fix_user_permissions.py | 17 +++++++++++++++++ 5 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 bench/patches/v5/__init__.py create mode 100644 bench/patches/v5/fix_user_permissions.py diff --git a/bench/cli.py b/bench/cli.py index ad1c90fa..68c96f45 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -8,6 +8,7 @@ from bench.commands import bench_command logger = logging.getLogger('bench') from_command_line = False +change_uid_msg = "You should not run this command as root" def cli(): global from_command_line @@ -72,7 +73,7 @@ def change_uid(): drop_privileges(uid_name=frappe_user, gid_name=frappe_user) os.environ['HOME'] = pwd.getpwnam(frappe_user).pw_dir else: - log('You should not run this command as root', level=3) + log(change_uid_msg, level=3) sys.exit(1) def old_frappe_cli(bench_path='.'): diff --git a/bench/config/templates/frappe_sudoers b/bench/config/templates/frappe_sudoers index f87e01d5..567ccf06 100644 --- a/bench/config/templates/frappe_sudoers +++ b/bench/config/templates/frappe_sudoers @@ -1,3 +1,6 @@ +# This file is auto-generated by frappe/bench +# To re-generate this file, run "bench setup sudoers" + {% if service %} {{ user }} ALL = (root) {{ service }} {{ user }} ALL = (root) NOPASSWD: {{ service }} nginx * @@ -14,3 +17,4 @@ {{ user }} ALL = (root) NOPASSWD: /opt/certbot-auto Defaults:{{ user }} !requiretty + diff --git a/bench/patches/patches.txt b/bench/patches/patches.txt index 754068c5..7cb1a076 100644 --- a/bench/patches/patches.txt +++ b/bench/patches/patches.txt @@ -4,3 +4,4 @@ bench.patches.v3.redis_bind_ip bench.patches.v4.update_node bench.patches.v4.update_socketio bench.patches.v4.install_yarn #2 +bench.patches.v5.fix_user_permissions diff --git a/bench/patches/v5/__init__.py b/bench/patches/v5/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/bench/patches/v5/fix_user_permissions.py b/bench/patches/v5/fix_user_permissions.py new file mode 100644 index 00000000..3bea08a6 --- /dev/null +++ b/bench/patches/v5/fix_user_permissions.py @@ -0,0 +1,17 @@ +# imports - standard imports +import subprocess + +# imports - module imports +from bench.utils import log, get_cmd_output, exec_cmd +from bench.cli import change_uid_msg + + +def execute(bench_path): + """fix supervisor using root then remove bench sudo later + chronology samajhiye""" + cmd = ["sudo", "-n", "bench"] + is_bench_sudoers_set = (not subprocess.call(cmd)) or (change_uid_msg in get_cmd_output(cmd)) + + if is_bench_sudoers_set: + exec_cmd("sudo bench setup supervisor --yes") + exec_cmd("sudo bench setup sudoers") From f6292bba774da6c6d1f85f733cfb257cacb16901 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sun, 15 Mar 2020 19:32:46 +0530 Subject: [PATCH 08/47] chore: added logging and removed unnecessary print in get_cmd_output --- bench/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/utils.py b/bench/utils.py index 464d03a9..bfc4cda0 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -437,6 +437,7 @@ def setup_sudoers(user): f.write(frappe_sudoers) os.chmod(sudoers_file, 0o440) + log("Sudoers was set up for user {}".format(user), level=1) def setup_logging(bench_path='.'): @@ -510,7 +511,6 @@ def check_git_for_shallow_clone(): def get_cmd_output(cmd, cwd='.'): - print("{0}$ {1}{2}".format(color.silver, cmd, color.nc)) try: output = subprocess.check_output(cmd, cwd=cwd, shell=True, stderr=subprocess.PIPE).strip() except subprocess.CalledProcessError as e: From 395b8df895f22e4ca1d225744ab8c44b5bde896a Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sun, 15 Mar 2020 19:40:46 +0530 Subject: [PATCH 09/47] fix: install supervisor on user if doesnt exist --- bench/patches/v5/fix_user_permissions.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/bench/patches/v5/fix_user_permissions.py b/bench/patches/v5/fix_user_permissions.py index 3bea08a6..98a8e327 100644 --- a/bench/patches/v5/fix_user_permissions.py +++ b/bench/patches/v5/fix_user_permissions.py @@ -1,16 +1,20 @@ # imports - standard imports import subprocess +import sys # imports - module imports -from bench.utils import log, get_cmd_output, exec_cmd +from bench.utils import log, get_cmd_output, exec_cmd, which from bench.cli import change_uid_msg def execute(bench_path): - """fix supervisor using root then remove bench sudo later - chronology samajhiye""" cmd = ["sudo", "-n", "bench"] + is_bench_sudoers_set = (not subprocess.call(cmd)) or (change_uid_msg in get_cmd_output(cmd)) + is_supervisor_installed = which('supervisorctl') + + if not is_supervisor_installed: + exec_cmd("{} -m pip install supervisor".format(sys.executable)) if is_bench_sudoers_set: exec_cmd("sudo bench setup supervisor --yes") From 1f0b78f2fe3ed1448b64ecbbe61b1635f574aceb Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sun, 15 Mar 2020 19:41:09 +0530 Subject: [PATCH 10/47] chore: update tests with updated sudoers conf --- bench/tests/test_setup_production.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bench/tests/test_setup_production.py b/bench/tests/test_setup_production.py index 8d0bbc7a..e0a60b17 100644 --- a/bench/tests/test_setup_production.py +++ b/bench/tests/test_setup_production.py @@ -61,6 +61,9 @@ class TestSetupProduction(TestBenchBase): def assert_sudoers(self, user): sudoers_file = '/etc/sudoers.d/frappe' + service = bench.utils.which("service") + nginx = bench.utils.which("nginx") + self.assertTrue(self.file_exists(sudoers_file)) if os.environ.get("CI"): @@ -69,9 +72,8 @@ class TestSetupProduction(TestBenchBase): with open(sudoers_file, 'r') as f: sudoers = f.read() - self.assertTrue('{user} ALL = (root) NOPASSWD: /usr/sbin/service nginx *'.format(user=user) in sudoers) - self.assertTrue('{user} ALL = (root) NOPASSWD: /usr/bin/supervisorctl'.format(user=user) in sudoers) - self.assertTrue('{user} ALL = (root) NOPASSWD: /usr/sbin/nginx'.format(user=user) in sudoers) + self.assertTrue('{user} ALL = (root) NOPASSWD: {service} nginx *'.format(service=service, user=user) in sudoers) + self.assertTrue('{user} ALL = (root) NOPASSWD: {nginx}'.format(nginx=nginx, user=user) in sudoers) def assert_supervisor_config(self, bench_name, use_rq=True): From 9e7b8d1f2e570a477bbdda22e68c0dd49ef3519a Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sun, 15 Mar 2020 22:50:52 +0530 Subject: [PATCH 11/47] fix: avoid circular imports and optimization --- bench/config/supervisor.py | 3 ++- bench/patches/v5/fix_user_permissions.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 141e97a6..7b4ff227 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -8,7 +8,6 @@ import bench from bench.app import get_current_frappe_version, use_rq from bench.utils import get_bench_name, find_executable from bench.config.common_site_config import get_config, update_config, get_gunicorn_workers -from bench.config.production_setup import service # imports - third party imports import click @@ -65,6 +64,8 @@ def get_supervisord_conf(): def update_supervisord_conf(user): """From bench v5.0, we're moving to supervisor running as user""" + from bench.config.production_setup import service + supervisord_conf = get_supervisord_conf() or "supervisord.conf" section = "unix_http_server" diff --git a/bench/patches/v5/fix_user_permissions.py b/bench/patches/v5/fix_user_permissions.py index 98a8e327..fba8d9f0 100644 --- a/bench/patches/v5/fix_user_permissions.py +++ b/bench/patches/v5/fix_user_permissions.py @@ -3,7 +3,7 @@ import subprocess import sys # imports - module imports -from bench.utils import log, get_cmd_output, exec_cmd, which +from bench.utils import get_cmd_output, exec_cmd, which from bench.cli import change_uid_msg From c1507c284868cf5f325229f19a6f184a18fe81f9 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 16 Mar 2020 00:52:04 +0530 Subject: [PATCH 12/47] chore: py2-3 compatible configparser --- bench/config/supervisor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 7b4ff227..a1168d87 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -1,5 +1,4 @@ # imports - standard imports -import configparser import getpass import os @@ -11,6 +10,7 @@ from bench.config.common_site_config import get_config, update_config, get_gunic # imports - third party imports import click +from six.moves import configparser def generate_supervisor_config(bench_path, user=None, yes=False): From 849c751e93afb231bcfba354f4eb90e12485448a Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 16 Mar 2020 11:46:11 +0530 Subject: [PATCH 13/47] chore: dont create supervisord conf if doesnt exist via bench.config.supervisor.update_supervisord_conf --- bench/config/supervisor.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index a1168d87..40b8a92f 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -66,9 +66,12 @@ def update_supervisord_conf(user): """From bench v5.0, we're moving to supervisor running as user""" from bench.config.production_setup import service - supervisord_conf = get_supervisord_conf() or "supervisord.conf" + supervisord_conf = get_supervisord_conf() section = "unix_http_server" + if not supervisord_conf: + return + config = configparser.ConfigParser() config.read(supervisord_conf) config[section]["chmod"] = "0760" From fe02844c69a849b4c1a826dbd16a4f5dc28127bc Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 16 Mar 2020 12:04:18 +0530 Subject: [PATCH 14/47] fix: run fix_user_permissions only if production or sudoers is set up --- bench/config/supervisor.py | 3 -- bench/patches/v5/fix_user_permissions.py | 39 +++++++++++++++++------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 40b8a92f..eadd0a7f 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -69,9 +69,6 @@ def update_supervisord_conf(user): supervisord_conf = get_supervisord_conf() section = "unix_http_server" - if not supervisord_conf: - return - config = configparser.ConfigParser() config.read(supervisord_conf) config[section]["chmod"] = "0760" diff --git a/bench/patches/v5/fix_user_permissions.py b/bench/patches/v5/fix_user_permissions.py index fba8d9f0..9da2ee99 100644 --- a/bench/patches/v5/fix_user_permissions.py +++ b/bench/patches/v5/fix_user_permissions.py @@ -1,21 +1,38 @@ # imports - standard imports +import os import subprocess -import sys # imports - module imports -from bench.utils import get_cmd_output, exec_cmd, which from bench.cli import change_uid_msg +from bench.config.production_setup import get_supervisor_confdir, is_centos7 +from bench.utils import exec_cmd, get_bench_name, get_cmd_output + + +def is_sudoers_set(): + cmd = ["sudo", "-n", "bench"] + return (not subprocess.call(cmd)) or (change_uid_msg in get_cmd_output(cmd)) + + +def is_production_set(bench_path): + production_setup = False + bench_name = get_bench_name(bench_path) + + supervisor_conf_extn = "ini" if is_centos7() else "conf" + supervisor_conf_file_name = '{bench_name}.{extn}'.format(bench_name=bench_name, extn=supervisor_conf_extn) + supervisor_conf = os.path.join(get_supervisor_confdir(), supervisor_conf_file_name) + + if os.path.exists(supervisor_conf): + production_setup = production_setup or True + + nginx_conf = '/etc/nginx/conf.d/{bench_name}.conf'.format(bench_name=bench_name) + + if os.path.exists(nginx_conf): + production_setup = production_setup or True + + return production_setup def execute(bench_path): - cmd = ["sudo", "-n", "bench"] - - is_bench_sudoers_set = (not subprocess.call(cmd)) or (change_uid_msg in get_cmd_output(cmd)) - is_supervisor_installed = which('supervisorctl') - - if not is_supervisor_installed: - exec_cmd("{} -m pip install supervisor".format(sys.executable)) - - if is_bench_sudoers_set: + if is_sudoers_set() or is_production_set(bench_path): exec_cmd("sudo bench setup supervisor --yes") exec_cmd("sudo bench setup sudoers") From 78dbdcb3f18193ba03a54722a9e4a244896907b9 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 16 Mar 2020 12:25:55 +0530 Subject: [PATCH 15/47] fix: run patches for current user --- bench/config/supervisor.py | 5 ++++- bench/patches/v5/fix_user_permissions.py | 11 ++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index eadd0a7f..8b043daf 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -69,10 +69,13 @@ def update_supervisord_conf(user): supervisord_conf = get_supervisord_conf() section = "unix_http_server" + if not supervisord_conf: + return + config = configparser.ConfigParser() config.read(supervisord_conf) config[section]["chmod"] = "0760" - config[section]["chown"] = user + config[section]["chown"] = "{user}:{user}".format(user=user) with open(supervisord_conf, "w") as f: config.write(f) diff --git a/bench/patches/v5/fix_user_permissions.py b/bench/patches/v5/fix_user_permissions.py index 9da2ee99..41322472 100644 --- a/bench/patches/v5/fix_user_permissions.py +++ b/bench/patches/v5/fix_user_permissions.py @@ -1,4 +1,5 @@ # imports - standard imports +import getpass import os import subprocess @@ -33,6 +34,10 @@ def is_production_set(bench_path): def execute(bench_path): - if is_sudoers_set() or is_production_set(bench_path): - exec_cmd("sudo bench setup supervisor --yes") - exec_cmd("sudo bench setup sudoers") + user = getpass.getuser() + + if is_sudoers_set(): + exec_cmd("sudo bench setup sudoers {user}".format(user=user)) + + if is_production_set(bench_path): + exec_cmd("sudo bench setup supervisor --yes --user {user}".format(user=user)) From 15b01e9451f8155a308fd1d047958c6f67b556a3 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 25 Feb 2020 12:45:49 +0530 Subject: [PATCH 16/47] fix: force use PYPI packaged bench --- bench/__init__.py | 10 +---- bench/app.py | 8 +--- bench/commands/__init__.py | 1 - bench/utils.py | 83 +++++++++++++++++++++----------------- 4 files changed, 49 insertions(+), 53 deletions(-) diff --git a/bench/__init__.py b/bench/__init__.py index e120c37b..d3484fe9 100644 --- a/bench/__init__.py +++ b/bench/__init__.py @@ -1,13 +1,7 @@ +from bench.utils import install_checker from jinja2 import Environment, PackageLoader __version__ = "4.1.0" env = Environment(loader=PackageLoader('bench.config')) - -FRAPPE_VERSION = None - -def set_frappe_version(bench_path='.'): - from .app import get_current_frappe_version - global FRAPPE_VERSION - if not FRAPPE_VERSION: - FRAPPE_VERSION = get_current_frappe_version(bench_path=bench_path) \ No newline at end of file +install_checker() \ No newline at end of file diff --git a/bench/app.py b/bench/app.py index ea5d2a22..804c3515 100755 --- a/bench/app.py +++ b/bench/app.py @@ -151,13 +151,7 @@ def new_app(app, bench_path='.'): app = app.lower().replace(" ", "_").replace("-", "_") logger.info('creating new app {}'.format(app)) apps = os.path.abspath(os.path.join(bench_path, 'apps')) - bench.set_frappe_version(bench_path=bench_path) - - if bench.FRAPPE_VERSION == 4: - exec_cmd("{frappe} --make_app {apps} {app}".format(frappe=get_frappe(bench_path=bench_path), - apps=apps, app=app)) - else: - run_frappe_cmd('make-app', apps, app, bench_path=bench_path) + run_frappe_cmd('make-app', apps, app, bench_path=bench_path) install_app(app, bench_path=bench_path) def install_app(app, bench_path=".", verbose=False, no_cache=False): diff --git a/bench/commands/__init__.py b/bench/commands/__init__.py index 7e514012..594e39c7 100755 --- a/bench/commands/__init__.py +++ b/bench/commands/__init__.py @@ -17,7 +17,6 @@ def bench_command(bench_path='.'): import bench from bench.utils import setup_logging - bench.set_frappe_version(bench_path=bench_path) setup_logging(bench_path=bench_path) diff --git a/bench/utils.py b/bench/utils.py index 10a4f801..ce4cfd64 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -8,7 +8,6 @@ from six import iteritems from six.moves.urllib.parse import urlparse import bench -from bench import env class PatchError(Exception): @@ -110,15 +109,11 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, no_auto_upda if apps_path: install_apps_from_path(apps_path, bench_path=path) - - bench.set_frappe_version(bench_path=path) - if bench.FRAPPE_VERSION > 5: - if not skip_assets: - update_node_packages(bench_path=path) + if not skip_assets: + update_node_packages(bench_path=path) + build_assets(bench_path=path) set_all_patches_executed(bench_path=path) - if not skip_assets: - build_assets(bench_path=path) if not skip_redis_config_generation: redis.generate_config(path) @@ -217,26 +212,16 @@ def setup_socketio(bench_path='.'): babel-cli babel-preset-es2015 babel-preset-es2016 babel-preset-es2017 babel-preset-babili", cwd=bench_path) def patch_sites(bench_path='.'): - bench.set_frappe_version(bench_path=bench_path) - try: - if bench.FRAPPE_VERSION == 4: - exec_cmd("{frappe} --latest all".format(frappe=get_frappe(bench_path=bench_path)), cwd=os.path.join(bench_path, 'sites')) - else: - run_frappe_cmd('--site', 'all', 'migrate', bench_path=bench_path) + run_frappe_cmd('--site', 'all', 'migrate', bench_path=bench_path) except subprocess.CalledProcessError: raise PatchError def build_assets(bench_path='.', app=None): - bench.set_frappe_version(bench_path=bench_path) - - if bench.FRAPPE_VERSION == 4: - exec_cmd("{frappe} --build".format(frappe=get_frappe(bench_path=bench_path)), cwd=os.path.join(bench_path, 'sites')) - else: - command = 'bench build' - if app: - command += ' --app {}'.format(app) - exec_cmd(command, cwd=bench_path) + command = 'bench build' + if app: + command += ' --app {}'.format(app) + exec_cmd(command, cwd=bench_path) def get_sites(bench_path='.'): sites_path = os.path.join(bench_path, 'sites') @@ -255,12 +240,7 @@ def setup_auto_update(bench_path='.'): def setup_backups(bench_path='.'): logger.info('setting up backups') bench_dir = get_bench_dir(bench_path=bench_path) - bench.set_frappe_version(bench_path=bench_path) - - if bench.FRAPPE_VERSION == 4: - backup_command = "cd {sites_dir} && {frappe} --backup all".format(frappe=get_frappe(bench_path=bench_path),) - else: - backup_command = "cd {bench_dir} && {bench} --site all backup".format(bench_dir=bench_dir, bench=sys.argv[0]) + backup_command = "cd {bench_dir} && {bench} --site all backup".format(bench_dir=bench_dir, bench=sys.argv[0]) add_to_crontab('0 */6 * * * {backup_command} >> {logfile} 2>&1'.format(backup_command=backup_command, logfile=os.path.join(get_bench_dir(bench_path=bench_path), 'logs', 'backup.log'))) @@ -302,6 +282,8 @@ def update_bench(bench_repo=True, requirements=True): logger.info("Bench Updated!") def setup_sudoers(user): + from bench import env + if not os.path.exists('/etc/sudoers.d'): os.makedirs('/etc/sudoers.d') @@ -557,13 +539,7 @@ def install_requirements(req_file, user=False): exec_cmd("{python} -m pip install {user_flag} -q -U -r {req_file}".format(python=python, user_flag=user_flag, req_file=req_file)) def backup_site(site, bench_path='.'): - bench.set_frappe_version(bench_path=bench_path) - - if bench.FRAPPE_VERSION == 4: - exec_cmd("{frappe} --backup {site}".format(frappe=get_frappe(bench_path=bench_path), site=site), - cwd=os.path.join(bench_path, 'sites')) - else: - run_frappe_cmd('--site', site, 'backup', bench_path=bench_path) + run_frappe_cmd('--site', site, 'backup', bench_path=bench_path) def backup_all_sites(bench_path='.'): for site in get_sites(bench_path=bench_path): @@ -1042,4 +1018,37 @@ def migrate_env(python, backup=False): log.debug('Migration Successful to {}'.format(python)) except: log.debug('Migration Error') - raise \ No newline at end of file + raise + +def is_dist_editable(dist): + """Is distribution an editable install?""" + for path_item in sys.path: + egg_link = os.path.join(path_item, dist + '.egg-link') + if os.path.isfile(egg_link): + return True + return False + +def install_checker(): + development_mode = os.path.exists(os.path.expanduser("~/.bench.dev")) + bench_editable = is_dist_editable(dist="bench") + + if development_mode and bench_editable: + log("bench is setup for development mode") + + if development_mode and not bench_editable: + log("Clone bench's git repository from https://github.com/frappe/bench to develop using bench") + + if not development_mode and not bench_editable: + """Ideal scenario for bench users""" + + if not development_mode and bench_editable: + log("Installing bench in editable mode is not recommended for production", level=3) + import click + # breaking change!!! + if click.confirm("Do you wish to uninstall editable install and install bench from PYPI?"): + log("Uninstalling bench") + os.system("pip uninstall bench -y") + log("Uninstalled bench", level=2) + log("Installing bench via PYPI") + os.system("pip install frappe-bench") + log("Installed bench via PYPI", level=2) From 5e7ad22be9d7fde9ae4a9dfc746567fb9d3543a1 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 16 Mar 2020 12:53:47 +0530 Subject: [PATCH 17/47] chore: throw warning that bench is being run in editable mode --- bench/cli.py | 6 +++++- bench/utils.py | 25 ------------------------- 2 files changed, 5 insertions(+), 26 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index 733a0dd6..55b7fe2e 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -1,6 +1,6 @@ import click import os, sys, logging, json, pwd, subprocess -from bench.utils import is_root, PatchError, drop_privileges, get_env_cmd, get_cmd_output, get_frappe, log, is_bench_directory +from bench.utils import is_root, PatchError, drop_privileges, get_env_cmd, get_cmd_output, get_frappe, log, is_bench_directory, is_dist_editable from bench.app import get_apps from bench.config.common_site_config import get_config from bench.commands import bench_command @@ -18,6 +18,10 @@ def cli(): change_dir() change_uid() + if is_dist_editable("bench"): + log("bench is installed in editable mode!") + print("Install bench via PYPI for production instead, using `pip install frappe-bench`") + if len(sys.argv) > 2 and sys.argv[1] == "frappe": return old_frappe_cli() diff --git a/bench/utils.py b/bench/utils.py index ce4cfd64..02d76e77 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -1027,28 +1027,3 @@ def is_dist_editable(dist): if os.path.isfile(egg_link): return True return False - -def install_checker(): - development_mode = os.path.exists(os.path.expanduser("~/.bench.dev")) - bench_editable = is_dist_editable(dist="bench") - - if development_mode and bench_editable: - log("bench is setup for development mode") - - if development_mode and not bench_editable: - log("Clone bench's git repository from https://github.com/frappe/bench to develop using bench") - - if not development_mode and not bench_editable: - """Ideal scenario for bench users""" - - if not development_mode and bench_editable: - log("Installing bench in editable mode is not recommended for production", level=3) - import click - # breaking change!!! - if click.confirm("Do you wish to uninstall editable install and install bench from PYPI?"): - log("Uninstalling bench") - os.system("pip uninstall bench -y") - log("Uninstalled bench", level=2) - log("Installing bench via PYPI") - os.system("pip install frappe-bench") - log("Installed bench via PYPI", level=2) From cdd7d3403448fa1fdaa94507dffad2c09d4506ef Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 16 Mar 2020 12:59:28 +0530 Subject: [PATCH 18/47] Revert "fix: force use PYPI packaged bench" This reverts commit 74bb1805b2d024ac32c4952c90c408c5b58b4969. --- bench/__init__.py | 10 ++++++-- bench/app.py | 8 ++++++- bench/commands/__init__.py | 1 + bench/utils.py | 48 ++++++++++++++++++++++++++++---------- 4 files changed, 52 insertions(+), 15 deletions(-) diff --git a/bench/__init__.py b/bench/__init__.py index d3484fe9..e120c37b 100644 --- a/bench/__init__.py +++ b/bench/__init__.py @@ -1,7 +1,13 @@ -from bench.utils import install_checker from jinja2 import Environment, PackageLoader __version__ = "4.1.0" env = Environment(loader=PackageLoader('bench.config')) -install_checker() \ No newline at end of file + +FRAPPE_VERSION = None + +def set_frappe_version(bench_path='.'): + from .app import get_current_frappe_version + global FRAPPE_VERSION + if not FRAPPE_VERSION: + FRAPPE_VERSION = get_current_frappe_version(bench_path=bench_path) \ No newline at end of file diff --git a/bench/app.py b/bench/app.py index 804c3515..ea5d2a22 100755 --- a/bench/app.py +++ b/bench/app.py @@ -151,7 +151,13 @@ def new_app(app, bench_path='.'): app = app.lower().replace(" ", "_").replace("-", "_") logger.info('creating new app {}'.format(app)) apps = os.path.abspath(os.path.join(bench_path, 'apps')) - run_frappe_cmd('make-app', apps, app, bench_path=bench_path) + bench.set_frappe_version(bench_path=bench_path) + + if bench.FRAPPE_VERSION == 4: + exec_cmd("{frappe} --make_app {apps} {app}".format(frappe=get_frappe(bench_path=bench_path), + apps=apps, app=app)) + else: + run_frappe_cmd('make-app', apps, app, bench_path=bench_path) install_app(app, bench_path=bench_path) def install_app(app, bench_path=".", verbose=False, no_cache=False): diff --git a/bench/commands/__init__.py b/bench/commands/__init__.py index 594e39c7..7e514012 100755 --- a/bench/commands/__init__.py +++ b/bench/commands/__init__.py @@ -17,6 +17,7 @@ def bench_command(bench_path='.'): import bench from bench.utils import setup_logging + bench.set_frappe_version(bench_path=bench_path) setup_logging(bench_path=bench_path) diff --git a/bench/utils.py b/bench/utils.py index 02d76e77..0cc6fedf 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -8,6 +8,7 @@ from six import iteritems from six.moves.urllib.parse import urlparse import bench +from bench import env class PatchError(Exception): @@ -109,11 +110,15 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, no_auto_upda if apps_path: install_apps_from_path(apps_path, bench_path=path) - if not skip_assets: - update_node_packages(bench_path=path) - build_assets(bench_path=path) + + bench.set_frappe_version(bench_path=path) + if bench.FRAPPE_VERSION > 5: + if not skip_assets: + update_node_packages(bench_path=path) set_all_patches_executed(bench_path=path) + if not skip_assets: + build_assets(bench_path=path) if not skip_redis_config_generation: redis.generate_config(path) @@ -212,16 +217,26 @@ def setup_socketio(bench_path='.'): babel-cli babel-preset-es2015 babel-preset-es2016 babel-preset-es2017 babel-preset-babili", cwd=bench_path) def patch_sites(bench_path='.'): + bench.set_frappe_version(bench_path=bench_path) + try: - run_frappe_cmd('--site', 'all', 'migrate', bench_path=bench_path) + if bench.FRAPPE_VERSION == 4: + exec_cmd("{frappe} --latest all".format(frappe=get_frappe(bench_path=bench_path)), cwd=os.path.join(bench_path, 'sites')) + else: + run_frappe_cmd('--site', 'all', 'migrate', bench_path=bench_path) except subprocess.CalledProcessError: raise PatchError def build_assets(bench_path='.', app=None): - command = 'bench build' - if app: - command += ' --app {}'.format(app) - exec_cmd(command, cwd=bench_path) + bench.set_frappe_version(bench_path=bench_path) + + if bench.FRAPPE_VERSION == 4: + exec_cmd("{frappe} --build".format(frappe=get_frappe(bench_path=bench_path)), cwd=os.path.join(bench_path, 'sites')) + else: + command = 'bench build' + if app: + command += ' --app {}'.format(app) + exec_cmd(command, cwd=bench_path) def get_sites(bench_path='.'): sites_path = os.path.join(bench_path, 'sites') @@ -240,7 +255,12 @@ def setup_auto_update(bench_path='.'): def setup_backups(bench_path='.'): logger.info('setting up backups') bench_dir = get_bench_dir(bench_path=bench_path) - backup_command = "cd {bench_dir} && {bench} --site all backup".format(bench_dir=bench_dir, bench=sys.argv[0]) + bench.set_frappe_version(bench_path=bench_path) + + if bench.FRAPPE_VERSION == 4: + backup_command = "cd {sites_dir} && {frappe} --backup all".format(frappe=get_frappe(bench_path=bench_path),) + else: + backup_command = "cd {bench_dir} && {bench} --site all backup".format(bench_dir=bench_dir, bench=sys.argv[0]) add_to_crontab('0 */6 * * * {backup_command} >> {logfile} 2>&1'.format(backup_command=backup_command, logfile=os.path.join(get_bench_dir(bench_path=bench_path), 'logs', 'backup.log'))) @@ -282,8 +302,6 @@ def update_bench(bench_repo=True, requirements=True): logger.info("Bench Updated!") def setup_sudoers(user): - from bench import env - if not os.path.exists('/etc/sudoers.d'): os.makedirs('/etc/sudoers.d') @@ -539,7 +557,13 @@ def install_requirements(req_file, user=False): exec_cmd("{python} -m pip install {user_flag} -q -U -r {req_file}".format(python=python, user_flag=user_flag, req_file=req_file)) def backup_site(site, bench_path='.'): - run_frappe_cmd('--site', site, 'backup', bench_path=bench_path) + bench.set_frappe_version(bench_path=bench_path) + + if bench.FRAPPE_VERSION == 4: + exec_cmd("{frappe} --backup {site}".format(frappe=get_frappe(bench_path=bench_path), site=site), + cwd=os.path.join(bench_path, 'sites')) + else: + run_frappe_cmd('--site', site, 'backup', bench_path=bench_path) def backup_all_sites(bench_path='.'): for site in get_sites(bench_path=bench_path): From 3e99fbc46b04f5493684884dae98b94c56c1d4bf Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 16 Mar 2020 13:16:43 +0530 Subject: [PATCH 19/47] fix: add section in conf if doesnt exist --- bench/config/supervisor.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 8b043daf..d97f3559 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -74,6 +74,10 @@ def update_supervisord_conf(user): config = configparser.ConfigParser() config.read(supervisord_conf) + + if section not in config.sections(): + config.add_section(section) + config[section]["chmod"] = "0760" config[section]["chown"] = "{user}:{user}".format(user=user) From 9f091383535a09e634b944b048d0fd1e18ad14ee Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 16 Mar 2020 14:31:20 +0530 Subject: [PATCH 20/47] chore: use set instead of get in config object --- bench/config/supervisor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index d97f3559..253a1a21 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -78,8 +78,8 @@ def update_supervisord_conf(user): if section not in config.sections(): config.add_section(section) - config[section]["chmod"] = "0760" - config[section]["chown"] = "{user}:{user}".format(user=user) + config.set(section, "chmod", "0760") + config.set(section, "chown", "{user}:{user}".format(user=user)) with open(supervisord_conf, "w") as f: config.write(f) From 2fe5cce6d445612c0a26b2bfbb0ea3a7dc20a66f Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 16 Mar 2020 17:31:08 +0530 Subject: [PATCH 21/47] fix: dont drop permissions for setting up supervisor --- bench/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/cli.py b/bench/cli.py index 68c96f45..eea1acc7 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -49,7 +49,7 @@ def check_uid(): sys.exit(1) def cmd_requires_root(): - if len(sys.argv) > 2 and sys.argv[2] in ('production', 'sudoers', 'lets-encrypt', 'fonts', + if len(sys.argv) > 2 and sys.argv[2] in ('production', 'sudoers', 'supervisor', 'lets-encrypt', 'fonts', 'print', 'firewall', 'ssh-port', 'role', 'fail2ban', 'wildcard-ssl'): return True if len(sys.argv) >= 2 and sys.argv[1] in ('patch', 'renew-lets-encrypt', 'disable-production', From 5dd3f5ce12380e8cbde5ff707efe5353df1681ff Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 16 Mar 2020 17:40:10 +0530 Subject: [PATCH 22/47] chore: show PYPI message as warning --- bench/cli.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index 890dd69b..6c1668f7 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -19,8 +19,7 @@ def cli(): change_uid() if is_dist_editable("bench"): - log("bench is installed in editable mode!") - print("Install bench via PYPI for production instead, using `pip install frappe-bench`") + log("bench is installed in editable mode!\n\nThis is not the recommended mode of install for production. Use the PYPI package using `pip install frappe-bench` instead\n", level=3) if len(sys.argv) > 2 and sys.argv[1] == "frappe": return old_frappe_cli() From a6f72c770f821944aa9cf45654416044f833c44c Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 17 Mar 2020 17:25:30 +0530 Subject: [PATCH 23/47] fix: use frappe_user from site_config and use getpass as fallback --- bench/patches/v5/fix_user_permissions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bench/patches/v5/fix_user_permissions.py b/bench/patches/v5/fix_user_permissions.py index 41322472..13bac4cb 100644 --- a/bench/patches/v5/fix_user_permissions.py +++ b/bench/patches/v5/fix_user_permissions.py @@ -6,6 +6,7 @@ import subprocess # imports - module imports from bench.cli import change_uid_msg from bench.config.production_setup import get_supervisor_confdir, is_centos7 +from bench.config.common_site_config import get_config from bench.utils import exec_cmd, get_bench_name, get_cmd_output @@ -34,7 +35,7 @@ def is_production_set(bench_path): def execute(bench_path): - user = getpass.getuser() + user = get_config('.').get("frappe_user") or getpass.getuser() if is_sudoers_set(): exec_cmd("sudo bench setup sudoers {user}".format(user=user)) From 6bb30e314848d33ebc595b2e32745caf64b89a6e Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 18 Mar 2020 20:16:41 +0530 Subject: [PATCH 24/47] feat: allow raise option in get_cmd_output reason: if command exits without output, it raises a CalledProcessError --- bench/patches/v5/fix_user_permissions.py | 2 +- bench/utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bench/patches/v5/fix_user_permissions.py b/bench/patches/v5/fix_user_permissions.py index 13bac4cb..346c78b7 100644 --- a/bench/patches/v5/fix_user_permissions.py +++ b/bench/patches/v5/fix_user_permissions.py @@ -12,7 +12,7 @@ from bench.utils import exec_cmd, get_bench_name, get_cmd_output def is_sudoers_set(): cmd = ["sudo", "-n", "bench"] - return (not subprocess.call(cmd)) or (change_uid_msg in get_cmd_output(cmd)) + return (not subprocess.call(cmd)) or (change_uid_msg in get_cmd_output(cmd, _raise=False)) def is_production_set(bench_path): diff --git a/bench/utils.py b/bench/utils.py index bfc4cda0..7b1a4742 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -510,13 +510,13 @@ def check_git_for_shallow_clone(): return True -def get_cmd_output(cmd, cwd='.'): +def get_cmd_output(cmd, cwd='.', _raise=True): try: output = subprocess.check_output(cmd, cwd=cwd, shell=True, stderr=subprocess.PIPE).strip() except subprocess.CalledProcessError as e: if e.output: output = e.output - else: + elif _raise: raise return safe_decode(output) From d51311b7f7caedec77c3474f3232358bd2aba0f5 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 19 Mar 2020 18:52:49 +0530 Subject: [PATCH 25/47] fix: better exception handling for checking sudoers set --- bench/patches/v5/fix_user_permissions.py | 14 +++++++++++++- bench/utils.py | 1 + 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/bench/patches/v5/fix_user_permissions.py b/bench/patches/v5/fix_user_permissions.py index 346c78b7..c11a9480 100644 --- a/bench/patches/v5/fix_user_permissions.py +++ b/bench/patches/v5/fix_user_permissions.py @@ -12,7 +12,19 @@ from bench.utils import exec_cmd, get_bench_name, get_cmd_output def is_sudoers_set(): cmd = ["sudo", "-n", "bench"] - return (not subprocess.call(cmd)) or (change_uid_msg in get_cmd_output(cmd, _raise=False)) + + with open(os.devnull, "wb") as f: + return_code_check = not subprocess.call(cmd, stdout=f) + + if return_code_check: + try: + bench_warn = change_uid_msg in get_cmd_output(cmd, _raise=False) + except Exception: + bench_warn = False + finally: + return_code_check = return_code_check and bench_warn + + return return_code_check def is_production_set(bench_path): diff --git a/bench/utils.py b/bench/utils.py index 7b1a4742..19fe9036 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -511,6 +511,7 @@ def check_git_for_shallow_clone(): def get_cmd_output(cmd, cwd='.', _raise=True): + output = "" try: output = subprocess.check_output(cmd, cwd=cwd, shell=True, stderr=subprocess.PIPE).strip() except subprocess.CalledProcessError as e: From bdc20a54356f429f723f0b87599f5283e134b237 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 24 Mar 2020 21:35:40 +0530 Subject: [PATCH 26/47] fix: Message per app when installing node deps --- bench/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bench/utils.py b/bench/utils.py index 6a19347e..2204f511 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -505,6 +505,7 @@ def update_yarn_packages(bench_path='.'): for app in os.listdir(apps_dir): app_path = os.path.join(apps_dir, app) if os.path.exists(os.path.join(app_path, 'package.json')): + print('\nRunning "yarn install" for {0}'.format(app)) exec_cmd('yarn install', cwd=app_path) From 35d1166868f4a1c67d9d6d1b8faeaf9ceb3032ae Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 25 Mar 2020 13:07:55 +0530 Subject: [PATCH 27/47] style: updated message log for app wise deps setup --- bench/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/utils.py b/bench/utils.py index 9cc1ee37..98734ed5 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -591,7 +591,7 @@ def update_yarn_packages(bench_path='.'): for app in os.listdir(apps_dir): app_path = os.path.join(apps_dir, app) if os.path.exists(os.path.join(app_path, 'package.json')): - print('\nRunning "yarn install" for {0}'.format(app)) + print('\n{0}Installing node dependencies for {1}{2}'.format(color.yellow, app, color.nc)) exec_cmd('yarn install', cwd=app_path) From 8c86ef576e72d7c4f57f3375e4b425fad4265744 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 25 Mar 2020 13:37:24 +0530 Subject: [PATCH 28/47] fix: bench_path during setup_env --- bench/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bench/utils.py b/bench/utils.py index 98734ed5..d2c0f1a1 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -295,11 +295,13 @@ def get_venv_path(): def setup_env(bench_path='.', python='python3'): frappe = os.path.join(bench_path, "apps", "frappe") - pip = os.path.join(".", "env", "bin", "pip") + pip = os.path.join(bench_path, "env", "bin", "pip") virtualenv = get_venv_path() exec_cmd('{} -q env -p {}'.format(virtualenv, python), cwd=bench_path) - exec_cmd('{} install -q -U -e {}'.format(pip, frappe), cwd=bench_path) + + if os.path.exists(frappe): + exec_cmd('{} install -q -U -e {}'.format(pip, frappe), cwd=bench_path) def setup_socketio(bench_path='.'): From af8f74db469dd3ffdce7f2c002e47d34b136d6bc Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 26 Mar 2020 13:09:23 +0530 Subject: [PATCH 29/47] chore(deepsource): update docstrings for public functions --- bench/config/production_setup.py | 1 + bench/config/supervisor.py | 2 ++ bench/patches/v5/fix_user_permissions.py | 3 +++ 3 files changed, 6 insertions(+) diff --git a/bench/config/production_setup.py b/bench/config/production_setup.py index 42729a22..9dc4af69 100755 --- a/bench/config/production_setup.py +++ b/bench/config/production_setup.py @@ -12,6 +12,7 @@ from bench.utils import CommandFailedError, exec_cmd, fix_prod_setup_perms, get_ def setup_production_prerequisites(): + """Installs ansible, fail2banc, NGINX and supervisor""" if not find_executable("ansible"): exec_cmd("sudo {0} -m pip install ansible".format(sys.executable)) if not find_executable("fail2ban-client"): diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 253a1a21..43560cf0 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -14,6 +14,7 @@ from six.moves import configparser def generate_supervisor_config(bench_path, user=None, yes=False): + """Generate supervisor config for respective bench path""" if not user: user = getpass.getuser() @@ -55,6 +56,7 @@ def generate_supervisor_config(bench_path, user=None, yes=False): def get_supervisord_conf(): + """Returns path of supervisord config from possible paths""" possibilities = ("supervisord.conf", "etc/supervisord.conf", "/etc/supervisord.conf", "/etc/supervisor/supervisord.conf", "/etc/supervisord.conf") for possibility in possibilities: diff --git a/bench/patches/v5/fix_user_permissions.py b/bench/patches/v5/fix_user_permissions.py index c11a9480..87d804d4 100644 --- a/bench/patches/v5/fix_user_permissions.py +++ b/bench/patches/v5/fix_user_permissions.py @@ -11,6 +11,7 @@ from bench.utils import exec_cmd, get_bench_name, get_cmd_output def is_sudoers_set(): + """Check if bench sudoers is set""" cmd = ["sudo", "-n", "bench"] with open(os.devnull, "wb") as f: @@ -28,6 +29,7 @@ def is_sudoers_set(): def is_production_set(bench_path): + """Check if production is set for current bench""" production_setup = False bench_name = get_bench_name(bench_path) @@ -47,6 +49,7 @@ def is_production_set(bench_path): def execute(bench_path): + """This patch checks if bench sudoers is set and regenerate supervisor and sudoers files""" user = get_config('.').get("frappe_user") or getpass.getuser() if is_sudoers_set(): From cd77b4225583687922f74c48b186d7589e5b412b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 26 Mar 2020 14:15:59 +0530 Subject: [PATCH 30/47] refactor: avoid redefining name from different scopes ref: deepsource https://deepsource.io/gh/frappe/bench/run/c9ee328a-6f59-49a9-b21c-8e770bd14091/python/PYL-W0621/ --- bench/commands/setup.py | 75 ++++++++++++-------------------- bench/config/production_setup.py | 12 +++-- bench/utils.py | 24 +++++----- 3 files changed, 49 insertions(+), 62 deletions(-) diff --git a/bench/commands/setup.py b/bench/commands/setup.py index 84e267c7..96360bea 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -2,12 +2,21 @@ import os import sys -# imports - module imports -from bench.utils import exec_cmd - # imports - third party imports -from six import PY3 import click +from six import PY3 + +# imports - module imports +import bench.config.lets_encrypt +import bench.config.nginx +import bench.config.procfile +import bench.config.production_setup +import bench.config.redis +import bench.config.site_config +import bench.config.supervisor + +import bench.utils +from bench.utils import exec_cmd, run_playbook @click.group(help="Setup command group for enabling setting up a Frappe environment") @@ -18,70 +27,59 @@ def setup(): @click.command("sudoers", help="Add commands to sudoers list for execution without password") @click.argument("user") def setup_sudoers(user): - from bench.utils import setup_sudoers - setup_sudoers(user) + bench.utils.setup_sudoers(user) @click.command("nginx", help="Generate configuration files for NGINX") @click.option("--yes", help="Yes to regeneration of nginx config file", default=False, is_flag=True) def setup_nginx(yes=False): - from bench.config.nginx import make_nginx_conf - make_nginx_conf(bench_path=".", yes=yes) + bench.config.nginx.make_nginx_conf(bench_path=".", yes=yes) @click.command("reload-nginx", help="Checks NGINX config file and reloads service") def reload_nginx(): - from bench.config.production_setup import reload_nginx - reload_nginx() + bench.config.production_setup.reload_nginx() @click.command("supervisor", help="Generate configuration for supervisor") @click.option("--user", help="optional user argument") @click.option("--yes", help="Yes to regeneration of supervisor config", is_flag=True, default=False) def setup_supervisor(user=None, yes=False): - from bench.config.supervisor import generate_supervisor_config - generate_supervisor_config(bench_path=".", user=user, yes=yes) + bench.config.supervisor.generate_supervisor_config(bench_path=".", user=user, yes=yes) @click.command("redis", help="Generates configuration for Redis") def setup_redis(): - from bench.config.redis import generate_config - generate_config(".") + bench.config.redis.generate_config(".") @click.command("fonts", help="Add Frappe fonts to system") def setup_fonts(): - from bench.utils import setup_fonts - setup_fonts() + bench.utils.setup_fonts() @click.command("production", help="Setup Frappe production environment for specific user") @click.argument("user") @click.option("--yes", help="Yes to regeneration config", is_flag=True, default=False) def setup_production(user, yes=False): - from bench.config.production_setup import setup_production - setup_production(user=user, yes=yes) + bench.config.production_setup.setup_production(user=user, yes=yes) @click.command("backups", help="Add cronjob for bench backups") def setup_backups(): - from bench.utils import setup_backups - setup_backups() + bench.utils.setup_backups() @click.command("env", help="Setup virtualenv for bench") @click.option("--python", type = str, default = "python3", help = "Path to Python Executable.") def setup_env(python="python3"): - from bench.utils import setup_env - setup_env(python=python) + bench.utils.setup_env(python=python) @click.command("firewall", help="Setup firewall for system") @click.option("--ssh_port") @click.option("--force") def setup_firewall(ssh_port=None, force=False): - from bench.utils import run_playbook - if not force: click.confirm("Setting up the firewall will block all ports except 80, 443 and {0}\nDo you want to continue?".format(ssh_port), abort=True) @@ -95,8 +93,6 @@ def setup_firewall(ssh_port=None, force=False): @click.argument("port") @click.option("--force") def set_ssh_port(port, force=False): - from bench.utils import run_playbook - if not force: click.confirm("This will change your SSH Port to {}\nDo you want to continue?".format(port), abort=True) @@ -108,8 +104,7 @@ def set_ssh_port(port, force=False): @click.option("--custom-domain") @click.option('-n', '--non-interactive', default=False, is_flag=True, help="Run command non-interactively. This flag restarts nginx and runs certbot non interactively. Shouldn't be used on 1'st attempt") def setup_letsencrypt(site, custom_domain, non_interactive): - from bench.config.lets_encrypt import setup_letsencrypt - setup_letsencrypt(site, custom_domain, bench_path=".", interactive=not non_interactive) + bench.config.lets_encrypt.setup_letsencrypt(site, custom_domain, bench_path=".", interactive=not non_interactive) @click.command("wildcard-ssl", help="Setup wildcard SSL certificate for multi-tenant bench") @@ -117,20 +112,17 @@ def setup_letsencrypt(site, custom_domain, non_interactive): @click.option("--email") @click.option("--exclude-base-domain", default=False, is_flag=True, help="SSL Certificate not applicable for base domain") def setup_wildcard_ssl(domain, email, exclude_base_domain): - from bench.config.lets_encrypt import setup_wildcard_ssl - setup_wildcard_ssl(domain, email, bench_path=".", exclude_base_domain=exclude_base_domain) + bench.config.lets_encrypt.setup_wildcard_ssl(domain, email, bench_path=".", exclude_base_domain=exclude_base_domain) @click.command("procfile", help="Generate Procfile for bench start") def setup_procfile(): - from bench.config.procfile import setup_procfile - setup_procfile(".") + bench.config.procfile.setup_procfile(".") @click.command("socketio", help="Setup node dependencies for socketio server") def setup_socketio(): - from bench.utils import setup_socketio - setup_socketio() + bench.utils.setup_socketio() @click.command("requirements", help="Setup Python and Node dependencies") @@ -203,34 +195,28 @@ def setup_config(): @click.option("--ssl-certificate-key", help="Absolute path to SSL Certificate Key") def add_domain(domain, site=None, ssl_certificate=None, ssl_certificate_key=None): """Add custom domain to site""" - from bench.config.site_config import add_domain - if not site: print("Please specify site") sys.exit(1) - add_domain(site, domain, ssl_certificate, ssl_certificate_key, bench_path=".") + bench.config.site_config.add_domain(site, domain, ssl_certificate, ssl_certificate_key, bench_path=".") @click.command("remove-domain", help="Remove custom domain from a site") @click.argument("domain") @click.option("--site", prompt=True) def remove_domain(domain, site=None): - from bench.config.site_config import remove_domain - if not site: print("Please specify site") sys.exit(1) - remove_domain(site, domain, bench_path=".") + bench.config.site_config.remove_domain(site, domain, bench_path=".") @click.command("sync-domains", help="Check if there is a change in domains. If yes, updates the domains list.") @click.option("--domain", multiple=True) @click.option("--site", prompt=True) def sync_domains(domain=None, site=None): - from bench.config.site_config import sync_domains - if not site: print("Please specify site") sys.exit(1) @@ -241,7 +227,7 @@ def sync_domains(domain=None, site=None): print("Domains should be a json list of strings or dictionaries") sys.exit(1) - changed = sync_domains(site, domains, bench_path=".") + changed = bench.config.site_config.sync_domains(site, domains, bench_path=".") # if changed, success, else failure sys.exit(0 if changed else 1) @@ -253,8 +239,6 @@ def sync_domains(domain=None, site=None): @click.option("--mysql_root_password") @click.option("--container", is_flag=True, default=False) def setup_roles(role, **kwargs): - from bench.utils import run_playbook - extra_vars = {"production": True} extra_vars.update(kwargs) @@ -269,7 +253,6 @@ def setup_roles(role, **kwargs): @click.option("--bantime", default=600, help="The counter is set to zero if no match is found within 'findtime' seconds. Default is 600 seconds") @click.option("--findtime", default=600, help="Duration (in seconds) for IP to be banned for. Negative number for 'permanent' ban. Default is 600 seconds") def setup_nginx_proxy_jail(**kwargs): - from bench.utils import run_playbook run_playbook("roles/fail2ban/tasks/configure_nginx_jail.yml", extra_vars=kwargs) diff --git a/bench/config/production_setup.py b/bench/config/production_setup.py index 9dc4af69..dff6a402 100755 --- a/bench/config/production_setup.py +++ b/bench/config/production_setup.py @@ -83,17 +83,21 @@ def disable_production(bench_path='.'): reload_nginx() -def service(service, option): +def service(service_name, service_option): if os.path.basename(find_executable('systemctl') or '') == 'systemctl' and is_running_systemd(): - exec_cmd("sudo {service_manager} {option} {service}".format(service_manager='systemctl', option=option, service=service)) + systemctl_cmd = "sudo {service_manager} {service_option} {service_name}" + exec_cmd(systemctl_cmd.format(service_manager='systemctl', service_option=service_option, service_name=service_name)) + elif os.path.basename(find_executable('service') or '') == 'service': - exec_cmd("sudo {service_manager} {service} {option} ".format(service_manager='service', service=service, option=option)) + service_cmd = "sudo {service_manager} {service_name} {service_option}" + exec_cmd(service_cmd.format(service_manager='service', service_name=service_name, service_option=service_option)) + else: # look for 'service_manager' and 'service_manager_command' in environment service_manager = os.environ.get("BENCH_SERVICE_MANAGER") if service_manager: service_manager_command = (os.environ.get("BENCH_SERVICE_MANAGER_COMMAND") - or "{service_manager} {option} {service}").format(service_manager=service_manager, service=service, option=option) + or "{service_manager} {service_option} {service}").format(service_manager=service_manager, service=service, service_option=service_option) exec_cmd(service_manager_command) else: diff --git a/bench/utils.py b/bench/utils.py index 19fe9036..6362f305 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -66,10 +66,10 @@ def log(message, level=0): 2: color.red + 'ERROR', # fail 3: color.yellow + 'WARN' # warn/suggest } - start = (levels.get(level) + ': ') if level in levels else '' - end = '\033[0m' + start_line = (levels.get(level) + ': ') if level in levels else '' + end_line = '\033[0m' - print(start + message + end) + print(start_line + message + end_line) def safe_decode(string, encoding = 'utf-8'): @@ -1110,8 +1110,8 @@ def migrate_env(python, backup=False): from bench.config.common_site_config import get_config from bench.app import get_apps - log = logging.getLogger(__name__) - log.setLevel(logging.DEBUG) + logger = logging.getLogger(__name__) + logger.setLevel(logging.DEBUG) nvenv = 'env' path = os.getcwd() @@ -1127,12 +1127,12 @@ def migrate_env(python, backup=False): redis = '{redis} -p {port}'.format(redis=which('redis-cli'), port=rredis.port) - log.debug('Clearing Redis Cache...') + logger.debug('Clearing Redis Cache...') exec_cmd('{redis} FLUSHALL'.format(redis = redis)) - log.debug('Clearing Redis DataBase...') + logger.debug('Clearing Redis DataBase...') exec_cmd('{redis} FLUSHDB'.format(redis = redis)) except: - log.warn('Please ensure Redis Connections are running or Daemonized.') + logger.warn('Please ensure Redis Connections are running or Daemonized.') # Backup venv: restore using `virtualenv --relocatable` if needed if backup: @@ -1143,7 +1143,7 @@ def migrate_env(python, backup=False): source = os.path.join(path, 'env') target = parch - log.debug('Backing up Virtual Environment') + logger.debug('Backing up Virtual Environment') stamp = datetime.now().strftime('%Y%m%d_%H%M%S') dest = os.path.join(path, str(stamp)) @@ -1152,15 +1152,15 @@ def migrate_env(python, backup=False): # Create virtualenv using specified python try: - log.debug('Setting up a New Virtual {} Environment'.format(python)) + logger.debug('Setting up a New Virtual {} Environment'.format(python)) exec_cmd('{virtualenv} --python {python} {pvenv}'.format(virtualenv=virtualenv, python=python, pvenv=pvenv)) apps = ' '.join(["-e {}".format(os.path.join("apps", app)) for app in get_apps()]) exec_cmd('{0} install -q -U {1}'.format(pip, apps)) - log.debug('Migration Successful to {}'.format(python)) + logger.debug('Migration Successful to {}'.format(python)) except: - log.debug('Migration Error') + logger.debug('Migration Error') raise From d5f8ad8e20380c42f24a87e29453e1475864919c Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 26 Mar 2020 14:18:39 +0530 Subject: [PATCH 31/47] chore: specific Exception handled ref: https://deepsource.io/gh/frappe/bench/run/c9ee328a-6f59-49a9-b21c-8e770bd14091/python/PYL-W0703 --- bench/patches/v5/fix_user_permissions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/patches/v5/fix_user_permissions.py b/bench/patches/v5/fix_user_permissions.py index 87d804d4..bdccf53c 100644 --- a/bench/patches/v5/fix_user_permissions.py +++ b/bench/patches/v5/fix_user_permissions.py @@ -20,7 +20,7 @@ def is_sudoers_set(): if return_code_check: try: bench_warn = change_uid_msg in get_cmd_output(cmd, _raise=False) - except Exception: + except subprocess.CalledProcessError: bench_warn = False finally: return_code_check = return_code_check and bench_warn From 7f792a86031078198ffd9004a0390276a0549863 Mon Sep 17 00:00:00 2001 From: Chinmay Pai Date: Tue, 31 Mar 2020 00:32:30 +0530 Subject: [PATCH 32/47] chore: warning text improvement highly opinionated improvement. --- bench/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/cli.py b/bench/cli.py index 6c1668f7..dc8c1d3a 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -19,7 +19,7 @@ def cli(): change_uid() if is_dist_editable("bench"): - log("bench is installed in editable mode!\n\nThis is not the recommended mode of install for production. Use the PYPI package using `pip install frappe-bench` instead\n", level=3) + log("bench is installed in editable mode!\n\nThis is not the recommended mode of installation for production. Instead, install the package from PyPI with: `pip install frappe-bench`\n", level=3) if len(sys.argv) > 2 and sys.argv[1] == "frappe": return old_frappe_cli() From 62f666c481cda9b9032fa42bc8ae31ae3bcf4224 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 2 Apr 2020 15:19:13 +0530 Subject: [PATCH 33/47] feat: check for newer versions on PyPI --- bench/cli.py | 3 ++- bench/utils.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/bench/cli.py b/bench/cli.py index f5891d0a..39b60954 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -1,6 +1,6 @@ import click import os, sys, logging, json, pwd, subprocess -from bench.utils import is_root, PatchError, drop_privileges, get_env_cmd, get_cmd_output, get_frappe, log, is_dist_editable, find_parent_bench +from bench.utils import is_root, PatchError, drop_privileges, get_env_cmd, get_cmd_output, get_frappe, log, is_dist_editable, find_parent_bench, check_latest_version from bench.app import get_apps from bench.config.common_site_config import get_config from bench.commands import bench_command @@ -42,6 +42,7 @@ def cli(): else: try: # NOTE: this is the main bench command + check_latest_version() bench_command() except PatchError: sys.exit(1) diff --git a/bench/utils.py b/bench/utils.py index 6fb985fc..ec49201f 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -21,6 +21,7 @@ from distutils.spawn import find_executable # imports - third party imports import click import requests +from semantic_version import Version from six import iteritems from six.moves.urllib.parse import urlparse @@ -80,6 +81,18 @@ def safe_decode(string, encoding = 'utf-8'): return string +def check_latest_version(): + pypi_request = requests.get("https://pypi.org/pypi/frappe-bench/json") + + if pypi_request.status_code == 200: + pypi_version_str = pypi_request.json().get('info').get('version') + pypi_version = Version(pypi_version_str) + local_version = Version(bench.__version__) + + if pypi_version > local_version: + log("A newer version of bench is available: {0} → {1}".format(local_version, pypi_version)) + + def get_frappe(bench_path='.'): frappe = get_env_cmd('frappe', bench_path=bench_path) if not os.path.exists(frappe): From 6ea7d75193e31561d047731ed6a8bb5022319759 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 2 Apr 2020 16:10:38 +0530 Subject: [PATCH 34/47] fix: check for updates after command execution and handle requests Exceptions --- bench/cli.py | 4 ++-- bench/utils.py | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index 39b60954..c1c75bc5 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -1,3 +1,4 @@ +import atexit import click import os, sys, logging, json, pwd, subprocess from bench.utils import is_root, PatchError, drop_privileges, get_env_cmd, get_cmd_output, get_frappe, log, is_dist_editable, find_parent_bench, check_latest_version @@ -41,8 +42,7 @@ def cli(): else: try: - # NOTE: this is the main bench command - check_latest_version() + atexit.register(check_latest_version) bench_command() except PatchError: sys.exit(1) diff --git a/bench/utils.py b/bench/utils.py index ec49201f..235d5df7 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -82,7 +82,12 @@ def safe_decode(string, encoding = 'utf-8'): def check_latest_version(): - pypi_request = requests.get("https://pypi.org/pypi/frappe-bench/json") + try: + pypi_request = requests.get("https://pypi.org/pypi/frappe-bench/json") + except Exception: + # Exceptions thrown are defined in requests.exceptions + # ignore checking on all Exceptions + return if pypi_request.status_code == 200: pypi_version_str = pypi_request.json().get('info').get('version') From 9b6155dcf31344981339a5256b661cb448e92379 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 2 Apr 2020 17:00:20 +0530 Subject: [PATCH 35/47] fix: added python source code encoding --- bench/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bench/utils.py b/bench/utils.py index 235d5df7..3fa61d9c 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -1,3 +1,6 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + # imports - standard imports import errno import glob From 5f03d30df4d4596d981a79c117cc58cf83a5b16c Mon Sep 17 00:00:00 2001 From: gavin Date: Wed, 8 Apr 2020 18:54:45 +0530 Subject: [PATCH 36/47] chore: changed LOG to INFO --- bench/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/utils.py b/bench/utils.py index 6fb985fc..e783a5af 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -61,7 +61,7 @@ def is_bench_directory(directory=os.path.curdir): def log(message, level=0): levels = { - 0: color.blue + 'LOG', # normal + 0: color.blue + 'INFO', # normal 1: color.green + 'SUCCESS', # success 2: color.red + 'ERROR', # fail 3: color.yellow + 'WARN' # warn/suggest From c0afa041c83af699c87d9915ef73c5749abbdb94 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 11 Apr 2020 21:17:13 +0530 Subject: [PATCH 37/47] feat: bench apps commands caching Port of https://github.com/frappe/bench/pull/888 --- bench/commands/__init__.py | 6 ++++-- bench/commands/utils.py | 12 ++++++++++++ bench/utils.py | 39 +++++++++++++++++++++++++++++++++++++- 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/bench/commands/__init__.py b/bench/commands/__init__.py index 090013c2..da4eb8c8 100755 --- a/bench/commands/__init__.py +++ b/bench/commands/__init__.py @@ -41,7 +41,8 @@ bench_command.add_command(switch_to_develop) from bench.commands.utils import (start, restart, set_nginx_port, set_ssl_certificate, set_ssl_certificate_key, set_url_root, set_mariadb_host, set_default_site, download_translations, backup_site, backup_all_sites, release, renew_lets_encrypt, - disable_production, bench_src, prepare_beta_release, set_redis_cache_host, set_redis_queue_host, set_redis_socketio_host, find_benches, migrate_env) + disable_production, bench_src, prepare_beta_release, set_redis_cache_host, set_redis_queue_host, set_redis_socketio_host, find_benches, migrate_env, + generate_command_cache, clear_command_cache) bench_command.add_command(start) bench_command.add_command(restart) bench_command.add_command(set_nginx_port) @@ -63,7 +64,8 @@ bench_command.add_command(bench_src) bench_command.add_command(prepare_beta_release) bench_command.add_command(find_benches) bench_command.add_command(migrate_env) - +bench_command.add_command(generate_command_cache) +bench_command.add_command(clear_command_cache) from bench.commands.setup import setup bench_command.add_command(setup) diff --git a/bench/commands/utils.py b/bench/commands/utils.py index 822086a7..fcb11512 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -180,3 +180,15 @@ def find_benches(location): def migrate_env(python, backup=True): from bench.utils import migrate_env migrate_env(python=python, backup=backup) + + +@click.command('generate-command-cache', help="Caches Frappe Framework commands") +def generate_command_cache(bench_path='.'): + from bench.utils import generate_command_cache + return generate_command_cache(bench_path=bench_path) + + +@click.command('clear-command-cache', help="Clears Frappe Framework cached commands") +def clear_command_cache(bench_path='.'): + from bench.utils import clear_command_cache + return clear_command_cache(bench_path=bench_path) diff --git a/bench/utils.py b/bench/utils.py index 897acb64..33a9a6cb 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -39,7 +39,7 @@ class CommandFailedError(Exception): pass logger = logging.getLogger(__name__) - +bench_cache_file = '.bench.cmd' folders_in_bench = ('apps', 'sites', 'config', 'logs', 'config/pids') sudoers_file = '/etc/sudoers.d/frappe' @@ -187,6 +187,8 @@ def update(pull=False, patch=False, build=False, requirements=False, backup=True patches.run(bench_path=bench_path) conf = get_config(bench_path) + clear_command_cache(bench_path='.') + if conf.get('release_bench'): print('Release bench detected, cannot update!') sys.exit(1) @@ -1137,3 +1139,38 @@ def find_parent_bench(path): # NOTE: the os.path.split assumes that given path is absolute parent_dir = os.path.split(path)[0] return find_parent_bench(parent_dir) + + +def generate_command_cache(bench_path='.'): + """ + Caches all available commands (even custom apps) via Frappe + Default caching behaviour: generated the first time any command (for a specific bench directory) + """ + + python = get_env_cmd('python', bench_path=bench_path) + sites_path = os.path.join(bench_path, 'sites') + + if os.path.exists(bench_cache_file): + os.remove(bench_cache_file) + + try: + output = get_cmd_output("{0} -m frappe.utils.bench_helper get-frappe-commands".format(python), cwd=sites_path) + with open(bench_cache_file, 'w') as f: + json.dump(eval(output), f) + return json.loads(output) + + except subprocess.CalledProcessError as e: + if hasattr(e, "stderr"): + print(e.stderr.decode('utf-8')) + + +def clear_command_cache(bench_path='.'): + """ + Clears commands cached + Default invalidation behaviour: destroyed on each run of `bench update` + """ + + if os.path.exists(bench_cache_file): + os.remove(bench_cache_file) + else: + print("Bench command cache doesn't exist in this folder!") \ No newline at end of file From e39ae1e0b5699e41dd552f48707299ccda4e86e3 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 11 Apr 2020 21:20:24 +0530 Subject: [PATCH 38/47] fix: better warning and cache handling --- bench/cli.py | 89 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 54 insertions(+), 35 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index c1c75bc5..723c596e 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -1,16 +1,27 @@ +# imports - standard imports import atexit +import json +import logging +import os +import pwd +import subprocess +import sys + +# imports - third party imports import click -import os, sys, logging, json, pwd, subprocess -from bench.utils import is_root, PatchError, drop_privileges, get_env_cmd, get_cmd_output, get_frappe, log, is_dist_editable, find_parent_bench, check_latest_version + +# imports - module imports from bench.app import get_apps -from bench.config.common_site_config import get_config from bench.commands import bench_command +from bench.config.common_site_config import get_config +from bench.utils import PatchError, bench_cache_file, check_latest_version, drop_privileges, find_parent_bench, generate_command_cache, get_cmd_output, get_env_cmd, get_frappe, is_bench_directory, is_dist_editable, is_root, log logger = logging.getLogger('bench') from_command_line = False change_uid_msg = "You should not run this command as root" + def cli(): global from_command_line from_command_line = True @@ -20,46 +31,50 @@ def cli(): change_dir() change_uid() - if is_dist_editable("bench"): + if is_dist_editable("bench") and len(sys.argv) > 1 and sys.argv[1] != "src": log("bench is installed in editable mode!\n\nThis is not the recommended mode of installation for production. Instead, install the package from PyPI with: `pip install frappe-bench`\n", level=3) + if not is_bench_directory() and not cmd_requires_root() and len(sys.argv) > 1 and sys.argv[1] not in ("init", "find", "src"): + log("Command not being executed in bench directory", level=3) + if len(sys.argv) > 2 and sys.argv[1] == "frappe": return old_frappe_cli() - elif len(sys.argv) > 1 and sys.argv[1] in get_frappe_commands(): - return frappe_cmd() + elif len(sys.argv) > 1: + if sys.argv[1] in get_frappe_commands() + ["--site", "--verbose", "--force", "--profile"]: + return frappe_cmd() - elif len(sys.argv) > 1 and sys.argv[1] in ("--site", "--verbose", "--force", "--profile"): - return frappe_cmd() + elif sys.argv[1] == "--help": + print(click.Context(bench_command).get_help()) + print(get_frappe_help()) + return - elif len(sys.argv) > 1 and sys.argv[1]=="--help": - print(click.Context(bench_command).get_help()) - print(get_frappe_help()) - return + elif sys.argv[1] in get_apps(): + return app_cmd() - elif len(sys.argv) > 1 and sys.argv[1] in get_apps(): - return app_cmd() + if not (len(sys.argv) > 1 and sys.argv[1] == "src"): + atexit.register(check_latest_version) + + try: + bench_command() + except PatchError: + sys.exit(1) - else: - try: - atexit.register(check_latest_version) - bench_command() - except PatchError: - sys.exit(1) def check_uid(): if cmd_requires_root() and not is_root(): log('superuser privileges required for this command', level=3) sys.exit(1) + def cmd_requires_root(): if len(sys.argv) > 2 and sys.argv[2] in ('production', 'sudoers', 'supervisor', 'lets-encrypt', 'fonts', - 'print', 'firewall', 'ssh-port', 'role', 'fail2ban', 'wildcard-ssl'): + 'print', 'firewall', 'ssh-port', 'role', 'fail2ban', 'wildcard-ssl', 'install'): return True - if len(sys.argv) >= 2 and sys.argv[1] in ('patch', 'renew-lets-encrypt', 'disable-production', - 'install'): + if len(sys.argv) >= 2 and sys.argv[1] in ('patch', 'renew-lets-encrypt', 'disable-production'): return True + def change_dir(): if os.path.exists('config.json') or "init" in sys.argv: return @@ -70,6 +85,7 @@ def change_dir(): if os.path.exists(dir_path): os.chdir(dir_path) + def change_uid(): if is_root() and not cmd_requires_root(): frappe_user = get_config(".").get('frappe_user') @@ -80,35 +96,37 @@ def change_uid(): log(change_uid_msg, level=3) sys.exit(1) + def old_frappe_cli(bench_path='.'): f = get_frappe(bench_path=bench_path) os.chdir(os.path.join(bench_path, 'sites')) os.execv(f, [f] + sys.argv[2:]) + def app_cmd(bench_path='.'): f = get_env_cmd('python', bench_path=bench_path) os.chdir(os.path.join(bench_path, 'sites')) os.execv(f, [f] + ['-m', 'frappe.utils.bench_helper'] + sys.argv[1:]) + def frappe_cmd(bench_path='.'): f = get_env_cmd('python', bench_path=bench_path) os.chdir(os.path.join(bench_path, 'sites')) os.execv(f, [f] + ['-m', 'frappe.utils.bench_helper', 'frappe'] + sys.argv[1:]) -def get_frappe_commands(bench_path='.'): - python = get_env_cmd('python', bench_path=bench_path) - sites_path = os.path.join(bench_path, 'sites') - if not os.path.exists(sites_path): - log("Command not being executed in bench directory", level=3) - return [] - try: - output = get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-commands".format(python=python), cwd=sites_path) - return json.loads(output) - except subprocess.CalledProcessError as e: - if hasattr(e, "stderr"): - print(e.stderr.decode('utf-8')) + +def get_frappe_commands(): + if not is_bench_directory(): return [] + if os.path.exists(bench_cache_file): + command_dump = open(bench_cache_file, 'r').read() or '[]' + return json.loads(command_dump) + + else: + return generate_command_cache() + + def get_frappe_help(bench_path='.'): python = get_env_cmd('python', bench_path=bench_path) sites_path = os.path.join(bench_path, 'sites') @@ -118,6 +136,7 @@ def get_frappe_help(bench_path='.'): except: return "" + def change_working_directory(): """Allows bench commands to be run from anywhere inside a bench directory""" cur_dir = os.path.abspath(".") From 7e7c3969a764faffe814e72a0940103c4bb69d67 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 11 Apr 2020 21:21:07 +0530 Subject: [PATCH 39/47] fix: set translations download limit based on cpu cores --- bench/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/utils.py b/bench/utils.py index 33a9a6cb..332bda2a 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -844,7 +844,7 @@ def update_translations_p(args): def download_translations_p(): - pool = multiprocessing.Pool(4) + pool = multiprocessing.Pool(multiprocessing.cpu_count()) langs = get_langs() apps = ('frappe', 'erpnext') From 2d436126fd4c1e187bbb9b2685f2b7575dfe19d4 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 11 Apr 2020 21:36:34 +0530 Subject: [PATCH 40/47] fix: help for bench install without sudo --- bench/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bench/cli.py b/bench/cli.py index 723c596e..a3235382 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -69,10 +69,12 @@ def check_uid(): def cmd_requires_root(): if len(sys.argv) > 2 and sys.argv[2] in ('production', 'sudoers', 'supervisor', 'lets-encrypt', 'fonts', - 'print', 'firewall', 'ssh-port', 'role', 'fail2ban', 'wildcard-ssl', 'install'): + 'print', 'firewall', 'ssh-port', 'role', 'fail2ban', 'wildcard-ssl'): return True if len(sys.argv) >= 2 and sys.argv[1] in ('patch', 'renew-lets-encrypt', 'disable-production'): return True + if len(sys.argv) > 2 and sys.argv[1] in ('install'): + return True def change_dir(): From 279073f1c0e3d1b4ee30ad81572b2b5c86ffce01 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 13 Apr 2020 13:02:51 +0530 Subject: [PATCH 41/47] style: fixed docstring and unused imports --- bench/cli.py | 1 - bench/utils.py | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index a3235382..628f2413 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -4,7 +4,6 @@ import json import logging import os import pwd -import subprocess import sys # imports - third party imports diff --git a/bench/utils.py b/bench/utils.py index 332bda2a..38786545 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -1142,8 +1142,7 @@ def find_parent_bench(path): def generate_command_cache(bench_path='.'): - """ - Caches all available commands (even custom apps) via Frappe + """Caches all available commands (even custom apps) via Frappe Default caching behaviour: generated the first time any command (for a specific bench directory) """ @@ -1165,8 +1164,7 @@ def generate_command_cache(bench_path='.'): def clear_command_cache(bench_path='.'): - """ - Clears commands cached + """Clears commands cached Default invalidation behaviour: destroyed on each run of `bench update` """ From 6dd01184462054c47de36884cf7e4ec735d50b44 Mon Sep 17 00:00:00 2001 From: Abhishek Kedar <44434910+AKedar21@users.noreply.github.com> Date: Tue, 14 Apr 2020 14:02:56 +0530 Subject: [PATCH 42/47] fix: typo cash to crash Fixed typo in "Open source..." sentence --- bench/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/utils.py b/bench/utils.py index 897acb64..e4a6c08b 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -240,7 +240,7 @@ def update(pull=False, patch=False, build=False, requirements=False, backup=True conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 }) update_config(conf, bench_path=bench_path) - print("_" * 80 + "\nBench: Deployment tool for Frappe and Frappe Applications (https://frappe.io/bench).\nOpen source depends on your contributions, so please contribute bug reports, patches, fixes or cash and be a part of the community") + print("_" * 80 + "\nBench: Deployment tool for Frappe and Frappe Applications (https://frappe.io/bench).\nOpen source depends on your contributions, so please contribute bug reports, patches, fixes or crash and be a part of the community") def copy_patches_txt(bench_path): From e1f1b0972dda9dc1ed8ca490bdf3966820d5cb92 Mon Sep 17 00:00:00 2001 From: gavin Date: Tue, 14 Apr 2020 15:21:30 +0530 Subject: [PATCH 43/47] docs: mellow-er contribution message post update --- bench/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/utils.py b/bench/utils.py index e4a6c08b..35c985ad 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -240,7 +240,7 @@ def update(pull=False, patch=False, build=False, requirements=False, backup=True conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 }) update_config(conf, bench_path=bench_path) - print("_" * 80 + "\nBench: Deployment tool for Frappe and Frappe Applications (https://frappe.io/bench).\nOpen source depends on your contributions, so please contribute bug reports, patches, fixes or crash and be a part of the community") + print("_" * 80 + "\nBench: Deployment tool for Frappe and Frappe Applications (https://frappe.io/bench).\nOpen source depends on your contributions, so do give back by submitting bug reports, patches and fixes and be a part of the community :)") def copy_patches_txt(bench_path): From 90f951c6ec3465b780f24f08cd87c35631cfd0dd Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 15 Apr 2020 11:28:19 +0530 Subject: [PATCH 44/47] fix(backups): setup backups for bench sites Using python-crontab library instead of manual processing --- bench/utils.py | 39 ++++++++++++--------------------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/bench/utils.py b/bench/utils.py index 35c985ad..55ca544b 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -23,6 +23,7 @@ from distutils.spawn import find_executable # imports - third party imports import click +from crontab import CronTab import requests from semantic_version import Version from six import iteritems @@ -361,42 +362,27 @@ def get_sites(bench_path='.'): return sites -def get_bench_dir(bench_path='.'): - return os.path.abspath(bench_path) - - def setup_backups(bench_path='.'): + from bench.config.common_site_config import get_config logger.info('setting up backups') - bench_dir = get_bench_dir(bench_path=bench_path) + + bench_dir = os.path.abspath(bench_path) + user = get_config('.').get('frappe_user') + logfile = os.path.join(bench_dir, 'logs', 'backup.log') bench.set_frappe_version(bench_path=bench_path) + system_crontab = CronTab(user=user) if bench.FRAPPE_VERSION == 4: backup_command = "cd {sites_dir} && {frappe} --backup all".format(frappe=get_frappe(bench_path=bench_path),) else: backup_command = "cd {bench_dir} && {bench} --site all backup".format(bench_dir=bench_dir, bench=sys.argv[0]) - add_to_crontab('0 */6 * * * {backup_command} >> {logfile} 2>&1'.format(backup_command=backup_command, - logfile=os.path.join(get_bench_dir(bench_path=bench_path), 'logs', 'backup.log'))) + job_command = "{backup_command} >> {logfile} 2>&1".format(backup_command=backup_command, logfile=logfile) - -def add_to_crontab(line): - current_crontab = read_crontab() - line = str.encode(line) - if not line in current_crontab: - cmd = ["crontab"] - if platform.system() == 'FreeBSD': - cmd = ["crontab", "-"] - s = subprocess.Popen(cmd, stdin=subprocess.PIPE) - s.stdin.write(current_crontab) - s.stdin.write(line + b'\n') - s.stdin.close() - - -def read_crontab(): - s = subprocess.Popen(["crontab", "-l"], stdin=subprocess.PIPE, stdout=subprocess.PIPE) - out = s.stdout.read() - s.stdout.close() - return out + if job_command not in str(system_crontab): + job = system_crontab.new(command=job_command, comment="bench auto backups set for every 6 hours") + job.hour.every(6) + system_crontab.write() def setup_sudoers(user): @@ -553,7 +539,6 @@ def restart_supervisor_processes(bench_path='.', web_workers=False): def restart_systemd_processes(bench_path='.', web_workers=False): - from .config.common_site_config import get_config bench_name = get_bench_name(bench_path) exec_cmd('sudo systemctl stop -- $(systemctl show -p Requires {bench_name}.target | cut -d= -f2)'.format(bench_name=bench_name)) exec_cmd('sudo systemctl start -- $(systemctl show -p Requires {bench_name}.target | cut -d= -f2)'.format(bench_name=bench_name)) From f28c3765d843d8118ab22e208857f98551e7ed3b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 16 Apr 2020 14:17:29 +0530 Subject: [PATCH 45/47] fix(backup): make auto backup logs verbose --- bench/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/utils.py b/bench/utils.py index 55ca544b..49e277c9 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -375,7 +375,7 @@ def setup_backups(bench_path='.'): if bench.FRAPPE_VERSION == 4: backup_command = "cd {sites_dir} && {frappe} --backup all".format(frappe=get_frappe(bench_path=bench_path),) else: - backup_command = "cd {bench_dir} && {bench} --site all backup".format(bench_dir=bench_dir, bench=sys.argv[0]) + backup_command = "cd {bench_dir} && {bench} --verbose --site all backup".format(bench_dir=bench_dir, bench=sys.argv[0]) job_command = "{backup_command} >> {logfile} 2>&1".format(backup_command=backup_command, logfile=logfile) From 6881c9338829c0a00ff1384f1bddfc77ea568cfd Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 17 Apr 2020 11:26:44 +0530 Subject: [PATCH 46/47] fix: bench path used for config --- bench/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/utils.py b/bench/utils.py index 49e277c9..773beb01 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -367,7 +367,7 @@ def setup_backups(bench_path='.'): logger.info('setting up backups') bench_dir = os.path.abspath(bench_path) - user = get_config('.').get('frappe_user') + user = get_config(bench_path=bench_dir).get('frappe_user') logfile = os.path.join(bench_dir, 'logs', 'backup.log') bench.set_frappe_version(bench_path=bench_path) system_crontab = CronTab(user=user) From a7c52b4059e5f4d07b9dfce4ed65c52a8ed10c5d Mon Sep 17 00:00:00 2001 From: lasalesi Date: Fri, 17 Apr 2020 15:01:07 +0200 Subject: [PATCH 47/47] fix(playbooks): wkhtmltopdf checksum string for debian 9 (#969) * Update main.yml Correcting checksum string for Debian 9 * fix(playbook): yaml syntax for checksum in wkhtmltopdf Co-authored-by: gavin --- playbooks/roles/wkhtmltopdf/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/wkhtmltopdf/tasks/main.yml b/playbooks/roles/wkhtmltopdf/tasks/main.yml index 16a6e99d..1656ffd2 100644 --- a/playbooks/roles/wkhtmltopdf/tasks/main.yml +++ b/playbooks/roles/wkhtmltopdf/tasks/main.yml @@ -72,7 +72,7 @@ get_url: url: https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.stretch_{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.deb dest: /tmp/wkhtmltox.deb - checksum: '{{ 1140b0ab02aa6e17346af2f14ed0de807376de475ba90e1db3975f112fbd20bb if ansible_architecture == "x86_64" else 5b2d15e738ac479e7a8ca6fd765f406c3684a48091813520f87878278d6dd22a }}' + checksum: "sha256:{{ '1140b0ab02aa6e17346af2f14ed0de807376de475ba90e1db3975f112fbd20bb' if ansible_architecture == 'x86_64' else '5b2d15e738ac479e7a8ca6fd765f406c3684a48091813520f87878278d6dd22a' }}" when: ansible_distribution == 'Debian' and ansible_distribution_major_version == '9' - name: download wkthmltox Debian 10 @@ -92,4 +92,4 @@ deb: /tmp/wkhtmltox.deb state: present when: ansible_os_family == 'Debian' -... \ No newline at end of file +...