From 51b81c399760581a57184e368f4a5e2bc4743b5f Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 13 Mar 2020 16:41:00 +0530 Subject: [PATCH 01/24] fix: remove bench and supervisor from sudoers chore: move production prerequisites into setup_production --- bench/commands/setup.py | 10 ---------- bench/config/production_setup.py | 21 ++++++++++++++++++--- bench/config/templates/frappe_sudoers | 5 ----- bench/tests/test_setup_production.py | 4 ++-- bench/utils.py | 9 +++------ 5 files changed, 23 insertions(+), 26 deletions(-) diff --git a/bench/commands/setup.py b/bench/commands/setup.py index 352a781b..84e267c7 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -60,16 +60,6 @@ def setup_fonts(): @click.option("--yes", help="Yes to regeneration config", is_flag=True, default=False) def setup_production(user, yes=False): from bench.config.production_setup import setup_production - # Install prereqs for production - from distutils.spawn import find_executable - if not find_executable("ansible"): - exec_cmd("sudo -H {0} -m pip install ansible".format(sys.executable)) - if not find_executable("fail2ban-client"): - exec_cmd("bench setup role fail2ban") - if not find_executable("nginx"): - exec_cmd("bench setup role nginx") - if not find_executable("supervisord"): - exec_cmd("bench setup role supervisor") setup_production(user=user, yes=yes) diff --git a/bench/config/production_setup.py b/bench/config/production_setup.py index 8b2e7801..e6d09098 100755 --- a/bench/config/production_setup.py +++ b/bench/config/production_setup.py @@ -4,8 +4,23 @@ from bench.config.systemd import generate_systemd_config from bench.config.nginx import make_nginx_conf from bench.config.common_site_config import get_config import os, subprocess +import sys +from distutils.spawn import find_executable + + +def setup_production_prerequisites(): + if not find_executable("ansible"): + exec_cmd("sudo {0} -m pip install ansible".format(sys.executable)) + if not find_executable("fail2ban-client"): + exec_cmd("bench setup role fail2ban") + if not find_executable("nginx"): + exec_cmd("bench setup role nginx") + if not find_executable("supervisord"): + exec_cmd("bench setup role supervisor") + def setup_production(user, bench_path='.', yes=False): + setup_production_prerequisites() if get_config(bench_path).get('restart_supervisor_on_update') and get_config(bench_path).get('restart_systemd_on_update'): raise Exception("You cannot use supervisor and systemd at the same time. Modify your common_site_config accordingly." ) @@ -109,15 +124,15 @@ def reload_supervisor(): try: # first try reread/update - exec_cmd('sudo {0} reread'.format(supervisorctl)) - exec_cmd('sudo {0} update'.format(supervisorctl)) + exec_cmd('{0} reread'.format(supervisorctl)) + exec_cmd('{0} update'.format(supervisorctl)) return except CommandFailedError: pass try: # something is wrong, so try reloading - exec_cmd('sudo {0} reload'.format(supervisorctl)) + exec_cmd('{0} reload'.format(supervisorctl)) return except CommandFailedError: pass diff --git a/bench/config/templates/frappe_sudoers b/bench/config/templates/frappe_sudoers index 5394444f..51595c8f 100644 --- a/bench/config/templates/frappe_sudoers +++ b/bench/config/templates/frappe_sudoers @@ -8,13 +8,8 @@ {{ user }} ALL = (root) NOPASSWD: {{ systemctl }} * nginx {{ user }} ALL = (root) NOPASSWD: {{ systemctl }} * supervisord {% endif %} -{% if supervisorctl %} -{{ user }} ALL = (root) NOPASSWD: {{ supervisorctl }} -{% endif %} {% if nginx %} {{ user }} ALL = (root) NOPASSWD: {{ nginx }} {% endif %} {{ user }} ALL = (root) NOPASSWD: /opt/certbot-auto -{{ user }} ALL = (root) NOPASSWD: {{ bench }} Defaults:{{ user }} !requiretty - diff --git a/bench/tests/test_setup_production.py b/bench/tests/test_setup_production.py index ca5b4abc..8d0bbc7a 100644 --- a/bench/tests/test_setup_production.py +++ b/bench/tests/test_setup_production.py @@ -126,12 +126,12 @@ class TestSetupProduction(TestBenchBase): def assert_supervisor_process(self, bench_name, use_rq=True, disable_production=False): - out = bench.utils.get_cmd_output("sudo supervisorctl status") + out = bench.utils.get_cmd_output("supervisorctl status") while "STARTING" in out: print ("Waiting for all processes to start...") time.sleep(10) - out = bench.utils.get_cmd_output("sudo supervisorctl status") + out = bench.utils.get_cmd_output("supervisorctl status") tests = [ "{bench_name}-web:{bench_name}-frappe-web[\s]+RUNNING", diff --git a/bench/utils.py b/bench/utils.py index bd42556f..0f534c7a 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -37,6 +37,7 @@ class CommandFailedError(Exception): logger = logging.getLogger(__name__) folders_in_bench = ('apps', 'sites', 'config', 'logs', 'config/pids') +sudoers_file = '/etc/sudoers.d/frappe' class color: @@ -423,16 +424,12 @@ def setup_sudoers(user): if set_permissions: os.chmod('/etc/sudoers', 0o440) - sudoers_file = '/etc/sudoers.d/frappe' - template = env.get_template('frappe_sudoers') frappe_sudoers = template.render(**{ 'user': user, 'service': find_executable('service'), 'systemctl': find_executable('systemctl'), - 'supervisorctl': find_executable('supervisorctl'), 'nginx': find_executable('nginx'), - 'bench': find_executable('bench') }) frappe_sudoers = safe_decode(frappe_sudoers) @@ -548,7 +545,7 @@ def restart_supervisor_processes(bench_path='.', web_workers=False): exec_cmd(cmd, cwd=bench_path) else: - supervisor_status = subprocess.check_output(['sudo', 'supervisorctl', 'status'], cwd=bench_path) + supervisor_status = subprocess.check_output(['supervisorctl', 'status'], cwd=bench_path) supervisor_status = safe_decode(supervisor_status) if web_workers and '{bench_name}-web:'.format(bench_name=bench_name) in supervisor_status: @@ -565,7 +562,7 @@ def restart_supervisor_processes(bench_path='.', web_workers=False): else: group = 'frappe:' - exec_cmd('sudo supervisorctl restart {group}'.format(group=group), cwd=bench_path) + exec_cmd('supervisorctl restart {group}'.format(group=group), cwd=bench_path) def restart_systemd_processes(bench_path='.', web_workers=False): From a03252881a0b0392a149eeb4963d651921783f22 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 13 Mar 2020 17:57:43 +0530 Subject: [PATCH 02/24] fix: get_cmd_output handles exit codes if no output eg: running "supervisorctl status" returns non zero codes in case certain processes its running are in failed state. get_cmd_output will now handle such situations chore: dropped bench.utils.get_program --- bench/utils.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/bench/utils.py b/bench/utils.py index 0f534c7a..464d03a9 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -450,17 +450,11 @@ def setup_logging(bench_path='.'): logger.setLevel(logging.DEBUG) -def get_program(programs): - program = None - for p in programs: - program = find_executable(p) - if program: - break - return program - - def get_process_manager(): - return get_program(['foreman', 'forego', 'honcho']) + for proc_man in ['honcho', 'foreman', 'forego']: + proc_man_path = find_executable(proc_man) + if proc_man_path: + return proc_man_path def start(no_dev=False, concurrency=None, procfile=None): @@ -516,14 +510,15 @@ def check_git_for_shallow_clone(): def get_cmd_output(cmd, cwd='.'): + print("{0}$ {1}{2}".format(color.silver, cmd, color.nc)) try: output = subprocess.check_output(cmd, cwd=cwd, shell=True, stderr=subprocess.PIPE).strip() - output = output.decode('utf-8') - return output except subprocess.CalledProcessError as e: if e.output: - print(e.output) - raise + output = e.output + else: + raise + return safe_decode(output) def safe_encode(what, encoding = 'utf-8'): @@ -545,7 +540,7 @@ def restart_supervisor_processes(bench_path='.', web_workers=False): exec_cmd(cmd, cwd=bench_path) else: - supervisor_status = subprocess.check_output(['supervisorctl', 'status'], cwd=bench_path) + supervisor_status = get_cmd_output('supervisorctl status', cwd=bench_path) supervisor_status = safe_decode(supervisor_status) if web_workers and '{bench_name}-web:'.format(bench_name=bench_name) in supervisor_status: From 39f8223d6d0b82c7b867789a5593f11e4d353bef Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 13 Mar 2020 18:47:05 +0530 Subject: [PATCH 03/24] fix: remove supervisord under service and systemctl in sudoers --- bench/config/templates/frappe_sudoers | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bench/config/templates/frappe_sudoers b/bench/config/templates/frappe_sudoers index 51595c8f..f87e01d5 100644 --- a/bench/config/templates/frappe_sudoers +++ b/bench/config/templates/frappe_sudoers @@ -1,15 +1,16 @@ {% if service %} {{ user }} ALL = (root) {{ service }} {{ user }} ALL = (root) NOPASSWD: {{ service }} nginx * -{{ user }} ALL = (root) NOPASSWD: {{ service }} supervisord * {% endif %} + {% if systemctl %} {{ user }} ALL = (root) {{ systemctl }} {{ user }} ALL = (root) NOPASSWD: {{ systemctl }} * nginx -{{ user }} ALL = (root) NOPASSWD: {{ systemctl }} * supervisord {% endif %} + {% if nginx %} {{ user }} ALL = (root) NOPASSWD: {{ nginx }} {% endif %} + {{ user }} ALL = (root) NOPASSWD: /opt/certbot-auto Defaults:{{ user }} !requiretty From d2a70badd3c61bd50b1fdc5dfd456ed77601f446 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 13 Mar 2020 18:48:22 +0530 Subject: [PATCH 04/24] chore: optimized and removed deprecated api usage and imports --- bench/config/production_setup.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/bench/config/production_setup.py b/bench/config/production_setup.py index e6d09098..42729a22 100755 --- a/bench/config/production_setup.py +++ b/bench/config/production_setup.py @@ -1,12 +1,15 @@ -from bench.utils import get_program, exec_cmd, get_cmd_output, fix_prod_setup_perms, get_bench_name, find_executable, CommandFailedError -from bench.config.supervisor import generate_supervisor_config -from bench.config.systemd import generate_systemd_config -from bench.config.nginx import make_nginx_conf -from bench.config.common_site_config import get_config -import os, subprocess +# imports - standard imports +import os import sys from distutils.spawn import find_executable +# imports - module imports +from bench.config.common_site_config import get_config +from bench.config.nginx import make_nginx_conf +from bench.config.supervisor import generate_supervisor_config +from bench.config.systemd import generate_systemd_config +from bench.utils import CommandFailedError, exec_cmd, fix_prod_setup_perms, get_bench_name, get_cmd_output + def setup_production_prerequisites(): if not find_executable("ansible"): @@ -55,6 +58,7 @@ def setup_production(user, bench_path='.', yes=False): reload_nginx() + def disable_production(bench_path='.'): bench_name = get_bench_name(bench_path) @@ -77,10 +81,11 @@ def disable_production(bench_path='.'): reload_nginx() + def service(service, option): - if os.path.basename(get_program(['systemctl']) or '') == 'systemctl' and is_running_systemd(): + if os.path.basename(find_executable('systemctl') or '') == 'systemctl' and is_running_systemd(): exec_cmd("sudo {service_manager} {option} {service}".format(service_manager='systemctl', option=option, service=service)) - elif os.path.basename(get_program(['service']) or '') == 'service': + elif os.path.basename(find_executable('service') or '') == 'service': exec_cmd("sudo {service_manager} {service} {option} ".format(service_manager='service', service=service, option=option)) else: # look for 'service_manager' and 'service_manager_command' in environment @@ -93,12 +98,14 @@ def service(service, option): else: raise Exception('No service manager found') + def get_supervisor_confdir(): possiblities = ('/etc/supervisor/conf.d', '/etc/supervisor.d/', '/etc/supervisord/conf.d', '/etc/supervisord.d') for possiblity in possiblities: if os.path.exists(possiblity): return possiblity + def remove_default_nginx_configs(): default_nginx_configs = ['/etc/nginx/conf.d/default.conf', '/etc/nginx/sites-enabled/default'] @@ -110,6 +117,7 @@ def remove_default_nginx_configs(): def is_centos7(): return os.path.exists('/etc/redhat-release') and get_cmd_output("cat /etc/redhat-release | sed 's/Linux\ //g' | cut -d' ' -f3 | cut -d. -f1").strip() == '7' + def is_running_systemd(): with open('/proc/1/comm') as f: comm = f.read().strip() @@ -119,6 +127,7 @@ def is_running_systemd(): return True return False + def reload_supervisor(): supervisorctl = find_executable('supervisorctl') @@ -153,7 +162,7 @@ def reload_supervisor(): def reload_nginx(): try: - subprocess.check_output(['sudo', find_executable('nginx'), '-t']) + exec_cmd('sudo {0} -t'.format(find_executable('nginx'))) except: raise From 1f0a5ce5569ae2846294c73565f82bd356ae2b1d Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 13 Mar 2020 19:38:13 +0530 Subject: [PATCH 05/24] fix: update supervisord.conf while generating supervisor.conf --- bench/config/supervisor.py | 44 ++++++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 60db0018..10e88916 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -1,17 +1,26 @@ -import os, getpass, click +# imports - standard imports +import configparser +import getpass +import os + +# imports - module imports import bench +from bench.app import get_current_frappe_version, use_rq +from bench.utils import get_bench_name, find_executable +from bench.config.common_site_config import get_config, update_config, get_gunicorn_workers + +# imports - third party imports +import click + def generate_supervisor_config(bench_path, user=None, yes=False): - from bench.app import get_current_frappe_version, use_rq - from bench.utils import get_bench_name, find_executable - from bench.config.common_site_config import get_config, update_config, get_gunicorn_workers - - template = bench.env.get_template('supervisor.conf') if not user: user = getpass.getuser() - config = get_config(bench_path=bench_path) + update_supervisord_conf(user=user) + template = bench.env.get_template('supervisor.conf') + config = get_config(bench_path=bench_path) bench_dir = os.path.abspath(bench_path) config = template.render(**{ @@ -44,3 +53,24 @@ def generate_supervisor_config(bench_path, user=None, yes=False): update_config({'restart_supervisor_on_update': True}, bench_path=bench_path) update_config({'restart_systemd_on_update': False}, bench_path=bench_path) + +def get_supervisord_conf(): + possibilities = ("supervisord.conf", "etc/supervisord.conf", "/etc/supervisord.conf", "/etc/supervisor/supervisord.conf", "/etc/supervisord.conf") + + for possibility in possibilities: + if os.path.exists(possibility): + return possibility + + +def update_supervisord_conf(user): + """From bench v5.0, we're moving to supervisor running as user""" + supervisord_conf = get_supervisord_conf() or "supervisord.conf" + section = "unix_http_server" + + config = configparser.ConfigParser() + config.read(supervisord_conf) + config[section]["chmod"] = "0760" + config[section]["chown"] = user + + with open(supervisord_conf, "w") as f: + config.write(f) From 48f70aca65b8627dce6ff124ad1509c9641f1895 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 13 Mar 2020 20:01:37 +0530 Subject: [PATCH 06/24] fix: restart supervisor service after conf update --- bench/config/supervisor.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 10e88916..141e97a6 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -8,6 +8,7 @@ import bench from bench.app import get_current_frappe_version, use_rq from bench.utils import get_bench_name, find_executable from bench.config.common_site_config import get_config, update_config, get_gunicorn_workers +from bench.config.production_setup import service # imports - third party imports import click @@ -74,3 +75,6 @@ def update_supervisord_conf(user): with open(supervisord_conf, "w") as f: config.write(f) + + # restart supervisor to take new changes into effect + service('supervisor', 'restart') From e12208dc25a3c034fc0a39dc17f52ce3e2e77d76 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sun, 15 Mar 2020 19:31:27 +0530 Subject: [PATCH 07/24] fix: update supervisord and sudoers for old benches --- bench/cli.py | 3 ++- bench/config/templates/frappe_sudoers | 4 ++++ bench/patches/patches.txt | 1 + bench/patches/v5/__init__.py | 0 bench/patches/v5/fix_user_permissions.py | 17 +++++++++++++++++ 5 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 bench/patches/v5/__init__.py create mode 100644 bench/patches/v5/fix_user_permissions.py diff --git a/bench/cli.py b/bench/cli.py index ad1c90fa..68c96f45 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -8,6 +8,7 @@ from bench.commands import bench_command logger = logging.getLogger('bench') from_command_line = False +change_uid_msg = "You should not run this command as root" def cli(): global from_command_line @@ -72,7 +73,7 @@ def change_uid(): drop_privileges(uid_name=frappe_user, gid_name=frappe_user) os.environ['HOME'] = pwd.getpwnam(frappe_user).pw_dir else: - log('You should not run this command as root', level=3) + log(change_uid_msg, level=3) sys.exit(1) def old_frappe_cli(bench_path='.'): diff --git a/bench/config/templates/frappe_sudoers b/bench/config/templates/frappe_sudoers index f87e01d5..567ccf06 100644 --- a/bench/config/templates/frappe_sudoers +++ b/bench/config/templates/frappe_sudoers @@ -1,3 +1,6 @@ +# This file is auto-generated by frappe/bench +# To re-generate this file, run "bench setup sudoers" + {% if service %} {{ user }} ALL = (root) {{ service }} {{ user }} ALL = (root) NOPASSWD: {{ service }} nginx * @@ -14,3 +17,4 @@ {{ user }} ALL = (root) NOPASSWD: /opt/certbot-auto Defaults:{{ user }} !requiretty + diff --git a/bench/patches/patches.txt b/bench/patches/patches.txt index 754068c5..7cb1a076 100644 --- a/bench/patches/patches.txt +++ b/bench/patches/patches.txt @@ -4,3 +4,4 @@ bench.patches.v3.redis_bind_ip bench.patches.v4.update_node bench.patches.v4.update_socketio bench.patches.v4.install_yarn #2 +bench.patches.v5.fix_user_permissions diff --git a/bench/patches/v5/__init__.py b/bench/patches/v5/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/bench/patches/v5/fix_user_permissions.py b/bench/patches/v5/fix_user_permissions.py new file mode 100644 index 00000000..3bea08a6 --- /dev/null +++ b/bench/patches/v5/fix_user_permissions.py @@ -0,0 +1,17 @@ +# imports - standard imports +import subprocess + +# imports - module imports +from bench.utils import log, get_cmd_output, exec_cmd +from bench.cli import change_uid_msg + + +def execute(bench_path): + """fix supervisor using root then remove bench sudo later + chronology samajhiye""" + cmd = ["sudo", "-n", "bench"] + is_bench_sudoers_set = (not subprocess.call(cmd)) or (change_uid_msg in get_cmd_output(cmd)) + + if is_bench_sudoers_set: + exec_cmd("sudo bench setup supervisor --yes") + exec_cmd("sudo bench setup sudoers") From f6292bba774da6c6d1f85f733cfb257cacb16901 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sun, 15 Mar 2020 19:32:46 +0530 Subject: [PATCH 08/24] chore: added logging and removed unnecessary print in get_cmd_output --- bench/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/utils.py b/bench/utils.py index 464d03a9..bfc4cda0 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -437,6 +437,7 @@ def setup_sudoers(user): f.write(frappe_sudoers) os.chmod(sudoers_file, 0o440) + log("Sudoers was set up for user {}".format(user), level=1) def setup_logging(bench_path='.'): @@ -510,7 +511,6 @@ def check_git_for_shallow_clone(): def get_cmd_output(cmd, cwd='.'): - print("{0}$ {1}{2}".format(color.silver, cmd, color.nc)) try: output = subprocess.check_output(cmd, cwd=cwd, shell=True, stderr=subprocess.PIPE).strip() except subprocess.CalledProcessError as e: From 395b8df895f22e4ca1d225744ab8c44b5bde896a Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sun, 15 Mar 2020 19:40:46 +0530 Subject: [PATCH 09/24] fix: install supervisor on user if doesnt exist --- bench/patches/v5/fix_user_permissions.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/bench/patches/v5/fix_user_permissions.py b/bench/patches/v5/fix_user_permissions.py index 3bea08a6..98a8e327 100644 --- a/bench/patches/v5/fix_user_permissions.py +++ b/bench/patches/v5/fix_user_permissions.py @@ -1,16 +1,20 @@ # imports - standard imports import subprocess +import sys # imports - module imports -from bench.utils import log, get_cmd_output, exec_cmd +from bench.utils import log, get_cmd_output, exec_cmd, which from bench.cli import change_uid_msg def execute(bench_path): - """fix supervisor using root then remove bench sudo later - chronology samajhiye""" cmd = ["sudo", "-n", "bench"] + is_bench_sudoers_set = (not subprocess.call(cmd)) or (change_uid_msg in get_cmd_output(cmd)) + is_supervisor_installed = which('supervisorctl') + + if not is_supervisor_installed: + exec_cmd("{} -m pip install supervisor".format(sys.executable)) if is_bench_sudoers_set: exec_cmd("sudo bench setup supervisor --yes") From 1f0b78f2fe3ed1448b64ecbbe61b1635f574aceb Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sun, 15 Mar 2020 19:41:09 +0530 Subject: [PATCH 10/24] chore: update tests with updated sudoers conf --- bench/tests/test_setup_production.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bench/tests/test_setup_production.py b/bench/tests/test_setup_production.py index 8d0bbc7a..e0a60b17 100644 --- a/bench/tests/test_setup_production.py +++ b/bench/tests/test_setup_production.py @@ -61,6 +61,9 @@ class TestSetupProduction(TestBenchBase): def assert_sudoers(self, user): sudoers_file = '/etc/sudoers.d/frappe' + service = bench.utils.which("service") + nginx = bench.utils.which("nginx") + self.assertTrue(self.file_exists(sudoers_file)) if os.environ.get("CI"): @@ -69,9 +72,8 @@ class TestSetupProduction(TestBenchBase): with open(sudoers_file, 'r') as f: sudoers = f.read() - self.assertTrue('{user} ALL = (root) NOPASSWD: /usr/sbin/service nginx *'.format(user=user) in sudoers) - self.assertTrue('{user} ALL = (root) NOPASSWD: /usr/bin/supervisorctl'.format(user=user) in sudoers) - self.assertTrue('{user} ALL = (root) NOPASSWD: /usr/sbin/nginx'.format(user=user) in sudoers) + self.assertTrue('{user} ALL = (root) NOPASSWD: {service} nginx *'.format(service=service, user=user) in sudoers) + self.assertTrue('{user} ALL = (root) NOPASSWD: {nginx}'.format(nginx=nginx, user=user) in sudoers) def assert_supervisor_config(self, bench_name, use_rq=True): From 9e7b8d1f2e570a477bbdda22e68c0dd49ef3519a Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sun, 15 Mar 2020 22:50:52 +0530 Subject: [PATCH 11/24] fix: avoid circular imports and optimization --- bench/config/supervisor.py | 3 ++- bench/patches/v5/fix_user_permissions.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 141e97a6..7b4ff227 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -8,7 +8,6 @@ import bench from bench.app import get_current_frappe_version, use_rq from bench.utils import get_bench_name, find_executable from bench.config.common_site_config import get_config, update_config, get_gunicorn_workers -from bench.config.production_setup import service # imports - third party imports import click @@ -65,6 +64,8 @@ def get_supervisord_conf(): def update_supervisord_conf(user): """From bench v5.0, we're moving to supervisor running as user""" + from bench.config.production_setup import service + supervisord_conf = get_supervisord_conf() or "supervisord.conf" section = "unix_http_server" diff --git a/bench/patches/v5/fix_user_permissions.py b/bench/patches/v5/fix_user_permissions.py index 98a8e327..fba8d9f0 100644 --- a/bench/patches/v5/fix_user_permissions.py +++ b/bench/patches/v5/fix_user_permissions.py @@ -3,7 +3,7 @@ import subprocess import sys # imports - module imports -from bench.utils import log, get_cmd_output, exec_cmd, which +from bench.utils import get_cmd_output, exec_cmd, which from bench.cli import change_uid_msg From c1507c284868cf5f325229f19a6f184a18fe81f9 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 16 Mar 2020 00:52:04 +0530 Subject: [PATCH 12/24] chore: py2-3 compatible configparser --- bench/config/supervisor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 7b4ff227..a1168d87 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -1,5 +1,4 @@ # imports - standard imports -import configparser import getpass import os @@ -11,6 +10,7 @@ from bench.config.common_site_config import get_config, update_config, get_gunic # imports - third party imports import click +from six.moves import configparser def generate_supervisor_config(bench_path, user=None, yes=False): From 849c751e93afb231bcfba354f4eb90e12485448a Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 16 Mar 2020 11:46:11 +0530 Subject: [PATCH 13/24] chore: dont create supervisord conf if doesnt exist via bench.config.supervisor.update_supervisord_conf --- bench/config/supervisor.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index a1168d87..40b8a92f 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -66,9 +66,12 @@ def update_supervisord_conf(user): """From bench v5.0, we're moving to supervisor running as user""" from bench.config.production_setup import service - supervisord_conf = get_supervisord_conf() or "supervisord.conf" + supervisord_conf = get_supervisord_conf() section = "unix_http_server" + if not supervisord_conf: + return + config = configparser.ConfigParser() config.read(supervisord_conf) config[section]["chmod"] = "0760" From fe02844c69a849b4c1a826dbd16a4f5dc28127bc Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 16 Mar 2020 12:04:18 +0530 Subject: [PATCH 14/24] fix: run fix_user_permissions only if production or sudoers is set up --- bench/config/supervisor.py | 3 -- bench/patches/v5/fix_user_permissions.py | 39 +++++++++++++++++------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 40b8a92f..eadd0a7f 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -69,9 +69,6 @@ def update_supervisord_conf(user): supervisord_conf = get_supervisord_conf() section = "unix_http_server" - if not supervisord_conf: - return - config = configparser.ConfigParser() config.read(supervisord_conf) config[section]["chmod"] = "0760" diff --git a/bench/patches/v5/fix_user_permissions.py b/bench/patches/v5/fix_user_permissions.py index fba8d9f0..9da2ee99 100644 --- a/bench/patches/v5/fix_user_permissions.py +++ b/bench/patches/v5/fix_user_permissions.py @@ -1,21 +1,38 @@ # imports - standard imports +import os import subprocess -import sys # imports - module imports -from bench.utils import get_cmd_output, exec_cmd, which from bench.cli import change_uid_msg +from bench.config.production_setup import get_supervisor_confdir, is_centos7 +from bench.utils import exec_cmd, get_bench_name, get_cmd_output + + +def is_sudoers_set(): + cmd = ["sudo", "-n", "bench"] + return (not subprocess.call(cmd)) or (change_uid_msg in get_cmd_output(cmd)) + + +def is_production_set(bench_path): + production_setup = False + bench_name = get_bench_name(bench_path) + + supervisor_conf_extn = "ini" if is_centos7() else "conf" + supervisor_conf_file_name = '{bench_name}.{extn}'.format(bench_name=bench_name, extn=supervisor_conf_extn) + supervisor_conf = os.path.join(get_supervisor_confdir(), supervisor_conf_file_name) + + if os.path.exists(supervisor_conf): + production_setup = production_setup or True + + nginx_conf = '/etc/nginx/conf.d/{bench_name}.conf'.format(bench_name=bench_name) + + if os.path.exists(nginx_conf): + production_setup = production_setup or True + + return production_setup def execute(bench_path): - cmd = ["sudo", "-n", "bench"] - - is_bench_sudoers_set = (not subprocess.call(cmd)) or (change_uid_msg in get_cmd_output(cmd)) - is_supervisor_installed = which('supervisorctl') - - if not is_supervisor_installed: - exec_cmd("{} -m pip install supervisor".format(sys.executable)) - - if is_bench_sudoers_set: + if is_sudoers_set() or is_production_set(bench_path): exec_cmd("sudo bench setup supervisor --yes") exec_cmd("sudo bench setup sudoers") From 78dbdcb3f18193ba03a54722a9e4a244896907b9 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 16 Mar 2020 12:25:55 +0530 Subject: [PATCH 15/24] fix: run patches for current user --- bench/config/supervisor.py | 5 ++++- bench/patches/v5/fix_user_permissions.py | 11 ++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index eadd0a7f..8b043daf 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -69,10 +69,13 @@ def update_supervisord_conf(user): supervisord_conf = get_supervisord_conf() section = "unix_http_server" + if not supervisord_conf: + return + config = configparser.ConfigParser() config.read(supervisord_conf) config[section]["chmod"] = "0760" - config[section]["chown"] = user + config[section]["chown"] = "{user}:{user}".format(user=user) with open(supervisord_conf, "w") as f: config.write(f) diff --git a/bench/patches/v5/fix_user_permissions.py b/bench/patches/v5/fix_user_permissions.py index 9da2ee99..41322472 100644 --- a/bench/patches/v5/fix_user_permissions.py +++ b/bench/patches/v5/fix_user_permissions.py @@ -1,4 +1,5 @@ # imports - standard imports +import getpass import os import subprocess @@ -33,6 +34,10 @@ def is_production_set(bench_path): def execute(bench_path): - if is_sudoers_set() or is_production_set(bench_path): - exec_cmd("sudo bench setup supervisor --yes") - exec_cmd("sudo bench setup sudoers") + user = getpass.getuser() + + if is_sudoers_set(): + exec_cmd("sudo bench setup sudoers {user}".format(user=user)) + + if is_production_set(bench_path): + exec_cmd("sudo bench setup supervisor --yes --user {user}".format(user=user)) From 3e99fbc46b04f5493684884dae98b94c56c1d4bf Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 16 Mar 2020 13:16:43 +0530 Subject: [PATCH 16/24] fix: add section in conf if doesnt exist --- bench/config/supervisor.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 8b043daf..d97f3559 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -74,6 +74,10 @@ def update_supervisord_conf(user): config = configparser.ConfigParser() config.read(supervisord_conf) + + if section not in config.sections(): + config.add_section(section) + config[section]["chmod"] = "0760" config[section]["chown"] = "{user}:{user}".format(user=user) From 9f091383535a09e634b944b048d0fd1e18ad14ee Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 16 Mar 2020 14:31:20 +0530 Subject: [PATCH 17/24] chore: use set instead of get in config object --- bench/config/supervisor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index d97f3559..253a1a21 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -78,8 +78,8 @@ def update_supervisord_conf(user): if section not in config.sections(): config.add_section(section) - config[section]["chmod"] = "0760" - config[section]["chown"] = "{user}:{user}".format(user=user) + config.set(section, "chmod", "0760") + config.set(section, "chown", "{user}:{user}".format(user=user)) with open(supervisord_conf, "w") as f: config.write(f) From 2fe5cce6d445612c0a26b2bfbb0ea3a7dc20a66f Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 16 Mar 2020 17:31:08 +0530 Subject: [PATCH 18/24] fix: dont drop permissions for setting up supervisor --- bench/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/cli.py b/bench/cli.py index 68c96f45..eea1acc7 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -49,7 +49,7 @@ def check_uid(): sys.exit(1) def cmd_requires_root(): - if len(sys.argv) > 2 and sys.argv[2] in ('production', 'sudoers', 'lets-encrypt', 'fonts', + if len(sys.argv) > 2 and sys.argv[2] in ('production', 'sudoers', 'supervisor', 'lets-encrypt', 'fonts', 'print', 'firewall', 'ssh-port', 'role', 'fail2ban', 'wildcard-ssl'): return True if len(sys.argv) >= 2 and sys.argv[1] in ('patch', 'renew-lets-encrypt', 'disable-production', From a6f72c770f821944aa9cf45654416044f833c44c Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 17 Mar 2020 17:25:30 +0530 Subject: [PATCH 19/24] fix: use frappe_user from site_config and use getpass as fallback --- bench/patches/v5/fix_user_permissions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bench/patches/v5/fix_user_permissions.py b/bench/patches/v5/fix_user_permissions.py index 41322472..13bac4cb 100644 --- a/bench/patches/v5/fix_user_permissions.py +++ b/bench/patches/v5/fix_user_permissions.py @@ -6,6 +6,7 @@ import subprocess # imports - module imports from bench.cli import change_uid_msg from bench.config.production_setup import get_supervisor_confdir, is_centos7 +from bench.config.common_site_config import get_config from bench.utils import exec_cmd, get_bench_name, get_cmd_output @@ -34,7 +35,7 @@ def is_production_set(bench_path): def execute(bench_path): - user = getpass.getuser() + user = get_config('.').get("frappe_user") or getpass.getuser() if is_sudoers_set(): exec_cmd("sudo bench setup sudoers {user}".format(user=user)) From 6bb30e314848d33ebc595b2e32745caf64b89a6e Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 18 Mar 2020 20:16:41 +0530 Subject: [PATCH 20/24] feat: allow raise option in get_cmd_output reason: if command exits without output, it raises a CalledProcessError --- bench/patches/v5/fix_user_permissions.py | 2 +- bench/utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bench/patches/v5/fix_user_permissions.py b/bench/patches/v5/fix_user_permissions.py index 13bac4cb..346c78b7 100644 --- a/bench/patches/v5/fix_user_permissions.py +++ b/bench/patches/v5/fix_user_permissions.py @@ -12,7 +12,7 @@ from bench.utils import exec_cmd, get_bench_name, get_cmd_output def is_sudoers_set(): cmd = ["sudo", "-n", "bench"] - return (not subprocess.call(cmd)) or (change_uid_msg in get_cmd_output(cmd)) + return (not subprocess.call(cmd)) or (change_uid_msg in get_cmd_output(cmd, _raise=False)) def is_production_set(bench_path): diff --git a/bench/utils.py b/bench/utils.py index bfc4cda0..7b1a4742 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -510,13 +510,13 @@ def check_git_for_shallow_clone(): return True -def get_cmd_output(cmd, cwd='.'): +def get_cmd_output(cmd, cwd='.', _raise=True): try: output = subprocess.check_output(cmd, cwd=cwd, shell=True, stderr=subprocess.PIPE).strip() except subprocess.CalledProcessError as e: if e.output: output = e.output - else: + elif _raise: raise return safe_decode(output) From d51311b7f7caedec77c3474f3232358bd2aba0f5 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 19 Mar 2020 18:52:49 +0530 Subject: [PATCH 21/24] fix: better exception handling for checking sudoers set --- bench/patches/v5/fix_user_permissions.py | 14 +++++++++++++- bench/utils.py | 1 + 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/bench/patches/v5/fix_user_permissions.py b/bench/patches/v5/fix_user_permissions.py index 346c78b7..c11a9480 100644 --- a/bench/patches/v5/fix_user_permissions.py +++ b/bench/patches/v5/fix_user_permissions.py @@ -12,7 +12,19 @@ from bench.utils import exec_cmd, get_bench_name, get_cmd_output def is_sudoers_set(): cmd = ["sudo", "-n", "bench"] - return (not subprocess.call(cmd)) or (change_uid_msg in get_cmd_output(cmd, _raise=False)) + + with open(os.devnull, "wb") as f: + return_code_check = not subprocess.call(cmd, stdout=f) + + if return_code_check: + try: + bench_warn = change_uid_msg in get_cmd_output(cmd, _raise=False) + except Exception: + bench_warn = False + finally: + return_code_check = return_code_check and bench_warn + + return return_code_check def is_production_set(bench_path): diff --git a/bench/utils.py b/bench/utils.py index 7b1a4742..19fe9036 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -511,6 +511,7 @@ def check_git_for_shallow_clone(): def get_cmd_output(cmd, cwd='.', _raise=True): + output = "" try: output = subprocess.check_output(cmd, cwd=cwd, shell=True, stderr=subprocess.PIPE).strip() except subprocess.CalledProcessError as e: From af8f74db469dd3ffdce7f2c002e47d34b136d6bc Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 26 Mar 2020 13:09:23 +0530 Subject: [PATCH 22/24] chore(deepsource): update docstrings for public functions --- bench/config/production_setup.py | 1 + bench/config/supervisor.py | 2 ++ bench/patches/v5/fix_user_permissions.py | 3 +++ 3 files changed, 6 insertions(+) diff --git a/bench/config/production_setup.py b/bench/config/production_setup.py index 42729a22..9dc4af69 100755 --- a/bench/config/production_setup.py +++ b/bench/config/production_setup.py @@ -12,6 +12,7 @@ from bench.utils import CommandFailedError, exec_cmd, fix_prod_setup_perms, get_ def setup_production_prerequisites(): + """Installs ansible, fail2banc, NGINX and supervisor""" if not find_executable("ansible"): exec_cmd("sudo {0} -m pip install ansible".format(sys.executable)) if not find_executable("fail2ban-client"): diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 253a1a21..43560cf0 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -14,6 +14,7 @@ from six.moves import configparser def generate_supervisor_config(bench_path, user=None, yes=False): + """Generate supervisor config for respective bench path""" if not user: user = getpass.getuser() @@ -55,6 +56,7 @@ def generate_supervisor_config(bench_path, user=None, yes=False): def get_supervisord_conf(): + """Returns path of supervisord config from possible paths""" possibilities = ("supervisord.conf", "etc/supervisord.conf", "/etc/supervisord.conf", "/etc/supervisor/supervisord.conf", "/etc/supervisord.conf") for possibility in possibilities: diff --git a/bench/patches/v5/fix_user_permissions.py b/bench/patches/v5/fix_user_permissions.py index c11a9480..87d804d4 100644 --- a/bench/patches/v5/fix_user_permissions.py +++ b/bench/patches/v5/fix_user_permissions.py @@ -11,6 +11,7 @@ from bench.utils import exec_cmd, get_bench_name, get_cmd_output def is_sudoers_set(): + """Check if bench sudoers is set""" cmd = ["sudo", "-n", "bench"] with open(os.devnull, "wb") as f: @@ -28,6 +29,7 @@ def is_sudoers_set(): def is_production_set(bench_path): + """Check if production is set for current bench""" production_setup = False bench_name = get_bench_name(bench_path) @@ -47,6 +49,7 @@ def is_production_set(bench_path): def execute(bench_path): + """This patch checks if bench sudoers is set and regenerate supervisor and sudoers files""" user = get_config('.').get("frappe_user") or getpass.getuser() if is_sudoers_set(): From cd77b4225583687922f74c48b186d7589e5b412b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 26 Mar 2020 14:15:59 +0530 Subject: [PATCH 23/24] refactor: avoid redefining name from different scopes ref: deepsource https://deepsource.io/gh/frappe/bench/run/c9ee328a-6f59-49a9-b21c-8e770bd14091/python/PYL-W0621/ --- bench/commands/setup.py | 75 ++++++++++++-------------------- bench/config/production_setup.py | 12 +++-- bench/utils.py | 24 +++++----- 3 files changed, 49 insertions(+), 62 deletions(-) diff --git a/bench/commands/setup.py b/bench/commands/setup.py index 84e267c7..96360bea 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -2,12 +2,21 @@ import os import sys -# imports - module imports -from bench.utils import exec_cmd - # imports - third party imports -from six import PY3 import click +from six import PY3 + +# imports - module imports +import bench.config.lets_encrypt +import bench.config.nginx +import bench.config.procfile +import bench.config.production_setup +import bench.config.redis +import bench.config.site_config +import bench.config.supervisor + +import bench.utils +from bench.utils import exec_cmd, run_playbook @click.group(help="Setup command group for enabling setting up a Frappe environment") @@ -18,70 +27,59 @@ def setup(): @click.command("sudoers", help="Add commands to sudoers list for execution without password") @click.argument("user") def setup_sudoers(user): - from bench.utils import setup_sudoers - setup_sudoers(user) + bench.utils.setup_sudoers(user) @click.command("nginx", help="Generate configuration files for NGINX") @click.option("--yes", help="Yes to regeneration of nginx config file", default=False, is_flag=True) def setup_nginx(yes=False): - from bench.config.nginx import make_nginx_conf - make_nginx_conf(bench_path=".", yes=yes) + bench.config.nginx.make_nginx_conf(bench_path=".", yes=yes) @click.command("reload-nginx", help="Checks NGINX config file and reloads service") def reload_nginx(): - from bench.config.production_setup import reload_nginx - reload_nginx() + bench.config.production_setup.reload_nginx() @click.command("supervisor", help="Generate configuration for supervisor") @click.option("--user", help="optional user argument") @click.option("--yes", help="Yes to regeneration of supervisor config", is_flag=True, default=False) def setup_supervisor(user=None, yes=False): - from bench.config.supervisor import generate_supervisor_config - generate_supervisor_config(bench_path=".", user=user, yes=yes) + bench.config.supervisor.generate_supervisor_config(bench_path=".", user=user, yes=yes) @click.command("redis", help="Generates configuration for Redis") def setup_redis(): - from bench.config.redis import generate_config - generate_config(".") + bench.config.redis.generate_config(".") @click.command("fonts", help="Add Frappe fonts to system") def setup_fonts(): - from bench.utils import setup_fonts - setup_fonts() + bench.utils.setup_fonts() @click.command("production", help="Setup Frappe production environment for specific user") @click.argument("user") @click.option("--yes", help="Yes to regeneration config", is_flag=True, default=False) def setup_production(user, yes=False): - from bench.config.production_setup import setup_production - setup_production(user=user, yes=yes) + bench.config.production_setup.setup_production(user=user, yes=yes) @click.command("backups", help="Add cronjob for bench backups") def setup_backups(): - from bench.utils import setup_backups - setup_backups() + bench.utils.setup_backups() @click.command("env", help="Setup virtualenv for bench") @click.option("--python", type = str, default = "python3", help = "Path to Python Executable.") def setup_env(python="python3"): - from bench.utils import setup_env - setup_env(python=python) + bench.utils.setup_env(python=python) @click.command("firewall", help="Setup firewall for system") @click.option("--ssh_port") @click.option("--force") def setup_firewall(ssh_port=None, force=False): - from bench.utils import run_playbook - if not force: click.confirm("Setting up the firewall will block all ports except 80, 443 and {0}\nDo you want to continue?".format(ssh_port), abort=True) @@ -95,8 +93,6 @@ def setup_firewall(ssh_port=None, force=False): @click.argument("port") @click.option("--force") def set_ssh_port(port, force=False): - from bench.utils import run_playbook - if not force: click.confirm("This will change your SSH Port to {}\nDo you want to continue?".format(port), abort=True) @@ -108,8 +104,7 @@ def set_ssh_port(port, force=False): @click.option("--custom-domain") @click.option('-n', '--non-interactive', default=False, is_flag=True, help="Run command non-interactively. This flag restarts nginx and runs certbot non interactively. Shouldn't be used on 1'st attempt") def setup_letsencrypt(site, custom_domain, non_interactive): - from bench.config.lets_encrypt import setup_letsencrypt - setup_letsencrypt(site, custom_domain, bench_path=".", interactive=not non_interactive) + bench.config.lets_encrypt.setup_letsencrypt(site, custom_domain, bench_path=".", interactive=not non_interactive) @click.command("wildcard-ssl", help="Setup wildcard SSL certificate for multi-tenant bench") @@ -117,20 +112,17 @@ def setup_letsencrypt(site, custom_domain, non_interactive): @click.option("--email") @click.option("--exclude-base-domain", default=False, is_flag=True, help="SSL Certificate not applicable for base domain") def setup_wildcard_ssl(domain, email, exclude_base_domain): - from bench.config.lets_encrypt import setup_wildcard_ssl - setup_wildcard_ssl(domain, email, bench_path=".", exclude_base_domain=exclude_base_domain) + bench.config.lets_encrypt.setup_wildcard_ssl(domain, email, bench_path=".", exclude_base_domain=exclude_base_domain) @click.command("procfile", help="Generate Procfile for bench start") def setup_procfile(): - from bench.config.procfile import setup_procfile - setup_procfile(".") + bench.config.procfile.setup_procfile(".") @click.command("socketio", help="Setup node dependencies for socketio server") def setup_socketio(): - from bench.utils import setup_socketio - setup_socketio() + bench.utils.setup_socketio() @click.command("requirements", help="Setup Python and Node dependencies") @@ -203,34 +195,28 @@ def setup_config(): @click.option("--ssl-certificate-key", help="Absolute path to SSL Certificate Key") def add_domain(domain, site=None, ssl_certificate=None, ssl_certificate_key=None): """Add custom domain to site""" - from bench.config.site_config import add_domain - if not site: print("Please specify site") sys.exit(1) - add_domain(site, domain, ssl_certificate, ssl_certificate_key, bench_path=".") + bench.config.site_config.add_domain(site, domain, ssl_certificate, ssl_certificate_key, bench_path=".") @click.command("remove-domain", help="Remove custom domain from a site") @click.argument("domain") @click.option("--site", prompt=True) def remove_domain(domain, site=None): - from bench.config.site_config import remove_domain - if not site: print("Please specify site") sys.exit(1) - remove_domain(site, domain, bench_path=".") + bench.config.site_config.remove_domain(site, domain, bench_path=".") @click.command("sync-domains", help="Check if there is a change in domains. If yes, updates the domains list.") @click.option("--domain", multiple=True) @click.option("--site", prompt=True) def sync_domains(domain=None, site=None): - from bench.config.site_config import sync_domains - if not site: print("Please specify site") sys.exit(1) @@ -241,7 +227,7 @@ def sync_domains(domain=None, site=None): print("Domains should be a json list of strings or dictionaries") sys.exit(1) - changed = sync_domains(site, domains, bench_path=".") + changed = bench.config.site_config.sync_domains(site, domains, bench_path=".") # if changed, success, else failure sys.exit(0 if changed else 1) @@ -253,8 +239,6 @@ def sync_domains(domain=None, site=None): @click.option("--mysql_root_password") @click.option("--container", is_flag=True, default=False) def setup_roles(role, **kwargs): - from bench.utils import run_playbook - extra_vars = {"production": True} extra_vars.update(kwargs) @@ -269,7 +253,6 @@ def setup_roles(role, **kwargs): @click.option("--bantime", default=600, help="The counter is set to zero if no match is found within 'findtime' seconds. Default is 600 seconds") @click.option("--findtime", default=600, help="Duration (in seconds) for IP to be banned for. Negative number for 'permanent' ban. Default is 600 seconds") def setup_nginx_proxy_jail(**kwargs): - from bench.utils import run_playbook run_playbook("roles/fail2ban/tasks/configure_nginx_jail.yml", extra_vars=kwargs) diff --git a/bench/config/production_setup.py b/bench/config/production_setup.py index 9dc4af69..dff6a402 100755 --- a/bench/config/production_setup.py +++ b/bench/config/production_setup.py @@ -83,17 +83,21 @@ def disable_production(bench_path='.'): reload_nginx() -def service(service, option): +def service(service_name, service_option): if os.path.basename(find_executable('systemctl') or '') == 'systemctl' and is_running_systemd(): - exec_cmd("sudo {service_manager} {option} {service}".format(service_manager='systemctl', option=option, service=service)) + systemctl_cmd = "sudo {service_manager} {service_option} {service_name}" + exec_cmd(systemctl_cmd.format(service_manager='systemctl', service_option=service_option, service_name=service_name)) + elif os.path.basename(find_executable('service') or '') == 'service': - exec_cmd("sudo {service_manager} {service} {option} ".format(service_manager='service', service=service, option=option)) + service_cmd = "sudo {service_manager} {service_name} {service_option}" + exec_cmd(service_cmd.format(service_manager='service', service_name=service_name, service_option=service_option)) + else: # look for 'service_manager' and 'service_manager_command' in environment service_manager = os.environ.get("BENCH_SERVICE_MANAGER") if service_manager: service_manager_command = (os.environ.get("BENCH_SERVICE_MANAGER_COMMAND") - or "{service_manager} {option} {service}").format(service_manager=service_manager, service=service, option=option) + or "{service_manager} {service_option} {service}").format(service_manager=service_manager, service=service, service_option=service_option) exec_cmd(service_manager_command) else: diff --git a/bench/utils.py b/bench/utils.py index 19fe9036..6362f305 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -66,10 +66,10 @@ def log(message, level=0): 2: color.red + 'ERROR', # fail 3: color.yellow + 'WARN' # warn/suggest } - start = (levels.get(level) + ': ') if level in levels else '' - end = '\033[0m' + start_line = (levels.get(level) + ': ') if level in levels else '' + end_line = '\033[0m' - print(start + message + end) + print(start_line + message + end_line) def safe_decode(string, encoding = 'utf-8'): @@ -1110,8 +1110,8 @@ def migrate_env(python, backup=False): from bench.config.common_site_config import get_config from bench.app import get_apps - log = logging.getLogger(__name__) - log.setLevel(logging.DEBUG) + logger = logging.getLogger(__name__) + logger.setLevel(logging.DEBUG) nvenv = 'env' path = os.getcwd() @@ -1127,12 +1127,12 @@ def migrate_env(python, backup=False): redis = '{redis} -p {port}'.format(redis=which('redis-cli'), port=rredis.port) - log.debug('Clearing Redis Cache...') + logger.debug('Clearing Redis Cache...') exec_cmd('{redis} FLUSHALL'.format(redis = redis)) - log.debug('Clearing Redis DataBase...') + logger.debug('Clearing Redis DataBase...') exec_cmd('{redis} FLUSHDB'.format(redis = redis)) except: - log.warn('Please ensure Redis Connections are running or Daemonized.') + logger.warn('Please ensure Redis Connections are running or Daemonized.') # Backup venv: restore using `virtualenv --relocatable` if needed if backup: @@ -1143,7 +1143,7 @@ def migrate_env(python, backup=False): source = os.path.join(path, 'env') target = parch - log.debug('Backing up Virtual Environment') + logger.debug('Backing up Virtual Environment') stamp = datetime.now().strftime('%Y%m%d_%H%M%S') dest = os.path.join(path, str(stamp)) @@ -1152,15 +1152,15 @@ def migrate_env(python, backup=False): # Create virtualenv using specified python try: - log.debug('Setting up a New Virtual {} Environment'.format(python)) + logger.debug('Setting up a New Virtual {} Environment'.format(python)) exec_cmd('{virtualenv} --python {python} {pvenv}'.format(virtualenv=virtualenv, python=python, pvenv=pvenv)) apps = ' '.join(["-e {}".format(os.path.join("apps", app)) for app in get_apps()]) exec_cmd('{0} install -q -U {1}'.format(pip, apps)) - log.debug('Migration Successful to {}'.format(python)) + logger.debug('Migration Successful to {}'.format(python)) except: - log.debug('Migration Error') + logger.debug('Migration Error') raise From d5f8ad8e20380c42f24a87e29453e1475864919c Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 26 Mar 2020 14:18:39 +0530 Subject: [PATCH 24/24] chore: specific Exception handled ref: https://deepsource.io/gh/frappe/bench/run/c9ee328a-6f59-49a9-b21c-8e770bd14091/python/PYL-W0703 --- bench/patches/v5/fix_user_permissions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/patches/v5/fix_user_permissions.py b/bench/patches/v5/fix_user_permissions.py index 87d804d4..bdccf53c 100644 --- a/bench/patches/v5/fix_user_permissions.py +++ b/bench/patches/v5/fix_user_permissions.py @@ -20,7 +20,7 @@ def is_sudoers_set(): if return_code_check: try: bench_warn = change_uid_msg in get_cmd_output(cmd, _raise=False) - except Exception: + except subprocess.CalledProcessError: bench_warn = False finally: return_code_check = return_code_check and bench_warn