From c43da5be68b770cea30385b5f2169deb69f2caef Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 10 Jun 2020 18:15:01 +0530 Subject: [PATCH 01/10] fix: update supervisord conf and reload, log everything --- bench/commands/setup.py | 1 + bench/config/production_setup.py | 3 ++- bench/config/supervisor.py | 39 ++++++++++++++++++++++++-------- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/bench/commands/setup.py b/bench/commands/setup.py index a7fb886f..08e2d4f2 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -43,6 +43,7 @@ def reload_nginx(): @click.option("--user", help="optional user argument") @click.option("--yes", help="Yes to regeneration of supervisor config", is_flag=True, default=False) def setup_supervisor(user=None, yes=False): + bench.config.supervisor.update_supervisord_config(user=user, yes=yes) bench.config.supervisor.generate_supervisor_config(bench_path=".", user=user, yes=yes) diff --git a/bench/config/production_setup.py b/bench/config/production_setup.py index 4b5fc774..e22d4a14 100755 --- a/bench/config/production_setup.py +++ b/bench/config/production_setup.py @@ -5,7 +5,7 @@ import sys # imports - module imports from bench.config.common_site_config import get_config from bench.config.nginx import make_nginx_conf -from bench.config.supervisor import generate_supervisor_config +from bench.config.supervisor import generate_supervisor_config, update_supervisord_config from bench.config.systemd import generate_systemd_config from bench.utils import CommandFailedError, exec_cmd, find_executable, fix_prod_setup_perms, get_bench_name, get_cmd_output @@ -30,6 +30,7 @@ def setup_production(user, bench_path='.', yes=False): if get_config(bench_path).get('restart_systemd_on_update'): generate_systemd_config(bench_path=bench_path, user=user, yes=yes) else: + update_supervisord_config(user=user, yes=yes) generate_supervisor_config(bench_path=bench_path, user=user, yes=yes) make_nginx_conf(bench_path=bench_path, yes=yes) fix_prod_setup_perms(bench_path, frappe_user=user) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 9e9b7a89..c57b86ff 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -1,5 +1,6 @@ # imports - standard imports import getpass +import logging import os # imports - module imports @@ -13,13 +14,14 @@ import click from six.moves import configparser +logger = logging.getLogger(bench.PROJECT_NAME) + + def generate_supervisor_config(bench_path, user=None, yes=False): """Generate supervisor config for respective bench path""" if not user: user = getpass.getuser() - update_supervisord_conf(user=user) - template = bench.config.env.get_template('supervisor.conf') config = get_config(bench_path=bench_path) bench_dir = os.path.abspath(bench_path) @@ -64,12 +66,17 @@ def get_supervisord_conf(): return possibility -def update_supervisord_conf(user): - """From bench v5.0, we're moving to supervisor running as user""" +def update_supervisord_config(user=None, yes=False): + """From bench v5.x, we're moving to supervisor running as user""" from bench.config.production_setup import service supervisord_conf = get_supervisord_conf() section = "unix_http_server" + updated_values = { + "chmod": "0760", + "chown": "{user}:{user}".format(user=user) + } + supervisord_conf_updated = False if not supervisord_conf: return @@ -79,12 +86,24 @@ def update_supervisord_conf(user): if section not in config.sections(): config.add_section(section) + supervisord_conf_updated = True - config.set(section, "chmod", "0760") - config.set(section, "chown", "{user}:{user}".format(user=user)) + for key, value in updated_values.items(): + current_value = config[section].get(key, "") + if current_value.strip() != value: + config.set(section, key, value) + supervisord_conf_updated = True + logger.log("Updated supervisord config: '{0}' changed from '{1}' to '{2}'".format(key, current_value, value)) - with open(supervisord_conf, "w") as f: - config.write(f) + if not supervisord_conf_updated: + return - # restart supervisor to take new changes into effect - service('supervisor', 'restart') + try: + with open(supervisord_conf, "w") as f: + config.write(f) + logger.log("Updated supervisord config at '{0}'".format(supervisord_conf)) + except Exception as e: + logger.log("Updating supervisord config failed due to '{0}'".format(e)) + + # Reread supervisor configuration, reload supervisord and supervisorctl, restart services that were started + service('supervisor', 'reload') From deb854b9d92304102f3daf33a37e0367621dc359 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 10 Jun 2020 18:22:15 +0530 Subject: [PATCH 02/10] fix: add confirmation for supervisord conf update --- bench/config/supervisor.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index c57b86ff..1625d08f 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -76,7 +76,7 @@ def update_supervisord_config(user=None, yes=False): "chmod": "0760", "chown": "{user}:{user}".format(user=user) } - supervisord_conf_updated = False + supervisord_conf_changes = "" if not supervisord_conf: return @@ -86,18 +86,24 @@ def update_supervisord_config(user=None, yes=False): if section not in config.sections(): config.add_section(section) - supervisord_conf_updated = True + action = "Section {0} Added".format(section) + logger.log(action) + supervisord_conf_changes += '\n' + action for key, value in updated_values.items(): current_value = config[section].get(key, "") if current_value.strip() != value: config.set(section, key, value) - supervisord_conf_updated = True - logger.log("Updated supervisord config: '{0}' changed from '{1}' to '{2}'".format(key, current_value, value)) + action = "Updated supervisord config: '{0}' changed from '{1}' to '{2}'".format(key, current_value, value) + logger.log(action) + supervisord_conf_changes += '\n' + action - if not supervisord_conf_updated: + if not supervisord_conf_changes: return + if not yes: + click.confirm("{0} will be updated with the following values:\n{1}\nDo you want to continue?".format(supervisord_conf, supervisord_conf_changes), abort=True) + try: with open(supervisord_conf, "w") as f: config.write(f) From 26ddfe73e2d8ea3c983f3004b11368add46ebc5d Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 10 Jun 2020 19:22:21 +0530 Subject: [PATCH 03/10] chore: use print statement for editable warning instead --- bench/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/cli.py b/bench/cli.py index bf0b5231..0b7fd692 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -33,7 +33,7 @@ def cli(): change_uid() if is_dist_editable(bench.PROJECT_NAME) and len(sys.argv) > 1 and sys.argv[1] != "src": - log("bench is installed in editable mode!\n\nThis is not the recommended mode of installation for production. Instead, install the package from PyPI with: `pip install frappe-bench`\n", level=3) + print("\033[93mWARN: bench is installed in editable mode!\n\nThis is not the recommended mode of installation for production. Instead, install the package from PyPI with: `pip install frappe-bench`\033[0m\n") if not is_bench_directory() and not cmd_requires_root() and len(sys.argv) > 1 and sys.argv[1] not in ("init", "find", "src"): log("Command not being executed in bench directory", level=3) From 629842a042be936e9bc1d42cdb9247ad94a2c2ae Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 10 Jun 2020 19:23:10 +0530 Subject: [PATCH 04/10] fix: show if service manager not found, also log it --- bench/config/production_setup.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bench/config/production_setup.py b/bench/config/production_setup.py index e22d4a14..3d4b826c 100755 --- a/bench/config/production_setup.py +++ b/bench/config/production_setup.py @@ -1,13 +1,18 @@ # imports - standard imports import os +import logging import sys # imports - module imports +import bench from bench.config.common_site_config import get_config from bench.config.nginx import make_nginx_conf from bench.config.supervisor import generate_supervisor_config, update_supervisord_config from bench.config.systemd import generate_systemd_config -from bench.utils import CommandFailedError, exec_cmd, find_executable, fix_prod_setup_perms, get_bench_name, get_cmd_output +from bench.utils import CommandFailedError, exec_cmd, find_executable, fix_prod_setup_perms, get_bench_name, get_cmd_output, log + + +logger = logging.getLogger(bench.PROJECT_NAME) def setup_production_prerequisites(): @@ -101,7 +106,7 @@ def service(service_name, service_option): exec_cmd(service_manager_command) else: - raise Exception('No service manager found') + log("No service manager found: '{0} {1}' failed to execute".format(service_name, service_option), level=2) def get_supervisor_confdir(): From fc4b90acdb576da1516afd8a2234742e2a7f090e Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 10 Jun 2020 19:25:35 +0530 Subject: [PATCH 05/10] fix: add logging and set user if not passed --- bench/config/supervisor.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 1625d08f..82fb364d 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -70,6 +70,9 @@ def update_supervisord_config(user=None, yes=False): """From bench v5.x, we're moving to supervisor running as user""" from bench.config.production_setup import service + if not user: + user = getpass.getuser() + supervisord_conf = get_supervisord_conf() section = "unix_http_server" updated_values = { @@ -79,6 +82,7 @@ def update_supervisord_config(user=None, yes=False): supervisord_conf_changes = "" if not supervisord_conf: + logger.log("supervisord.conf not found") return config = configparser.ConfigParser() @@ -94,11 +98,12 @@ def update_supervisord_config(user=None, yes=False): current_value = config[section].get(key, "") if current_value.strip() != value: config.set(section, key, value) - action = "Updated supervisord config: '{0}' changed from '{1}' to '{2}'".format(key, current_value, value) + action = "Updated supervisord.conf: '{0}' changed from '{1}' to '{2}'".format(key, current_value, value) logger.log(action) supervisord_conf_changes += '\n' + action if not supervisord_conf_changes: + logger.log("supervisord.conf not updated") return if not yes: @@ -107,9 +112,9 @@ def update_supervisord_config(user=None, yes=False): try: with open(supervisord_conf, "w") as f: config.write(f) - logger.log("Updated supervisord config at '{0}'".format(supervisord_conf)) + logger.log("Updated supervisord.conf at '{0}'".format(supervisord_conf)) except Exception as e: - logger.log("Updating supervisord config failed due to '{0}'".format(e)) + logger.log("Updating supervisord.conf failed due to '{0}'".format(e)) # Reread supervisor configuration, reload supervisord and supervisorctl, restart services that were started service('supervisor', 'reload') From fb5e1501057e4e0b62cd508175830d2558b8c309 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 13 Jul 2020 11:39:14 +0530 Subject: [PATCH 06/10] fix: added verbosity --- bench/config/production_setup.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bench/config/production_setup.py b/bench/config/production_setup.py index 3d4b826c..6809b733 100755 --- a/bench/config/production_setup.py +++ b/bench/config/production_setup.py @@ -28,15 +28,20 @@ def setup_production_prerequisites(): def setup_production(user, bench_path='.', yes=False): + print("Setting Up prerequisites...") setup_production_prerequisites() if get_config(bench_path).get('restart_supervisor_on_update') and get_config(bench_path).get('restart_systemd_on_update'): raise Exception("You cannot use supervisor and systemd at the same time. Modify your common_site_config accordingly." ) if get_config(bench_path).get('restart_systemd_on_update'): + print("Setting Up systemd...") generate_systemd_config(bench_path=bench_path, user=user, yes=yes) else: + print("Setting Up supervisor...") update_supervisord_config(user=user, yes=yes) generate_supervisor_config(bench_path=bench_path, user=user, yes=yes) + + print("Setting Up NGINX...") make_nginx_conf(bench_path=bench_path, yes=yes) fix_prod_setup_perms(bench_path, frappe_user=user) remove_default_nginx_configs() @@ -44,6 +49,7 @@ def setup_production(user, bench_path='.', yes=False): bench_name = get_bench_name(bench_path) nginx_conf = '/etc/nginx/conf.d/{bench_name}.conf'.format(bench_name=bench_name) + print("Setting Up symlinks and reloading services...") if get_config(bench_path).get('restart_supervisor_on_update'): supervisor_conf_extn = "ini" if is_centos7() else "conf" supervisor_conf = os.path.join(get_supervisor_confdir(), '{bench_name}.{extn}'.format( From 851b4c3275bde7608d26804b9134d93944be3b7e Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 13 Jul 2020 12:04:21 +0530 Subject: [PATCH 07/10] fix(tests): add --yes flag to production setup --- bench/tests/test_setup_production.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/tests/test_setup_production.py b/bench/tests/test_setup_production.py index e0a60b17..5f983c71 100644 --- a/bench/tests/test_setup_production.py +++ b/bench/tests/test_setup_production.py @@ -19,7 +19,7 @@ class TestSetupProduction(TestBenchBase): for bench_name in ("test-bench-1", "test-bench-2"): bench_path = os.path.join(os.path.abspath(self.benches_path), bench_name) self.init_bench(bench_name) - bench.utils.exec_cmd("sudo bench setup production {0}".format(user), cwd=bench_path) + bench.utils.exec_cmd("sudo bench setup production {0} --yes".format(user), cwd=bench_path) self.assert_nginx_config(bench_name) self.assert_supervisor_config(bench_name) self.assert_supervisor_process(bench_name) From 115d88faf3c246e224ab25b94e134c0cf0c20063 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 13 Jul 2020 18:20:05 +0530 Subject: [PATCH 08/10] debug: check if there's an exception being raised silently --- bench/config/production_setup.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/bench/config/production_setup.py b/bench/config/production_setup.py index 6809b733..4d6f241a 100755 --- a/bench/config/production_setup.py +++ b/bench/config/production_setup.py @@ -38,7 +38,15 @@ def setup_production(user, bench_path='.', yes=False): generate_systemd_config(bench_path=bench_path, user=user, yes=yes) else: print("Setting Up supervisor...") - update_supervisord_config(user=user, yes=yes) + try: + update_supervisord_config(user=user, yes=yes) + except Exception: + import sys, traceback + exc_type, exc_value, exc_tb = sys.exc_info() + trace_list = traceback.format_exception(exc_type, exc_value, exc_tb) + body = "".join(str(t) for t in trace_list) + print(body) + generate_supervisor_config(bench_path=bench_path, user=user, yes=yes) print("Setting Up NGINX...") From 312a92d0caf0bfb76803c0071eba9cc55abd52c7 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 13 Jul 2020 18:36:21 +0530 Subject: [PATCH 09/10] fix: compatible configparser get --- bench/config/supervisor.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 82fb364d..8788757f 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -95,7 +95,11 @@ def update_supervisord_config(user=None, yes=False): supervisord_conf_changes += '\n' + action for key, value in updated_values.items(): - current_value = config[section].get(key, "") + try: + current_value = config.get(section, key) + except configparser.NoOptionError: + current_value = "" + if current_value.strip() != value: config.set(section, key, value) action = "Updated supervisord.conf: '{0}' changed from '{1}' to '{2}'".format(key, current_value, value) From 8173b009f61a5177affff4fdcbb80a44ede58a87 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 13 Jul 2020 18:43:18 +0530 Subject: [PATCH 10/10] Revert "debug: check if there's an exception being raised silently" This reverts commit 115d88faf3c246e224ab25b94e134c0cf0c20063. --- bench/config/production_setup.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/bench/config/production_setup.py b/bench/config/production_setup.py index 4d6f241a..6809b733 100755 --- a/bench/config/production_setup.py +++ b/bench/config/production_setup.py @@ -38,15 +38,7 @@ def setup_production(user, bench_path='.', yes=False): generate_systemd_config(bench_path=bench_path, user=user, yes=yes) else: print("Setting Up supervisor...") - try: - update_supervisord_config(user=user, yes=yes) - except Exception: - import sys, traceback - exc_type, exc_value, exc_tb = sys.exc_info() - trace_list = traceback.format_exception(exc_type, exc_value, exc_tb) - body = "".join(str(t) for t in trace_list) - print(body) - + update_supervisord_config(user=user, yes=yes) generate_supervisor_config(bench_path=bench_path, user=user, yes=yes) print("Setting Up NGINX...")