From 2c5d1188b8a197dae5a45ac882d059afa5f69220 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 22 May 2020 17:17:35 +0530 Subject: [PATCH 01/45] chore: bump bench to v5.1.0 --- bench/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/__init__.py b/bench/__init__.py index f4ceb0c0..dd6d2c46 100644 --- a/bench/__init__.py +++ b/bench/__init__.py @@ -1,4 +1,4 @@ -VERSION = "5.0.0" +VERSION = "5.1.0" PROJECT_NAME = "frappe-bench" FRAPPE_VERSION = None From cac837547c67fdfd89d3c7f4d916000600732e2b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 27 May 2020 19:56:45 +0530 Subject: [PATCH 02/45] fix: maintain worker and worker.error logfile in dev mode --- bench/config/templates/Procfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bench/config/templates/Procfile b/bench/config/templates/Procfile index e557f561..f810506e 100644 --- a/bench/config/templates/Procfile +++ b/bench/config/templates/Procfile @@ -11,9 +11,9 @@ watch: bench watch {% endif %} {% if use_rq -%} schedule: bench schedule -worker_short: bench worker --queue short --quiet -worker_long: bench worker --queue long --quiet -worker_default: bench worker --queue default --quiet +worker_short: bench worker --queue short 1>> logs/worker.log 2>> logs/worker.error.log +worker_long: bench worker --queue long 1>> logs/worker.log 2>> logs/worker.error.log +worker_default: bench worker --queue default 1>> logs/worker.log 2>> logs/worker.error.log {% else %} workerbeat: sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app beat -s scheduler.schedule' worker: sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app worker -n jobs@%h -Ofair --soft-time-limit 360 --time-limit 390' From 649997efba90d3f1bbaa7364a4647a6a6106d5be Mon Sep 17 00:00:00 2001 From: gavin Date: Thu, 28 May 2020 20:15:04 +0530 Subject: [PATCH 03/45] docs: re-link installation docs --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 0b2d03da..6d4be94d 100755 --- a/README.md +++ b/README.md @@ -147,6 +147,11 @@ For more information and advanced setup instructions, check out the [Easy Instal ### Manual Installation Although not recommended, some might want to manually setup a bench instance locally for development. To quickly get started on installing bench the hard way, you can follow [Installing Bench and Frappe](https://frappe.io/docs/user/en/installation). + + ```sh + $ pip install frappe-bench + ``` +Next you'll have to set up the pre-requisites for setting up a Frappe Environment. Checkout [docs/installation](https://github.com/frappe/bench/blob/master/docs/installation.md) for more information on this. For more extensive distribution-dependent documentation, check out the following guides: From a728416c08450e19c6e251d23d5429ee4fb36a68 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 29 May 2020 10:31:52 +0530 Subject: [PATCH 04/45] fix: update bench install methods --- README.md | 13 +++++++------ docs/installation.md | 7 ++----- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 6d4be94d..9c521edb 100755 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@
- +

Bench

@@ -141,17 +141,18 @@ In case the setup fails, the log file is saved under `/tmp/logs/install_bench.lo - Create an Issue in this repository with the log file attached. - Search for an existing issue or post the log file on the [Frappe/ERPNext Discuss Forum](https://discuss.erpnext.com/c/bench) with the tag `installation_problem` under "Install/Update" category. -For more information and advanced setup instructions, check out the [Easy Install Documentation](https://github.com/frappe/bench/blob/master/docs/easy_install.md). +For more information and advanced setup instructions, check out the [Easy Install Documentation](https://github.com/frappe/bench/blob/develop/docs/easy_install.md). ### Manual Installation Although not recommended, some might want to manually setup a bench instance locally for development. To quickly get started on installing bench the hard way, you can follow [Installing Bench and Frappe](https://frappe.io/docs/user/en/installation). - + ```sh $ pip install frappe-bench ``` -Next you'll have to set up the pre-requisites for setting up a Frappe Environment. Checkout [docs/installation](https://github.com/frappe/bench/blob/master/docs/installation.md) for more information on this. + +Next you'll have to set up the pre-requisites for setting up a Frappe Environment. Checkout [docs/installation](https://github.com/frappe/bench/blob/develop/docs/installation.md) for more information on this. For more extensive distribution-dependent documentation, check out the following guides: @@ -205,12 +206,12 @@ For more extensive distribution-dependent documentation, check out the following ``` -For more in-depth information on commands and their usage, follow [Commands and Usage](https://github.com/frappe/bench/blob/master/docs/commands_and_usage.md). As for a consolidated list of bench commands, check out [Bench Usage](https://github.com/frappe/bench/blob/master/docs/bench_usage.md). +For more in-depth information on commands and their usage, follow [Commands and Usage](https://github.com/frappe/bench/blob/develop/docs/commands_and_usage.md). As for a consolidated list of bench commands, check out [Bench Usage](https://github.com/frappe/bench/blob/develop/docs/bench_usage.md). ## Custom Bench Commands -If you wish to extend the capabilities of bench with your own custom Frappe Application, you may follow [Adding Custom Bench Commands](https://github.com/frappe/bench/blob/master/docs/bench_custom_cmd.md). +If you wish to extend the capabilities of bench with your own custom Frappe Application, you may follow [Adding Custom Bench Commands](https://github.com/frappe/bench/blob/develop/docs/bench_custom_cmd.md). ## Bench Manager diff --git a/docs/installation.md b/docs/installation.md index a7e0f06a..3d4d004e 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -30,9 +30,6 @@ To manually install frappe/erpnext, you can follow this [this wiki](https://gith #### 2. Install Bench -Install bench as a *non root* user, +Install the latest bench using pip - git clone https://github.com/frappe/bench ~/.bench - pip3 install --user -e ~/.bench - -Note: Please do not remove the bench directory the above commands will create + pip3 install frappe-bench From 5a6c8262a18914509a16ee7f58e98db7cd7e1ac7 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 29 May 2020 10:39:15 +0530 Subject: [PATCH 05/45] docs: minor restructure for manual installation --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9c521edb..51ecdade 100755 --- a/README.md +++ b/README.md @@ -146,13 +146,14 @@ For more information and advanced setup instructions, check out the [Easy Instal ### Manual Installation -Although not recommended, some might want to manually setup a bench instance locally for development. To quickly get started on installing bench the hard way, you can follow [Installing Bench and Frappe](https://frappe.io/docs/user/en/installation). +Some might want to manually setup a bench instance locally for development. To quickly get started on installing bench the hard way, you can follow the guide on [Installing Bench and the Frappe Framework](https://frappe.io/docs/user/en/installation). - ```sh - $ pip install frappe-bench - ``` +You'll have to set up the system dependencies required for setting up a Frappe Environment. Checkout [docs/installation](https://github.com/frappe/bench/blob/develop/docs/installation.md) for more information on this. If you've already set up, install bench via pip: -Next you'll have to set up the pre-requisites for setting up a Frappe Environment. Checkout [docs/installation](https://github.com/frappe/bench/blob/develop/docs/installation.md) for more information on this. + +```sh +$ pip install frappe-bench +``` For more extensive distribution-dependent documentation, check out the following guides: From 9eca22bac406dfb1cc0b8f9bfb4fe89f4a0e2576 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 29 May 2020 11:15:34 +0530 Subject: [PATCH 06/45] chore: dont run travis for docs updates --- .travis.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index aa4416c6..768e8c14 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ dist: bionic sudo: true git: - depth: 1 + depth: 10 cache: - pip @@ -70,6 +70,16 @@ matrix: env: TEST=easy_install script: sudo python $TRAVIS_BUILD_DIR/install.py --user travis --run-travis --production --verbose +before_install: +- | + if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then + TRAVIS_COMMIT_RANGE="FETCH_HEAD...$TRAVIS_BRANCH" + fi + git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '\.md$|^docs' || { + echo "Only docs were updated, stopping build process." + exit + } + install: - pip install urllib3 pyOpenSSL ndg-httpsclient pyasn1 From 45684ca685a38f46b97c5d7139336dccb138e538 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 29 May 2020 11:32:55 +0530 Subject: [PATCH 07/45] chore: depth 2 should suffice --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 768e8c14..19efc21b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ dist: bionic sudo: true git: - depth: 10 + depth: 2 cache: - pip From e26a0a9b37237bcf97a3a0df9bb66937e56b6439 Mon Sep 17 00:00:00 2001 From: gavin Date: Tue, 2 Jun 2020 13:32:20 +0530 Subject: [PATCH 08/45] chore: fixed image src --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 51ecdade..d9c2d1bb 100755 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@
- +

Bench

From fdcf01d45ca5ec8e63b06b4ba224360080c0802e Mon Sep 17 00:00:00 2001 From: Nikunj Patel Date: Tue, 9 Jun 2020 12:25:26 +0200 Subject: [PATCH 09/45] For ubuntu >=18.04 use libssl1.1 to support upgrade of OS For ubuntu >=18.04 use libssl1.1 to support upgrade of OS since libssl1.0-dev is not supported by Ubuntu20.04 LTS. --- bench/playbooks/roles/common/tasks/ubuntu.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bench/playbooks/roles/common/tasks/ubuntu.yml b/bench/playbooks/roles/common/tasks/ubuntu.yml index 527e42c2..cdc56065 100644 --- a/bench/playbooks/roles/common/tasks/ubuntu.yml +++ b/bench/playbooks/roles/common/tasks/ubuntu.yml @@ -33,9 +33,9 @@ - name: install pdf prerequisites for Ubuntu >= 18.04 apt: pkg: - - libssl1.0-dev + - libssl1.1 state: present force: yes when: ansible_distribution_version is version_compare('18.04', 'ge') -... \ No newline at end of file +... From 52ee67f298adb9939d9cee1266250968f35e8c0b Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Wed, 10 Jun 2020 14:11:33 +0530 Subject: [PATCH 10/45] fix: trailing slash deleting apps folder --- bench/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/app.py b/bench/app.py index 8b165b2b..d20e5a8c 100755 --- a/bench/app.py +++ b/bench/app.py @@ -116,7 +116,7 @@ def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=Fal sys.exit(1) # Gets repo name from URL - repo_name = git_url.rsplit('/', 1)[1].rsplit('.', 1)[0] + repo_name = git_url.rstrip('/').rsplit('/', 1)[1].rsplit('.', 1)[0] shallow_clone = '--depth 1' if check_git_for_shallow_clone() else '' branch = '--branch {branch}'.format(branch=branch) if branch else '' else: From c43da5be68b770cea30385b5f2169deb69f2caef Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 10 Jun 2020 18:15:01 +0530 Subject: [PATCH 11/45] fix: update supervisord conf and reload, log everything --- bench/commands/setup.py | 1 + bench/config/production_setup.py | 3 ++- bench/config/supervisor.py | 39 ++++++++++++++++++++++++-------- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/bench/commands/setup.py b/bench/commands/setup.py index a7fb886f..08e2d4f2 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -43,6 +43,7 @@ def reload_nginx(): @click.option("--user", help="optional user argument") @click.option("--yes", help="Yes to regeneration of supervisor config", is_flag=True, default=False) def setup_supervisor(user=None, yes=False): + bench.config.supervisor.update_supervisord_config(user=user, yes=yes) bench.config.supervisor.generate_supervisor_config(bench_path=".", user=user, yes=yes) diff --git a/bench/config/production_setup.py b/bench/config/production_setup.py index 4b5fc774..e22d4a14 100755 --- a/bench/config/production_setup.py +++ b/bench/config/production_setup.py @@ -5,7 +5,7 @@ import sys # imports - module imports from bench.config.common_site_config import get_config from bench.config.nginx import make_nginx_conf -from bench.config.supervisor import generate_supervisor_config +from bench.config.supervisor import generate_supervisor_config, update_supervisord_config from bench.config.systemd import generate_systemd_config from bench.utils import CommandFailedError, exec_cmd, find_executable, fix_prod_setup_perms, get_bench_name, get_cmd_output @@ -30,6 +30,7 @@ def setup_production(user, bench_path='.', yes=False): if get_config(bench_path).get('restart_systemd_on_update'): generate_systemd_config(bench_path=bench_path, user=user, yes=yes) else: + update_supervisord_config(user=user, yes=yes) generate_supervisor_config(bench_path=bench_path, user=user, yes=yes) make_nginx_conf(bench_path=bench_path, yes=yes) fix_prod_setup_perms(bench_path, frappe_user=user) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 9e9b7a89..c57b86ff 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -1,5 +1,6 @@ # imports - standard imports import getpass +import logging import os # imports - module imports @@ -13,13 +14,14 @@ import click from six.moves import configparser +logger = logging.getLogger(bench.PROJECT_NAME) + + def generate_supervisor_config(bench_path, user=None, yes=False): """Generate supervisor config for respective bench path""" if not user: user = getpass.getuser() - update_supervisord_conf(user=user) - template = bench.config.env.get_template('supervisor.conf') config = get_config(bench_path=bench_path) bench_dir = os.path.abspath(bench_path) @@ -64,12 +66,17 @@ def get_supervisord_conf(): return possibility -def update_supervisord_conf(user): - """From bench v5.0, we're moving to supervisor running as user""" +def update_supervisord_config(user=None, yes=False): + """From bench v5.x, we're moving to supervisor running as user""" from bench.config.production_setup import service supervisord_conf = get_supervisord_conf() section = "unix_http_server" + updated_values = { + "chmod": "0760", + "chown": "{user}:{user}".format(user=user) + } + supervisord_conf_updated = False if not supervisord_conf: return @@ -79,12 +86,24 @@ def update_supervisord_conf(user): if section not in config.sections(): config.add_section(section) + supervisord_conf_updated = True - config.set(section, "chmod", "0760") - config.set(section, "chown", "{user}:{user}".format(user=user)) + for key, value in updated_values.items(): + current_value = config[section].get(key, "") + if current_value.strip() != value: + config.set(section, key, value) + supervisord_conf_updated = True + logger.log("Updated supervisord config: '{0}' changed from '{1}' to '{2}'".format(key, current_value, value)) - with open(supervisord_conf, "w") as f: - config.write(f) + if not supervisord_conf_updated: + return - # restart supervisor to take new changes into effect - service('supervisor', 'restart') + try: + with open(supervisord_conf, "w") as f: + config.write(f) + logger.log("Updated supervisord config at '{0}'".format(supervisord_conf)) + except Exception as e: + logger.log("Updating supervisord config failed due to '{0}'".format(e)) + + # Reread supervisor configuration, reload supervisord and supervisorctl, restart services that were started + service('supervisor', 'reload') From deb854b9d92304102f3daf33a37e0367621dc359 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 10 Jun 2020 18:22:15 +0530 Subject: [PATCH 12/45] fix: add confirmation for supervisord conf update --- bench/config/supervisor.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index c57b86ff..1625d08f 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -76,7 +76,7 @@ def update_supervisord_config(user=None, yes=False): "chmod": "0760", "chown": "{user}:{user}".format(user=user) } - supervisord_conf_updated = False + supervisord_conf_changes = "" if not supervisord_conf: return @@ -86,18 +86,24 @@ def update_supervisord_config(user=None, yes=False): if section not in config.sections(): config.add_section(section) - supervisord_conf_updated = True + action = "Section {0} Added".format(section) + logger.log(action) + supervisord_conf_changes += '\n' + action for key, value in updated_values.items(): current_value = config[section].get(key, "") if current_value.strip() != value: config.set(section, key, value) - supervisord_conf_updated = True - logger.log("Updated supervisord config: '{0}' changed from '{1}' to '{2}'".format(key, current_value, value)) + action = "Updated supervisord config: '{0}' changed from '{1}' to '{2}'".format(key, current_value, value) + logger.log(action) + supervisord_conf_changes += '\n' + action - if not supervisord_conf_updated: + if not supervisord_conf_changes: return + if not yes: + click.confirm("{0} will be updated with the following values:\n{1}\nDo you want to continue?".format(supervisord_conf, supervisord_conf_changes), abort=True) + try: with open(supervisord_conf, "w") as f: config.write(f) From 26ddfe73e2d8ea3c983f3004b11368add46ebc5d Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 10 Jun 2020 19:22:21 +0530 Subject: [PATCH 13/45] chore: use print statement for editable warning instead --- bench/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/cli.py b/bench/cli.py index bf0b5231..0b7fd692 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -33,7 +33,7 @@ def cli(): change_uid() if is_dist_editable(bench.PROJECT_NAME) and len(sys.argv) > 1 and sys.argv[1] != "src": - log("bench is installed in editable mode!\n\nThis is not the recommended mode of installation for production. Instead, install the package from PyPI with: `pip install frappe-bench`\n", level=3) + print("\033[93mWARN: bench is installed in editable mode!\n\nThis is not the recommended mode of installation for production. Instead, install the package from PyPI with: `pip install frappe-bench`\033[0m\n") if not is_bench_directory() and not cmd_requires_root() and len(sys.argv) > 1 and sys.argv[1] not in ("init", "find", "src"): log("Command not being executed in bench directory", level=3) From 629842a042be936e9bc1d42cdb9247ad94a2c2ae Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 10 Jun 2020 19:23:10 +0530 Subject: [PATCH 14/45] fix: show if service manager not found, also log it --- bench/config/production_setup.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bench/config/production_setup.py b/bench/config/production_setup.py index e22d4a14..3d4b826c 100755 --- a/bench/config/production_setup.py +++ b/bench/config/production_setup.py @@ -1,13 +1,18 @@ # imports - standard imports import os +import logging import sys # imports - module imports +import bench from bench.config.common_site_config import get_config from bench.config.nginx import make_nginx_conf from bench.config.supervisor import generate_supervisor_config, update_supervisord_config from bench.config.systemd import generate_systemd_config -from bench.utils import CommandFailedError, exec_cmd, find_executable, fix_prod_setup_perms, get_bench_name, get_cmd_output +from bench.utils import CommandFailedError, exec_cmd, find_executable, fix_prod_setup_perms, get_bench_name, get_cmd_output, log + + +logger = logging.getLogger(bench.PROJECT_NAME) def setup_production_prerequisites(): @@ -101,7 +106,7 @@ def service(service_name, service_option): exec_cmd(service_manager_command) else: - raise Exception('No service manager found') + log("No service manager found: '{0} {1}' failed to execute".format(service_name, service_option), level=2) def get_supervisor_confdir(): From fc4b90acdb576da1516afd8a2234742e2a7f090e Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 10 Jun 2020 19:25:35 +0530 Subject: [PATCH 15/45] fix: add logging and set user if not passed --- bench/config/supervisor.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 1625d08f..82fb364d 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -70,6 +70,9 @@ def update_supervisord_config(user=None, yes=False): """From bench v5.x, we're moving to supervisor running as user""" from bench.config.production_setup import service + if not user: + user = getpass.getuser() + supervisord_conf = get_supervisord_conf() section = "unix_http_server" updated_values = { @@ -79,6 +82,7 @@ def update_supervisord_config(user=None, yes=False): supervisord_conf_changes = "" if not supervisord_conf: + logger.log("supervisord.conf not found") return config = configparser.ConfigParser() @@ -94,11 +98,12 @@ def update_supervisord_config(user=None, yes=False): current_value = config[section].get(key, "") if current_value.strip() != value: config.set(section, key, value) - action = "Updated supervisord config: '{0}' changed from '{1}' to '{2}'".format(key, current_value, value) + action = "Updated supervisord.conf: '{0}' changed from '{1}' to '{2}'".format(key, current_value, value) logger.log(action) supervisord_conf_changes += '\n' + action if not supervisord_conf_changes: + logger.log("supervisord.conf not updated") return if not yes: @@ -107,9 +112,9 @@ def update_supervisord_config(user=None, yes=False): try: with open(supervisord_conf, "w") as f: config.write(f) - logger.log("Updated supervisord config at '{0}'".format(supervisord_conf)) + logger.log("Updated supervisord.conf at '{0}'".format(supervisord_conf)) except Exception as e: - logger.log("Updating supervisord config failed due to '{0}'".format(e)) + logger.log("Updating supervisord.conf failed due to '{0}'".format(e)) # Reread supervisor configuration, reload supervisord and supervisorctl, restart services that were started service('supervisor', 'reload') From 2fd86c9eb6220ea565daf7a5be5a8a1e22750c60 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 10 Jun 2020 19:27:58 +0530 Subject: [PATCH 16/45] fix: populate bench.log via bench.utils.log --- bench/utils.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bench/utils.py b/bench/utils.py index 7914eecd..d94fc26b 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -70,9 +70,16 @@ def log(message, level=0): 2: color.red + 'ERROR', # fail 3: color.yellow + 'WARN' # warn/suggest } + loggers = { + 2: logger.error, + 3: logger.warning + } + start_line = (levels.get(level) + ': ') if level in levels else '' + level_logger = loggers.get(level, logger.info) end_line = '\033[0m' + level_logger(message) print(start_line + message + end_line) From a6bb58717d3a1ae0fbe735f30d3e7baa1a2e7923 Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Thu, 11 Jun 2020 18:30:00 +0530 Subject: [PATCH 17/45] fix: use remote_addr instead of user-defined header ip $proxy_add_x_forwarded_for sets the user defined ip in the request. changing it to $remote_addr ensures that nginx sets the remote ip itself, disregarding user-specified ip from the request. Signed-off-by: Chinmay D. Pai --- bench/config/templates/nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/config/templates/nginx.conf b/bench/config/templates/nginx.conf index 9df365f3..06fc0b2b 100644 --- a/bench/config/templates/nginx.conf +++ b/bench/config/templates/nginx.conf @@ -85,7 +85,7 @@ server { } location @webserver { - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Frappe-Site-Name {{ site_name }}; proxy_set_header Host $host; From 3336cff2b401f97bcf98dfed3d50601fb7c78ae3 Mon Sep 17 00:00:00 2001 From: Nikunj Patel Date: Wed, 24 Jun 2020 09:10:03 +0200 Subject: [PATCH 18/45] feat(playbooks): wkthmltox for Ubuntu 20 (#1019) * Updating file to support Ubuntu20.04 LTS * Update main.yml * Update bench/playbooks/roles/wkhtmltopdf/tasks/main.yml * Removed checksum from Ubuntu 20 Co-authored-by: gavin --- bench/playbooks/roles/wkhtmltopdf/tasks/main.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bench/playbooks/roles/wkhtmltopdf/tasks/main.yml b/bench/playbooks/roles/wkhtmltopdf/tasks/main.yml index 1656ffd2..aea2f55f 100644 --- a/bench/playbooks/roles/wkhtmltopdf/tasks/main.yml +++ b/bench/playbooks/roles/wkhtmltopdf/tasks/main.yml @@ -20,6 +20,12 @@ force: yes when: ansible_os_family == 'Debian' +- name: download wkthmltox Ubuntu 20 + get_url: + url: https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.focal_amd64.deb + dest: /tmp/wkhtmltox.deb + when: ansible_distribution == 'Ubuntu' and ansible_distribution_major_version == '20' + - name: download wkthmltox Ubuntu 18 get_url: url: https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.deb From 691687edbef00238166cd209717e66cd9fb74944 Mon Sep 17 00:00:00 2001 From: Rohan Bansal Date: Wed, 1 Jul 2020 18:54:41 +0530 Subject: [PATCH 19/45] fix: run backups every 6 hours, rather than every minute per 6 hours --- bench/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/utils.py b/bench/utils.py index d94fc26b..948d62ea 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -400,7 +400,7 @@ def setup_backups(bench_path='.'): if job_command not in str(system_crontab): job = system_crontab.new(command=job_command, comment="bench auto backups set for every 6 hours") - job.hour.every(6) + job.every(6).hours() system_crontab.write() From ce0a3de259d8f90749ff54ef1161b58ebc4f3d8f Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 3 Jul 2020 16:17:49 +0530 Subject: [PATCH 20/45] fix: dont show warning message in developer mode --- bench/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/cli.py b/bench/cli.py index bf0b5231..63369a1d 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -32,7 +32,7 @@ def cli(): change_dir() change_uid() - if is_dist_editable(bench.PROJECT_NAME) and len(sys.argv) > 1 and sys.argv[1] != "src": + if is_dist_editable(bench.PROJECT_NAME) and len(sys.argv) > 1 and sys.argv[1] != "src" and not get_config(".").get("developer_mode"): log("bench is installed in editable mode!\n\nThis is not the recommended mode of installation for production. Instead, install the package from PyPI with: `pip install frappe-bench`\n", level=3) if not is_bench_directory() and not cmd_requires_root() and len(sys.argv) > 1 and sys.argv[1] not in ("init", "find", "src"): From c76df683a149ae1fb2ae5cb9cb77f53a7f8a5157 Mon Sep 17 00:00:00 2001 From: Kenneth Sequeira <33246109+kennethsequeira@users.noreply.github.com> Date: Mon, 6 Jul 2020 11:18:36 +0530 Subject: [PATCH 21/45] chore: typo fix (#1029) Co-authored-by: gavin --- docs/easy_install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/easy_install.md b/docs/easy_install.md index 7bf525d0..3c59f308 100644 --- a/docs/easy_install.md +++ b/docs/easy_install.md @@ -88,4 +88,4 @@ TLDR; Save the logs! 3. A lot of things can go wrong in setting up the environment due to prior settings, company protocols or even breaking changes in system packages and their dependencies. -4. Sharing your logfile in any issues opened related to this can help us find solutions to it faster and make the sript better! +4. Sharing your logfile in any issues opened related to this can help us find solutions to it faster and make the script better! From b237a0d2cf163e62e33bfb7bd964da5d711f3be3 Mon Sep 17 00:00:00 2001 From: gavin Date: Mon, 6 Jul 2020 11:23:16 +0530 Subject: [PATCH 22/45] Revert "chore: dont run travis for docs updates" --- .travis.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 19efc21b..aa4416c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ dist: bionic sudo: true git: - depth: 2 + depth: 1 cache: - pip @@ -70,16 +70,6 @@ matrix: env: TEST=easy_install script: sudo python $TRAVIS_BUILD_DIR/install.py --user travis --run-travis --production --verbose -before_install: -- | - if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then - TRAVIS_COMMIT_RANGE="FETCH_HEAD...$TRAVIS_BRANCH" - fi - git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '\.md$|^docs' || { - echo "Only docs were updated, stopping build process." - exit - } - install: - pip install urllib3 pyOpenSSL ndg-httpsclient pyasn1 From e5c3b591f7c7da9ab2aad62e2996831371ec0890 Mon Sep 17 00:00:00 2001 From: Rohan Bansal Date: Mon, 6 Jul 2020 16:52:09 +0530 Subject: [PATCH 23/45] patch: fix old auto-backup cron jobs --- bench/patches/patches.txt | 1 + bench/patches/v5/fix_backup_cronjob.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 bench/patches/v5/fix_backup_cronjob.py diff --git a/bench/patches/patches.txt b/bench/patches/patches.txt index 7cb1a076..55a65019 100644 --- a/bench/patches/patches.txt +++ b/bench/patches/patches.txt @@ -5,3 +5,4 @@ bench.patches.v4.update_node bench.patches.v4.update_socketio bench.patches.v4.install_yarn #2 bench.patches.v5.fix_user_permissions +bench.patches.v5.fix_backup_cronjob diff --git a/bench/patches/v5/fix_backup_cronjob.py b/bench/patches/v5/fix_backup_cronjob.py new file mode 100644 index 00000000..15bbfcf3 --- /dev/null +++ b/bench/patches/v5/fix_backup_cronjob.py @@ -0,0 +1,15 @@ +from bench.config.common_site_config import get_config +from crontab import CronTab + + +def execute(bench_path): + """ + This patch fixes a cron job that would backup sites every minute per 6 hours + """ + + user = get_config(bench_path=bench_path).get('frappe_user') + user_crontab = CronTab(user=user) + + for job in user_crontab.find_comment("bench auto backups set for every 6 hours"): + job.every(6).hours() + user_crontab.write() From fb5e1501057e4e0b62cd508175830d2558b8c309 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 13 Jul 2020 11:39:14 +0530 Subject: [PATCH 24/45] fix: added verbosity --- bench/config/production_setup.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bench/config/production_setup.py b/bench/config/production_setup.py index 3d4b826c..6809b733 100755 --- a/bench/config/production_setup.py +++ b/bench/config/production_setup.py @@ -28,15 +28,20 @@ def setup_production_prerequisites(): def setup_production(user, bench_path='.', yes=False): + print("Setting Up prerequisites...") setup_production_prerequisites() if get_config(bench_path).get('restart_supervisor_on_update') and get_config(bench_path).get('restart_systemd_on_update'): raise Exception("You cannot use supervisor and systemd at the same time. Modify your common_site_config accordingly." ) if get_config(bench_path).get('restart_systemd_on_update'): + print("Setting Up systemd...") generate_systemd_config(bench_path=bench_path, user=user, yes=yes) else: + print("Setting Up supervisor...") update_supervisord_config(user=user, yes=yes) generate_supervisor_config(bench_path=bench_path, user=user, yes=yes) + + print("Setting Up NGINX...") make_nginx_conf(bench_path=bench_path, yes=yes) fix_prod_setup_perms(bench_path, frappe_user=user) remove_default_nginx_configs() @@ -44,6 +49,7 @@ def setup_production(user, bench_path='.', yes=False): bench_name = get_bench_name(bench_path) nginx_conf = '/etc/nginx/conf.d/{bench_name}.conf'.format(bench_name=bench_name) + print("Setting Up symlinks and reloading services...") if get_config(bench_path).get('restart_supervisor_on_update'): supervisor_conf_extn = "ini" if is_centos7() else "conf" supervisor_conf = os.path.join(get_supervisor_confdir(), '{bench_name}.{extn}'.format( From 851b4c3275bde7608d26804b9134d93944be3b7e Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 13 Jul 2020 12:04:21 +0530 Subject: [PATCH 25/45] fix(tests): add --yes flag to production setup --- bench/tests/test_setup_production.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/tests/test_setup_production.py b/bench/tests/test_setup_production.py index e0a60b17..5f983c71 100644 --- a/bench/tests/test_setup_production.py +++ b/bench/tests/test_setup_production.py @@ -19,7 +19,7 @@ class TestSetupProduction(TestBenchBase): for bench_name in ("test-bench-1", "test-bench-2"): bench_path = os.path.join(os.path.abspath(self.benches_path), bench_name) self.init_bench(bench_name) - bench.utils.exec_cmd("sudo bench setup production {0}".format(user), cwd=bench_path) + bench.utils.exec_cmd("sudo bench setup production {0} --yes".format(user), cwd=bench_path) self.assert_nginx_config(bench_name) self.assert_supervisor_config(bench_name) self.assert_supervisor_process(bench_name) From 115d88faf3c246e224ab25b94e134c0cf0c20063 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 13 Jul 2020 18:20:05 +0530 Subject: [PATCH 26/45] debug: check if there's an exception being raised silently --- bench/config/production_setup.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/bench/config/production_setup.py b/bench/config/production_setup.py index 6809b733..4d6f241a 100755 --- a/bench/config/production_setup.py +++ b/bench/config/production_setup.py @@ -38,7 +38,15 @@ def setup_production(user, bench_path='.', yes=False): generate_systemd_config(bench_path=bench_path, user=user, yes=yes) else: print("Setting Up supervisor...") - update_supervisord_config(user=user, yes=yes) + try: + update_supervisord_config(user=user, yes=yes) + except Exception: + import sys, traceback + exc_type, exc_value, exc_tb = sys.exc_info() + trace_list = traceback.format_exception(exc_type, exc_value, exc_tb) + body = "".join(str(t) for t in trace_list) + print(body) + generate_supervisor_config(bench_path=bench_path, user=user, yes=yes) print("Setting Up NGINX...") From 312a92d0caf0bfb76803c0071eba9cc55abd52c7 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 13 Jul 2020 18:36:21 +0530 Subject: [PATCH 27/45] fix: compatible configparser get --- bench/config/supervisor.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 82fb364d..8788757f 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -95,7 +95,11 @@ def update_supervisord_config(user=None, yes=False): supervisord_conf_changes += '\n' + action for key, value in updated_values.items(): - current_value = config[section].get(key, "") + try: + current_value = config.get(section, key) + except configparser.NoOptionError: + current_value = "" + if current_value.strip() != value: config.set(section, key, value) action = "Updated supervisord.conf: '{0}' changed from '{1}' to '{2}'".format(key, current_value, value) From 8173b009f61a5177affff4fdcbb80a44ede58a87 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 13 Jul 2020 18:43:18 +0530 Subject: [PATCH 28/45] Revert "debug: check if there's an exception being raised silently" This reverts commit 115d88faf3c246e224ab25b94e134c0cf0c20063. --- bench/config/production_setup.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/bench/config/production_setup.py b/bench/config/production_setup.py index 4d6f241a..6809b733 100755 --- a/bench/config/production_setup.py +++ b/bench/config/production_setup.py @@ -38,15 +38,7 @@ def setup_production(user, bench_path='.', yes=False): generate_systemd_config(bench_path=bench_path, user=user, yes=yes) else: print("Setting Up supervisor...") - try: - update_supervisord_config(user=user, yes=yes) - except Exception: - import sys, traceback - exc_type, exc_value, exc_tb = sys.exc_info() - trace_list = traceback.format_exception(exc_type, exc_value, exc_tb) - body = "".join(str(t) for t in trace_list) - print(body) - + update_supervisord_config(user=user, yes=yes) generate_supervisor_config(bench_path=bench_path, user=user, yes=yes) print("Setting Up NGINX...") From 6dc784fe4d33188e4095b1ce6480ff4e746c8eda Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 13 Jul 2020 19:01:51 +0530 Subject: [PATCH 29/45] fix(tests): print traceback from errors --- .travis.yml | 16 ++++++++-------- bench/tests/test_base.py | 6 ++++++ bench/tests/test_init.py | 7 +++++++ bench/tests/test_setup_production.py | 7 +++++++ 4 files changed, 28 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index aa4416c6..0d77579b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,42 +18,42 @@ matrix: - name: "Python 2.7 Basic Setup" python: 2.7 env: TEST=bench - script: python -m unittest -v bench.tests.test_init + script: python -m unittest -v bench.tests.basic - name: "Python 3.6 Basic Setup" python: 3.6 env: TEST=bench - script: python -m unittest -v bench.tests.test_init + script: python -m unittest -v bench.tests.basic - name: "Python 3.7 Basic Setup" python: 3.7 env: TEST=bench - script: python -m unittest -v bench.tests.test_init + script: python -m unittest -v bench.tests.basic - name: "Python 3.8 Production Setup" python: 3.8 env: TEST=bench - script: python -m unittest -v bench.tests.test_setup_production + script: python -m unittest -v bench.tests.production_setup - name: "Python 2.7 Production Setup" python: 2.7 env: TEST=bench - script: python -m unittest -v bench.tests.test_setup_production + script: python -m unittest -v bench.tests.production_setup - name: "Python 3.6 Production Setup" python: 3.6 env: TEST=bench - script: python -m unittest -v bench.tests.test_setup_production + script: python -m unittest -v bench.tests.production_setup - name: "Python 3.7 Production Setup" python: 3.7 env: TEST=bench - script: python -m unittest -v bench.tests.test_setup_production + script: python -m unittest -v bench.tests.production_setup - name: "Python 3.8 Production Setup" python: 3.8 env: TEST=bench - script: python -m unittest -v bench.tests.test_setup_production + script: python -m unittest -v bench.tests.production_setup - name: "Python 3.6 Easy Install" python: 3.6 diff --git a/bench/tests/test_base.py b/bench/tests/test_base.py index 44dc131d..5807ec44 100644 --- a/bench/tests/test_base.py +++ b/bench/tests/test_base.py @@ -102,3 +102,9 @@ class TestBenchBase(unittest.TestCase): if os.environ.get("CI"): return not subprocess.call(["sudo", "test", "-f", path]) return os.path.isfile(path) + + def get_traceback(self): + exc_type, exc_value, exc_tb = sys.exc_info() + trace_list = traceback.format_exception(exc_type, exc_value, exc_tb) + body = "".join(str(t) for t in trace_list) + return body diff --git a/bench/tests/test_init.py b/bench/tests/test_init.py index cf92d630..07b0f96a 100755 --- a/bench/tests/test_init.py +++ b/bench/tests/test_init.py @@ -34,6 +34,13 @@ class TestBenchInit(TestBenchBase): self.assert_config(bench_name) + def basic(self): + try: + self.test_init() + except Exception: + print(self.get_traceback()) + + def test_multiple_benches(self): for bench_name in ("test-bench-1", "test-bench-2"): self.init_bench(bench_name) diff --git a/bench/tests/test_setup_production.py b/bench/tests/test_setup_production.py index e0a60b17..9e9f0bd2 100644 --- a/bench/tests/test_setup_production.py +++ b/bench/tests/test_setup_production.py @@ -33,6 +33,13 @@ class TestSetupProduction(TestBenchBase): bench.utils.exec_cmd("sudo bench disable-production", cwd=bench_path) + def production_setup(self): + try: + self.test_setup_production() + except Exception: + print(self.get_traceback()) + + def assert_nginx_config(self, bench_name): conf_src = os.path.join(os.path.abspath(self.benches_path), bench_name, 'config', 'nginx.conf') conf_dest = "/etc/nginx/conf.d/{bench_name}.conf".format(bench_name=bench_name) From 87bc347b5cc683b13a9a15549e379f4e045e79df Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 13 Jul 2020 19:02:30 +0530 Subject: [PATCH 30/45] style: sorted imports + trailing spaces --- bench/tests/test_base.py | 10 +++++----- bench/tests/test_init.py | 4 ++-- bench/tests/test_setup_production.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bench/tests/test_base.py b/bench/tests/test_base.py index 5807ec44..73cc90f5 100644 --- a/bench/tests/test_base.py +++ b/bench/tests/test_base.py @@ -1,19 +1,19 @@ # imports - standard imports +import getpass import json import os import shutil import subprocess import sys +import traceback import unittest -import getpass - -# imports - module imports -import bench -import bench.utils # imports - third party imports from six import PY2 +# imports - module imports +import bench +import bench.utils if PY2: FRAPPE_BRANCH = "version-12" diff --git a/bench/tests/test_init.py b/bench/tests/test_init.py index 07b0f96a..69d49a45 100755 --- a/bench/tests/test_init.py +++ b/bench/tests/test_init.py @@ -11,7 +11,7 @@ import git import bench import bench.utils from bench.release import get_bumped_version -from bench.tests.test_base import TestBenchBase, FRAPPE_BRANCH +from bench.tests.test_base import FRAPPE_BRANCH, TestBenchBase class TestBenchInit(TestBenchBase): @@ -155,4 +155,4 @@ class TestBenchInit(TestBenchBase): if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/bench/tests/test_setup_production.py b/bench/tests/test_setup_production.py index 9e9f0bd2..272db54d 100644 --- a/bench/tests/test_setup_production.py +++ b/bench/tests/test_setup_production.py @@ -176,4 +176,4 @@ class TestSetupProduction(TestBenchBase): if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() From 939d3d5c6d1dd6f2391ef0bb3dfe3344f3dd6fb8 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 13 Jul 2020 19:12:23 +0530 Subject: [PATCH 31/45] chore: run specified test alone --- .travis.yml | 16 ++++++++-------- bench/tests/test_setup_production.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0d77579b..4ad7e0f6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,42 +18,42 @@ matrix: - name: "Python 2.7 Basic Setup" python: 2.7 env: TEST=bench - script: python -m unittest -v bench.tests.basic + script: python bench/tests/test_init.py TestBenchInit.basic - name: "Python 3.6 Basic Setup" python: 3.6 env: TEST=bench - script: python -m unittest -v bench.tests.basic + script: python bench/tests/test_init.py TestBenchInit.basic - name: "Python 3.7 Basic Setup" python: 3.7 env: TEST=bench - script: python -m unittest -v bench.tests.basic + script: python bench/tests/test_init.py TestBenchInit.basic - name: "Python 3.8 Production Setup" python: 3.8 env: TEST=bench - script: python -m unittest -v bench.tests.production_setup + script: python bench/tests/test_setup_production.py TestSetupProduction.production - name: "Python 2.7 Production Setup" python: 2.7 env: TEST=bench - script: python -m unittest -v bench.tests.production_setup + script: python bench/tests/test_setup_production.py TestSetupProduction.production - name: "Python 3.6 Production Setup" python: 3.6 env: TEST=bench - script: python -m unittest -v bench.tests.production_setup + script: python bench/tests/test_setup_production.py TestSetupProduction.production - name: "Python 3.7 Production Setup" python: 3.7 env: TEST=bench - script: python -m unittest -v bench.tests.production_setup + script: python bench/tests/test_setup_production.py TestSetupProduction.production - name: "Python 3.8 Production Setup" python: 3.8 env: TEST=bench - script: python -m unittest -v bench.tests.production_setup + script: python bench/tests/test_setup_production.py TestSetupProduction.production - name: "Python 3.6 Easy Install" python: 3.6 diff --git a/bench/tests/test_setup_production.py b/bench/tests/test_setup_production.py index 272db54d..f371e454 100644 --- a/bench/tests/test_setup_production.py +++ b/bench/tests/test_setup_production.py @@ -33,7 +33,7 @@ class TestSetupProduction(TestBenchBase): bench.utils.exec_cmd("sudo bench disable-production", cwd=bench_path) - def production_setup(self): + def production(self): try: self.test_setup_production() except Exception: From ba7abd11a3689da188bf50c568afe8e4d6435428 Mon Sep 17 00:00:00 2001 From: gavin Date: Wed, 15 Jul 2020 10:49:48 +0530 Subject: [PATCH 32/45] chore(issue): updated bug report template --- .github/ISSUE_TEMPLATE/bug-report.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index dbe2664f..ec6ea6d3 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -6,8 +6,14 @@ labels: bug Issue: Bug report +Please make sure your issue is reproducible on the latest bench version. The currently supported branches are: + - PyPI [latest](https://pypi.org/project/frappe-bench/) + - master (minor bug fixes) + - v5.x (Merged with develop on every release) + - develop (all updates) + **Do the checklist before filing an issue:** - - [ ] Can you replicate the issue? + - [ ] Can you replicate the issue on the supported bench versions? - [ ] Is this something you can debug and fix? Send a pull request! Bug fixes and documentation fixes are welcome **Describe the bug** :chart_with_downwards_trend: @@ -34,9 +40,8 @@ If applicable, add screenshots to help explain your problem. **Version Information** -Can be found out by running `bench version` in your respective bench folder - -- Bench Branch: `master` _(Only master is supported)_ +Can be found out by running `bench version` in your respective bench folder. +- Bench Branch: - Frappe Version: - ERPNext Version: From d6a12221da0ceaa0c30a1a2c55fde101ed41bef0 Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Wed, 29 Jul 2020 16:33:47 +0530 Subject: [PATCH 33/45] fix: updated URL for MacOS guide (#1037) * fix: updated URL for MacOS guide * Update README.md Co-authored-by: gavin Co-authored-by: gavin --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d9c2d1bb..8bdc2af4 100755 --- a/README.md +++ b/README.md @@ -158,7 +158,7 @@ $ pip install frappe-bench For more extensive distribution-dependent documentation, check out the following guides: - [Hitchhiker's Guide to Installing Frappe on Linux](https://github.com/frappe/frappe/wiki/The-Hitchhiker%27s-Guide-to-Installing-Frappe-on-Linux) - - [Hitchhiker's Guide to Installing Frappe on MacOS](https://github.com/frappe/frappe/wiki/The-Hitchhiker%27s-Guide-to-Installing-Frappe-on-Mac-OS-X) + - [Hitchhiker's Guide to Installing Frappe on MacOS](https://github.com/frappe/bench/wiki/Setting-up-a-Mac-for-Frappe-ERPNext-Development) ## Basic Usage From 547f2bae82d8114f9bb7ef3260483895952cb289 Mon Sep 17 00:00:00 2001 From: ABHISHEK KEDAR <44434910+ABHISHEK-KEDAR-21@users.noreply.github.com> Date: Mon, 3 Aug 2020 17:48:00 +0530 Subject: [PATCH 34/45] feat: bench start --no-prefix (#1040) * bench start --no-prefix option * help added for --no-prefix option --- bench/commands/utils.py | 5 +++-- bench/utils.py | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/bench/commands/utils.py b/bench/commands/utils.py index fcb11512..63ab9bb1 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -8,11 +8,12 @@ import click @click.command('start', help="Start Frappe development processes") @click.option('--no-dev', is_flag=True, default=False) +@click.option('--no-prefix', is_flag=True, default=False, help="Hide process name from bench start log") @click.option('--concurrency', '-c', type=str) @click.option('--procfile', '-p', type=str) -def start(no_dev, concurrency, procfile): +def start(no_dev, concurrency, procfile, no_prefix): from bench.utils import start - start(no_dev=no_dev, concurrency=concurrency, procfile=procfile) + start(no_dev=no_dev, concurrency=concurrency, procfile=procfile, no_prefix=no_prefix) @click.command('restart', help="Restart supervisor processes or systemd units") diff --git a/bench/utils.py b/bench/utils.py index 948d62ea..4a4fa813 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -461,7 +461,7 @@ def get_process_manager(): return proc_man_path -def start(no_dev=False, concurrency=None, procfile=None): +def start(no_dev=False, concurrency=None, procfile=None, no_prefix=False): program = get_process_manager() if not program: raise Exception("No process manager found") @@ -476,6 +476,9 @@ def start(no_dev=False, concurrency=None, procfile=None): if procfile: command.extend(['-f', procfile]) + if no_prefix: + command.extend(['--no-prefix']) + os.execv(program, command) From 2b2783368a643afb65b10dd39877799e67c28798 Mon Sep 17 00:00:00 2001 From: gavin Date: Tue, 4 Aug 2020 16:34:19 +0530 Subject: [PATCH 35/45] chore: fix outdated update help --- bench/commands/update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/commands/update.py b/bench/commands/update.py index cac2ffdd..dee57d28 100755 --- a/bench/commands/update.py +++ b/bench/commands/update.py @@ -6,7 +6,7 @@ from bench.app import pull_apps from bench.utils import post_upgrade, patch_sites, build_assets -@click.command('update', help="Updates bench tool and if executed in a bench directory, without any flags will backup, pull, setup requirements, build, run patches and restart bench. Using specific flags will only do certain tasks instead of all") +@click.command('update', help="Performs an update operation on current bench. Without any flags will backup, pull, setup requirements, build, run patches and restart bench. Using specific flags will only do certain tasks instead of all") @click.option('--pull', is_flag=True, help="Pull updates for all the apps in bench") @click.option('--apps', type=str) @click.option('--patch', is_flag=True, help="Run migrations for all sites in the bench") From de5e2789122e219a73c889508fa6100ad77b58c8 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 5 Aug 2020 13:07:12 +0530 Subject: [PATCH 36/45] fix: Bump Nodejs v8 => v12 --- bench/playbooks/roles/nodejs/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/playbooks/roles/nodejs/defaults/main.yml b/bench/playbooks/roles/nodejs/defaults/main.yml index f991faa1..a09f08cb 100644 --- a/bench/playbooks/roles/nodejs/defaults/main.yml +++ b/bench/playbooks/roles/nodejs/defaults/main.yml @@ -1,3 +1,3 @@ --- -node_version: 8 +node_version: 12 ... \ No newline at end of file From 3ea66343284e45b72bbe1e2558f7cc6baadc274c Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 5 Aug 2020 13:59:00 +0530 Subject: [PATCH 37/45] fix: run test builds for py2.7 and py3.7 --- .travis.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.travis.yml b/.travis.yml index 4ad7e0f6..295e057d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,6 +55,16 @@ matrix: env: TEST=bench script: python bench/tests/test_setup_production.py TestSetupProduction.production + - name: "Python 2.7 Tests" + python: 2.7 + env: TEST=bench + script: python -m unittest -v bench.tests.test_init + + - name: "Python 3.7 Tests" + python: 3.7 + env: TEST=bench + script: python -m unittest -v bench.tests.test_init + - name: "Python 3.6 Easy Install" python: 3.6 env: TEST=easy_install From fadf44d747fee577a6e81eac19c56d6b81a0f83b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 5 Aug 2020 16:31:27 +0530 Subject: [PATCH 38/45] fix(install): SHow Debian 10 is supported, update MariaDB defaults --- install.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install.py b/install.py index 31ee5ca9..bb3ac75f 100644 --- a/install.py +++ b/install.py @@ -79,7 +79,7 @@ def check_distribution_compatibility(): supported_dists = { 'macos': [10.9, 10.10, 10.11, 10.12], 'ubuntu': [14, 15, 16, 18, 19], - 'debian': [8, 9], + 'debian': [8, 9, 10], 'centos': [7] } @@ -396,7 +396,7 @@ def parse_commandline_args(): parser.add_argument('--overwrite', dest='overwrite', action='store_true', default=False, help='Whether to overwrite an existing bench') # set passwords parser.add_argument('--mysql-root-password', dest='mysql_root_password', help='Set mysql root password') - parser.add_argument('--mariadb-version', dest='mariadb_version', default='10.2', help='Specify mariadb version') + parser.add_argument('--mariadb-version', dest='mariadb_version', default='10.4', help='Specify mariadb version') parser.add_argument('--admin-password', dest='admin_password', help='Set admin password') parser.add_argument('--bench-name', dest='bench_name', help='Create bench with specified name. Default name is frappe-bench') # Python interpreter to be used From 967d1498f6cb7a41bd02a324b12899f23aca93e9 Mon Sep 17 00:00:00 2001 From: Adam Tang <37611541+adamtang79@users.noreply.github.com> Date: Fri, 14 Aug 2020 13:20:36 +0800 Subject: [PATCH 39/45] fix: using distro instead of platform do such a way that platform is still perserved and sure to be removed said v13 for python3 era. --- install.py | 50 +++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/install.py b/install.py index bb3ac75f..56d514b3 100644 --- a/install.py +++ b/install.py @@ -74,11 +74,11 @@ def check_system_package_managers(): raise Exception('Cannot find any compatible package manager!') -def check_distribution_compatibility(): - dist_name, dist_version = get_distribution_info() +def check_distribution_compatibility(args): + dist_name, dist_version = get_distribution_info(args) supported_dists = { 'macos': [10.9, 10.10, 10.11, 10.12], - 'ubuntu': [14, 15, 16, 18, 19], + 'ubuntu': [14, 15, 16, 18, 19, 20], 'debian': [8, 9, 10], 'centos': [7] } @@ -93,11 +93,36 @@ def check_distribution_compatibility(): else: log("Sorry, the installer doesn't support {0}. Aborting installation!".format(dist_name), level=2) +def import_with_install(package): + # copied from https://discuss.erpnext.com/u/nikunj_patel + # https://discuss.erpnext.com/t/easy-install-setup-guide-for-erpnext-installation-on-ubuntu-20-04-lts-with-some-modification-of-course/62375/5 -def get_distribution_info(): + # need to move to top said v13 for fully python3 era + import importlib + + try: + importlib.import_module(package) + except ImportError: + # caveat : pip3 must be installed + + import pip + + pip.main(['install', package]) + finally: + globals()[package] = importlib.import_module(package) + + +def get_distribution_info(args): # return distribution name and major version if platform.system() == "Linux": - current_dist = platform.dist() + if args.python_version == "3": + install_package('pip3', 'python3-pip') + + import_with_install('distro') + + current_dist = distro.linux_distribution(full_distribution_name=True) + else: + current_dist = platform.dist() return current_dist[0].lower(), current_dist[1].rsplit('.')[0] elif platform.system() == "Darwin": @@ -191,7 +216,7 @@ def install_bench(args): raise Exception('Please run this script as a non-root user with sudo privileges, but without using sudo or pass --user=USER') # Python executable - dist_name, dist_version = get_distribution_info() + dist_name, dist_version = get_distribution_info(args) if dist_name=='centos': args.python = 'python3.6' else: @@ -403,6 +428,16 @@ def parse_commandline_args(): parser.add_argument('--python', dest='python', default='python3', help=argparse.SUPPRESS) # LXC Support parser.add_argument('--container', dest='container', default=False, action='store_true', help='Use if you\'re creating inside LXC') + + # for detecting dist; backward compatible if specify 2 + + parser.add_argument( + '--python-version', + dest='python_version', + default='3', + help='For detecting dist. Backward compatible if specify 2. Default is 3.' + ) + args = parser.parse_args() return args @@ -433,10 +468,11 @@ if __name__ == '__main__': with warnings.catch_warnings(): warnings.simplefilter("ignore") setup_log_stream(args) - check_distribution_compatibility() + check_distribution_compatibility(args) check_system_package_managers() check_environment() install_prerequisites() install_bench(args) log("Bench + Frappe + ERPNext has been successfully installed!") + \ No newline at end of file From 9bdff524cd8fe87681548369b2fc8fde600e6bd5 Mon Sep 17 00:00:00 2001 From: Adam Tang <37611541+adamtang79@users.noreply.github.com> Date: Fri, 14 Aug 2020 13:39:10 +0800 Subject: [PATCH 40/45] fix: dns caching handle name not match --- bench/playbooks/roles/dns_caching/handlers/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/playbooks/roles/dns_caching/handlers/main.yml b/bench/playbooks/roles/dns_caching/handlers/main.yml index afe7ac3e..f3b1d4c4 100644 --- a/bench/playbooks/roles/dns_caching/handlers/main.yml +++ b/bench/playbooks/roles/dns_caching/handlers/main.yml @@ -1,4 +1,4 @@ --- -- name: Restart network manager +- name: restart network manager service: name=NetworkManager state=restarted ... \ No newline at end of file From ff48ceda915c011f91f56fac58b607fa119155b8 Mon Sep 17 00:00:00 2001 From: Adam Tang <37611541+adamtang79@users.noreply.github.com> Date: Fri, 14 Aug 2020 16:11:40 +0800 Subject: [PATCH 41/45] fix: use sys version info instead of user entry --- install.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/install.py b/install.py index 56d514b3..d62f6ddf 100644 --- a/install.py +++ b/install.py @@ -74,8 +74,8 @@ def check_system_package_managers(): raise Exception('Cannot find any compatible package manager!') -def check_distribution_compatibility(args): - dist_name, dist_version = get_distribution_info(args) +def check_distribution_compatibility(): + dist_name, dist_version = get_distribution_info() supported_dists = { 'macos': [10.9, 10.10, 10.11, 10.12], 'ubuntu': [14, 15, 16, 18, 19, 20], @@ -112,10 +112,10 @@ def import_with_install(package): globals()[package] = importlib.import_module(package) -def get_distribution_info(args): +def get_distribution_info(): # return distribution name and major version if platform.system() == "Linux": - if args.python_version == "3": + if sys.version_info.major == 3 and sys.version_info.minor > 7: install_package('pip3', 'python3-pip') import_with_install('distro') @@ -216,7 +216,7 @@ def install_bench(args): raise Exception('Please run this script as a non-root user with sudo privileges, but without using sudo or pass --user=USER') # Python executable - dist_name, dist_version = get_distribution_info(args) + dist_name, dist_version = get_distribution_info() if dist_name=='centos': args.python = 'python3.6' else: @@ -429,15 +429,6 @@ def parse_commandline_args(): # LXC Support parser.add_argument('--container', dest='container', default=False, action='store_true', help='Use if you\'re creating inside LXC') - # for detecting dist; backward compatible if specify 2 - - parser.add_argument( - '--python-version', - dest='python_version', - default='3', - help='For detecting dist. Backward compatible if specify 2. Default is 3.' - ) - args = parser.parse_args() return args @@ -468,7 +459,7 @@ if __name__ == '__main__': with warnings.catch_warnings(): warnings.simplefilter("ignore") setup_log_stream(args) - check_distribution_compatibility(args) + check_distribution_compatibility() check_system_package_managers() check_environment() install_prerequisites() From f032fc0b86f6891399d3dba41cca782319667ed3 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 14 Aug 2020 20:29:46 +0530 Subject: [PATCH 42/45] fix: Check if Py3.5+ before installing distro --- install.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/install.py b/install.py index d62f6ddf..2b70d7f4 100644 --- a/install.py +++ b/install.py @@ -20,6 +20,7 @@ execution_time = "{:%H:%M}".format(execution_timestamp) log_file_name = "easy-install__{0}__{1}.log".format(execution_day, execution_time.replace(':', '-')) log_path = os.path.join(tmp_log_folder, log_file_name) log_stream = sys.stdout +distro_required = not ((sys.version_info.major < 3) or (sys.version_info.major == 3 and sys.version_info.minor < 5)) def log(message, level=0): @@ -93,10 +94,10 @@ def check_distribution_compatibility(): else: log("Sorry, the installer doesn't support {0}. Aborting installation!".format(dist_name), level=2) + def import_with_install(package): # copied from https://discuss.erpnext.com/u/nikunj_patel # https://discuss.erpnext.com/t/easy-install-setup-guide-for-erpnext-installation-on-ubuntu-20-04-lts-with-some-modification-of-course/62375/5 - # need to move to top said v13 for fully python3 era import importlib @@ -115,14 +116,11 @@ def import_with_install(package): def get_distribution_info(): # return distribution name and major version if platform.system() == "Linux": - if sys.version_info.major == 3 and sys.version_info.minor > 7: - install_package('pip3', 'python3-pip') - - import_with_install('distro') - + if distro_required: current_dist = distro.linux_distribution(full_distribution_name=True) else: current_dist = platform.dist() + return current_dist[0].lower(), current_dist[1].rsplit('.')[0] elif platform.system() == "Darwin": @@ -391,6 +389,12 @@ def run_playbook(playbook_name, sudo=False, extra_vars=None): return success +def setup_script_requirements(): + if distro_required: + install_package('pip3', 'python3-pip') + import_with_install('distro') + + def parse_commandline_args(): import argparse @@ -433,6 +437,7 @@ def parse_commandline_args(): return args + if __name__ == '__main__': if sys.version[0] == '2': if not os.environ.get('CI'): @@ -459,6 +464,7 @@ if __name__ == '__main__': with warnings.catch_warnings(): warnings.simplefilter("ignore") setup_log_stream(args) + setup_script_requirements() check_distribution_compatibility() check_system_package_managers() check_environment() @@ -466,4 +472,3 @@ if __name__ == '__main__': install_bench(args) log("Bench + Frappe + ERPNext has been successfully installed!") - \ No newline at end of file From 892b449cd607d570c7b3da7f262f35b988b17a02 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 14 Aug 2020 20:45:50 +0530 Subject: [PATCH 43/45] chore: Use 3.4, 3.6 and 3.8 to test install.py Previously, tests were run for 3.6, 3.7 and 3.8; Changed the targets to have more range of the versions --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 295e057d..379add08 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,13 +65,13 @@ matrix: env: TEST=bench script: python -m unittest -v bench.tests.test_init - - name: "Python 3.6 Easy Install" - python: 3.6 + - name: "Python 3.4 Easy Install" + python: 3.4 env: TEST=easy_install script: sudo python $TRAVIS_BUILD_DIR/install.py --user travis --run-travis --production --verbose - - name: "Python 3.7 Easy Install" - python: 3.7 + - name: "Python 3.6 Easy Install" + python: 3.6 env: TEST=easy_install script: sudo python $TRAVIS_BUILD_DIR/install.py --user travis --run-travis --production --verbose From 5c8e9f2e556fddff55f26063bad249a8babf6d5a Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 15 Aug 2020 11:20:19 +0530 Subject: [PATCH 44/45] chore: Use 3.5, 3.7 and 3.8 to test install.py 3.4 isn't available so let's not! --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 379add08..8bc96c6e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,13 +65,13 @@ matrix: env: TEST=bench script: python -m unittest -v bench.tests.test_init - - name: "Python 3.4 Easy Install" - python: 3.4 + - name: "Python 3.5 Easy Install" + python: 3.5 env: TEST=easy_install script: sudo python $TRAVIS_BUILD_DIR/install.py --user travis --run-travis --production --verbose - - name: "Python 3.6 Easy Install" - python: 3.6 + - name: "Python 3.7 Easy Install" + python: 3.7 env: TEST=easy_install script: sudo python $TRAVIS_BUILD_DIR/install.py --user travis --run-travis --production --verbose From 4f73fb4b1fd2753ede031847c155d7990f39c37c Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 18 Aug 2020 15:37:50 +0530 Subject: [PATCH 45/45] chore: bump bench to v5.2.0 --- bench/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/__init__.py b/bench/__init__.py index dd6d2c46..acb504df 100644 --- a/bench/__init__.py +++ b/bench/__init__.py @@ -1,4 +1,4 @@ -VERSION = "5.1.0" +VERSION = "5.2.0" PROJECT_NAME = "frappe-bench" FRAPPE_VERSION = None