From 31e4aeeab8bca4a8d58ad72db735128670a7480b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 24 Sep 2020 12:56:11 +0530 Subject: [PATCH 01/51] fix: Handle no args passed in CLI --- bench/cli.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index 95ca545d..e0d94f1a 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -30,11 +30,10 @@ def cli(): logger = setup_logging() or logging.getLogger(bench.PROJECT_NAME) logger.info(command) - if sys.argv[1] not in ("src", ): + if len(sys.argv) > 1 and sys.argv[1] not in ("src", ): check_uid() change_uid() - - change_dir() + change_dir() if is_dist_editable(bench.PROJECT_NAME) and len(sys.argv) > 1 and sys.argv[1] != "src" and not get_config(".").get("developer_mode"): log("bench is installed in editable mode!\n\nThis is not the recommended mode of installation for production. Instead, install the package from PyPI with: `pip install frappe-bench`\n", level=3) From 28067f6f9c68037afe43160b71d59b6508e067b0 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 24 Sep 2020 13:09:58 +0530 Subject: [PATCH 02/51] test: Add bench utils tests --- bench/tests/test_init.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bench/tests/test_init.py b/bench/tests/test_init.py index 69d49a45..77e4a3a0 100755 --- a/bench/tests/test_init.py +++ b/bench/tests/test_init.py @@ -27,6 +27,11 @@ class TestBenchInit(TestBenchBase): self.assertEqual( get_bumped_version('11.0.5-beta.22', 'prerelease'), '11.0.5-beta.23' ) + def test_utils(self): + self.assertEqual(subprocess.call("bench"), 0) + self.assertEqual(os.path.dirname(bench.cli.src), subprocess.check_output(["bench", "src"], cwd=".").decode('utf8')) + + def test_init(self, bench_name="test-bench", **kwargs): self.init_bench(bench_name, **kwargs) self.assert_folders(bench_name) From 2f0b508d2ef1d4935ccd4a74d374ee6be28a7bec Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 24 Sep 2020 13:35:47 +0530 Subject: [PATCH 03/51] fix: Add bench.cli namespace --- bench/tests/test_init.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bench/tests/test_init.py b/bench/tests/test_init.py index 77e4a3a0..4ebcae15 100755 --- a/bench/tests/test_init.py +++ b/bench/tests/test_init.py @@ -9,6 +9,7 @@ import git # imports - module imports import bench +import bench.cli import bench.utils from bench.release import get_bumped_version from bench.tests.test_base import FRAPPE_BRANCH, TestBenchBase From b8ed4f6a4bc35f7a8935793c3616fc1258ff2685 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 24 Sep 2020 16:16:31 +0530 Subject: [PATCH 04/51] fix: Remove src test --- bench/tests/test_init.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bench/tests/test_init.py b/bench/tests/test_init.py index 4ebcae15..2f409899 100755 --- a/bench/tests/test_init.py +++ b/bench/tests/test_init.py @@ -30,7 +30,6 @@ class TestBenchInit(TestBenchBase): def test_utils(self): self.assertEqual(subprocess.call("bench"), 0) - self.assertEqual(os.path.dirname(bench.cli.src), subprocess.check_output(["bench", "src"], cwd=".").decode('utf8')) def test_init(self, bench_name="test-bench", **kwargs): From df0562d59b423f1d833c929a38ea5e380bbcae4f Mon Sep 17 00:00:00 2001 From: Hendy Irawan Date: Sun, 4 Oct 2020 00:13:23 +0700 Subject: [PATCH 05/51] fix(install): Install dev packages for Arm 64-bit Until psycopg2-binary is available for aarch64 (Arm 64-bit), we'll need libpq and libssl dev packages to build psycopg2 from source Fixes https://github.com/frappe/bench/issues/1084 --- install.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/install.py b/install.py index 46ce4247..4831d52b 100644 --- a/install.py +++ b/install.py @@ -156,6 +156,14 @@ def install_prerequisites(): 'sudo yum install -y epel-release redhat-lsb-core git python-setuptools python-devel openssl-devel libffi-devel' ] }) + + # until psycopg2-binary is available for aarch64 (Arm 64-bit), we'll need libpq and libssl dev packages to build psycopg2 from source + if platform.machine() == 'aarch64': + log("Installing libpq and libssl dev packages to build psycopg2 for aarch64...") + run_os_command({ + 'apt-get': ['sudo apt-get install -y libpq-dev libssl-dev'], + 'yum': ['sudo yum install -y libpq-devel openssl-devel'] + }) install_package('curl') install_package('wget') From e146919f9d642b7d28ef2e9d905fc394f04f19f8 Mon Sep 17 00:00:00 2001 From: gavin Date: Tue, 27 Oct 2020 13:57:05 +0530 Subject: [PATCH 06/51] chore: Added semantic config Installed https://github.com/zeke/semantic-pull-requests and added config for repo --- .github/semantic.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/semantic.yml diff --git a/.github/semantic.yml b/.github/semantic.yml new file mode 100644 index 00000000..e1e53bc1 --- /dev/null +++ b/.github/semantic.yml @@ -0,0 +1,13 @@ +# Always validate the PR title AND all the commits +titleAndCommits: true + +# Allow use of Merge commits (eg on github: "Merge branch 'master' into feature/ride-unicorns") +# this is only relevant when using commitsOnly: true (or titleAndCommits: true) +allowMergeCommits: true + +# Allow use of Revert commits (eg on github: "Revert "feat: ride unicorns"") +# this is only relevant when using commitsOnly: true (or titleAndCommits: true) +allowRevertCommits: true + +# For allowed PR types: https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json +# Tool Reference: https://github.com/zeke/semantic-pull-requests From f17c579ffdbb82526acff00dea03e0b624c2bd81 Mon Sep 17 00:00:00 2001 From: Karan Sharma Date: Wed, 24 Oct 2018 16:41:05 +0530 Subject: [PATCH 07/51] build(config): Change default worker count --- bench/config/common_site_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/config/common_site_config.py b/bench/config/common_site_config.py index 02c6778a..d71d9efd 100644 --- a/bench/config/common_site_config.py +++ b/bench/config/common_site_config.py @@ -55,7 +55,7 @@ def get_gunicorn_workers(): '''This function will return the maximum workers that can be started depending upon number of cpu's present on the machine''' return { - "gunicorn_workers": multiprocessing.cpu_count() + "gunicorn_workers": multiprocessing.cpu_count() * 2 + 1 } def update_config_for_frappe(config, bench_path): From 482bb2e59182c11f66b45539d12414a366b5bb72 Mon Sep 17 00:00:00 2001 From: Hendy Irawan Date: Sat, 3 Oct 2020 23:58:18 +0700 Subject: [PATCH 08/51] build(install): Upgrade Ansible to 2.8.15 (or later patch version) --- install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.py b/install.py index 46ce4247..b5bd43f5 100644 --- a/install.py +++ b/install.py @@ -163,7 +163,7 @@ def install_prerequisites(): install_package('pip3', 'python3-pip') success = run_os_command({ - 'python3': "sudo -H python3 -m pip install --upgrade setuptools wheel cryptography ansible==2.8.5 pip" + 'python3': "sudo -H python3 -m pip install --upgrade setuptools wheel cryptography ansible~=2.8.15 pip" }) if not (success or shutil.which('ansible')): From 608b28c2460adf2b6ee1504b21a7e69dfdb06a1e Mon Sep 17 00:00:00 2001 From: Hendy Irawan Date: Sun, 4 Oct 2020 07:07:41 +0000 Subject: [PATCH 09/51] fix(install): Obey --without-site --- bench/playbooks/roles/bench/tasks/setup_erpnext.yml | 5 +++-- install.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/bench/playbooks/roles/bench/tasks/setup_erpnext.yml b/bench/playbooks/roles/bench/tasks/setup_erpnext.yml index 32e498d1..9db153ab 100644 --- a/bench/playbooks/roles/bench/tasks/setup_erpnext.yml +++ b/bench/playbooks/roles/bench/tasks/setup_erpnext.yml @@ -13,16 +13,17 @@ - name: Check whether the site already exists stat: path="{{ bench_path }}/sites/{{ site }}" register: site_folder + when: not without_site - name: Create a new site command: "bench new-site {{ site }} --admin-password '{{ admin_password }}' --mariadb-root-password '{{ mysql_root_password }}'" args: chdir: "{{ bench_path }}" - when: not site_folder.stat.exists + when: not without_site and not site_folder.stat.exists - name: Install ERPNext to default site command: "bench --site {{ site }} install-app erpnext" args: chdir: "{{ bench_path }}" - when: not without_erpnext + when: not without_site and not without_erpnext ... \ No newline at end of file diff --git a/install.py b/install.py index 46ce4247..8a08c1db 100644 --- a/install.py +++ b/install.py @@ -327,8 +327,8 @@ def get_passwords(args): mysql_root_password = '' continue - # admin password - if not admin_password: + # admin password, only needed if we're also creating a site + if not admin_password and not args.without_site: admin_password = getpass.unix_getpass(prompt='Please enter the default Administrator user password: ') conf_admin_passswd = getpass.unix_getpass(prompt='Re-enter Administrator password: ') From f2e7702fdefe30ee8d0546da5dc09e9042f7f570 Mon Sep 17 00:00:00 2001 From: Adam Tang <37611541+adamtang79@users.noreply.github.com> Date: Wed, 28 Oct 2020 09:04:41 +0800 Subject: [PATCH 10/51] fix: utf8mb4_general_ci fix: prevent error while bench new-site --- bench/playbooks/roles/mariadb/tasks/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bench/playbooks/roles/mariadb/tasks/main.yml b/bench/playbooks/roles/mariadb/tasks/main.yml index 1f119005..8079583d 100644 --- a/bench/playbooks/roles/mariadb/tasks/main.yml +++ b/bench/playbooks/roles/mariadb/tasks/main.yml @@ -48,6 +48,10 @@ [mysqld] pid-file = /var/run/mysqld/mysqld.pid socket = /var/run/mysqld/mysqld.sock + + # setting appeared inside mysql but overwritten by mariadb inside mariadb.conf.d/xx-server.cnf valued as utf8mb4_general_ci + + collation-server = utf8mb4_unicode_ci create: yes become: yes become_user: root From 6a207ca57ab36d3d43b169b8a7609ba87dc8611b Mon Sep 17 00:00:00 2001 From: Hendy Irawan Date: Sat, 3 Oct 2020 16:12:39 +0700 Subject: [PATCH 11/51] fix(install): Allow override of frappe_branch and erpnext_branch Regardless of args.version (which always has a default set). Additionally, print useful log of (non-credentials) variables used. --- install.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/install.py b/install.py index 7a33ba11..6dcf30a6 100644 --- a/install.py +++ b/install.py @@ -247,12 +247,11 @@ def install_bench(args): else: frappe_branch = "version-{0}".format(args.version) erpnext_branch = "version-{0}".format(args.version) - else: - if args.frappe_branch: - frappe_branch = args.frappe_branch - - if args.erpnext_branch: - erpnext_branch = args.erpnext_branch + # Allow override of frappe_branch and erpnext_branch, regardless of args.version (which always has a default set) + if args.frappe_branch: + frappe_branch = args.frappe_branch + if args.erpnext_branch: + erpnext_branch = args.erpnext_branch extra_vars.update(frappe_branch=frappe_branch) extra_vars.update(erpnext_branch=erpnext_branch) @@ -261,6 +260,7 @@ def install_bench(args): extra_vars.update(bench_name=bench_name) # Will install ERPNext production setup by default + log("Initializing bench using bench_name=%s frappe_branch=%s erpnext_branch=%s ..." % (bench_name, frappe_branch, erpnext_branch)) run_playbook('site.yml', sudo=True, extra_vars=extra_vars) if os.path.exists(tmp_bench_repo): From 2c4659a7aab2c4c8ed7734438ffc42360c9c6023 Mon Sep 17 00:00:00 2001 From: Hendy Irawan Date: Fri, 30 Oct 2020 17:35:19 +0700 Subject: [PATCH 12/51] fix(install): Helpful logging Co-authored-by: gavin --- install.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/install.py b/install.py index 6dcf30a6..40dba36e 100644 --- a/install.py +++ b/install.py @@ -260,7 +260,10 @@ def install_bench(args): extra_vars.update(bench_name=bench_name) # Will install ERPNext production setup by default - log("Initializing bench using bench_name=%s frappe_branch=%s erpnext_branch=%s ..." % (bench_name, frappe_branch, erpnext_branch)) + if args.without_erpnext: + log("Initializing bench {bench_name}:\n\tFrappe Branch: {frappe_branch}\n\tERPNext will not be installed due to --without-erpnext".format(bench_name=bench_name, frappe_branch=frappe_branch)) + else: + log("Initializing bench {bench_name}:\n\tFrappe Branch: {frappe_branch}\n\tERPNext Branch: {erpnext_branch}".format(bench_name=bench_name, frappe_branch=frappe_branch, erpnext_branch=erpnext_branch)) run_playbook('site.yml', sudo=True, extra_vars=extra_vars) if os.path.exists(tmp_bench_repo): @@ -273,11 +276,15 @@ def clone_bench_repo(args): repo_url = args.repo_url or 'https://github.com/frappe/bench' if os.path.exists(tmp_bench_repo): + log('Not cloning already existing Bench repository at {tmp_bench_repo}'.format(tmp_bench_repo=tmp_bench_repo)) return 0 elif args.without_bench_setup: clone_path = os.path.join(os.path.expanduser('~'), 'bench') + log('--without-bench-setup specified, clone path is: {clone_path}'.format(clone_path=clone_path)) else: clone_path = tmp_bench_repo + # Not logging repo_url to avoid accidental credential leak in case credential is embedded in URL + log('Cloning bench repository branch {branch} into {clone_path}'.format(branch=branch, clone_path=clone_path)) success = run_os_command( {'git': 'git clone --quiet {repo_url} {bench_repo} --depth 1 --branch {branch}'.format( @@ -336,6 +343,8 @@ def get_passwords(args): passwords_didnt_match("Administrator") admin_password = '' continue + elif args.without_site: + log("Not creating a new site due to --without-site") pass_set = False else: @@ -405,8 +414,8 @@ def parse_commandline_args(): args_group.add_argument('--develop', dest='develop', action='store_true', default=False, help='Install developer setup') args_group.add_argument('--production', dest='production', action='store_true', default=False, help='Setup Production environment for bench') - parser.add_argument('--site', dest='site', action='store', default='site1.local', help='Specifiy name for your first ERPNext site') - parser.add_argument('--without-site', dest='without_site', action='store_true', default=False) + parser.add_argument('--site', dest='site', action='store', default='site1.local', help='Specify name for your first ERPNext site') + parser.add_argument('--without-site', dest='without_site', action='store_true', default=False, help='Do not create a new site') parser.add_argument('--verbose', dest='verbose', action='store_true', default=False, help='Run the script in verbose mode') parser.add_argument('--user', dest='user', help='Install frappe-bench for this user') parser.add_argument('--bench-branch', dest='bench_branch', help='Clone a particular branch of bench repository') From d6d1b91a79b3b74a5595a2b943041659ff5928bd Mon Sep 17 00:00:00 2001 From: Hendy Irawan Date: Sat, 3 Oct 2020 14:33:12 +0700 Subject: [PATCH 13/51] fix(playbooks): Support wkhtmltox arm64 architecture For Ubuntu 20.04 LTS and later --- bench/playbooks/roles/wkhtmltopdf/tasks/main.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/bench/playbooks/roles/wkhtmltopdf/tasks/main.yml b/bench/playbooks/roles/wkhtmltopdf/tasks/main.yml index aea2f55f..a2c6086d 100644 --- a/bench/playbooks/roles/wkhtmltopdf/tasks/main.yml +++ b/bench/playbooks/roles/wkhtmltopdf/tasks/main.yml @@ -20,11 +20,17 @@ force: yes when: ansible_os_family == 'Debian' -- name: download wkthmltox Ubuntu 20 +- name: download wkthmltox Ubuntu 20 amd64 get_url: url: https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.focal_amd64.deb dest: /tmp/wkhtmltox.deb - when: ansible_distribution == 'Ubuntu' and ansible_distribution_major_version == '20' + when: ansible_distribution == 'Ubuntu' and ansible_distribution_major_version == '20' and ansible_architecture == 'x86_64' +- name: download wkthmltox Ubuntu 20 arm64 + get_url: + # wkhtmltox supports arm64 starting from 0.12.6 + url: https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_arm64.deb + dest: /tmp/wkhtmltox.deb + when: ansible_distribution == 'Ubuntu' and ansible_distribution_major_version == '20' and ansible_architecture == 'aarch64' - name: download wkthmltox Ubuntu 18 get_url: From 6ef6e37b80fd08ff45d6ae0171646b07c38b5641 Mon Sep 17 00:00:00 2001 From: Hendy Irawan Date: Fri, 30 Oct 2020 18:00:26 +0700 Subject: [PATCH 14/51] style: Apply suggestions from code review Co-authored-by: gavin --- bench/playbooks/roles/wkhtmltopdf/tasks/main.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bench/playbooks/roles/wkhtmltopdf/tasks/main.yml b/bench/playbooks/roles/wkhtmltopdf/tasks/main.yml index a2c6086d..2a6a89ea 100644 --- a/bench/playbooks/roles/wkhtmltopdf/tasks/main.yml +++ b/bench/playbooks/roles/wkhtmltopdf/tasks/main.yml @@ -20,11 +20,12 @@ force: yes when: ansible_os_family == 'Debian' -- name: download wkthmltox Ubuntu 20 amd64 +- name: download wkthmltox Ubuntu 20 get_url: url: https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.focal_amd64.deb dest: /tmp/wkhtmltox.deb - when: ansible_distribution == 'Ubuntu' and ansible_distribution_major_version == '20' and ansible_architecture == 'x86_64' + when: ansible_distribution == 'Ubuntu' and ansible_distribution_major_version == '20' and ansible_architecture != 'aarch64' + - name: download wkthmltox Ubuntu 20 arm64 get_url: # wkhtmltox supports arm64 starting from 0.12.6 From 54e6bcfe9c967137e0be00cefe93a5ddaec30056 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 30 Oct 2020 17:22:02 +0530 Subject: [PATCH 15/51] fix: Run compileall in bench after update --- bench/commands/update.py | 5 +++-- bench/utils.py | 12 ++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/bench/commands/update.py b/bench/commands/update.py index dee57d28..e1113b40 100755 --- a/bench/commands/update.py +++ b/bench/commands/update.py @@ -15,11 +15,12 @@ from bench.utils import post_upgrade, patch_sites, build_assets @click.option('--restart-supervisor', is_flag=True, help="Restart supervisor processes after update") @click.option('--restart-systemd', is_flag=True, help="Restart systemd units after update") @click.option('--no-backup', is_flag=True, help="If this flag is set, sites won't be backed up prior to updates. Note: This is not recommended in production.") +@click.option('--no-compile', is_flag=True, help="If set, Python bytecode won't be compiled before restarting the processes") @click.option('--force', is_flag=True, help="Forces major version upgrades") @click.option('--reset', is_flag=True, help="Hard resets git branch's to their new states overriding any changes and overriding rebase on pull") -def update(pull, apps, patch, build, requirements, restart_supervisor, restart_systemd, no_backup, force, reset): +def update(pull, apps, patch, build, requirements, restart_supervisor, restart_systemd, no_backup, no_compile, force, reset): from bench.utils import update - update(pull=pull, apps=apps, patch=patch, build=build, requirements=requirements, restart_supervisor=restart_supervisor, restart_systemd=restart_systemd, backup=not no_backup, force=force, reset=reset) + update(pull=pull, apps=apps, patch=patch, build=build, requirements=requirements, restart_supervisor=restart_supervisor, restart_systemd=restart_systemd, backup=not no_backup, compile=not no_compile, force=force, reset=reset) @click.command('retry-upgrade', help="Retry a failed upgrade") diff --git a/bench/utils.py b/bench/utils.py index 8e76903f..bb1b55e4 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- # imports - standard imports +import compileall import errno import glob import grp @@ -183,8 +184,8 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, copy_patches_txt(path) -def update(pull=False, apps=None, patch=False, build=False, requirements=False, backup=True, force=False, reset=False, - restart_supervisor=False, restart_systemd=False): +def update(pull=False, apps=None, patch=False, build=False, requirements=False, backup=True, compile=True, + force=False, reset=False, restart_supervisor=False, restart_systemd=False): """command: bench update""" from bench import patches from bench.app import is_version_upgrade, pull_apps, validate_branch @@ -218,7 +219,6 @@ def update(pull=False, apps=None, patch=False, build=False, requirements=False, if version_upgrade[0] or (not version_upgrade[0] and force): validate_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) - conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 }) update_config(conf, bench_path=bench_path) @@ -246,6 +246,10 @@ def update(pull=False, apps=None, patch=False, build=False, requirements=False, if version_upgrade[0] or (not version_upgrade[0] and force): post_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) + if pull and compile: + print("Compiling Python files...") + compileall.compile_dir('../apps', quiet=1, rx=re.compile('.*node_modules.*')) + if restart_supervisor or conf.get('restart_supervisor_on_update'): restart_supervisor_processes(bench_path=bench_path) @@ -481,7 +485,7 @@ def start(no_dev=False, concurrency=None, procfile=None, no_prefix=False): if no_prefix: command.extend(['--no-prefix']) - + os.execv(program, command) From aae7ec4ca65d95d0033e9f4a264e14128f98af22 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 24 Nov 2020 15:46:51 +0530 Subject: [PATCH 16/51] chore: Remove extra whitespaces --- install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.py b/install.py index d4f963f5..69b9a3a4 100644 --- a/install.py +++ b/install.py @@ -156,7 +156,7 @@ def install_prerequisites(): 'sudo yum install -y epel-release redhat-lsb-core git python-setuptools python-devel openssl-devel libffi-devel' ] }) - + # until psycopg2-binary is available for aarch64 (Arm 64-bit), we'll need libpq and libssl dev packages to build psycopg2 from source if platform.machine() == 'aarch64': log("Installing libpq and libssl dev packages to build psycopg2 for aarch64...") From e9c046d56b15e39630490bc64ebaaa6727623186 Mon Sep 17 00:00:00 2001 From: Samuel Danieli <23150094+scdanieli@users.noreply.github.com> Date: Tue, 1 Dec 2020 16:04:57 +0100 Subject: [PATCH 17/51] fix: wrong flag description --- bench/commands/make.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bench/commands/make.py b/bench/commands/make.py index 7fc799e6..2f9f8ac0 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -11,8 +11,8 @@ import click @click.option('--frappe-branch', default=None, help="path to frappe repo") @click.option('--clone-from', default=None, help="copy repos from path") @click.option('--clone-without-update', is_flag=True, help="copy repos from path without update") -@click.option('--no-procfile', is_flag=True, help="Pull changes in all the apps in bench") -@click.option('--no-backups',is_flag=True, help="Run migrations for all sites in the bench") +@click.option('--no-procfile', is_flag=True, help="Do not create a Procfile") +@click.option('--no-backups',is_flag=True, help="Do not set up automatic backup creation for every (new) site") @click.option('--skip-redis-config-generation', is_flag=True, help="Skip redis config generation if already specifying the common-site-config file") @click.option('--skip-assets',is_flag=True, default=False, help="Do not build assets") @click.option('--verbose',is_flag=True, help="Verbose output during install") From 9de383a760a604a73509d72c230f59e5a0fac11e Mon Sep 17 00:00:00 2001 From: Samuel Danieli <23150094+scdanieli@users.noreply.github.com> Date: Wed, 2 Dec 2020 12:01:25 +0100 Subject: [PATCH 18/51] feat: better flag description Co-authored-by: gavin --- bench/commands/make.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/commands/make.py b/bench/commands/make.py index 2f9f8ac0..554b0603 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -12,7 +12,7 @@ import click @click.option('--clone-from', default=None, help="copy repos from path") @click.option('--clone-without-update', is_flag=True, help="copy repos from path without update") @click.option('--no-procfile', is_flag=True, help="Do not create a Procfile") -@click.option('--no-backups',is_flag=True, help="Do not set up automatic backup creation for every (new) site") +@click.option('--no-backups',is_flag=True, help="Do not set up automatic periodic backups for all sites on this bench") @click.option('--skip-redis-config-generation', is_flag=True, help="Skip redis config generation if already specifying the common-site-config file") @click.option('--skip-assets',is_flag=True, default=False, help="Do not build assets") @click.option('--verbose',is_flag=True, help="Verbose output during install") From 7f9bed4c8e18114a54432e426e958281da7428ec Mon Sep 17 00:00:00 2001 From: Raffael Meyer <14891507+barredterra@users.noreply.github.com> Date: Mon, 25 Jan 2021 19:36:56 +0100 Subject: [PATCH 19/51] fix: nginx proxy buffer --- bench/config/templates/nginx.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bench/config/templates/nginx.conf b/bench/config/templates/nginx.conf index 06fc0b2b..82f16d46 100644 --- a/bench/config/templates/nginx.conf +++ b/bench/config/templates/nginx.conf @@ -29,6 +29,10 @@ server { {% if allow_rate_limiting %} limit_conn per_host_{{ bench_name_hash }} 8; {% endif %} + + proxy_buffer_size 128k; + proxy_buffers 4 256k; + proxy_busy_buffers_size 256k; {% if ssl_certificate and ssl_certificate_key %} ssl on; From 0b925a383816911835bff3a3cf5522ce79e2e87a Mon Sep 17 00:00:00 2001 From: "hasnain2808@gmail.com" Date: Wed, 3 Feb 2021 10:21:40 +0530 Subject: [PATCH 20/51] chore: Add Semantic Release --- .github/workflows/release.yml | 28 ++++++++++++++++++++++++++++ .releaserc | 25 +++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 .github/workflows/release.yml create mode 100644 .releaserc diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..ff2f7879 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,28 @@ +name: Release +on: + push: + branches: + - v5.x +jobs: + release: + name: Release + runs-on: ubuntu-18.04 + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Setup Node.js + uses: actions/setup-node@v1 + with: + node-version: 12 + - name: Set-up semantic-release plugins + run: npm install @semantic-release/git @semantic-release/exec --no-save + - name: Set-up python build env + run: pip install wheel twine + - name: Release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TEST_PYPI_PASSWORD: ${{ secrets.TEST_PYPI_PASSWORD }} + TEST_PYPI_USERNAME: ${{ secrets.TEST_PYPI_USERNAME }} + run: npx semantic-release diff --git a/.releaserc b/.releaserc new file mode 100644 index 00000000..7d97606d --- /dev/null +++ b/.releaserc @@ -0,0 +1,25 @@ +{ + "branches": ["v5.x"], + "plugins": [ + "@semantic-release/commit-analyzer", + "@semantic-release/release-notes-generator", + ["@semantic-release/exec", { + "prepareCmd": 'sed -ir "s/[0-9]*\.[0-9]*\.[0-9]*/${nextRelease.version}/" bench/__init__.py' + }], + ["@semantic-release/exec", { + "prepareCmd": "python setup.py bdist_wheel --universal" + }], + ["@semantic-release/git", { + "assets": ["bench/__init__.py"], + "message": "chore(release): Bumped to Version ${nextRelease.version}\n\n${nextRelease.notes}" + }], + ["@semantic-release/github", { + "assets": [ + {"path": "dist/*"}, + ] + }], + ["@semantic-release/exec", { + "publishCmd": "python -m twine upload -r testpypi dist/* -u $TEST_PYPI_USERNAME -p $TEST_PYPI_PASSWORD" + }] + ] +} From 392bfb48f061b3e4f23400dedec9ff4c420cec33 Mon Sep 17 00:00:00 2001 From: "hasnain2808@gmail.com" Date: Thu, 4 Feb 2021 09:48:56 +0530 Subject: [PATCH 21/51] chore: testpypi to pypi --- .github/workflows/release.yml | 4 ++-- .releaserc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ff2f7879..0f5c3ef9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,6 +23,6 @@ jobs: - name: Release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TEST_PYPI_PASSWORD: ${{ secrets.TEST_PYPI_PASSWORD }} - TEST_PYPI_USERNAME: ${{ secrets.TEST_PYPI_USERNAME }} + TEST_PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + TEST_PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }} run: npx semantic-release diff --git a/.releaserc b/.releaserc index 7d97606d..7b4a238f 100644 --- a/.releaserc +++ b/.releaserc @@ -19,7 +19,7 @@ ] }], ["@semantic-release/exec", { - "publishCmd": "python -m twine upload -r testpypi dist/* -u $TEST_PYPI_USERNAME -p $TEST_PYPI_PASSWORD" + "publishCmd": "python -m twine upload dist/* -u $PYPI_USERNAME -p $PYPI_PASSWORD" }] ] } From 34a56287b51c17b8d0642dd3ea1ac69806a4103a Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 4 Feb 2021 12:57:17 +0530 Subject: [PATCH 22/51] style: Re-worded tasks and fixed inconsistent spacing --- .github/workflows/release.yml | 20 +++++++------- .releaserc | 50 +++++++++++++++++++++-------------- 2 files changed, 40 insertions(+), 30 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0f5c3ef9..18103def 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,4 @@ -name: Release +name: Generate Semantic Release and publish on PyPI on: push: branches: @@ -8,21 +8,21 @@ jobs: name: Release runs-on: ubuntu-18.04 steps: - - name: Checkout + - name: Checkout Entire Repository uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Setup Node.js + - name: Setup Node.js v12 uses: actions/setup-node@v1 with: node-version: 12 - - name: Set-up semantic-release plugins - run: npm install @semantic-release/git @semantic-release/exec --no-save - - name: Set-up python build env - run: pip install wheel twine - - name: Release + - name: Setup dependencies + run: | + npm install @semantic-release/git @semantic-release/exec --no-save + pip install wheel twine + - name: Create Release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TEST_PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - TEST_PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }} + TEST_PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + TEST_PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }} run: npx semantic-release diff --git a/.releaserc b/.releaserc index 7b4a238f..43cc0109 100644 --- a/.releaserc +++ b/.releaserc @@ -1,25 +1,35 @@ { "branches": ["v5.x"], - "plugins": [ + "plugins": [ "@semantic-release/commit-analyzer", "@semantic-release/release-notes-generator", - ["@semantic-release/exec", { - "prepareCmd": 'sed -ir "s/[0-9]*\.[0-9]*\.[0-9]*/${nextRelease.version}/" bench/__init__.py' - }], - ["@semantic-release/exec", { - "prepareCmd": "python setup.py bdist_wheel --universal" - }], - ["@semantic-release/git", { - "assets": ["bench/__init__.py"], - "message": "chore(release): Bumped to Version ${nextRelease.version}\n\n${nextRelease.notes}" - }], - ["@semantic-release/github", { - "assets": [ - {"path": "dist/*"}, - ] - }], - ["@semantic-release/exec", { - "publishCmd": "python -m twine upload dist/* -u $PYPI_USERNAME -p $PYPI_PASSWORD" - }] - ] + [ + "@semantic-release/exec", { + "prepareCmd": 'sed -ir "s/[0-9]*\.[0-9]*\.[0-9]*/${nextRelease.version}/" bench/__init__.py' + } + ], + [ + "@semantic-release/exec", { + "prepareCmd": "python setup.py bdist_wheel --universal" + } + ], + [ + "@semantic-release/git", { + "assets": ["bench/__init__.py"], + "message": "chore(release): Bumped to Version ${nextRelease.version}\n\n${nextRelease.notes}" + } + ], + [ + "@semantic-release/github", { + "assets": [ + {"path": "dist/*"}, + ] + } + ], + [ + "@semantic-release/exec", { + "publishCmd": "python -m twine upload dist/* -u $PYPI_USERNAME -p $PYPI_PASSWORD" + } + ] + ] } From 4211f5d750b5f6130e5479ea263154d3cd87bb84 Mon Sep 17 00:00:00 2001 From: Mohammad Hasnain Mohsin Rajan Date: Thu, 4 Feb 2021 16:09:42 +0530 Subject: [PATCH 23/51] fix: change TEST_PYPI to PYPI --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 18103def..fbb96207 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,6 +23,6 @@ jobs: - name: Create Release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TEST_PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - TEST_PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }} + PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }} run: npx semantic-release From 7e171967b3773446e231dfe56221e107eda0c157 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 9 Feb 2021 11:06:04 +0530 Subject: [PATCH 24/51] docs: Update help message for --frappe-branch --- bench/commands/make.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/commands/make.py b/bench/commands/make.py index 554b0603..c20b25b3 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -8,7 +8,7 @@ import click @click.option('--ignore-exist', is_flag = True, default = False, help = "Ignore if Bench instance exists.") @click.option('--apps_path', default=None, help="path to json files with apps to install after init") @click.option('--frappe-path', default=None, help="path to frappe repo") -@click.option('--frappe-branch', default=None, help="path to frappe repo") +@click.option('--frappe-branch', default=None, help="Clone a particular branch of frappe") @click.option('--clone-from', default=None, help="copy repos from path") @click.option('--clone-without-update', is_flag=True, help="copy repos from path without update") @click.option('--no-procfile', is_flag=True, help="Do not create a Procfile") From 17ef5da5ce3a7061a01c890ae8ca90dcbb6c7957 Mon Sep 17 00:00:00 2001 From: DANY ROBERT Date: Tue, 16 Feb 2021 20:17:49 +0530 Subject: [PATCH 25/51] fix: Upgrade pip first (#1120) * fix: Upgrade pip first `pip` needs to be upgraded before `cryptography` to avoid version compatibility error. * fix: Add setuptools-rust in upgrade. * fix: Install setuptools-rust * fix: Add setuptools-rust in requirments.txt * fix: Revert unwanted addition to README * fix: Install setuptools-rust beforehand. * fix: Upgrade pip beforehand. * fix: Upgrade pip separately beforehand. * fix: Upgrade setuptools-rust with pip. * chore: Remove unused dependency Co-authored-by: gavin --- install.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/install.py b/install.py index 71647824..1abd2655 100644 --- a/install.py +++ b/install.py @@ -170,8 +170,11 @@ def install_prerequisites(): install_package('git') install_package('pip3', 'python3-pip') + run_os_command({ + 'python3': "sudo -H python3 -m pip install --upgrade pip setuptools-rust" + }) success = run_os_command({ - 'python3': "sudo -H python3 -m pip install --upgrade setuptools wheel cryptography ansible~=2.8.15 pip" + 'python3': "sudo -H python3 -m pip install --upgrade setuptools wheel cryptography ansible~=2.8.15" }) if not (success or shutil.which('ansible')): From 5c5d2b57dfaf0e43b56b7140717649e356e2cad6 Mon Sep 17 00:00:00 2001 From: Nicolas Streng Date: Tue, 16 Feb 2021 16:45:07 +0100 Subject: [PATCH 26/51] fix: create bench folder in user directory --- bench/playbooks/create_user.yml | 2 +- bench/playbooks/roles/bench/tasks/main.yml | 4 ++-- bench/playbooks/roles/bench/tasks/setup_inputrc.yml | 4 ++-- bench/playbooks/site.yml | 4 ++-- install.py | 8 +++++++- 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/bench/playbooks/create_user.yml b/bench/playbooks/create_user.yml index 2991afb3..10695833 100644 --- a/bench/playbooks/create_user.yml +++ b/bench/playbooks/create_user.yml @@ -11,7 +11,7 @@ - name: Set home folder perms file: - path: '/home/{{ frappe_user }}' + path: '{{ user_directory }}' mode: 'o+rx' owner: '{{ frappe_user }}' group: '{{ frappe_user }}' diff --git a/bench/playbooks/roles/bench/tasks/main.yml b/bench/playbooks/roles/bench/tasks/main.yml index e01d18b7..164a216e 100644 --- a/bench/playbooks/roles/bench/tasks/main.yml +++ b/bench/playbooks/roles/bench/tasks/main.yml @@ -33,7 +33,7 @@ - name: Fix permissions become_user: root - command: chown {{ frappe_user }} -R /home/{{ frappe_user }} + command: chown {{ frappe_user }} -R {{ user_directory }} - name: python3 bench init for develop command: bench init {{ bench_path }} --frappe-path {{ frappe_repo_url }} --frappe-branch {{ frappe_branch }} --python {{ python }} @@ -77,6 +77,6 @@ # Setup Bench for production environment - include_tasks: setup_bench_production.yml vars: - bench_path: "/home/{{ frappe_user }}/{{ bench_name }}" + bench_path: "{{ user_directory }}/{{ bench_name }}" when: not run_travis and production ... diff --git a/bench/playbooks/roles/bench/tasks/setup_inputrc.yml b/bench/playbooks/roles/bench/tasks/setup_inputrc.yml index 9c88b933..14b47a34 100644 --- a/bench/playbooks/roles/bench/tasks/setup_inputrc.yml +++ b/bench/playbooks/roles/bench/tasks/setup_inputrc.yml @@ -1,11 +1,11 @@ --- - name: insert/update inputrc for history blockinfile: - dest: "/home/{{ frappe_user }}/.inputrc" + dest: "{{ user_directory }}/.inputrc" create: yes block: | ## arrow up "\e[A":history-search-backward ## arrow down "\e[B":history-search-forward -... \ No newline at end of file +... diff --git a/bench/playbooks/site.yml b/bench/playbooks/site.yml index 328cfcba..7a3e106c 100644 --- a/bench/playbooks/site.yml +++ b/bench/playbooks/site.yml @@ -40,8 +40,8 @@ - name: setup bench and dev environment hosts: localhost vars: - bench_repo_path: "/home/{{ frappe_user }}/.bench" - bench_path: "/home/{{ frappe_user }}/{{ bench_name }}" + bench_repo_path: "{{ user_directory }}/.bench" + bench_path: "{{ user_directory }}/{{ bench_name }}" roles: # setup frappe-bench - { role: bench, tags: "bench", when: not run_travis and not without_bench_setup } diff --git a/install.py b/install.py index 71647824..0b154093 100644 --- a/install.py +++ b/install.py @@ -231,10 +231,11 @@ def install_bench(args): # create user if not exists extra_vars = vars(args) extra_vars.update(frappe_user=args.user) + + extra_vars.update(user_directory=get_user_home_directory(args.user)) if os.path.exists(tmp_bench_repo): repo_path = tmp_bench_repo - else: repo_path = os.path.join(os.path.expanduser('~'), 'bench') @@ -383,6 +384,11 @@ def get_extra_vars_json(extra_args): return ('@' + json_path) +def get_user_home_directory(user): + # Return home directory /home/USERNAME or anything else defined as home directory in + # passwd for user. + return os.path.expanduser('~'+user) + def run_playbook(playbook_name, sudo=False, extra_vars=None): args = ['ansible-playbook', '-c', 'local', playbook_name , '-vvvv'] From 294b399942e3b143fdb235efe9176460f12ad9d7 Mon Sep 17 00:00:00 2001 From: Nicolas Streng Date: Wed, 17 Feb 2021 08:18:37 +0100 Subject: [PATCH 27/51] fix: Update documentation for user directory --- docs/easy_install.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/easy_install.md b/docs/easy_install.md index 3c59f308..91ee5566 100644 --- a/docs/easy_install.md +++ b/docs/easy_install.md @@ -34,6 +34,8 @@ If you are on a fresh server and logged in as root, at first create a dedicated *(it is very common to use "frappe" as frappe-username, but this comes with the security flaw of ["frappe" ranking very high](https://www.reddit.com/r/dataisbeautiful/comments/b3sirt/i_deployed_over_a_dozen_cyber_honeypots_all_over/?st=JTJ0SC0Q&sh=76e05240) in as a username challenged in hacking attempts. So, for production sites it is highly recommended to use a custom username harder to guess)* +*(you can specify the flag --home to specify a directory for your [frappe-user]. Bench will follow the home directory specified by the user's home directory e.g. /data/[frappe-user]/frappe-bench)* + Switch to `[frappe-user]` (using `su [frappe-user]`) and start the setup wget https://raw.githubusercontent.com/frappe/bench/develop/install.py @@ -71,7 +73,7 @@ use --python flag to specify virtual environments python version, by default scr ## How do I start ERPNext -1. For development: Go to your bench folder (`frappe-bench` by default) and start the bench with `bench start` +1. For development: Go to your bench folder (`~[frappe-user]/frappe-bench` by default) and start the bench with `bench start` 2. For production: Your process will be setup and managed by `nginx` and `supervisor`. Checkout [Setup Production](https://frappe.io/docs/user/en/bench/guides/setup-production.html) for more information. --- From fec89212143d122c647292195fd4b1f8024a55bd Mon Sep 17 00:00:00 2001 From: DANY ROBERT Date: Thu, 18 Feb 2021 11:31:38 +0530 Subject: [PATCH 28/51] fix: fail2ban help message --- bench/commands/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bench/commands/setup.py b/bench/commands/setup.py index 08e2d4f2..328fc74d 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -253,8 +253,8 @@ def setup_roles(role, **kwargs): @click.command("fail2ban", help="Setup fail2ban, an intrusion prevention software framework that protects computer servers from brute-force attacks") @click.option("--maxretry", default=6, help="Number of matches (i.e. value of the counter) which triggers ban action on the IP. Default is 6 seconds" ) -@click.option("--bantime", default=600, help="The counter is set to zero if no match is found within 'findtime' seconds. Default is 600 seconds") -@click.option("--findtime", default=600, help="Duration (in seconds) for IP to be banned for. Negative number for 'permanent' ban. Default is 600 seconds") +@click.option("--bantime", default=600, help="Duration (in seconds) for IP to be banned for. Negative number for 'permanent' ban. Default is 600 seconds") +@click.option("--findtime", default=600, help="The counter is set to zero if match found within 'findtime' seconds doesn't exceed 'maxretry'. Default is 600 seconds") def setup_nginx_proxy_jail(**kwargs): run_playbook("roles/fail2ban/tasks/configure_nginx_jail.yml", extra_vars=kwargs) From cc9258492784ed0fc7fa31f283b97f2e4e260836 Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Sat, 20 Feb 2021 13:14:56 +0530 Subject: [PATCH 29/51] perf: Remove requests from the import tree --- bench/app.py | 5 ++++- bench/release.py | 9 ++++++--- bench/utils.py | 7 ++++++- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/bench/app.py b/bench/app.py index 2c075985..e70d51ed 100755 --- a/bench/app.py +++ b/bench/app.py @@ -13,7 +13,6 @@ import sys # imports - third party imports import click import git -import requests import semantic_version from six.moves import reload_module @@ -91,6 +90,8 @@ def remove_from_excluded_apps_txt(app, bench_path='.'): return write_excluded_apps_txt(apps, bench_path=bench_path) def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=False, restart_bench=True, overwrite=False): + import requests + if not os.path.exists(git_url): if not is_git_url(git_url): orgs = ['frappe', 'erpnext'] @@ -433,6 +434,8 @@ def install_apps_from_path(path, bench_path='.'): get_app(app['url'], branch=app.get('branch'), bench_path=bench_path, skip_assets=True) def get_apps_json(path): + import requests + if path.startswith('http'): r = requests.get(path) return r.json() diff --git a/bench/release.py b/bench/release.py index f026b379..81efdc02 100755 --- a/bench/release.py +++ b/bench/release.py @@ -4,11 +4,8 @@ import os import sys import semantic_version import git -import requests import getpass import re -from requests.auth import HTTPBasicAuth -import requests.exceptions from time import sleep from .config.common_site_config import get_config import click @@ -47,6 +44,9 @@ def release(bench_path, app, bump_type, from_branch, to_branch, repo_name=repo_name, remote=remote, frontport=frontport) def validate(bench_path, config): + import requests + from requests.auth import HTTPBasicAuth + global github_username, github_password github_username = config.get('github_username') @@ -306,6 +306,9 @@ def push_release(repo_path, from_branch, to_branch, remote='upstream'): def create_github_release(repo_path, tag_name, message, remote='upstream', owner='frappe', repo_name=None, gh_username=None, gh_password=None, prerelease=False): + import requests + import requests.exceptions + from requests.auth import HTTPBasicAuth print('creating release on github') diff --git a/bench/utils.py b/bench/utils.py index bb1b55e4..7a08c346 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -24,7 +24,6 @@ from distutils.spawn import find_executable # imports - third party imports import click from crontab import CronTab -import requests from semantic_version import Version from six import iteritems from six.moves.urllib.parse import urlparse @@ -93,6 +92,8 @@ def safe_decode(string, encoding = 'utf-8'): def check_latest_version(): + import requests + try: pypi_request = requests.get("https://pypi.org/pypi/frappe-bench/json") except Exception: @@ -815,6 +816,8 @@ sudo supervisorctl reload def update_translations_p(args): + import requests + try: update_translations(*args) except requests.exceptions.HTTPError: @@ -846,6 +849,8 @@ def get_langs(): def update_translations(app, lang): + import requests + translations_dir = os.path.join('apps', app, app, 'translations') csv_file = os.path.join(translations_dir, lang + '.csv') url = "https://translate.erpnext.com/files/{}-{}.csv".format(app, lang) From df773d4f9f60f6d8b566f29ee44bc3769b895e8d Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Sat, 20 Feb 2021 13:22:56 +0530 Subject: [PATCH 30/51] perf: Remove git from the import tree --- bench/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/app.py b/bench/app.py index e70d51ed..84070c28 100755 --- a/bench/app.py +++ b/bench/app.py @@ -12,7 +12,6 @@ import sys # imports - third party imports import click -import git import semantic_version from six.moves import reload_module @@ -359,6 +358,7 @@ def get_repo_dir(app, bench_path='.'): return os.path.join(bench_path, 'apps', app) def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrade=True): + import git from bench.utils import update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets, post_upgrade apps_dir = os.path.join(bench_path, 'apps') version_upgrade = (False,) From ad90d9485f389b168b96f1de161a0bd281a936cf Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Sat, 20 Feb 2021 13:52:39 +0530 Subject: [PATCH 31/51] perf: Remove Jinja and PackageLoader from the import tree --- bench/config/__init__.py | 5 +++-- bench/config/lets_encrypt.py | 2 +- bench/config/nginx.py | 4 ++-- bench/config/procfile.py | 2 +- bench/config/redis.py | 2 +- bench/config/supervisor.py | 2 +- bench/config/systemd.py | 26 +++++++++++++------------- bench/utils.py | 2 +- 8 files changed, 23 insertions(+), 22 deletions(-) diff --git a/bench/config/__init__.py b/bench/config/__init__.py index 301b515b..5a1b0c60 100644 --- a/bench/config/__init__.py +++ b/bench/config/__init__.py @@ -1,6 +1,7 @@ """Module for setting up system and respective bench configurations""" # imports - third party imports -from jinja2 import Environment, PackageLoader -env = Environment(loader=PackageLoader('bench.config')) +def env(): + from jinja2 import Environment, PackageLoader + return Environment(loader=PackageLoader('bench.config')) diff --git a/bench/config/lets_encrypt.py b/bench/config/lets_encrypt.py index 32ca3380..9f285199 100755 --- a/bench/config/lets_encrypt.py +++ b/bench/config/lets_encrypt.py @@ -48,7 +48,7 @@ def setup_letsencrypt(site, custom_domain, bench_path, interactive): def create_config(site, custom_domain): - config = bench.config.env.get_template('letsencrypt.cfg').render(domain=custom_domain or site) + config = bench.config.env().get_template('letsencrypt.cfg').render(domain=custom_domain or site) config_path = '/etc/letsencrypt/configs/{site}.cfg'.format(site=custom_domain or site) create_dir_if_missing(config_path) diff --git a/bench/config/nginx.py b/bench/config/nginx.py index fbb73680..4ed4233d 100644 --- a/bench/config/nginx.py +++ b/bench/config/nginx.py @@ -20,7 +20,7 @@ def make_nginx_conf(bench_path, yes=False): if not click.confirm('nginx.conf already exists and this will overwrite it. Do you want to continue?'): return - template = bench.config.env.get_template('nginx.conf') + template = bench.config.env().get_template('nginx.conf') bench_path = os.path.abspath(bench_path) sites_path = os.path.join(bench_path, "sites") @@ -59,7 +59,7 @@ def make_bench_manager_nginx_conf(bench_path, yes=False, port=23624, domain=None from bench.config.site_config import get_site_config from bench.config.common_site_config import get_config - template = bench.config.env.get_template('bench_manager_nginx.conf') + template = bench.config.env().get_template('bench_manager_nginx.conf') bench_path = os.path.abspath(bench_path) sites_path = os.path.join(bench_path, "sites") diff --git a/bench/config/procfile.py b/bench/config/procfile.py index 1b3f20ed..e57e579d 100755 --- a/bench/config/procfile.py +++ b/bench/config/procfile.py @@ -18,7 +18,7 @@ def setup_procfile(bench_path, yes=False, skip_redis=False): click.confirm('A Procfile already exists and this will overwrite it. Do you want to continue?', abort=True) - procfile = bench.config.env.get_template('Procfile').render( + procfile = bench.config.env().get_template('Procfile').render( node=find_executable("node") or find_executable("nodejs"), use_rq=use_rq(bench_path), webserver_port=config.get('webserver_port'), diff --git a/bench/config/redis.py b/bench/config/redis.py index 42005b07..9d8f3ee4 100644 --- a/bench/config/redis.py +++ b/bench/config/redis.py @@ -52,7 +52,7 @@ def generate_config(bench_path): os.makedirs(pid_path) def write_redis_config(template_name, context, bench_path): - template = bench.config.env.get_template(template_name) + template = bench.config.env().get_template(template_name) if "pid_path" not in context: context["pid_path"] = os.path.abspath(os.path.join(bench_path, "config", "pids")) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 8788757f..8a3d07fa 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -22,7 +22,7 @@ def generate_supervisor_config(bench_path, user=None, yes=False): if not user: user = getpass.getuser() - template = bench.config.env.get_template('supervisor.conf') + template = bench.config.env().get_template('supervisor.conf') config = get_config(bench_path=bench_path) bench_dir = os.path.abspath(bench_path) diff --git a/bench/config/systemd.py b/bench/config/systemd.py index 4a414825..574f857e 100644 --- a/bench/config/systemd.py +++ b/bench/config/systemd.py @@ -85,7 +85,7 @@ def setup_systemd_directory(bench_path): def setup_main_config(bench_info, bench_path): # Main config - bench_template = bench.config.env.get_template('systemd/frappe-bench.target') + bench_template = bench.config.env().get_template('systemd/frappe-bench.target') bench_config = bench_template.render(**bench_info) bench_config_path = os.path.join(bench_path, 'config', 'systemd' , bench_info.get("bench_name") + '.target') @@ -94,11 +94,11 @@ def setup_main_config(bench_info, bench_path): def setup_workers_config(bench_info, bench_path): # Worker Group - bench_workers_target_template = bench.config.env.get_template('systemd/frappe-bench-workers.target') - bench_default_worker_template = bench.config.env.get_template('systemd/frappe-bench-frappe-default-worker.service') - bench_short_worker_template = bench.config.env.get_template('systemd/frappe-bench-frappe-short-worker.service') - bench_long_worker_template = bench.config.env.get_template('systemd/frappe-bench-frappe-long-worker.service') - bench_schedule_worker_template = bench.config.env.get_template('systemd/frappe-bench-frappe-schedule.service') + bench_workers_target_template = bench.config.env().get_template('systemd/frappe-bench-workers.target') + bench_default_worker_template = bench.config.env().get_template('systemd/frappe-bench-frappe-default-worker.service') + bench_short_worker_template = bench.config.env().get_template('systemd/frappe-bench-frappe-short-worker.service') + bench_long_worker_template = bench.config.env().get_template('systemd/frappe-bench-frappe-long-worker.service') + bench_schedule_worker_template = bench.config.env().get_template('systemd/frappe-bench-frappe-schedule.service') bench_workers_target_config = bench_workers_target_template.render(**bench_info) bench_default_worker_config = bench_default_worker_template.render(**bench_info) @@ -129,9 +129,9 @@ def setup_workers_config(bench_info, bench_path): def setup_web_config(bench_info, bench_path): # Web Group - bench_web_target_template = bench.config.env.get_template('systemd/frappe-bench-web.target') - bench_web_service_template = bench.config.env.get_template('systemd/frappe-bench-frappe-web.service') - bench_node_socketio_template = bench.config.env.get_template('systemd/frappe-bench-node-socketio.service') + bench_web_target_template = bench.config.env().get_template('systemd/frappe-bench-web.target') + bench_web_service_template = bench.config.env().get_template('systemd/frappe-bench-frappe-web.service') + bench_node_socketio_template = bench.config.env().get_template('systemd/frappe-bench-node-socketio.service') bench_web_target_config = bench_web_target_template.render(**bench_info) bench_web_service_config = bench_web_service_template.render(**bench_info) @@ -152,10 +152,10 @@ def setup_web_config(bench_info, bench_path): def setup_redis_config(bench_info, bench_path): # Redis Group - bench_redis_target_template = bench.config.env.get_template('systemd/frappe-bench-redis.target') - bench_redis_cache_template = bench.config.env.get_template('systemd/frappe-bench-redis-cache.service') - bench_redis_queue_template = bench.config.env.get_template('systemd/frappe-bench-redis-queue.service') - bench_redis_socketio_template = bench.config.env.get_template('systemd/frappe-bench-redis-socketio.service') + bench_redis_target_template = bench.config.env().get_template('systemd/frappe-bench-redis.target') + bench_redis_cache_template = bench.config.env().get_template('systemd/frappe-bench-redis-cache.service') + bench_redis_queue_template = bench.config.env().get_template('systemd/frappe-bench-redis-queue.service') + bench_redis_socketio_template = bench.config.env().get_template('systemd/frappe-bench-redis-socketio.service') bench_redis_target_config = bench_redis_target_template.render(**bench_info) bench_redis_cache_config = bench_redis_cache_template.render(**bench_info) diff --git a/bench/utils.py b/bench/utils.py index 7a08c346..4e7a24db 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -423,7 +423,7 @@ def setup_sudoers(user): if set_permissions: os.chmod('/etc/sudoers', 0o440) - template = bench.config.env.get_template('frappe_sudoers') + template = bench.config.env().get_template('frappe_sudoers') frappe_sudoers = template.render(**{ 'user': user, 'service': find_executable('service'), From ad37665e8b9520a22d0a624104a547afa83a40b8 Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Sun, 21 Feb 2021 10:01:44 +0530 Subject: [PATCH 32/51] perf: Remove multiprocessing from the import tree --- bench/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bench/utils.py b/bench/utils.py index 4e7a24db..f6ec6b54 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -9,7 +9,6 @@ import grp import itertools import json import logging -import multiprocessing import os import pwd import re @@ -825,6 +824,8 @@ def update_translations_p(args): def download_translations_p(): + import multiprocessing + pool = multiprocessing.Pool(multiprocessing.cpu_count()) langs = get_langs() From 2a4377ac6f22f53dc687f222e2e6c7bb2f7879e6 Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Sun, 21 Feb 2021 10:02:02 +0530 Subject: [PATCH 33/51] perf: Remove crontab from the import tree --- bench/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/utils.py b/bench/utils.py index f6ec6b54..1428d19d 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -22,7 +22,6 @@ from distutils.spawn import find_executable # imports - third party imports import click -from crontab import CronTab from semantic_version import Version from six import iteritems from six.moves.urllib.parse import urlparse @@ -386,6 +385,7 @@ def get_sites(bench_path='.'): def setup_backups(bench_path='.'): + from crontab import CronTab from bench.config.common_site_config import get_config logger.log('setting up backups') From 104efd349bcc9cf1ed599279778976322272a065 Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Sun, 21 Feb 2021 10:02:20 +0530 Subject: [PATCH 34/51] perf: Remove ast from the import tree --- bench/commands/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bench/commands/config.py b/bench/commands/config.py index 4d4bec59..196e7877 100644 --- a/bench/commands/config.py +++ b/bench/commands/config.py @@ -1,5 +1,4 @@ # imports - standard imports -import ast # imports - module imports from bench.config.common_site_config import update_config, get_config, put_config @@ -52,6 +51,8 @@ def config_http_timeout(seconds): @click.command('set-common-config', help='Set value in common config') @click.option('configs', '-c', '--config', multiple=True, type=(str, str)) def set_common_config(configs): + import ast + common_site_config = {} for key, value in configs: if value in ('true', 'false'): From 8527b106c39c445b9fb8eee3d14fa8beaf9828cf Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Sun, 21 Feb 2021 10:02:41 +0530 Subject: [PATCH 35/51] perf: Remove multiprocessing from the import tree --- bench/config/common_site_config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bench/config/common_site_config.py b/bench/config/common_site_config.py index d71d9efd..26f1040b 100644 --- a/bench/config/common_site_config.py +++ b/bench/config/common_site_config.py @@ -1,7 +1,6 @@ # imports - standard imports import getpass import json -import multiprocessing import os # imports - third party imports @@ -54,6 +53,8 @@ def get_config_path(bench_path): def get_gunicorn_workers(): '''This function will return the maximum workers that can be started depending upon number of cpu's present on the machine''' + import multiprocessing + return { "gunicorn_workers": multiprocessing.cpu_count() * 2 + 1 } From 6c0e1a359ebd19bd3833c3e9d7c5d1ef8948419d Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Sun, 21 Feb 2021 10:03:03 +0530 Subject: [PATCH 36/51] perf: Remove siz.moves.configparser from the import tree --- bench/config/supervisor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 8a3d07fa..b2f26d67 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -11,7 +11,6 @@ from bench.config.common_site_config import get_config, update_config, get_gunic # imports - third party imports import click -from six.moves import configparser logger = logging.getLogger(bench.PROJECT_NAME) @@ -68,6 +67,8 @@ def get_supervisord_conf(): def update_supervisord_config(user=None, yes=False): """From bench v5.x, we're moving to supervisor running as user""" + from six.moves import configparser + from bench.config.production_setup import service if not user: From e12717d63ac410f14f292dab62b398b37b5e48a3 Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Sun, 21 Feb 2021 10:03:36 +0530 Subject: [PATCH 37/51] perf: Remove crontab from the import tree --- bench/config/lets_encrypt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bench/config/lets_encrypt.py b/bench/config/lets_encrypt.py index 9f285199..c771226d 100755 --- a/bench/config/lets_encrypt.py +++ b/bench/config/lets_encrypt.py @@ -3,7 +3,6 @@ import os # imports - third party imports import click -from crontab import CronTab from six.moves.urllib.request import urlretrieve # imports - module imports @@ -86,6 +85,8 @@ def run_certbot_and_setup_ssl(site, custom_domain, bench_path, interactive=True) def setup_crontab(): + from crontab import CronTab + job_command = '/opt/certbot-auto renew -a nginx --post-hook "systemctl reload nginx"' job_comment = 'Renew lets-encrypt every month' print("Setting Up cron job to {0}".format(job_comment)) From 5525548ced5a3c9666e30f22682f7d7cef362240 Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Sun, 21 Feb 2021 10:04:03 +0530 Subject: [PATCH 38/51] perf: Remove six.moves.urllib from the import tree --- bench/config/lets_encrypt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bench/config/lets_encrypt.py b/bench/config/lets_encrypt.py index c771226d..752d361c 100755 --- a/bench/config/lets_encrypt.py +++ b/bench/config/lets_encrypt.py @@ -3,7 +3,6 @@ import os # imports - third party imports import click -from six.moves.urllib.request import urlretrieve # imports - module imports import bench @@ -107,6 +106,8 @@ def create_dir_if_missing(path): def get_certbot(): + from six.moves.urllib.request import urlretrieve + certbot_path = get_certbot_path() create_dir_if_missing(certbot_path) From 36e48c7dd7625edb4dded617992576faa01c96e6 Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Sun, 21 Feb 2021 10:05:40 +0530 Subject: [PATCH 39/51] perf: Remove bench.config.nginx from the import tree --- bench/commands/setup.py | 3 ++- bench/config/site_config.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/bench/commands/setup.py b/bench/commands/setup.py index 08e2d4f2..87b0d5b7 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -7,7 +7,6 @@ import click # imports - module imports import bench.config.lets_encrypt -import bench.config.nginx import bench.config.procfile import bench.config.production_setup import bench.config.redis @@ -31,6 +30,8 @@ def setup_sudoers(user): @click.command("nginx", help="Generate configuration files for NGINX") @click.option("--yes", help="Yes to regeneration of nginx config file", default=False, is_flag=True) def setup_nginx(yes=False): + import bench.config.nginx + bench.config.nginx.make_nginx_conf(bench_path=".", yes=yes) diff --git a/bench/config/site_config.py b/bench/config/site_config.py index 696185c8..9ea67a86 100644 --- a/bench/config/site_config.py +++ b/bench/config/site_config.py @@ -4,7 +4,6 @@ import os from collections import defaultdict # imports - module imports -from bench.config.nginx import make_nginx_conf from bench.utils import get_sites @@ -35,6 +34,8 @@ def set_ssl_certificate_key(site, ssl_certificate_key, bench_path='.', gen_confi set_site_config_nginx_property(site, {"ssl_certificate_key": ssl_certificate_key}, bench_path=bench_path, gen_config=gen_config) def set_site_config_nginx_property(site, config, bench_path='.', gen_config=True): + from bench.config.nginx import make_nginx_conf + if site not in get_sites(bench_path=bench_path): raise Exception("No such site") update_site_config(site, config, bench_path=bench_path) From cab2d44df5c1854d5f5dbce6e627a35565b4d8ef Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Sun, 21 Feb 2021 10:06:11 +0530 Subject: [PATCH 40/51] perf: Remove bench.config.letsencrypt from the import tree --- bench/commands/setup.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/bench/commands/setup.py b/bench/commands/setup.py index 87b0d5b7..21dfdafa 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -6,9 +6,7 @@ import sys import click # imports - module imports -import bench.config.lets_encrypt import bench.config.procfile -import bench.config.production_setup import bench.config.redis import bench.config.site_config import bench.config.supervisor @@ -37,6 +35,8 @@ def setup_nginx(yes=False): @click.command("reload-nginx", help="Checks NGINX config file and reloads service") def reload_nginx(): + import bench.config.production_setup + bench.config.production_setup.reload_nginx() @@ -62,6 +62,8 @@ def setup_fonts(): @click.argument("user") @click.option("--yes", help="Yes to regeneration config", is_flag=True, default=False) def setup_production(user, yes=False): + import bench.config.production_setup + bench.config.production_setup.setup_production(user=user, yes=yes) @@ -104,6 +106,8 @@ def set_ssh_port(port, force=False): @click.option("--custom-domain") @click.option('-n', '--non-interactive', default=False, is_flag=True, help="Run command non-interactively. This flag restarts nginx and runs certbot non interactively. Shouldn't be used on 1'st attempt") def setup_letsencrypt(site, custom_domain, non_interactive): + import bench.config.lets_encrypt + bench.config.lets_encrypt.setup_letsencrypt(site, custom_domain, bench_path=".", interactive=not non_interactive) @@ -112,6 +116,8 @@ def setup_letsencrypt(site, custom_domain, non_interactive): @click.option("--email") @click.option("--exclude-base-domain", default=False, is_flag=True, help="SSL Certificate not applicable for base domain") def setup_wildcard_ssl(domain, email, exclude_base_domain): + import bench.config.lets_encrypt + bench.config.lets_encrypt.setup_wildcard_ssl(domain, email, bench_path=".", exclude_base_domain=exclude_base_domain) From c5d2fb9255979a46fc4e29b9d04f44e8acf2f722 Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Sun, 21 Feb 2021 10:06:44 +0530 Subject: [PATCH 41/51] perf: Remove bench.config.common_site_config from the import tree --- bench/app.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bench/app.py b/bench/app.py index 84070c28..a3a63ec8 100755 --- a/bench/app.py +++ b/bench/app.py @@ -17,7 +17,6 @@ from six.moves import reload_module # imports - module imports import bench -from bench.config.common_site_config import get_config from bench.utils import color, CommandFailedError, build_assets, check_git_for_shallow_clone, exec_cmd, get_cmd_output, get_frappe, restart_supervisor_processes, restart_systemd_processes, run_frappe_cmd @@ -166,6 +165,8 @@ def new_app(app, bench_path='.'): def install_app(app, bench_path=".", verbose=False, no_cache=False, restart_bench=True, skip_assets=False): + from bench.config.common_site_config import get_config + print('\n{0}Installing {1}{2}'.format(color.yellow, app, color.nc)) logger.log("installing {}".format(app)) @@ -194,6 +195,8 @@ def install_app(app, bench_path=".", verbose=False, no_cache=False, restart_benc def remove_app(app, bench_path='.'): + from bench.config.common_site_config import get_config + if app not in get_apps(bench_path): print("No app named {0}".format(app)) sys.exit(1) @@ -221,6 +224,8 @@ def remove_app(app, bench_path='.'): def pull_apps(apps=None, bench_path='.', reset=False): '''Check all apps if there no local changes, pull''' + from bench.config.common_site_config import get_config + rebase = '--rebase' if get_config(bench_path).get('rebase_on_pull') else '' apps = apps or get_apps(bench_path=bench_path) From 6706b230da1bfb42dbd671c5524ee1ed62c6908c Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Sun, 21 Feb 2021 11:10:43 +0530 Subject: [PATCH 42/51] perf: Remove six.moves.urllib from the import tree --- bench/config/common_site_config.py | 3 ++- bench/config/redis.py | 3 ++- bench/utils.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/bench/config/common_site_config.py b/bench/config/common_site_config.py index 26f1040b..08f62c8f 100644 --- a/bench/config/common_site_config.py +++ b/bench/config/common_site_config.py @@ -4,7 +4,6 @@ import json import os # imports - third party imports -from six.moves.urllib.parse import urlparse default_config = { @@ -74,6 +73,8 @@ def update_config_for_frappe(config, bench_path): # TODO Optionally we need to add the host or domain name in case dns_multitenant is false def make_ports(bench_path): + from six.moves.urllib.parse import urlparse + benches_path = os.path.dirname(os.path.abspath(bench_path)) default_ports = { diff --git a/bench/config/redis.py b/bench/config/redis.py index 9d8f3ee4..8c707dcd 100644 --- a/bench/config/redis.py +++ b/bench/config/redis.py @@ -5,7 +5,6 @@ import subprocess # imports - third party imports import semantic_version -from six.moves.urllib.parse import urlparse # imports - module imports import bench @@ -13,6 +12,8 @@ from bench.config.common_site_config import get_config def generate_config(bench_path): + from six.moves.urllib.parse import urlparse + config = get_config(bench_path) ports = {} diff --git a/bench/utils.py b/bench/utils.py index 1428d19d..6a2614cf 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -24,7 +24,6 @@ from distutils.spawn import find_executable import click from semantic_version import Version from six import iteritems -from six.moves.urllib.parse import urlparse # imports - module imports import bench @@ -973,6 +972,7 @@ def find_benches(directory=None): def migrate_env(python, backup=False): + from six.moves.urllib.parse import urlparse from bench.config.common_site_config import get_config from bench.app import get_apps From 07ecaa1f776d15c98d50087e3757dc4910203e7d Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Sun, 21 Feb 2021 11:12:00 +0530 Subject: [PATCH 43/51] perf: Remove shutil from the import tree --- bench/app.py | 3 ++- bench/utils.py | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/bench/app.py b/bench/app.py index a3a63ec8..689a5921 100755 --- a/bench/app.py +++ b/bench/app.py @@ -6,7 +6,6 @@ import json import logging import os import re -import shutil import subprocess import sys @@ -89,6 +88,7 @@ def remove_from_excluded_apps_txt(app, bench_path='.'): def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=False, restart_bench=True, overwrite=False): import requests + import shutil if not os.path.exists(git_url): if not is_git_url(git_url): @@ -195,6 +195,7 @@ def install_app(app, bench_path=".", verbose=False, no_cache=False, restart_benc def remove_app(app, bench_path='.'): + import shutil from bench.config.common_site_config import get_config if app not in get_apps(bench_path): diff --git a/bench/utils.py b/bench/utils.py index 6a2614cf..a9f9a60c 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -13,7 +13,6 @@ import os import pwd import re import select -import shutil import site import subprocess import sys @@ -261,6 +260,8 @@ def update(pull=False, apps=None, patch=False, build=False, requirements=False, def copy_patches_txt(bench_path): + import shutil + shutil.copy(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'patches', 'patches.txt'), os.path.join(bench_path, 'patches.txt')) @@ -900,6 +901,8 @@ def get_bench_name(bench_path): def setup_fonts(): + import shutil + fonts_path = os.path.join('/tmp', 'fonts') if os.path.exists('/etc/fonts_backup'): @@ -972,6 +975,7 @@ def find_benches(directory=None): def migrate_env(python, backup=False): + import shutil from six.moves.urllib.parse import urlparse from bench.config.common_site_config import get_config from bench.app import get_apps From 2df0bf726d9c1fb5fb4188c4c745d918d1cf1774 Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Sun, 21 Feb 2021 11:12:40 +0530 Subject: [PATCH 44/51] perf: Remove semantic_version from the import tree --- bench/app.py | 3 ++- bench/config/redis.py | 5 ++--- bench/utils.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bench/app.py b/bench/app.py index 689a5921..0cf8f619 100755 --- a/bench/app.py +++ b/bench/app.py @@ -11,7 +11,6 @@ import sys # imports - third party imports import click -import semantic_version from six.moves import reload_module # imports - module imports @@ -432,6 +431,8 @@ def get_version_from_string(contents, field='__version__'): return match.group(2) def get_major_version(version): + import semantic_version + return semantic_version.Version(version).major def install_apps_from_path(path, bench_path='.'): diff --git a/bench/config/redis.py b/bench/config/redis.py index 8c707dcd..7af1bbf7 100644 --- a/bench/config/redis.py +++ b/bench/config/redis.py @@ -3,9 +3,6 @@ import os import re import subprocess -# imports - third party imports -import semantic_version - # imports - module imports import bench from bench.config.common_site_config import get_config @@ -62,6 +59,8 @@ def write_redis_config(template_name, context, bench_path): f.write(template.render(**context)) def get_redis_version(): + import semantic_version + version_string = subprocess.check_output('redis-server --version', shell=True) version_string = version_string.decode('utf-8').strip() # extract version number from string diff --git a/bench/utils.py b/bench/utils.py index a9f9a60c..66297ffc 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -21,7 +21,6 @@ from distutils.spawn import find_executable # imports - third party imports import click -from semantic_version import Version from six import iteritems # imports - module imports @@ -89,6 +88,7 @@ def safe_decode(string, encoding = 'utf-8'): def check_latest_version(): import requests + from semantic_version import Version try: pypi_request = requests.get("https://pypi.org/pypi/frappe-bench/json") From 5d903466604c354ede536ffeadbffba0306c7666 Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Sun, 21 Feb 2021 11:12:57 +0530 Subject: [PATCH 45/51] perf: Remove six.moves.reload_module from the import tree --- bench/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/app.py b/bench/app.py index 0cf8f619..0367ea9b 100755 --- a/bench/app.py +++ b/bench/app.py @@ -11,7 +11,6 @@ import sys # imports - third party imports import click -from six.moves import reload_module # imports - module imports import bench @@ -364,6 +363,7 @@ def get_repo_dir(app, bench_path='.'): def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrade=True): import git + from six.moves import reload_module from bench.utils import update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets, post_upgrade apps_dir = os.path.join(bench_path, 'apps') version_upgrade = (False,) From b70aa21a7394c6f323162ee754ca4ed7e76320cd Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Sun, 21 Feb 2021 11:24:32 +0530 Subject: [PATCH 46/51] chore: Remove unused import comment blocks --- bench/commands/config.py | 2 -- bench/config/__init__.py | 1 - bench/config/common_site_config.py | 1 - 3 files changed, 4 deletions(-) diff --git a/bench/commands/config.py b/bench/commands/config.py index 196e7877..b3691941 100644 --- a/bench/commands/config.py +++ b/bench/commands/config.py @@ -1,5 +1,3 @@ -# imports - standard imports - # imports - module imports from bench.config.common_site_config import update_config, get_config, put_config diff --git a/bench/config/__init__.py b/bench/config/__init__.py index 5a1b0c60..f79be532 100644 --- a/bench/config/__init__.py +++ b/bench/config/__init__.py @@ -1,6 +1,5 @@ """Module for setting up system and respective bench configurations""" -# imports - third party imports def env(): from jinja2 import Environment, PackageLoader diff --git a/bench/config/common_site_config.py b/bench/config/common_site_config.py index 08f62c8f..41d9cc9c 100644 --- a/bench/config/common_site_config.py +++ b/bench/config/common_site_config.py @@ -3,7 +3,6 @@ import getpass import json import os -# imports - third party imports default_config = { From 5cf1363481f0cde81d0c4163f4c5de0aff934360 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 26 Feb 2021 07:55:04 +0530 Subject: [PATCH 47/51] feat: skip redis config during supervisor setup --- bench/commands/setup.py | 5 +++-- bench/config/supervisor.py | 5 +++-- bench/config/templates/supervisor.conf | 6 ++++++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/bench/commands/setup.py b/bench/commands/setup.py index 08e2d4f2..860c8375 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -42,9 +42,10 @@ def reload_nginx(): @click.command("supervisor", help="Generate configuration for supervisor") @click.option("--user", help="optional user argument") @click.option("--yes", help="Yes to regeneration of supervisor config", is_flag=True, default=False) -def setup_supervisor(user=None, yes=False): +@click.option("--skip-redis", help="Skip redis configuration", is_flag=True, default=False) +def setup_supervisor(user=None, yes=False, skip_redis=False): bench.config.supervisor.update_supervisord_config(user=user, yes=yes) - bench.config.supervisor.generate_supervisor_config(bench_path=".", user=user, yes=yes) + bench.config.supervisor.generate_supervisor_config(bench_path=".", user=user, yes=yes, skip_redis=skip_redis) @click.command("redis", help="Generates configuration for Redis") diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 8788757f..08740e70 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -17,7 +17,7 @@ from six.moves import configparser logger = logging.getLogger(bench.PROJECT_NAME) -def generate_supervisor_config(bench_path, user=None, yes=False): +def generate_supervisor_config(bench_path, user=None, yes=False, skip_redis=False): """Generate supervisor config for respective bench path""" if not user: user = getpass.getuser() @@ -42,7 +42,8 @@ def generate_supervisor_config(bench_path, user=None, yes=False): "gunicorn_workers": config.get('gunicorn_workers', get_gunicorn_workers()["gunicorn_workers"]), "bench_name": get_bench_name(bench_path), "background_workers": config.get('background_workers') or 1, - "bench_cmd": find_executable('bench') + "bench_cmd": find_executable('bench'), + "skip_redis": skip_redis, }) conf_path = os.path.join(bench_path, 'config', 'supervisor.conf') diff --git a/bench/config/templates/supervisor.conf b/bench/config/templates/supervisor.conf index a9d06331..92aa8333 100644 --- a/bench/config/templates/supervisor.conf +++ b/bench/config/templates/supervisor.conf @@ -114,6 +114,7 @@ killasgroup=true {% endif %} +{% if not skip_redis %} [program:{{ bench_name }}-redis-cache] command={{ redis_server }} {{ redis_cache_config }} priority=1 @@ -133,8 +134,10 @@ stdout_logfile={{ bench_dir }}/logs/redis-queue.log stderr_logfile={{ bench_dir }}/logs/redis-queue.error.log user={{ user }} directory={{ sites_dir }} +{% endif %} {% if frappe_version > 5 %} +{% if not skip_redis %} [program:{{ bench_name }}-redis-socketio] command={{ redis_server }} {{ redis_socketio_config }} priority=1 @@ -144,6 +147,7 @@ stdout_logfile={{ bench_dir }}/logs/redis-socketio.log stderr_logfile={{ bench_dir }}/logs/redis-socketio.error.log user={{ user }} directory={{ sites_dir }} +{% endif %} {% if node %} [program:{{ bench_name }}-node-socketio] @@ -174,5 +178,7 @@ programs={{ bench_name }}-frappe-workerbeat,{{ bench_name }}-frappe-worker,{{ be {% endif %} +{% if not skip_redis %} [group:{{ bench_name }}-redis] programs={{ bench_name }}-redis-cache,{{ bench_name }}-redis-queue {%- if frappe_version > 5 -%} ,{{ bench_name }}-redis-socketio {%- endif %} +{% endif %} From 4c627e4531ace6324bee3e810b9329f278833f3d Mon Sep 17 00:00:00 2001 From: Mohammad Hasnain Mohsin Rajan Date: Thu, 18 Mar 2021 13:57:21 +0530 Subject: [PATCH 48/51] chore: update version (#1138) * chore: update version * chore: use dev version on develop branch * chore: move develop branch check before imports --- bench/__init__.py | 2 +- bench/utils.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bench/__init__.py b/bench/__init__.py index eec75ea4..19d73b97 100644 --- a/bench/__init__.py +++ b/bench/__init__.py @@ -1,4 +1,4 @@ -VERSION = "5.2.1" +VERSION = "5.0.0-dev" PROJECT_NAME = "frappe-bench" FRAPPE_VERSION = None diff --git a/bench/utils.py b/bench/utils.py index 66297ffc..2c22005b 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -87,6 +87,9 @@ def safe_decode(string, encoding = 'utf-8'): def check_latest_version(): + if bench.VERSION.endswith("dev"): + return + import requests from semantic_version import Version From 9b297284a676f11299ffa6bca25c3c4e2584774b Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 22 Mar 2021 16:23:26 +0530 Subject: [PATCH 49/51] build(deps): [security] bump jinja2 from 2.10.3 to 2.11.3 (#1142) Bumps [jinja2](https://github.com/pallets/jinja) from 2.10.3 to 2.11.3. **This update includes a security fix.** - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/master/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/2.10.3...2.11.3) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ea1bdd08..92df9331 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ Click==7.0 GitPython==2.1.15 honcho==1.0.1 -Jinja2==2.10.3 +Jinja2==2.11.3 python-crontab==2.4.0 requests==2.22.0 semantic-version==2.8.2 From 777b204490cd4bc96eea9fe9ef0cefccd17b4ba4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20HOUZET?= Date: Wed, 7 Apr 2021 09:43:33 +0200 Subject: [PATCH 50/51] docs: Change link on Menu to Basic Usage Fix link on menu to Basic Usage --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index abbbb7f4..f11b7879 100755 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Bench is a command-line utility that helps you to install, update, and manage mu - [Production Setup](#docker-installation-for-production) - [Easy Install Script](#easy-install-script) - [Manual Installation](#manual-installation) - - [Usage](#usage) + - [Usage](#basic-usage) - [Custom Bench commands](#custom-bench-commands) - [Bench Manager](#bench-manager) - [Guides](#guides) From ec2ce0f3b80d0e838b8ccaf1180313e5c47b8339 Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Tue, 13 Apr 2021 13:06:08 +0530 Subject: [PATCH 51/51] chore: remove support for older versions of frappe --- bench/app.py | 8 +---- bench/config/supervisor.py | 3 +- bench/config/systemd.py | 3 +- bench/config/templates/supervisor.conf | 5 +-- bench/utils.py | 48 +++++--------------------- 5 files changed, 13 insertions(+), 54 deletions(-) diff --git a/bench/app.py b/bench/app.py index 0367ea9b..18d768da 100755 --- a/bench/app.py +++ b/bench/app.py @@ -152,13 +152,7 @@ def new_app(app, bench_path='.'): app = app.lower().replace(" ", "_").replace("-", "_") logger.log('creating new app {}'.format(app)) apps = os.path.abspath(os.path.join(bench_path, 'apps')) - bench.set_frappe_version(bench_path=bench_path) - - if bench.FRAPPE_VERSION == 4: - exec_cmd("{frappe} --make_app {apps} {app}".format(frappe=get_frappe(bench_path=bench_path), - apps=apps, app=app)) - else: - run_frappe_cmd('make-app', apps, app, bench_path=bench_path) + run_frappe_cmd('make-app', apps, app, bench_path=bench_path) install_app(app, bench_path=bench_path) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index dd9cfa6e..76ffd877 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -5,7 +5,7 @@ import os # imports - module imports import bench -from bench.app import get_current_frappe_version, use_rq +from bench.app import use_rq from bench.utils import get_bench_name, find_executable from bench.config.common_site_config import get_config, update_config, get_gunicorn_workers @@ -29,7 +29,6 @@ def generate_supervisor_config(bench_path, user=None, yes=False, skip_redis=Fals "bench_dir": bench_dir, "sites_dir": os.path.join(bench_dir, 'sites'), "user": user, - "frappe_version": get_current_frappe_version(bench_path), "use_rq": use_rq(bench_path), "http_timeout": config.get("http_timeout", 120), "redis_server": find_executable('redis-server'), diff --git a/bench/config/systemd.py b/bench/config/systemd.py index 574f857e..c0a0053c 100644 --- a/bench/config/systemd.py +++ b/bench/config/systemd.py @@ -7,7 +7,7 @@ import click # imports - module imports import bench -from bench.app import get_current_frappe_version, use_rq +from bench.app import use_rq from bench.config.common_site_config import get_config, get_gunicorn_workers, update_config from bench.utils import exec_cmd, find_executable, get_bench_name @@ -51,7 +51,6 @@ def generate_systemd_config(bench_path, user=None, yes=False, "bench_dir": bench_dir, "sites_dir": os.path.join(bench_dir, 'sites'), "user": user, - "frappe_version": get_current_frappe_version(bench_path), "use_rq": use_rq(bench_path), "http_timeout": config.get("http_timeout", 120), "redis_server": find_executable('redis-server'), diff --git a/bench/config/templates/supervisor.conf b/bench/config/templates/supervisor.conf index 92aa8333..d24b1cb6 100644 --- a/bench/config/templates/supervisor.conf +++ b/bench/config/templates/supervisor.conf @@ -136,7 +136,6 @@ user={{ user }} directory={{ sites_dir }} {% endif %} -{% if frappe_version > 5 %} {% if not skip_redis %} [program:{{ bench_name }}-redis-socketio] command={{ redis_server }} {{ redis_socketio_config }} @@ -161,8 +160,6 @@ user={{ user }} directory={{ bench_dir }} {% endif %} -{% endif %} - [group:{{ bench_name }}-web] programs={{ bench_name }}-frappe-web {%- if node -%} ,{{ bench_name }}-node-socketio {%- endif%} @@ -180,5 +177,5 @@ programs={{ bench_name }}-frappe-workerbeat,{{ bench_name }}-frappe-worker,{{ be {% if not skip_redis %} [group:{{ bench_name }}-redis] -programs={{ bench_name }}-redis-cache,{{ bench_name }}-redis-queue {%- if frappe_version > 5 -%} ,{{ bench_name }}-redis-socketio {%- endif %} +programs={{ bench_name }}-redis-cache,{{ bench_name }}-redis-queue,{{ bench_name }}-redis-socketio {% endif %} diff --git a/bench/utils.py b/bench/utils.py index 2c22005b..54350ba9 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -163,11 +163,8 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, if apps_path: install_apps_from_path(apps_path, bench_path=path) - - bench.set_frappe_version(bench_path=path) - if bench.FRAPPE_VERSION > 5: - if not skip_assets: - update_node_packages(bench_path=path) + if not skip_assets: + update_node_packages(bench_path=path) set_all_patches_executed(bench_path=path) if not skip_assets: @@ -358,27 +355,17 @@ def setup_socketio(bench_path='.'): def patch_sites(bench_path='.'): - bench.set_frappe_version(bench_path=bench_path) - try: - if bench.FRAPPE_VERSION == 4: - exec_cmd("{frappe} --latest all".format(frappe=get_frappe(bench_path=bench_path)), cwd=os.path.join(bench_path, 'sites')) - else: - run_frappe_cmd('--site', 'all', 'migrate', bench_path=bench_path) + run_frappe_cmd('--site', 'all', 'migrate', bench_path=bench_path) except subprocess.CalledProcessError: raise PatchError def build_assets(bench_path='.', app=None): - bench.set_frappe_version(bench_path=bench_path) - - if bench.FRAPPE_VERSION == 4: - exec_cmd("{frappe} --build".format(frappe=get_frappe(bench_path=bench_path)), cwd=os.path.join(bench_path, 'sites')) - else: - command = 'bench build' - if app: - command += ' --app {}'.format(app) - exec_cmd(command, cwd=bench_path) + command = 'bench build' + if app: + command += ' --app {}'.format(app) + exec_cmd(command, cwd=bench_path) def get_sites(bench_path='.'): @@ -395,14 +382,8 @@ def setup_backups(bench_path='.'): bench_dir = os.path.abspath(bench_path) user = get_config(bench_path=bench_dir).get('frappe_user') logfile = os.path.join(bench_dir, 'logs', 'backup.log') - bench.set_frappe_version(bench_path=bench_path) system_crontab = CronTab(user=user) - - if bench.FRAPPE_VERSION == 4: - backup_command = "cd {sites_dir} && {frappe} --backup all".format(frappe=get_frappe(bench_path=bench_path),) - else: - backup_command = "cd {bench_dir} && {bench} --verbose --site all backup".format(bench_dir=bench_dir, bench=sys.argv[0]) - + backup_command = "cd {bench_dir} && {bench} --verbose --site all backup".format(bench_dir=bench_dir, bench=sys.argv[0]) job_command = "{backup_command} >> {logfile} 2>&1".format(backup_command=backup_command, logfile=logfile) if job_command not in str(system_crontab): @@ -662,13 +643,7 @@ def update_npm_packages(bench_path='.'): def backup_site(site, bench_path='.'): - bench.set_frappe_version(bench_path=bench_path) - - if bench.FRAPPE_VERSION == 4: - exec_cmd("{frappe} --backup {site}".format(frappe=get_frappe(bench_path=bench_path), site=site), - cwd=os.path.join(bench_path, 'sites')) - else: - run_frappe_cmd('--site', site, 'backup', bench_path=bench_path) + run_frappe_cmd('--site', site, 'backup', bench_path=bench_path) def backup_all_sites(bench_path='.'): @@ -754,11 +729,6 @@ def fix_prod_setup_perms(bench_path='.', frappe_user=None): os.chown(path, uid, gid) -def get_current_frappe_version(bench_path='.'): - from .app import get_current_frappe_version as fv - return fv(bench_path=bench_path) - - def run_frappe_cmd(*args, **kwargs): from .cli import from_command_line