From 89bc45e2d0f60f9defd3d7f2236148b1ec8eb9eb Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 23 Apr 2021 12:22:17 +0530 Subject: [PATCH 1/4] ci: Run all builds on 3.7, 3.8, 3.9 * Drop CI testing for older versions: 2.7, 3.5, 3.6 --- .travis.yml | 49 ++++++++++++++++++++++--------------------------- 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8bc96c6e..67c7f56b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,35 +15,20 @@ addons: matrix: include: - - name: "Python 2.7 Basic Setup" - python: 2.7 - env: TEST=bench - script: python bench/tests/test_init.py TestBenchInit.basic - - - name: "Python 3.6 Basic Setup" - python: 3.6 - env: TEST=bench - script: python bench/tests/test_init.py TestBenchInit.basic - - name: "Python 3.7 Basic Setup" python: 3.7 env: TEST=bench script: python bench/tests/test_init.py TestBenchInit.basic - - name: "Python 3.8 Production Setup" + - name: "Python 3.8 Basic Setup" python: 3.8 env: TEST=bench - script: python bench/tests/test_setup_production.py TestSetupProduction.production + script: python bench/tests/test_init.py TestBenchInit.basic - - name: "Python 2.7 Production Setup" - python: 2.7 + - name: "Python 3.9 Basic Setup" + python: 3.9 env: TEST=bench - script: python bench/tests/test_setup_production.py TestSetupProduction.production - - - name: "Python 3.6 Production Setup" - python: 3.6 - env: TEST=bench - script: python bench/tests/test_setup_production.py TestSetupProduction.production + script: python bench/tests/test_init.py TestBenchInit.basic - name: "Python 3.7 Production Setup" python: 3.7 @@ -55,20 +40,25 @@ matrix: env: TEST=bench script: python bench/tests/test_setup_production.py TestSetupProduction.production - - name: "Python 2.7 Tests" - python: 2.7 + - name: "Python 3.9 Production Setup" + python: 3.9 env: TEST=bench - script: python -m unittest -v bench.tests.test_init + script: python bench/tests/test_setup_production.py TestSetupProduction.production - name: "Python 3.7 Tests" python: 3.7 env: TEST=bench script: python -m unittest -v bench.tests.test_init - - name: "Python 3.5 Easy Install" - python: 3.5 - env: TEST=easy_install - script: sudo python $TRAVIS_BUILD_DIR/install.py --user travis --run-travis --production --verbose + - name: "Python 3.8 Tests" + python: 3.8 + env: TEST=bench + script: python -m unittest -v bench.tests.test_init + + - name: "Python 3.9 Tests" + python: 3.9 + env: TEST=bench + script: python -m unittest -v bench.tests.test_init - name: "Python 3.7 Easy Install" python: 3.7 @@ -80,6 +70,11 @@ matrix: env: TEST=easy_install script: sudo python $TRAVIS_BUILD_DIR/install.py --user travis --run-travis --production --verbose + - name: "Python 3.9 Easy Install" + python: 3.9 + env: TEST=easy_install + script: sudo python $TRAVIS_BUILD_DIR/install.py --user travis --run-travis --production --verbose + install: - pip install urllib3 pyOpenSSL ndg-httpsclient pyasn1 From b0ccb6efbea33acdb07b3500a07b37c8ffbeae23 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 23 Apr 2021 18:37:03 +0530 Subject: [PATCH 2/4] fix: Invoke pip via python Due to "WARNING: pip is being invoked by an old script wrapper. This will fail in a future version of pip. Please see https://github.com/pypa/pip/issues/5599 for advice on fixing the underlying issue. To avoid this problem you can invoke Python with '-m pip' instead of running pip directly." --- bench/app.py | 8 ++++---- bench/commands/make.py | 4 ++-- bench/utils.py | 15 +++++++-------- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/bench/app.py b/bench/app.py index 7ebaf8ff..872270ac 100755 --- a/bench/app.py +++ b/bench/app.py @@ -174,12 +174,12 @@ def install_app(app, bench_path=".", verbose=False, no_cache=False, restart_benc print('\n{0}Installing {1}{2}'.format(color.yellow, app, color.nc)) logger.log("installing {}".format(app)) - pip_path = os.path.join(bench_path, "env", "bin", "pip") + python_path = os.path.join(bench_path, "env", "bin", "python") quiet_flag = "-q" if not verbose else "" app_path = os.path.join(bench_path, "apps", app) cache_flag = "--no-cache-dir" if no_cache else "" - exec_cmd("{pip} install {quiet} -U -e {app} {no_cache}".format(pip=pip_path, quiet=quiet_flag, app=app_path, no_cache=cache_flag)) + exec_cmd("{py_path} -m pip install {quiet} -U -e {app} {no_cache}".format(py_path=python_path, quiet=quiet_flag, app=app_path, no_cache=cache_flag)) if os.path.exists(os.path.join(app_path, 'package.json')): exec_cmd("yarn install", cwd=app_path) @@ -208,7 +208,7 @@ def remove_app(app, bench_path='.'): app_path = os.path.join(bench_path, 'apps', app) site_path = os.path.join(bench_path, 'sites') - pip = os.path.join(bench_path, 'env', 'bin', 'pip') + py = os.path.join(bench_path, 'env', 'bin', 'python') for site in os.listdir(site_path): req_file = os.path.join(site_path, site, 'site_config.json') @@ -218,7 +218,7 @@ def remove_app(app, bench_path='.'): print("Cannot remove, app is installed on site: {0}".format(site)) sys.exit(1) - exec_cmd("{0} uninstall -y {1}".format(pip, app), cwd=bench_path) + exec_cmd("{0} -m pip uninstall -y {1}".format(py, app), cwd=bench_path) remove_from_appstxt(app, bench_path) shutil.rmtree(app_path) run_frappe_cmd("build", bench_path=bench_path) diff --git a/bench/commands/make.py b/bench/commands/make.py index c20b25b3..f47a72e0 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -98,5 +98,5 @@ def pip(ctx, args): "Run pip commands in bench env" import os from bench.utils import get_env_cmd - env_pip = get_env_cmd('pip') - os.execv(env_pip, (env_pip,) + args) + env_py = get_env_cmd('python') + os.execv(env_py, (env_py, '-m', 'pip') + args) diff --git a/bench/utils.py b/bench/utils.py index ede9b169..7bb312ba 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -339,13 +339,13 @@ def get_venv_path(): def setup_env(bench_path='.', python='python3'): frappe = os.path.join(bench_path, "apps", "frappe") - pip = os.path.join(bench_path, "env", "bin", "pip") + py = os.path.join(bench_path, "env", "bin", "python") virtualenv = get_venv_path() exec_cmd('{} -q env -p {}'.format(virtualenv, python), cwd=bench_path) if os.path.exists(frappe): - exec_cmd('{} install -q -U -e {}'.format(pip, frappe), cwd=bench_path) + exec_cmd('{} -m pip install -q -U -e {}'.format(py, frappe), cwd=bench_path) def setup_socketio(bench_path='.'): @@ -555,8 +555,8 @@ def set_default_site(site, bench_path='.'): def update_env_pip(bench_path): - env_pip = os.path.join(bench_path, 'env', 'bin', 'pip') - exec_cmd("{pip} install -q -U pip".format(pip=env_pip)) + env_py = os.path.join(bench_path, 'env', 'bin', 'python') + exec_cmd("{env_py} -m pip install -q -U pip".format(env_py=env_py)) def update_requirements(bench_path='.'): @@ -571,14 +571,14 @@ def update_requirements(bench_path='.'): def update_python_packages(bench_path='.'): from bench.app import get_apps - pip_path = os.path.join(bench_path, "env", "bin", "pip") + env_py = os.path.join(bench_path, "env", "bin", "python") print('Updating Python libraries...') update_env_pip(bench_path) for app in get_apps(): print('\n{0}Installing python dependencies for {1}{2}'.format(color.yellow, app, color.nc)) app_path = os.path.join(bench_path, "apps", app) - exec_cmd("{0} install -q -U -e {1}".format(pip_path, app_path), cwd=bench_path) + exec_cmd("{0} -m pip install -q -U -e {1}".format(env_py, app_path), cwd=bench_path) def update_node_packages(bench_path='.'): @@ -965,7 +965,6 @@ def migrate_env(python, backup=False): python = which(python) virtualenv = which('virtualenv') pvenv = os.path.join(path, nvenv) - pip = os.path.join(pvenv, 'bin', 'pip') # Clear Cache before Bench Dies. try: @@ -1006,7 +1005,7 @@ def migrate_env(python, backup=False): venv_creation = exec_cmd('{virtualenv} --python {python} {pvenv}'.format(virtualenv=virtualenv, python=python, pvenv=pvenv)) apps = ' '.join(["-e {}".format(os.path.join("apps", app)) for app in get_apps()]) - packages_setup = exec_cmd('{0} install -q -U {1}'.format(pip, apps)) + packages_setup = exec_cmd('{0} -m pip install -q -U {1}'.format(pvenv, apps)) logger.log('Migration Successful to {}'.format(python)) except: From 530a980ef74fd20a9de289284fde16875256b49b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 23 Apr 2021 18:38:27 +0530 Subject: [PATCH 3/4] fix: Use execve and set envvars instead of execv Changed from execv to execve with empty env dict because bench commands on macOS 10.14.6 PY3.9.0a4 fails with "Error while finding module specification for 'frappe.utils.bench_helper' (ModuleNotFoundError: No module named 'frappe')" This error has come up in the past too. I'm now suspecting it has something to do with the current process passing on it's environment variables to the process that's going to replace it, which causes these issues. Note that direct execution of the commands without the bench wrapper worked fine although via bench failed with the specified error --- bench/cli.py | 6 +++--- bench/utils.py | 9 ++++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/bench/cli.py b/bench/cli.py index 05c74d86..3b5d16b9 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -109,19 +109,19 @@ def change_uid(): def old_frappe_cli(bench_path='.'): f = get_frappe(bench_path=bench_path) os.chdir(os.path.join(bench_path, 'sites')) - os.execv(f, [f] + sys.argv[2:]) + os.execve(f, [f] + sys.argv[2:], {}) def app_cmd(bench_path='.'): f = get_env_cmd('python', bench_path=bench_path) os.chdir(os.path.join(bench_path, 'sites')) - os.execv(f, [f] + ['-m', 'frappe.utils.bench_helper'] + sys.argv[1:]) + os.execve(f, [f] + ['-m', 'frappe.utils.bench_helper'] + sys.argv[1:], {}) def frappe_cmd(bench_path='.'): f = get_env_cmd('python', bench_path=bench_path) os.chdir(os.path.join(bench_path, 'sites')) - os.execv(f, [f] + ['-m', 'frappe.utils.bench_helper', 'frappe'] + sys.argv[1:]) + os.execve(f, [f] + ['-m', 'frappe.utils.bench_helper', 'frappe'] + sys.argv[1:], {}) def get_frappe_commands(): diff --git a/bench/utils.py b/bench/utils.py index 7bb312ba..49529c10 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -453,12 +453,15 @@ def get_process_manager(): def start(no_dev=False, concurrency=None, procfile=None, no_prefix=False): + env = os.environ program = get_process_manager() + if not program: raise Exception("No process manager found") - os.environ['PYTHONUNBUFFERED'] = "true" + + env['PYTHONUNBUFFERED'] = "true" if not no_dev: - os.environ['DEV_SERVER'] = "true" + env['DEV_SERVER'] = "true" command = [program, 'start'] if concurrency: @@ -470,7 +473,7 @@ def start(no_dev=False, concurrency=None, procfile=None, no_prefix=False): if no_prefix: command.extend(['--no-prefix']) - os.execv(program, command) + os.execve(program, command, env=env) def get_git_version(): From 7e1db3fd74a9c610abf5c79e173d5faf4c4c24a4 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 23 Apr 2021 19:14:30 +0530 Subject: [PATCH 4/4] test: Update env test Env folder's include may be empty or non existent --- bench/tests/test_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/tests/test_base.py b/bench/tests/test_base.py index ec38c813..edda52a3 100644 --- a/bench/tests/test_base.py +++ b/bench/tests/test_base.py @@ -44,7 +44,7 @@ class TestBenchBase(unittest.TestCase): bench_path = os.path.abspath(bench_name) python_path = os.path.abspath(os.path.join(bench_path, "env", "bin", "python")) self.assertTrue(python_path.startswith(bench_path)) - for subdir in ("bin", "include", "lib", "share"): + for subdir in ("bin", "lib", "share"): self.assert_exists(bench_name, "env", subdir) def assert_config(self, bench_name):