From 3b06683ebf41d4bcf9c48eb342ff7a125fa69a2e Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 22 Jan 2020 12:04:27 +0530 Subject: [PATCH 1/4] fix: checks for bench requirements to be installed without the --user flag via pip references: https://stackoverflow.com/questions/1871549/determine-if-python-is-running-inside-virtualenv --- bench/utils.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bench/utils.py b/bench/utils.py index 3e291c5c..37b2bf40 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -537,10 +537,15 @@ def update_npm_packages(bench_path='.'): exec_cmd('npm install', cwd=bench_path) +def in_virtual_env(): + if sys.version_info.major == 2: + return hasattr(sys, 'real_prefix') + if sys.version_info.major == 3: + return sys.base_prefix != sys.prefix + def install_requirements(pip, req_file, user=False): if os.path.exists(req_file): - # sys.real_prefix exists only in a virtualenv - if hasattr(sys, 'real_prefix'): + if in_virtual_env(): user = False user_flag = "--user" if user else "" From b9bb9440d9cb335a0aa2ec7f807add634d7ff29f Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 22 Jan 2020 12:54:58 +0530 Subject: [PATCH 2/4] fix: update packages even if pip not in path --- bench/utils.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/bench/utils.py b/bench/utils.py index 37b2bf40..09b34f86 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -457,8 +457,7 @@ def set_default_site(site, bench_path='.'): def update_bench_requirements(): bench_req_file = os.path.join(os.path.dirname(bench.__path__[0]), 'requirements.txt') - user_pip = which("pip" if PY2 else "pip3") - install_requirements(user_pip, bench_req_file, user=True) + install_requirements(bench_req_file, user=True) def update_env_pip(bench_path): env_pip = os.path.join(bench_path, 'env', 'bin', 'pip') @@ -543,13 +542,19 @@ def in_virtual_env(): if sys.version_info.major == 3: return sys.base_prefix != sys.prefix -def install_requirements(pip, req_file, user=False): +def install_requirements(req_file, user=False): if os.path.exists(req_file): + if user: + python = sys.executable + else: + python = os.path.join("env", "bin", "python") + if in_virtual_env(): user = False user_flag = "--user" if user else "" - exec_cmd("{pip} install {user_flag} -q -U -r {req_file}".format(pip=pip, user_flag=user_flag, req_file=req_file)) + + exec_cmd("{python} -m pip install {user_flag} -q -U -r {req_file}".format(python=python, user_flag=user_flag, req_file=req_file)) def backup_site(site, bench_path='.'): bench.set_frappe_version(bench_path=bench_path) From 63128a21bf723a2b42336b0094fe341bd757aa28 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 22 Jan 2020 15:19:38 +0530 Subject: [PATCH 3/4] fix: check if virtual env (using pip refs) --- bench/utils.py | 72 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 65 insertions(+), 7 deletions(-) diff --git a/bench/utils.py b/bench/utils.py index 09b34f86..06a10571 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -1,4 +1,4 @@ -import os, sys, shutil, subprocess, logging, itertools, requests, json, platform, select, pwd, grp, multiprocessing, hashlib, glob, errno +import os, sys, shutil, subprocess, logging, itertools, requests, json, platform, select, pwd, grp, multiprocessing, hashlib, glob, re, site, errno from distutils.spawn import find_executable import bench import semantic_version @@ -536,12 +536,6 @@ def update_npm_packages(bench_path='.'): exec_cmd('npm install', cwd=bench_path) -def in_virtual_env(): - if sys.version_info.major == 2: - return hasattr(sys, 'real_prefix') - if sys.version_info.major == 3: - return sys.base_prefix != sys.prefix - def install_requirements(req_file, user=False): if os.path.exists(req_file): if user: @@ -907,3 +901,67 @@ def find_benches(directory=None): benches.extend(find_benches(sub)) return benches + +def in_virtual_env(): + # type: () -> bool + """Returns a boolean, whether running in venv with no system site-packages. + pip really does the best job at this: virtualenv_no_global at https://raw.githubusercontent.com/pypa/pip/master/src/pip/_internal/utils/virtualenv.py + """ + + def running_under_venv(): + # handles PEP 405 compliant virtual environments. + return sys.prefix != getattr(sys, "base_prefix", sys.prefix) + + def running_under_regular_virtualenv(): + # pypa/virtualenv case + return hasattr(sys, 'real_prefix') + + def _no_global_under_venv(): + # type: () -> bool + """Check `{sys.prefix}/pyvenv.cfg` for system site-packages inclusion + PEP 405 specifies that when system site-packages are not supposed to be + visible from a virtual environment, `pyvenv.cfg` must contain the following + line: + include-system-site-packages = false + Additionally, log a warning if accessing the file fails. + """ + def _get_pyvenv_cfg_lines(): + pyvenv_cfg_file = os.path.join(sys.prefix, 'pyvenv.cfg') + try: + with open(pyvenv_cfg_file) as f: + return f.read().splitlines() # avoids trailing newlines + except IOError: + return None + + _INCLUDE_SYSTEM_SITE_PACKAGES_REGEX = re.compile( + r"include-system-site-packages\s*=\s*(?Ptrue|false)" + ) + cfg_lines = _get_pyvenv_cfg_lines() + if cfg_lines is None: + # We're not in a "sane" venv, so assume there is no system + # site-packages access (since that's PEP 405's default state). + return True + + for line in cfg_lines: + match = _INCLUDE_SYSTEM_SITE_PACKAGES_REGEX.match(line) + if match is not None and match.group('value') == 'false': + return True + return False + + def _no_global_under_regular_virtualenv(): + # type: () -> bool + """Check if "no-global-site-packages.txt" exists beside site.py + This mirrors logic in pypa/virtualenv for determining whether system + site-packages are visible in the virtual environment. + """ + site_mod_dir = os.path.dirname(os.path.abspath(site.__file__)) + no_global_site_packages_file = os.path.join(site_mod_dir, 'no-global-site-packages.txt') + return os.path.exists(no_global_site_packages_file) + + if running_under_regular_virtualenv(): + return _no_global_under_regular_virtualenv() + + if running_under_venv(): + return _no_global_under_venv() + + return False From 3ea19e95336e46410e8919f3eb8b67fa8e7ad153 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 22 Jan 2020 15:26:58 +0530 Subject: [PATCH 4/4] chore: bench.utils format, imports sort, spaces --- bench/utils.py | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/bench/utils.py b/bench/utils.py index 06a10571..67217a19 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -1,9 +1,12 @@ -import os, sys, shutil, subprocess, logging, itertools, requests, json, platform, select, pwd, grp, multiprocessing, hashlib, glob, re, site, errno +import errno, glob, grp, itertools, json, logging, multiprocessing, os, platform, pwd, re, select, shutil, site, subprocess, sys from distutils.spawn import find_executable -import bench + +import requests import semantic_version +from six import iteritems + +import bench from bench import env -from six import iteritems, PY2 class PatchError(Exception): @@ -718,11 +721,13 @@ def post_upgrade(from_ver, to_ver, bench_path='.'): if from_ver <= 5 and to_ver == 6: setup_socketio(bench_path=bench_path) - print("As you have setup your bench for production, you will have to reload configuration for nginx and supervisor") - print("To complete the migration, please run the following commands") - print() - print("sudo service nginx restart") - print("sudo supervisorctl reload") + message = """ +As you have setup your bench for production, you will have to reload configuration for nginx and supervisor. To complete the migration, please run the following commands +sudo service nginx restart +sudo supervisorctl reload + """.strip() + print(message) + def update_translations_p(args): try: @@ -730,6 +735,7 @@ def update_translations_p(args): except requests.exceptions.HTTPError: print('Download failed for', args[0], args[1]) + def download_translations_p(): pool = multiprocessing.Pool(4) @@ -739,18 +745,21 @@ def download_translations_p(): pool.map(update_translations_p, args) + def download_translations(): langs = get_langs() apps = ('frappe', 'erpnext') for app, lang in itertools.product(apps, langs): update_translations(app, lang) + def get_langs(): lang_file = 'apps/frappe/frappe/geo/languages.json' with open(lang_file) as f: langs = json.loads(f.read()) return [d['code'] for d in langs] + def update_translations(app, lang): translations_dir = os.path.join('apps', app, app, 'translations') csv_file = os.path.join(translations_dir, lang + '.csv') @@ -795,15 +804,18 @@ def log_line(data, stream): return sys.stderr.write(data) return sys.stdout.write(data) + def get_output(*cmd): s = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) out = s.stdout.read() s.stdout.close() return out + def before_update(bench_path, requirements): validate_pillow_dependencies(bench_path, requirements) + def validate_pillow_dependencies(bench_path, requirements): if not requirements: return @@ -831,9 +843,11 @@ def validate_pillow_dependencies(bench_path, requirements): raise + def get_bench_name(bench_path): return os.path.basename(os.path.abspath(bench_path)) + def setup_fonts(): fonts_path = os.path.join('/tmp', 'fonts') @@ -848,6 +862,7 @@ def setup_fonts(): shutil.rmtree(fonts_path) exec_cmd("fc-cache -fv") + def set_git_remote_url(git_url, bench_path='.'): "Set app remote git url" app = git_url.rsplit('/', 1)[1].rsplit('.', 1)[0] @@ -860,6 +875,7 @@ def set_git_remote_url(git_url, bench_path='.'): if os.path.exists(os.path.join(app_dir, '.git')): exec_cmd("git remote set-url upstream {}".format(git_url), cwd=app_dir) + def run_playbook(playbook_name, extra_vars=None, tag=None): if not find_executable('ansible'): print("Ansible is needed to run this command, please install it using 'pip install ansible'") @@ -874,6 +890,7 @@ def run_playbook(playbook_name, extra_vars=None, tag=None): subprocess.check_call(args, cwd=os.path.join(os.path.dirname(bench.__path__[0]), 'playbooks')) + def find_benches(directory=None): if not directory: directory = os.path.expanduser("~") @@ -902,6 +919,7 @@ def find_benches(directory=None): return benches + def in_virtual_env(): # type: () -> bool """Returns a boolean, whether running in venv with no system site-packages.