diff --git a/bench/utils.py b/bench/utils.py index 3b5b2bb6..6a19347e 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -1,9 +1,12 @@ -import os, sys, shutil, subprocess, logging, itertools, requests, json, platform, select, pwd, grp, multiprocessing, hashlib, glob, errno +import errno, glob, grp, itertools, json, logging, multiprocessing, os, platform, pwd, re, select, shutil, site, subprocess, sys from distutils.spawn import find_executable -import bench + +import requests import semantic_version +from six import iteritems + +import bench from bench import env -from six import iteritems, PY2 class PatchError(Exception): @@ -458,8 +461,7 @@ def set_default_site(site, bench_path='.'): def update_bench_requirements(): bench_req_file = os.path.join(os.path.dirname(bench.__path__[0]), 'requirements.txt') - user_pip = which("pip" if PY2 else "pip3") - install_requirements(user_pip, bench_req_file, user=True) + install_requirements(bench_req_file, user=True) def update_env_pip(bench_path): env_pip = os.path.join(bench_path, 'env', 'bin', 'pip') @@ -538,14 +540,19 @@ def update_npm_packages(bench_path='.'): exec_cmd('npm install', cwd=bench_path) -def install_requirements(pip, req_file, user=False): +def install_requirements(req_file, user=False): if os.path.exists(req_file): - # sys.real_prefix exists only in a virtualenv - if hasattr(sys, 'real_prefix'): + if user: + python = sys.executable + else: + python = os.path.join("env", "bin", "python") + + if in_virtual_env(): user = False user_flag = "--user" if user else "" - exec_cmd("{pip} install {user_flag} -q -U -r {req_file}".format(pip=pip, user_flag=user_flag, req_file=req_file)) + + exec_cmd("{python} -m pip install {user_flag} -q -U -r {req_file}".format(python=python, user_flag=user_flag, req_file=req_file)) def backup_site(site, bench_path='.'): bench.set_frappe_version(bench_path=bench_path) @@ -715,11 +722,13 @@ def post_upgrade(from_ver, to_ver, bench_path='.'): if from_ver <= 5 and to_ver == 6: setup_socketio(bench_path=bench_path) - print("As you have setup your bench for production, you will have to reload configuration for nginx and supervisor") - print("To complete the migration, please run the following commands") - print() - print("sudo service nginx restart") - print("sudo supervisorctl reload") + message = """ +As you have setup your bench for production, you will have to reload configuration for nginx and supervisor. To complete the migration, please run the following commands +sudo service nginx restart +sudo supervisorctl reload + """.strip() + print(message) + def update_translations_p(args): try: @@ -727,6 +736,7 @@ def update_translations_p(args): except requests.exceptions.HTTPError: print('Download failed for', args[0], args[1]) + def download_translations_p(): pool = multiprocessing.Pool(4) @@ -736,18 +746,21 @@ def download_translations_p(): pool.map(update_translations_p, args) + def download_translations(): langs = get_langs() apps = ('frappe', 'erpnext') for app, lang in itertools.product(apps, langs): update_translations(app, lang) + def get_langs(): lang_file = 'apps/frappe/frappe/geo/languages.json' with open(lang_file) as f: langs = json.loads(f.read()) return [d['code'] for d in langs] + def update_translations(app, lang): translations_dir = os.path.join('apps', app, app, 'translations') csv_file = os.path.join(translations_dir, lang + '.csv') @@ -792,15 +805,18 @@ def log_line(data, stream): return sys.stderr.write(data) return sys.stdout.write(data) + def get_output(*cmd): s = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) out = s.stdout.read() s.stdout.close() return out + def before_update(bench_path, requirements): validate_pillow_dependencies(bench_path, requirements) + def validate_pillow_dependencies(bench_path, requirements): if not requirements: return @@ -828,9 +844,11 @@ def validate_pillow_dependencies(bench_path, requirements): raise + def get_bench_name(bench_path): return os.path.basename(os.path.abspath(bench_path)) + def setup_fonts(): fonts_path = os.path.join('/tmp', 'fonts') @@ -845,6 +863,7 @@ def setup_fonts(): shutil.rmtree(fonts_path) exec_cmd("fc-cache -fv") + def set_git_remote_url(git_url, bench_path='.'): "Set app remote git url" app = git_url.rsplit('/', 1)[1].rsplit('.', 1)[0] @@ -857,6 +876,7 @@ def set_git_remote_url(git_url, bench_path='.'): if os.path.exists(os.path.join(app_dir, '.git')): exec_cmd("git remote set-url upstream {}".format(git_url), cwd=app_dir) + def run_playbook(playbook_name, extra_vars=None, tag=None): if not find_executable('ansible'): print("Ansible is needed to run this command, please install it using 'pip install ansible'") @@ -871,6 +891,7 @@ def run_playbook(playbook_name, extra_vars=None, tag=None): subprocess.check_call(args, cwd=os.path.join(os.path.dirname(bench.__path__[0]), 'playbooks')) + def find_benches(directory=None): if not directory: directory = os.path.expanduser("~") @@ -898,3 +919,68 @@ def find_benches(directory=None): benches.extend(find_benches(sub)) return benches + + +def in_virtual_env(): + # type: () -> bool + """Returns a boolean, whether running in venv with no system site-packages. + pip really does the best job at this: virtualenv_no_global at https://raw.githubusercontent.com/pypa/pip/master/src/pip/_internal/utils/virtualenv.py + """ + + def running_under_venv(): + # handles PEP 405 compliant virtual environments. + return sys.prefix != getattr(sys, "base_prefix", sys.prefix) + + def running_under_regular_virtualenv(): + # pypa/virtualenv case + return hasattr(sys, 'real_prefix') + + def _no_global_under_venv(): + # type: () -> bool + """Check `{sys.prefix}/pyvenv.cfg` for system site-packages inclusion + PEP 405 specifies that when system site-packages are not supposed to be + visible from a virtual environment, `pyvenv.cfg` must contain the following + line: + include-system-site-packages = false + Additionally, log a warning if accessing the file fails. + """ + def _get_pyvenv_cfg_lines(): + pyvenv_cfg_file = os.path.join(sys.prefix, 'pyvenv.cfg') + try: + with open(pyvenv_cfg_file) as f: + return f.read().splitlines() # avoids trailing newlines + except IOError: + return None + + _INCLUDE_SYSTEM_SITE_PACKAGES_REGEX = re.compile( + r"include-system-site-packages\s*=\s*(?Ptrue|false)" + ) + cfg_lines = _get_pyvenv_cfg_lines() + if cfg_lines is None: + # We're not in a "sane" venv, so assume there is no system + # site-packages access (since that's PEP 405's default state). + return True + + for line in cfg_lines: + match = _INCLUDE_SYSTEM_SITE_PACKAGES_REGEX.match(line) + if match is not None and match.group('value') == 'false': + return True + return False + + def _no_global_under_regular_virtualenv(): + # type: () -> bool + """Check if "no-global-site-packages.txt" exists beside site.py + This mirrors logic in pypa/virtualenv for determining whether system + site-packages are visible in the virtual environment. + """ + site_mod_dir = os.path.dirname(os.path.abspath(site.__file__)) + no_global_site_packages_file = os.path.join(site_mod_dir, 'no-global-site-packages.txt') + return os.path.exists(no_global_site_packages_file) + + if running_under_regular_virtualenv(): + return _no_global_under_regular_virtualenv() + + if running_under_venv(): + return _no_global_under_venv() + + return False