From 0c29bba6599533a7191b0b97ef0929d25c045bfa Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 6 Feb 2020 01:32:50 +0530 Subject: [PATCH 1/2] refactor: bench migrate-env command * moved from __init__ to utils and updated sequence of installing applications in env * chanaged import namespaces --- bench/commands/__init__.py | 124 +++---------------------------------- bench/commands/utils.py | 9 +++ bench/utils.py | 67 ++++++++++++++++++++ 3 files changed, 83 insertions(+), 117 deletions(-) diff --git a/bench/commands/__init__.py b/bench/commands/__init__.py index 4572a42b..3c1c45ec 100755 --- a/bench/commands/__init__.py +++ b/bench/commands/__init__.py @@ -1,15 +1,5 @@ import click -import os, shutil -import os.path as osp -import logging - -from datetime import datetime - -from bench.utils import which, exec_cmd - -log = logging.getLogger(__name__) -log.setLevel(logging.DEBUG) def print_bench_version(ctx, param, value): """Prints current bench version""" @@ -31,11 +21,13 @@ def bench_command(bench_path='.'): setup_logging(bench_path=bench_path) -from bench.commands.make import init, get_app, new_app, remove_app +from bench.commands.make import init, get_app, new_app, remove_app, exclude_app_for_update, include_app_for_update bench_command.add_command(init) bench_command.add_command(get_app) bench_command.add_command(new_app) bench_command.add_command(remove_app) +bench_command.add_command(exclude_app_for_update) +bench_command.add_command(include_app_for_update) from bench.commands.update import update, retry_upgrade, switch_to_branch, switch_to_master, switch_to_develop @@ -45,9 +37,10 @@ bench_command.add_command(switch_to_branch) bench_command.add_command(switch_to_master) bench_command.add_command(switch_to_develop) + from bench.commands.utils import (start, restart, set_nginx_port, set_ssl_certificate, set_ssl_certificate_key, set_url_root, set_mariadb_host, set_default_site, download_translations, shell, backup_site, backup_all_sites, release, renew_lets_encrypt, - disable_production, bench_src, prepare_beta_release, set_redis_cache_host, set_redis_queue_host, set_redis_socketio_host, find_benches) + disable_production, bench_src, prepare_beta_release, set_redis_cache_host, set_redis_queue_host, set_redis_socketio_host, find_benches, migrate_env) bench_command.add_command(start) bench_command.add_command(restart) bench_command.add_command(set_nginx_port) @@ -69,6 +62,8 @@ bench_command.add_command(disable_production) bench_command.add_command(bench_src) bench_command.add_command(prepare_beta_release) bench_command.add_command(find_benches) +bench_command.add_command(migrate_env) + from bench.commands.setup import setup bench_command.add_command(setup) @@ -84,108 +79,3 @@ bench_command.add_command(remote_urls) from bench.commands.install import install bench_command.add_command(install) - -from bench.config.common_site_config import get_config -try: - from urlparse import urlparse -except ImportError: - from urllib.parse import urlparse - -@click.command('migrate-env') -@click.argument('python', type = str) -@click.option('--no-backup', is_flag=True) -def migrate_env(python, no_backup = False): - """ - Migrate Virtual Environment to desired Python Version. - """ - try: - # Clear Cache before Bench Dies. - config = get_config(bench_path = os.getcwd()) - rredis = urlparse(config['redis_cache']) - - redis = '{redis} -p {port}'.format( - redis = which('redis-cli'), - port = rredis.port - ) - - log.debug('Clearing Redis Cache...') - exec_cmd('{redis} FLUSHALL'.format(redis = redis)) - log.debug('Clearing Redis DataBase...') - exec_cmd('{redis} FLUSHDB'.format(redis = redis)) - except Exception: - log.warn('Please ensure Redis Connections are running or Daemonized.') - - try: - # This is with the assumption that a bench is set-up within path. - path = os.getcwd() - - # I know, bad name for a flag. Thanks, Ameya! :| - - if not no_backup: - # Back, the f*ck up. - parch = osp.join(path, 'archived_envs') - if not osp.exists(parch): - os.mkdir(parch) - - # Simply moving. Thanks, Ameya. - # I'm keen to zip. - source = osp.join(path, 'env') - target = parch - - log.debug('Backing up Virtual Environment') - stamp = datetime.now().strftime('%Y%m%d_%H%M%S') - dest = osp.join(path, str(stamp)) - - # WARNING: This is an archive, you might have to use virtualenv --relocate - # That's because virtualenv creates symlinks with shebangs pointing to executables. - # shebangs, shebangs - ricky martin. - - # ...and shutil.copytree is a f*cking mess. - os.rename(source, dest) - shutil.move(dest, target) - - log.debug('Setting up a New Virtual {python} Environment'.format( - python = python - )) - - # Path to Python Executable (Basically $PYTHONPTH) - python = which(python) - - - virtualenv = which('virtualenv') - - nvenv = 'env' - pvenv = osp.join(path, nvenv) - - exec_cmd('{virtualenv} --python {python} {pvenv}'.format( - virtualenv = virtualenv, - python = python, - pvenv = pvenv - ), cwd = path) - - pip = osp.join(pvenv, 'bin', 'pip') - exec_cmd('{pip} install --upgrade pip'.format(pip=pip)) - exec_cmd('{pip} install --upgrade setuptools'.format(pip=pip)) - # TODO: Options - - papps = osp.join(path, 'apps') - apps = ['frappe', 'erpnext'] + [app for app in os.listdir(papps) if app not in ['frappe', 'erpnext']] - - for app in apps: - papp = osp.join(papps, app) - if osp.isdir(papp) and osp.exists(osp.join(papp, 'setup.py')): - exec_cmd('{pip} install -e {app}'.format( - pip = pip, app = papp - )) - - log.debug('Migration Successful to {python}'.format( - python = python - )) - except: - log.debug('Migration Error') - raise - -bench_command.add_command(migrate_env) - -from bench.commands.make import exclude_app_for_update, include_app_for_update -bench_command.add_command(exclude_app_for_update) -bench_command.add_command(include_app_for_update) diff --git a/bench/commands/utils.py b/bench/commands/utils.py index 8cdd302d..7e7d5d24 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -198,3 +198,12 @@ def find_benches(location): """Finds benches recursively from location""" from bench.utils import find_benches find_benches(directory=location) + + +@click.command('migrate-env') +@click.argument('python', type=str) +@click.option('--no-backup', is_flag=True) +def migrate_env(python, no_backup=False): + """Migrate Virtual Environment to desired Python Version""" + from bench.utils import migrate_env + migrate_env(python=python, no_backup=no_backup) diff --git a/bench/utils.py b/bench/utils.py index 6a19347e..143b6b0b 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -1,9 +1,11 @@ import errno, glob, grp, itertools, json, logging, multiprocessing, os, platform, pwd, re, select, shutil, site, subprocess, sys +from datetime import datetime from distutils.spawn import find_executable import requests import semantic_version from six import iteritems +from six.moves.urllib.parse import urlparse import bench from bench import env @@ -984,3 +986,68 @@ def in_virtual_env(): return _no_global_under_venv() return False + +def migrate_env(python, no_backup=False): + from bench.config.common_site_config import get_config + from bench.app import get_apps + + log = logging.getLogger(__name__) + log.setLevel(logging.DEBUG) + + nvenv = 'env' + path = os.getcwd() + python = which(python) + virtualenv = which('virtualenv') + pvenv = os.path.join(path, nvenv) + pip = os.path.join(pvenv, 'bin', 'pip') + + # Clear Cache before Bench Dies. + try: + config = get_config(bench_path=os.getcwd()) + rredis = urlparse(config['redis_cache']) + + redis = '{redis} -p {port}'.format(redis=which('redis-cli'), port=rredis.port) + + log.debug('Clearing Redis Cache...') + exec_cmd('{redis} FLUSHALL'.format(redis = redis)) + log.debug('Clearing Redis DataBase...') + exec_cmd('{redis} FLUSHDB'.format(redis = redis)) + except: + log.warn('Please ensure Redis Connections are running or Daemonized.') + + # I know, bad name for a flag. Thanks, Ameya! :| - + if not no_backup: + # Back, the f*ck up. + parch = os.path.join(path, 'archived_envs') + if not os.path.exists(parch): + os.mkdir(parch) + + # Simply moving. Thanks, Ameya. + # I'm keen to zip. + source = os.path.join(path, 'env') + target = parch + + log.debug('Backing up Virtual Environment') + stamp = datetime.now().strftime('%Y%m%d_%H%M%S') + dest = os.path.join(path, str(stamp)) + + # WARNING: This is an archive, you might have to use virtualenv --relocate + # That's because virtualenv creates symlinks with shebangs pointing to executables. + # shebangs, shebangs - ricky martin. + + # ...and shutil.copytree is a f*cking mess. + os.rename(source, dest) + shutil.move(dest, target) + + try: + log.debug('Setting up a New Virtual {} Environment'.format(python)) + + exec_cmd('{virtualenv} --python {python} {pvenv}'.format(virtualenv=virtualenv, python=python, pvenv=pvenv)) + + apps = ' '.join(["-e {}".format(os.path.join("apps", app)) for app in get_apps()]) + exec_cmd('{0} install -q -U {1}'.format(pip, apps)) + + log.debug('Migration Successful to {}'.format(python)) + except: + log.debug('Migration Error') + raise \ No newline at end of file From 1259b089c8c58230250971e478e20ddb9adeeb2c Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sun, 9 Feb 2020 13:14:06 +0530 Subject: [PATCH 2/2] refactor: change "no_backup" variable and cleanup comments --- bench/commands/utils.py | 6 +++--- bench/utils.py | 16 ++++------------ 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/bench/commands/utils.py b/bench/commands/utils.py index 7e7d5d24..f2bac0af 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -202,8 +202,8 @@ def find_benches(location): @click.command('migrate-env') @click.argument('python', type=str) -@click.option('--no-backup', is_flag=True) -def migrate_env(python, no_backup=False): +@click.option('--no-backup', 'backup', is_flag=True, default=True) +def migrate_env(python, backup=True): """Migrate Virtual Environment to desired Python Version""" from bench.utils import migrate_env - migrate_env(python=python, no_backup=no_backup) + migrate_env(python=python, backup=backup) diff --git a/bench/utils.py b/bench/utils.py index 143b6b0b..fc7f4a1b 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -987,7 +987,7 @@ def in_virtual_env(): return False -def migrate_env(python, no_backup=False): +def migrate_env(python, backup=False): from bench.config.common_site_config import get_config from bench.app import get_apps @@ -1015,15 +1015,12 @@ def migrate_env(python, no_backup=False): except: log.warn('Please ensure Redis Connections are running or Daemonized.') - # I know, bad name for a flag. Thanks, Ameya! :| - - if not no_backup: - # Back, the f*ck up. + # Backup venv: restore using `virtualenv --relocatable` if needed + if backup: parch = os.path.join(path, 'archived_envs') if not os.path.exists(parch): os.mkdir(parch) - # Simply moving. Thanks, Ameya. - # I'm keen to zip. source = os.path.join(path, 'env') target = parch @@ -1031,17 +1028,12 @@ def migrate_env(python, no_backup=False): stamp = datetime.now().strftime('%Y%m%d_%H%M%S') dest = os.path.join(path, str(stamp)) - # WARNING: This is an archive, you might have to use virtualenv --relocate - # That's because virtualenv creates symlinks with shebangs pointing to executables. - # shebangs, shebangs - ricky martin. - - # ...and shutil.copytree is a f*cking mess. os.rename(source, dest) shutil.move(dest, target) + # Create virtualenv using specified python try: log.debug('Setting up a New Virtual {} Environment'.format(python)) - exec_cmd('{virtualenv} --python {python} {pvenv}'.format(virtualenv=virtualenv, python=python, pvenv=pvenv)) apps = ' '.join(["-e {}".format(os.path.join("apps", app)) for app in get_apps()])