From aea6bf5e4bffffabf5fdf754622477cafe946d2c Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Thu, 3 Aug 2017 16:33:57 +0530 Subject: [PATCH] Develop (#463) * Add arguments for mysql_root_password and admin_password Consider existing passwords.txt * Remove duplicated --overwrite after rebase * python 3 compatible changes for `update_npm_packages` (#434) * Fix travis (#446) * python 3 compatible changes for `update_npm_packages` * install urllib3 pyOpenSSL ndg-httpsclient pyasn1 to fix wkhtmltopdf download failure on Travis * use latest version of ansible * test should look for mysqlclient-1.3.10 not mysqlclient-1.3.8 * `exec_cmd` causes TypeError on Python 3 (#451) (#452) * python 3 compatible changes for `update_npm_packages` * add universal_newlines=True to Popen * Added docker specific changes to bench (#460) * Added skip_bench_mkdir flag * added skip_bench_mkdir and skip_redis_config_generation * Removed the brackets around not in the utils.py/redis.generate_config() * Replace lines 56-61 in utils.py since the tests failed. * Reinstated lines 56-61 * printing the dirname * Removed the rogue break * Update utils.py --- .travis.yml | 2 ++ bench/commands/make.py | 8 ++++---- bench/utils.py | 34 ++++++++++++++++++++++------------ playbooks/install.py | 38 ++++++++++++++++++++++++++++++++------ 4 files changed, 60 insertions(+), 22 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6ae5eba9..9c42c989 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,8 @@ python: - "2.7" install: + - sudo rm /etc/apt/sources.list.d/docker.list + - sudo pip install urllib3 pyOpenSSL ndg-httpsclient pyasn1 - sudo apt-get purge -y mysql-common mysql-server mysql-client - sudo apt-get install --only-upgrade -y git - mkdir -p ~/.bench diff --git a/bench/commands/make.py b/bench/commands/make.py index 8059f62f..18a06c92 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -10,13 +10,15 @@ import click @click.option('--no-backups',is_flag=True, help="Run migrations for all sites in the bench") @click.option('--no-auto-update',is_flag=True, help="Build JS and CSS artifacts for the bench") @click.option('--verbose',is_flag=True, help="Verbose output during install") +@click.option('--skip-bench-mkdir', is_flag=True, help="Skip mkdir frappe-bench") +@click.option('--skip-redis-config-generation', is_flag=True, help="Skip redis config generation if already specifying the common-site-config file") def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, - no_auto_update, clone_from, verbose): + no_auto_update, clone_from, verbose, skip_bench_mkdir, skip_redis_config_generation): "Create a new bench" from bench.utils import init init(path, apps_path=apps_path, no_procfile=no_procfile, no_backups=no_backups, no_auto_update=no_auto_update, frappe_path=frappe_path, frappe_branch=frappe_branch, - verbose=verbose, clone_from=clone_from) + verbose=verbose, clone_from=clone_from, skip_bench_mkdir=skip_bench_mkdir, skip_redis_config_generation=skip_redis_config_generation) click.echo('Bench {} initialized'.format(path)) @@ -54,5 +56,3 @@ def new_site(site, mariadb_root_password=None, admin_password=None): "Create a new site in the bench" from bench.utils import new_site new_site(site, mariadb_root_password=mariadb_root_password, admin_password=admin_password) - - diff --git a/bench/utils.py b/bench/utils.py index 34b97cb2..896fc0c0 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -2,6 +2,8 @@ import os, sys, shutil, subprocess, logging, itertools, requests, json, platform from distutils.spawn import find_executable import bench from bench import env +from six import iteritems + class PatchError(Exception): pass @@ -25,21 +27,27 @@ def get_env_cmd(cmd, bench_path='.'): def init(path, apps_path=None, no_procfile=False, no_backups=False, no_auto_update=False, frappe_path=None, frappe_branch=None, wheel_cache_dir=None, - verbose=False, clone_from=None): + verbose=False, clone_from=None, skip_bench_mkdir=False, skip_redis_config_generation=False): from .app import get_app, install_apps_from_path from .config.common_site_config import make_config from .config import redis from .config.procfile import setup_procfile from bench.patches import set_all_patches_executed - if os.path.exists(path): - print('Directory {} already exists!'.format(path)) - raise Exception("Site directory already exists") - # sys.exit(1) + if(skip_bench_mkdir): + pass + else: + if os.path.exists(path): + print('Directory {} already exists!'.format(path)) + raise Exception("Site directory already exists") + os.makedirs(path) - os.makedirs(path) for dirname in folders_in_bench: - os.mkdir(os.path.join(path, dirname)) + try: + os.makedirs(os.path.join(path, dirname)) + except OSError, e: + if e.errno != os.errno.EEXIST: + pass setup_logging() @@ -65,7 +73,9 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, set_all_patches_executed(bench_path=path) build_assets(bench_path=path) - redis.generate_config(path) + + if not skip_redis_config_generation: + redis.generate_config(path) if not no_procfile: setup_procfile(path) @@ -119,7 +129,7 @@ def exec_cmd(cmd, cwd='.'): logger.info(cmd) - p = subprocess.Popen(cmd, cwd=cwd, shell=True, stdout=stdout, stderr=stderr) + p = subprocess.Popen(cmd, cwd=cwd, shell=True, stdout=stdout, stderr=stderr, universal_newlines=True) if async: return_code = print_output(p) @@ -411,7 +421,7 @@ def update_npm_packages(bench_path='.'): with open(package_json_path, "r") as f: app_package_json = json.loads(f.read()) # package.json is usually a dict in a dict - for key, value in app_package_json.iteritems(): + for key, value in iteritems(app_package_json): if not key in package_json: package_json[key] = value else: @@ -421,7 +431,7 @@ def update_npm_packages(bench_path='.'): package_json[key].extend(value) else: package_json[key] = value - + if package_json is {}: with open(os.path.join(os.path.dirname(__file__), 'package.json'), 'r') as f: package_json = json.loads(f.read()) @@ -651,7 +661,7 @@ def update_translations(app, lang): f.flush() print('downloaded for', app, lang) - + def download_chart_of_accounts(): charts_dir = os.path.join('apps', "erpnext", "erpnext", 'accounts', 'chart_of_accounts', "submitted") csv_file = os.path.join(translations_dir, lang + '.csv') diff --git a/playbooks/install.py b/playbooks/install.py index 65e6f35d..1c6f7406 100755 --- a/playbooks/install.py +++ b/playbooks/install.py @@ -61,7 +61,7 @@ def install_bench(args): # Restricting ansible version due to following bug in ansible 2.1 # https://github.com/ansible/ansible-modules-core/issues/3752 success = run_os_command({ - 'pip': "sudo pip install 'ansible==2.0.2.0'" + 'pip': "sudo pip install ansible" }) if not success: @@ -97,7 +97,7 @@ def install_bench(args): extra_vars.update(repo_path=repo_path) run_playbook('develop/create_user.yml', extra_vars=extra_vars) - extra_vars.update(get_passwords(args.run_travis or args.without_bench_setup)) + extra_vars.update(get_passwords(args)) if args.production: extra_vars.update(max_worker_connections=multiprocessing.cpu_count() * 1024) @@ -229,9 +229,31 @@ def could_not_install(package): def is_sudo_user(): return os.geteuid() == 0 -def get_passwords(ignore_prompt=False): + +def get_passwords(args): + """ + Returns a dict of passwords for further use + and creates passwords.txt in the bench user's home directory + """ + + ignore_prompt = args.run_travis or args.without_bench_setup + mysql_root_password, admin_password = '', '' + passwords_file_path = os.path.join(os.path.expanduser('~' + args.user), 'passwords.txt') + if not ignore_prompt: - mysql_root_password, admin_password = '', '' + # set passwords from existing passwords.txt + if os.path.isfile(passwords_file_path): + with open(passwords_file_path, 'r') as f: + passwords = json.load(f) + mysql_root_password, admin_password = passwords['mysql_root_password'], passwords['admin_password'] + + # set passwords from cli args + if args.mysql_root_password: + mysql_root_password = args.mysql_root_password + if args.admin_password: + admin_password = args.admin_password + + # prompt for passwords pass_set = True while pass_set: # mysql root password @@ -262,7 +284,6 @@ def get_passwords(ignore_prompt=False): } if not ignore_prompt: - passwords_file_path = os.path.join(os.path.expanduser('~'), 'passwords.txt') with open(passwords_file_path, 'w') as f: json.dump(passwords, f, indent=1) @@ -270,6 +291,7 @@ def get_passwords(ignore_prompt=False): return passwords + def get_extra_vars_json(extra_args): # We need to pass production as extra_vars to the playbook to execute conditionals in the # playbook. Extra variables can passed as json or key=value pair. Here, we will use JSON. @@ -334,11 +356,15 @@ def parse_commandline_args(): parser.add_argument('--without-bench-setup', dest='without_bench_setup', action='store_true', default=False, help=argparse.SUPPRESS) - + # whether to overwrite an existing bench parser.add_argument('--overwrite', dest='overwrite', action='store_true', default=False, help='Whether to overwrite an existing bench') + # set passwords + parser.add_argument('--mysql-root-password', dest='mysql_root_password', help='Set mysql root password') + parser.add_argument('--admin-password', dest='admin_password', help='Set admin password') + args = parser.parse_args() return args