From b5a029033d58337c9833c0fb4aafc4b71b5c710d Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Tue, 23 May 2017 11:14:16 +0530 Subject: [PATCH] Develop (#411) * [fix] readme * Removed swtich-to-v4 and switch-to-v5 commands * consider passwords.txt (#408) * Fixes bench release github username error (#406) * ran 2to3 script * Correct urllib.parse import * Backward compatible urllib import * removed test_setup_production_v6 * Fix bug global hashlib is not defined (#399) * Don't need $query_string (#390) * removed semicolon, click.prompt adds it * Letsencrypt fix develop (#407) * [Fix] Letsencrypt error - Please specify user or filename to write. * Fix bug global hashlib is not defined (#399) * Don't need $query_string (#390) * [Fix] Issue #404 - write to system cron * letsencrypt.py - renamed user_crontab to system_crontab * Npm dependency management using package.json (#409) * Fix bug global hashlib is not defined (#399) * Don't need $query_string (#390) * Develop (#400) * [fix] readme * Removed swtich-to-v4 and switch-to-v5 commands * Add "bench setup babel" command * Add "less" package * Npm dependency management using package.json * Add default package.json --- bench/app.py | 3 +- bench/commands/setup.py | 7 +++ bench/commands/update.py | 7 ++- bench/config/lets_encrypt.py | 8 +-- bench/package.json | 18 ++++++ bench/release.py | 2 +- bench/utils.py | 59 ++++++++++++++++++-- playbooks/develop/includes/setup_dev_env.yml | 6 -- 8 files changed, 90 insertions(+), 20 deletions(-) create mode 100644 bench/package.json diff --git a/bench/app.py b/bench/app.py index 281a0f8f..f5fd8fa4 100755 --- a/bench/app.py +++ b/bench/app.py @@ -253,7 +253,7 @@ def get_repo_dir(app, bench_path='.'): return os.path.join(bench_path, 'apps', app) def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrade=True): - from .utils import update_requirements, backup_all_sites, patch_sites, build_assets, pre_upgrade, post_upgrade + from .utils import update_requirements, update_npm_packages, backup_all_sites, patch_sites, build_assets, pre_upgrade, post_upgrade from . import utils apps_dir = os.path.join(bench_path, 'apps') version_upgrade = (False,) @@ -293,6 +293,7 @@ def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrad if version_upgrade[0] and upgrade: update_requirements() + update_npm_packages() pre_upgrade(version_upgrade[1], version_upgrade[2]) reload(utils) backup_all_sites() diff --git a/bench/commands/setup.py b/bench/commands/setup.py index 76921023..e8975eb0 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -117,6 +117,12 @@ def setup_socketio(): from bench.utils import setup_socketio setup_socketio() +@click.command('requirements') +def setup_requirements(): + "Setup python and node requirements" + from bench.utils import update_requirements, update_npm_packages + update_requirements() + update_npm_packages() @click.command('config') def setup_config(): @@ -185,6 +191,7 @@ setup.add_command(setup_backups) setup.add_command(setup_env) setup.add_command(setup_procfile) setup.add_command(setup_socketio) +setup.add_command(setup_requirements) setup.add_command(setup_config) setup.add_command(setup_fonts) setup.add_command(add_domain) diff --git a/bench/commands/update.py b/bench/commands/update.py index b3bcc383..d38e82e0 100755 --- a/bench/commands/update.py +++ b/bench/commands/update.py @@ -3,7 +3,7 @@ import sys, os from bench.config.common_site_config import get_config from bench.app import pull_all_apps, is_version_upgrade from bench.utils import (update_bench, validate_upgrade, pre_upgrade, post_upgrade, before_update, - update_requirements, backup_all_sites, patch_sites, build_assets, restart_supervisor_processes) + update_requirements, update_npm_packages, backup_all_sites, patch_sites, build_assets, restart_supervisor_processes) from bench import patches #TODO: Not DRY @@ -62,7 +62,8 @@ def update(pull=False, patch=False, build=False, bench=False, auto=False, restar _update(pull, patch, build, bench, auto, restart_supervisor, requirements, no_backup, upgrade, force=force, reset=reset) -def _update(pull=False, patch=False, build=False, update_bench=False, auto=False, restart_supervisor=False, requirements=False, no_backup=False, upgrade=False, bench_path='.', force=False, reset=False): +def _update(pull=False, patch=False, build=False, update_bench=False, auto=False, restart_supervisor=False, + requirements=False, no_backup=False, upgrade=False, bench_path='.', force=False, reset=False): conf = get_config(bench_path=bench_path) version_upgrade = is_version_upgrade(bench_path=bench_path) @@ -78,8 +79,8 @@ def _update(pull=False, patch=False, build=False, update_bench=False, auto=False pull_all_apps(bench_path=bench_path, reset=reset) if requirements: - print('Updating Python libraries...') update_requirements(bench_path=bench_path) + update_npm_packages(bench_path=bench_path) if upgrade and (version_upgrade[0] or (not version_upgrade[0] and force)): pre_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) diff --git a/bench/config/lets_encrypt.py b/bench/config/lets_encrypt.py index d21ec580..f09a75bf 100755 --- a/bench/config/lets_encrypt.py +++ b/bench/config/lets_encrypt.py @@ -81,12 +81,12 @@ def run_certbot_and_setup_ssl(site, custom_domain, bench_path): def setup_crontab(): job_command = 'sudo service nginx stop && /opt/certbot-auto renew && sudo service nginx start' - user_crontab = CronTab() - if job_command not in str(user_crontab): - job = user_crontab.new(command=job_command, comment="Renew lets-encrypt every month") + system_crontab = CronTab(tabfile='/etc/crontab', user=True) + if job_command not in str(system_crontab): + job = system_crontab.new(command=job_command, comment="Renew lets-encrypt every month") job.every().month() job.enable() - user_crontab.write() + system_crontab.write() def create_dir_if_missing(path): diff --git a/bench/package.json b/bench/package.json new file mode 100644 index 00000000..31121283 --- /dev/null +++ b/bench/package.json @@ -0,0 +1,18 @@ +{ + "name": "frappe", + "description": "Default package.json for frappe apps", + "dependencies": { + "babel-core": "^6.24.1", + "babel-preset-babili": "0.0.12", + "babel-preset-es2015": "^6.24.1", + "babel-preset-es2016": "^6.24.1", + "babel-preset-es2017": "^6.24.1", + "chokidar": "^1.7.0", + "cookie": "^0.3.1", + "express": "^4.15.3", + "less": "^2.7.2", + "redis": "^2.7.1", + "socket.io": "^2.0.1", + "superagent": "^3.5.2" + } +} diff --git a/bench/release.py b/bench/release.py index caea605e..a82a0826 100755 --- a/bench/release.py +++ b/bench/release.py @@ -38,7 +38,7 @@ def validate(bench_path): github_password = config.get('github_password') if not github_username: - github_username = input('Username: ') + github_username = click.prompt('Username', type=str) if not github_password: github_password = getpass.getpass() diff --git a/bench/utils.py b/bench/utils.py index 8ef8b70a..50040f33 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -61,7 +61,7 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, bench.set_frappe_version(bench_path=path) if bench.FRAPPE_VERSION > 5: - setup_socketio(bench_path=path) + update_npm_packages(bench_path=path) set_all_patches_executed(bench_path=path) build_assets(bench_path=path) @@ -139,20 +139,36 @@ def setup_env(bench_path='.'): def setup_socketio(bench_path='.'): exec_cmd("npm install socket.io redis express superagent cookie", cwd=bench_path) + def new_site(site, mariadb_root_password=None, admin_password=None, bench_path='.'): + """ + Creates a new site in the specified bench, default is current bench + """ + logger.info('creating new site {}'.format(site)) - mariadb_root_password_fragment = '--root_password {}'.format(mariadb_root_password) if mariadb_root_password else '' + + # consider an existing passwords.txt file + passwords_file_path = os.path.join(os.path.expanduser('~'), 'passwords.txt') + if os.path.isfile(passwords_file_path): + with open(passwords_file_path, 'r') as f: + passwords = json.load(f) + mariadb_root_password, admin_password = passwords['mysql_root_password'], passwords['admin_password'] + + mysql_root_password_fragment = '--root_password {}'.format(mariadb_root_password) if mariadb_root_password else '' admin_password_fragment = '--admin_password {}'.format(admin_password) if admin_password else '' - exec_cmd("{frappe} {site} --install {db_name} {mariadb_root_password_fragment} {admin_password_fragment}".format( + + exec_cmd("{frappe} {site} --install {db_name} {mysql_root_password_fragment} {admin_password_fragment}".format( frappe=get_frappe(bench_path=bench_path), site=site, - db_name = hashlib.sha1(site).hexdigest()[:10], - mariadb_root_password_fragment=mariadb_root_password_fragment, + db_name=hashlib.sha1(site).hexdigest()[:10], + mysql_root_password_fragment=mysql_root_password_fragment, admin_password_fragment=admin_password_fragment ), cwd=os.path.join(bench_path, 'sites')) + if len(get_sites(bench_path=bench_path)) == 1: exec_cmd("{frappe} --use {site}".format(frappe=get_frappe(bench_path=bench_path), site=site), cwd=os.path.join(bench_path, 'sites')) + def patch_sites(bench_path='.'): bench.set_frappe_version(bench_path=bench_path) @@ -366,6 +382,7 @@ def set_default_site(site, bench_path='.'): cwd=os.path.join(bench_path, 'sites')) def update_requirements(bench_path='.'): + print('Updating Python libraries...') pip = os.path.join(bench_path, 'env', 'bin', 'pip') # upgrade pip to latest @@ -381,6 +398,38 @@ def update_requirements(bench_path='.'): req_file = os.path.join(apps_dir, app, 'requirements.txt') install_requirements(pip, req_file) +def update_npm_packages(bench_path='.'): + print('Updating node libraries...') + apps_dir = os.path.join(bench_path, 'apps') + package_json = {} + + for app in os.listdir(apps_dir): + package_json_path = os.path.join(apps_dir, app, 'package.json') + + if os.path.exists(package_json_path): + with open(package_json_path, "r") as f: + app_package_json = json.loads(f.read()) + # package.json is usually a dict in a dict + for key, value in app_package_json.iteritems(): + if not key in package_json: + package_json[key] = value + else: + if isinstance(value, dict): + package_json[key].update(value) + elif isinstance(value, list): + package_json[key].extend(value) + else: + package_json[key] = value + + if package_json is {}: + with open(os.path.join(os.path.dirname(__file__), 'package.json'), 'r') as f: + package_json = json.loads(f.read()) + + with open(os.path.join(bench_path, 'package.json'), 'w') as f: + f.write(json.dumps(package_json, indent=1, sort_keys=True)) + + exec_cmd('npm install', cwd=bench_path) + def install_requirements(pip, req_file): if os.path.exists(req_file): exec_cmd("{pip} install -q -r {req_file}".format(pip=pip, req_file=req_file)) diff --git a/playbooks/develop/includes/setup_dev_env.yml b/playbooks/develop/includes/setup_dev_env.yml index 3b54c2af..4e07360e 100644 --- a/playbooks/develop/includes/setup_dev_env.yml +++ b/playbooks/develop/includes/setup_dev_env.yml @@ -1,10 +1,4 @@ --- - # Setup Socketio - - name: setup procfile - command: bench setup socketio - args: - creates: "{{ bench_path }}/node_modules" - chdir: "{{ bench_path }}" # Setup Procfile - name: setup procfile