From 4c30071bcbf1b40ce168dee2257ae1a4372af703 Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Thu, 13 Oct 2016 12:58:07 +0530 Subject: [PATCH 1/6] [fix] readme --- README.md | 17 +++++++++++------ vm/ansible/vm.yml | 1 + 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index ac611727..aab80dda 100755 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Easy Setup - This is an opinionated setup with logging and SE Linux. So, it is best to setup on a blank server. - Works on Ubuntu 14.04 to 16.04, CentOS 7+, Debian 7 to 8 and MacOS X. -- You may have to install Python 2.7 (eg on Ubuntu 16.04+) by running `apt-get install python-minimal` +- You may have to install Python 2.7 (eg on Ubuntu 16.04+) by running `apt-get install python-minimal` - This script will install the pre-requisites, install bench and setup an ERPNext site - Passwords for Frappe Administrator and MariaDB (root) will be asked - You can then login as **Administrator** with the Administrator password @@ -31,18 +31,23 @@ Steps Open your Terminal and enter: - # Linux: +#### Linux: wget https://raw.githubusercontent.com/frappe/bench/master/playbooks/install.py - # Mac OSX: +#### MacOS: + +Install X Code (from App store) and HomeBrew (http://brew.sh/) - # install X Code (from App store) - # install HomeBrew (http://brew.sh/) brew install python brew install git + +Download the Script + curl "https://raw.githubusercontent.com/frappe/bench/master/playbooks/install.py" -o install.py +#### Run the Script + # For development sudo python install.py --develop @@ -50,7 +55,7 @@ Open your Terminal and enter: sudo python install.py --production # If you're logged in as root, use --user flag to create a user and install using that user - sudo python install.py --develop --user frappe + python install.py --develop --user frappe For development, you have to explicitly start services by running `bench start`. This script requires Python2.7+ installed on your machine. You will have to manually create a new site (`bench new-site`) and get apps that you need (`bench get-app`, `bench install-app`). diff --git a/vm/ansible/vm.yml b/vm/ansible/vm.yml index 42b638d7..18357509 100644 --- a/vm/ansible/vm.yml +++ b/vm/ansible/vm.yml @@ -7,6 +7,7 @@ - git: repo=https://github.com/frappe/bench dest=/home/frappe/bench-repo + depth=no tags: - bench_install From c1af9ca4e952672edf6b5df0c308d1179dfaf442 Mon Sep 17 00:00:00 2001 From: Nabin Hait Date: Tue, 18 Apr 2017 14:52:14 +0530 Subject: [PATCH 2/6] Removed swtich-to-v4 and switch-to-v5 commands --- bench/app.py | 6 ------ bench/commands/__init__.py | 5 +---- bench/commands/update.py | 26 ++------------------------ bench/utils.py | 7 +++++++ 4 files changed, 10 insertions(+), 34 deletions(-) diff --git a/bench/app.py b/bench/app.py index 9b4481cc..0d341b49 100755 --- a/bench/app.py +++ b/bench/app.py @@ -279,12 +279,6 @@ def switch_to_master(apps=None, bench_path='.', upgrade=False): def switch_to_develop(apps=None, bench_path='.', upgrade=False): switch_branch('develop', apps=apps, bench_path=bench_path, upgrade=upgrade) -def switch_to_v4(apps=None, bench_path='.', upgrade=False): - switch_branch('v4.x.x', apps=apps, bench_path=bench_path, upgrade=upgrade) - -def switch_to_v5(apps=None, bench_path='.', upgrade=False): - switch_branch('v5.x.x', apps=apps, bench_path=bench_path, upgrade=upgrade) - def get_version_from_string(contents, field='__version__'): match = re.search(r"^(\s*%s\s*=\s*['\\\"])(.+?)(['\"])(?sm)" % field, contents) diff --git a/bench/commands/__init__.py b/bench/commands/__init__.py index 1da94c30..1bf61806 100755 --- a/bench/commands/__init__.py +++ b/bench/commands/__init__.py @@ -29,15 +29,12 @@ bench_command.add_command(remove_app) bench_command.add_command(new_site) -from bench.commands.update import update, retry_upgrade, switch_to_branch, switch_to_master, switch_to_develop, switch_to_v4, switch_to_v5 +from bench.commands.update import update, retry_upgrade, switch_to_branch, switch_to_master, switch_to_develop bench_command.add_command(update) bench_command.add_command(retry_upgrade) bench_command.add_command(switch_to_branch) bench_command.add_command(switch_to_master) bench_command.add_command(switch_to_develop) -bench_command.add_command(switch_to_v4) -bench_command.add_command(switch_to_v5) - from bench.commands.utils import (start, restart, set_nginx_port, set_ssl_certificate, set_ssl_certificate_key, set_url_root, set_mariadb_host, set_default_site, download_translations, shell, backup_site, backup_all_sites, release, renew_lets_encrypt, diff --git a/bench/commands/update.py b/bench/commands/update.py index 7afc0765..afd1362f 100755 --- a/bench/commands/update.py +++ b/bench/commands/update.py @@ -54,7 +54,7 @@ def update(pull=False, patch=False, build=False, bench=False, auto=False, restar print "This update will cause a major version change in Frappe/ERPNext from {0} to {1}.".format(*version_upgrade[1:]) print "This would take significant time to migrate and might break custom apps. Please run `bench update --upgrade` to confirm." print - print "You can stay on the latest stable release by running `bench switch-to-master` or pin your bench to {0} by running `bench switch-to-v{0}`".format(version_upgrade[1]) + print "You can stay on the latest stable release by running `bench switch-to-master` or pin your bench to {0}".format(version_upgrade[1]) sys.exit(1) _update(pull, patch, build, bench, auto, restart_supervisor, requirements, no_backup, upgrade, force=force) @@ -145,26 +145,4 @@ def switch_to_develop(upgrade=False): switch_to_develop(upgrade=upgrade, apps=['frappe', 'erpnext']) print print 'Switched to develop' - print 'Please run `bench update --patch` to be safe from any differences in database schema' - - -@click.command('switch-to-v4') -@click.option('--upgrade',is_flag=True) -def switch_to_v4(upgrade=False): - "Switch frappe and erpnext to v4 branch" - from bench.app import switch_to_v4 - switch_to_v4(upgrade=upgrade) - print - print 'Switched to v4' - print 'Please run `bench update --patch` to be safe from any differences in database schema' - - -@click.command('switch-to-v5') -@click.option('--upgrade',is_flag=True) -def switch_to_v5(upgrade=False): - "Switch frappe and erpnext to v5 branch" - from bench.app import switch_to_v5 - switch_to_v5(upgrade=upgrade) - print - print 'Switched to v5' - print 'Please run `bench update --patch` to be safe from any differences in database schema' + print 'Please run `bench update --patch` to be safe from any differences in database schema' \ No newline at end of file diff --git a/bench/utils.py b/bench/utils.py index 5fed8246..ac282cec 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -593,6 +593,13 @@ def update_translations(app, lang): f.flush() print 'downloaded for', app, lang + +def download_chart_of_accounts(): + charts_dir = os.path.join('apps', "erpnext", "erpnext", 'accounts', 'chart_of_accounts', "submitted") + csv_file = os.path.join(translations_dir, lang + '.csv') + url = "https://translate.erpnext.com/files/{}-{}.csv".format(app, lang) + r = requests.get(url, stream=True) + r.raise_for_status() def print_output(p): while p.poll() is None: From 485d10089ad954c18317503216bbd505f1268f94 Mon Sep 17 00:00:00 2001 From: DRogue Date: Fri, 19 May 2017 12:56:52 +0200 Subject: [PATCH 3/6] consider passwords.txt (#408) --- bench/utils.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/bench/utils.py b/bench/utils.py index 1714f451..1ab43537 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -139,20 +139,36 @@ def setup_env(bench_path='.'): def setup_socketio(bench_path='.'): exec_cmd("npm install socket.io redis express superagent cookie", cwd=bench_path) + def new_site(site, mariadb_root_password=None, admin_password=None, bench_path='.'): + """ + Creates a new site in the specified bench, default is current bench + """ + logger.info('creating new site {}'.format(site)) - mariadb_root_password_fragment = '--root_password {}'.format(mariadb_root_password) if mariadb_root_password else '' + + # consider an existing passwords.txt file + passwords_file_path = os.path.join(os.path.expanduser('~'), 'passwords.txt') + if os.path.isfile(passwords_file_path): + with open(passwords_file_path, 'r') as f: + passwords = json.load(f) + mariadb_root_password, admin_password = passwords['mysql_root_password'], passwords['admin_password'] + + mysql_root_password_fragment = '--root_password {}'.format(mariadb_root_password) if mariadb_root_password else '' admin_password_fragment = '--admin_password {}'.format(admin_password) if admin_password else '' - exec_cmd("{frappe} {site} --install {db_name} {mariadb_root_password_fragment} {admin_password_fragment}".format( + + exec_cmd("{frappe} {site} --install {db_name} {mysql_root_password_fragment} {admin_password_fragment}".format( frappe=get_frappe(bench_path=bench_path), site=site, - db_name = hashlib.sha1(site).hexdigest()[:10], - mariadb_root_password_fragment=mariadb_root_password_fragment, + db_name=hashlib.sha1(site).hexdigest()[:10], + mysql_root_password_fragment=mysql_root_password_fragment, admin_password_fragment=admin_password_fragment ), cwd=os.path.join(bench_path, 'sites')) + if len(get_sites(bench_path=bench_path)) == 1: exec_cmd("{frappe} --use {site}".format(frappe=get_frappe(bench_path=bench_path), site=site), cwd=os.path.join(bench_path, 'sites')) + def patch_sites(bench_path='.'): bench.set_frappe_version(bench_path=bench_path) From de0d88c0fa5ae190951e5ebda1db12892fb09993 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Mon, 22 May 2017 15:07:21 +0530 Subject: [PATCH 4/6] Fixes bench release github username error (#406) * ran 2to3 script * Correct urllib.parse import * Backward compatible urllib import * removed test_setup_production_v6 * Fix bug global hashlib is not defined (#399) * Don't need $query_string (#390) * removed semicolon, click.prompt adds it --- bench/config/templates/nginx.conf | 2 +- bench/release.py | 2 +- bench/utils.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bench/config/templates/nginx.conf b/bench/config/templates/nginx.conf index bab700d4..589db7e3 100644 --- a/bench/config/templates/nginx.conf +++ b/bench/config/templates/nginx.conf @@ -128,7 +128,7 @@ server { {% endfor -%} ; - return 301 https://$host$request_uri?$query_string; + return 301 https://$host$request_uri; } {% endif %} diff --git a/bench/release.py b/bench/release.py index caea605e..a82a0826 100755 --- a/bench/release.py +++ b/bench/release.py @@ -38,7 +38,7 @@ def validate(bench_path): github_password = config.get('github_password') if not github_username: - github_username = input('Username: ') + github_username = click.prompt('Username', type=str) if not github_password: github_password = getpass.getpass() diff --git a/bench/utils.py b/bench/utils.py index 1ab43537..6cc26a1f 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -1,4 +1,4 @@ -import os, sys, shutil, subprocess, logging, itertools, requests, json, platform, select, pwd, grp, multiprocessing +import os, sys, shutil, subprocess, logging, itertools, requests, json, platform, select, pwd, grp, multiprocessing, hashlib from distutils.spawn import find_executable import bench from bench import env From d53f2b07fdcdc9fdf2c59b92123092245bfe16f4 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Mon, 22 May 2017 15:07:40 +0530 Subject: [PATCH 5/6] Letsencrypt fix develop (#407) * [Fix] Letsencrypt error - Please specify user or filename to write. * Fix bug global hashlib is not defined (#399) * Don't need $query_string (#390) * [Fix] Issue #404 - write to system cron * letsencrypt.py - renamed user_crontab to system_crontab --- bench/config/lets_encrypt.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bench/config/lets_encrypt.py b/bench/config/lets_encrypt.py index d21ec580..f09a75bf 100755 --- a/bench/config/lets_encrypt.py +++ b/bench/config/lets_encrypt.py @@ -81,12 +81,12 @@ def run_certbot_and_setup_ssl(site, custom_domain, bench_path): def setup_crontab(): job_command = 'sudo service nginx stop && /opt/certbot-auto renew && sudo service nginx start' - user_crontab = CronTab() - if job_command not in str(user_crontab): - job = user_crontab.new(command=job_command, comment="Renew lets-encrypt every month") + system_crontab = CronTab(tabfile='/etc/crontab', user=True) + if job_command not in str(system_crontab): + job = system_crontab.new(command=job_command, comment="Renew lets-encrypt every month") job.every().month() job.enable() - user_crontab.write() + system_crontab.write() def create_dir_if_missing(path): From b1e598f52835dba5194e85df38198ce1d577a360 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 23 May 2017 11:12:18 +0530 Subject: [PATCH 6/6] Npm dependency management using package.json (#409) * Fix bug global hashlib is not defined (#399) * Don't need $query_string (#390) * Develop (#400) * [fix] readme * Removed swtich-to-v4 and switch-to-v5 commands * Add "bench setup babel" command * Add "less" package * Npm dependency management using package.json * Add default package.json --- bench/app.py | 3 +- bench/commands/setup.py | 7 ++++ bench/commands/update.py | 7 ++-- bench/package.json | 18 ++++++++++ bench/utils.py | 35 +++++++++++++++++++- playbooks/develop/includes/setup_dev_env.yml | 6 ---- 6 files changed, 65 insertions(+), 11 deletions(-) create mode 100644 bench/package.json diff --git a/bench/app.py b/bench/app.py index 281a0f8f..f5fd8fa4 100755 --- a/bench/app.py +++ b/bench/app.py @@ -253,7 +253,7 @@ def get_repo_dir(app, bench_path='.'): return os.path.join(bench_path, 'apps', app) def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrade=True): - from .utils import update_requirements, backup_all_sites, patch_sites, build_assets, pre_upgrade, post_upgrade + from .utils import update_requirements, update_npm_packages, backup_all_sites, patch_sites, build_assets, pre_upgrade, post_upgrade from . import utils apps_dir = os.path.join(bench_path, 'apps') version_upgrade = (False,) @@ -293,6 +293,7 @@ def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrad if version_upgrade[0] and upgrade: update_requirements() + update_npm_packages() pre_upgrade(version_upgrade[1], version_upgrade[2]) reload(utils) backup_all_sites() diff --git a/bench/commands/setup.py b/bench/commands/setup.py index 76921023..e8975eb0 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -117,6 +117,12 @@ def setup_socketio(): from bench.utils import setup_socketio setup_socketio() +@click.command('requirements') +def setup_requirements(): + "Setup python and node requirements" + from bench.utils import update_requirements, update_npm_packages + update_requirements() + update_npm_packages() @click.command('config') def setup_config(): @@ -185,6 +191,7 @@ setup.add_command(setup_backups) setup.add_command(setup_env) setup.add_command(setup_procfile) setup.add_command(setup_socketio) +setup.add_command(setup_requirements) setup.add_command(setup_config) setup.add_command(setup_fonts) setup.add_command(add_domain) diff --git a/bench/commands/update.py b/bench/commands/update.py index b3bcc383..d38e82e0 100755 --- a/bench/commands/update.py +++ b/bench/commands/update.py @@ -3,7 +3,7 @@ import sys, os from bench.config.common_site_config import get_config from bench.app import pull_all_apps, is_version_upgrade from bench.utils import (update_bench, validate_upgrade, pre_upgrade, post_upgrade, before_update, - update_requirements, backup_all_sites, patch_sites, build_assets, restart_supervisor_processes) + update_requirements, update_npm_packages, backup_all_sites, patch_sites, build_assets, restart_supervisor_processes) from bench import patches #TODO: Not DRY @@ -62,7 +62,8 @@ def update(pull=False, patch=False, build=False, bench=False, auto=False, restar _update(pull, patch, build, bench, auto, restart_supervisor, requirements, no_backup, upgrade, force=force, reset=reset) -def _update(pull=False, patch=False, build=False, update_bench=False, auto=False, restart_supervisor=False, requirements=False, no_backup=False, upgrade=False, bench_path='.', force=False, reset=False): +def _update(pull=False, patch=False, build=False, update_bench=False, auto=False, restart_supervisor=False, + requirements=False, no_backup=False, upgrade=False, bench_path='.', force=False, reset=False): conf = get_config(bench_path=bench_path) version_upgrade = is_version_upgrade(bench_path=bench_path) @@ -78,8 +79,8 @@ def _update(pull=False, patch=False, build=False, update_bench=False, auto=False pull_all_apps(bench_path=bench_path, reset=reset) if requirements: - print('Updating Python libraries...') update_requirements(bench_path=bench_path) + update_npm_packages(bench_path=bench_path) if upgrade and (version_upgrade[0] or (not version_upgrade[0] and force)): pre_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) diff --git a/bench/package.json b/bench/package.json new file mode 100644 index 00000000..31121283 --- /dev/null +++ b/bench/package.json @@ -0,0 +1,18 @@ +{ + "name": "frappe", + "description": "Default package.json for frappe apps", + "dependencies": { + "babel-core": "^6.24.1", + "babel-preset-babili": "0.0.12", + "babel-preset-es2015": "^6.24.1", + "babel-preset-es2016": "^6.24.1", + "babel-preset-es2017": "^6.24.1", + "chokidar": "^1.7.0", + "cookie": "^0.3.1", + "express": "^4.15.3", + "less": "^2.7.2", + "redis": "^2.7.1", + "socket.io": "^2.0.1", + "superagent": "^3.5.2" + } +} diff --git a/bench/utils.py b/bench/utils.py index 6cc26a1f..50040f33 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -61,7 +61,7 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, bench.set_frappe_version(bench_path=path) if bench.FRAPPE_VERSION > 5: - setup_socketio(bench_path=path) + update_npm_packages(bench_path=path) set_all_patches_executed(bench_path=path) build_assets(bench_path=path) @@ -382,6 +382,7 @@ def set_default_site(site, bench_path='.'): cwd=os.path.join(bench_path, 'sites')) def update_requirements(bench_path='.'): + print('Updating Python libraries...') pip = os.path.join(bench_path, 'env', 'bin', 'pip') # upgrade pip to latest @@ -397,6 +398,38 @@ def update_requirements(bench_path='.'): req_file = os.path.join(apps_dir, app, 'requirements.txt') install_requirements(pip, req_file) +def update_npm_packages(bench_path='.'): + print('Updating node libraries...') + apps_dir = os.path.join(bench_path, 'apps') + package_json = {} + + for app in os.listdir(apps_dir): + package_json_path = os.path.join(apps_dir, app, 'package.json') + + if os.path.exists(package_json_path): + with open(package_json_path, "r") as f: + app_package_json = json.loads(f.read()) + # package.json is usually a dict in a dict + for key, value in app_package_json.iteritems(): + if not key in package_json: + package_json[key] = value + else: + if isinstance(value, dict): + package_json[key].update(value) + elif isinstance(value, list): + package_json[key].extend(value) + else: + package_json[key] = value + + if package_json is {}: + with open(os.path.join(os.path.dirname(__file__), 'package.json'), 'r') as f: + package_json = json.loads(f.read()) + + with open(os.path.join(bench_path, 'package.json'), 'w') as f: + f.write(json.dumps(package_json, indent=1, sort_keys=True)) + + exec_cmd('npm install', cwd=bench_path) + def install_requirements(pip, req_file): if os.path.exists(req_file): exec_cmd("{pip} install -q -r {req_file}".format(pip=pip, req_file=req_file)) diff --git a/playbooks/develop/includes/setup_dev_env.yml b/playbooks/develop/includes/setup_dev_env.yml index 3b54c2af..4e07360e 100644 --- a/playbooks/develop/includes/setup_dev_env.yml +++ b/playbooks/develop/includes/setup_dev_env.yml @@ -1,10 +1,4 @@ --- - # Setup Socketio - - name: setup procfile - command: bench setup socketio - args: - creates: "{{ bench_path }}/node_modules" - chdir: "{{ bench_path }}" # Setup Procfile - name: setup procfile