diff --git a/.deepsource.toml b/.deepsource.toml new file mode 100644 index 00000000..90b9b9bb --- /dev/null +++ b/.deepsource.toml @@ -0,0 +1,16 @@ +version = 1 + +exclude_patterns = [ + ".*" +] + +test_patterns = [ + "bench/tests/**" +] + +[[analyzers]] +name = "python" +enabled = true +dependency_file_paths = [ + "requirements.txt" +] \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 9df3b6d4..550aa53e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,26 +1,96 @@ language: python -dist: xenial +dist: bionic +sudo: true -python: - - "2.7" +git: + depth: 1 + +cache: + - pip + - npm + - yarn + +addons: + mariadb: '10.3' + +matrix: + include: + - name: "Python 2.7 Basic Setup" + python: 2.7 + env: TEST=bench + script: python -m unittest -v bench.tests.test_init + + - name: "Python 3.6 Basic Setup" + python: 3.6 + env: TEST=bench + script: python -m unittest -v bench.tests.test_init + + - name: "Python 3.7 Basic Setup" + python: 3.7 + env: TEST=bench + script: python -m unittest -v bench.tests.test_init + + - name: "Python 3.8 Production Setup" + python: 3.8 + env: TEST=bench + script: python -m unittest -v bench.tests.test_setup_production + + - name: "Python 2.7 Production Setup" + python: 2.7 + env: TEST=bench + script: python -m unittest -v bench.tests.test_setup_production + + - name: "Python 3.6 Production Setup" + python: 3.6 + env: TEST=bench + script: python -m unittest -v bench.tests.test_setup_production + + - name: "Python 3.7 Production Setup" + python: 3.7 + env: TEST=bench + script: python -m unittest -v bench.tests.test_setup_production + + - name: "Python 3.8 Production Setup" + python: 3.8 + env: TEST=bench + script: python -m unittest -v bench.tests.test_setup_production + + - name: "Python 3.6 Easy Install" + python: 3.6 + env: TEST=easy_install + script: sudo python $TRAVIS_BUILD_DIR/playbooks/install.py --user travis --run-travis --production --verbose + + - name: "Python 3.7 Easy Install" + python: 3.7 + env: TEST=easy_install + script: sudo python $TRAVIS_BUILD_DIR/playbooks/install.py --user travis --run-travis --production --verbose + + - name: "Python 3.8 Easy Install" + python: 3.8 + env: TEST=easy_install + script: sudo python $TRAVIS_BUILD_DIR/playbooks/install.py --user travis --run-travis --production --verbose install: - - sudo pip install urllib3 pyOpenSSL ndg-httpsclient pyasn1 - - sudo apt-get purge -y mysql-common mysql-server mysql-client - - sudo apt-get install --only-upgrade -y git - - sudo apt-get install hhvm && rm -rf /home/travis/.kiex/ - - mkdir -p ~/.bench - - mkdir -p /tmp/.bench - - cp -r $TRAVIS_BUILD_DIR/* ~/.bench - - cp -r $TRAVIS_BUILD_DIR/* /tmp/.bench + - pip install urllib3 pyOpenSSL ndg-httpsclient pyasn1 - - sudo python $TRAVIS_BUILD_DIR/playbooks/install.py --user travis --run-travis --production --verbose - # - sudo bash $TRAVIS_BUILD_DIR/install_scripts/setup_frappe.sh --skip-install-bench --mysql-root-password travis - # - cd ~ && sudo python bench-repo/installer/install.py --only-dependencies + - if [ $TEST == "bench" ];then + wget -q -O /tmp/wkhtmltox.tar.xz https://github.com/frappe/wkhtmltopdf/raw/master/wkhtmltox-0.12.3_linux-generic-amd64.tar.xz; + tar -xf /tmp/wkhtmltox.tar.xz -C /tmp; + sudo mv /tmp/wkhtmltox/bin/wkhtmltopdf /usr/local/bin/wkhtmltopdf; + sudo chmod o+x /usr/local/bin/wkhtmltopdf; -script: - - cd ~ - - sudo pip install --upgrade pip - - sudo pip install -e ~/.bench - # - sudo python -m unittest bench.tests.test_setup_production.TestSetupProduction.test_setup_production_v6 - - sudo python -m unittest -v bench.tests.test_setup_production + mkdir -p ~/.bench; + cp -r $TRAVIS_BUILD_DIR/* ~/.bench; + pip install -q -U -e ~/.bench; + sudo pip install -q -U -e ~/.bench; + + mysql -u root -e "SET GLOBAL character_set_server = 'utf8mb4'"; + mysql -u root -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'"; + mysql -u root -e "UPDATE mysql.user SET Password=PASSWORD('travis') WHERE User='root'"; + mysql -u root -e "FLUSH PRIVILEGES"; + fi + + - if [ $TEST == "easy_install" ];then + mkdir -p /tmp/.bench; + cp -r $TRAVIS_BUILD_DIR/* /tmp/.bench; + fi diff --git a/README.md b/README.md index 8eb96fed..294520d8 100755 --- a/README.md +++ b/README.md @@ -117,12 +117,11 @@ Releases can be made for [Frappe](https://github.com/frappe/frappe) apps using b --- -# Docker Install +# Docker -1. For developer setup, you can also use the official [Frappe Docker](https://github.com/frappe/frappe_docker/). -2. The app, mariadb and redis run on individual containers. -3. This setup supports multi-tenancy and exposes the frappe-bench volume as a external storage. -4. For more details, [ead the instructions on the [Frappe Docker README](https://github.com/frappe/frappe_docker/) +- For official images and resources [Frappe Docker](https://github.com/frappe/frappe_docker) +- Production Installation [README](https://github.com/frappe/frappe_docker/blob/develop/README.md) +- Developer Setup [README](https://github.com/frappe/frappe_docker/blob/develop/development/README.md) --- diff --git a/bench/app.py b/bench/app.py index ea5d2a22..211388db 100755 --- a/bench/app.py +++ b/bench/app.py @@ -1,23 +1,32 @@ +# imports - compatibility imports from __future__ import print_function -import os -from .utils import (exec_cmd, get_frappe, check_git_for_shallow_clone, build_assets, - restart_supervisor_processes, get_cmd_output, run_frappe_cmd, CommandFailedError, - restart_systemd_processes) -from .config.common_site_config import get_config +# imports - standard imports +import json import logging +import os +import re +import shutil +import subprocess +import sys + +# imports - third party imports +import click +import git import requests import semantic_version -import json -import re -import subprocess -import bench -import sys -import shutil +from six.moves import reload_module -logging.basicConfig(level="DEBUG") +# imports - module imports +import bench +from bench.config.common_site_config import get_config +from bench.utils import CommandFailedError, build_assets, check_git_for_shallow_clone, exec_cmd, get_cmd_output, get_frappe, restart_supervisor_processes, restart_systemd_processes, run_frappe_cmd + + +logging.basicConfig(level="INFO") logger = logging.getLogger(__name__) + class InvalidBranchException(Exception): pass class InvalidRemoteException(Exception): pass @@ -50,7 +59,7 @@ def write_appstxt(apps, bench_path='.'): with open(os.path.join(bench_path, 'sites', 'apps.txt'), 'w') as f: return f.write('\n'.join(apps)) -def check_url(url, raise_err = True): +def check_url(url, raise_err=True): try: from urlparse import urlparse except ImportError: @@ -59,7 +68,7 @@ def check_url(url, raise_err = True): parsed = urlparse(url) if not parsed.scheme: if raise_err: - raise TypeError('{url} Not a valid URL'.format(url = url)) + raise TypeError('{url} Not a valid URL'.format(url=url)) else: return False @@ -92,59 +101,61 @@ def remove_from_excluded_apps_txt(app, bench_path='.'): apps.remove(app) return write_excluded_apps_txt(apps, bench_path=bench_path) -def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=False, - postprocess = True): - # from bench.utils import check_url - try: - from urlparse import urljoin - except ImportError: - from urllib.parse import urljoin +def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=False, postprocess=True, overwrite=False): + if not os.path.exists(git_url): + if not check_url(git_url, raise_err=False): + orgs = ['frappe', 'erpnext'] + for org in orgs: + url = 'https://api.github.com/repos/{org}/{app}'.format(org=org, app=git_url) + res = requests.get(url) + if res.ok: + data = res.json() + if 'name' in data: + if git_url == data['name']: + git_url = 'https://github.com/{org}/{app}'.format(org=org, app=git_url) + break - if not check_url(git_url, raise_err = False): - orgs = ['frappe', 'erpnext'] - for org in orgs: - url = 'https://api.github.com/repos/{org}/{app}'.format(org = org, app = git_url) - res = requests.get(url) - if res.ok: - data = res.json() - if 'name' in data: - if git_url == data['name']: - git_url = 'https://github.com/{org}/{app}'.format(org = org, app = git_url) - break + # Gets repo name from URL + repo_name = git_url.rsplit('/', 1)[1].rsplit('.', 1)[0] + shallow_clone = '--depth 1' if check_git_for_shallow_clone() else '' + branch = '--branch {branch}'.format(branch=branch) if branch else '' + else: + repo_name = git_url.split(os.sep)[-1] + shallow_clone = '' + branch = '--branch {branch}'.format(branch=branch) if branch else '' - #Gets repo name from URL - repo_name = git_url.rsplit('/', 1)[1].rsplit('.', 1)[0] - logger.info('getting app {}'.format(repo_name)) - shallow_clone = '--depth 1' if check_git_for_shallow_clone() else '' - branch = '--branch {branch}'.format(branch=branch) if branch else '' + if os.path.isdir(os.path.join(bench_path, 'apps', repo_name)): + # application directory already exists + # prompt user to overwrite it + if overwrite or click.confirm('''A directory for the application "{0}" already exists. +Do you want to continue and overwrite it?'''.format(repo_name)): + shutil.rmtree(os.path.join(bench_path, 'apps', repo_name)) + elif click.confirm('''Do you want to reinstall the existing application?''', abort=True): + app_name = get_app_name(bench_path, repo_name) + install_app(app=app_name, bench_path=bench_path, verbose=verbose, skip_assets=skip_assets) + sys.exit() - exec_cmd("git clone -q {git_url} {branch} {shallow_clone} --origin upstream".format( - git_url=git_url, - shallow_clone=shallow_clone, - branch=branch), - cwd=os.path.join(bench_path, 'apps')) + logger.info('Getting app {0}'.format(repo_name)) + exec_cmd("git clone {git_url} {branch} {shallow_clone} --origin upstream".format( + git_url=git_url, + shallow_clone=shallow_clone, + branch=branch), + cwd=os.path.join(bench_path, 'apps')) - #Retrieves app name from setup.py + app_name = get_app_name(bench_path, repo_name) + install_app(app=app_name, bench_path=bench_path, verbose=verbose, skip_assets=skip_assets) + + +def get_app_name(bench_path, repo_name): + # retrieves app name from setup.py app_path = os.path.join(bench_path, 'apps', repo_name, 'setup.py') with open(app_path, 'rb') as f: app_name = re.search(r'name\s*=\s*[\'"](.*)[\'"]', f.read().decode('utf-8')).group(1) if repo_name != app_name: apps_path = os.path.join(os.path.abspath(bench_path), 'apps') os.rename(os.path.join(apps_path, repo_name), os.path.join(apps_path, app_name)) + return app_name - print('installing', app_name) - install_app(app=app_name, bench_path=bench_path, verbose=verbose) - - if postprocess: - - if not skip_assets: - build_assets(bench_path=bench_path, app=app_name) - conf = get_config(bench_path=bench_path) - - if conf.get('restart_supervisor_on_update'): - restart_supervisor_processes(bench_path=bench_path) - if conf.get('restart_systemd_on_update'): - restart_systemd_processes(bench_path=bench_path) def new_app(app, bench_path='.'): # For backwards compatibility @@ -160,7 +171,8 @@ def new_app(app, bench_path='.'): run_frappe_cmd('make-app', apps, app, bench_path=bench_path) install_app(app, bench_path=bench_path) -def install_app(app, bench_path=".", verbose=False, no_cache=False): + +def install_app(app, bench_path=".", verbose=False, no_cache=False, postprocess=True, skip_assets=False): logger.info("installing {}".format(app)) pip_path = os.path.join(bench_path, "env", "bin", "pip") @@ -168,11 +180,23 @@ def install_app(app, bench_path=".", verbose=False, no_cache=False): app_path = os.path.join(bench_path, "apps", app) cache_flag = "--no-cache-dir" if no_cache else "" - exec_cmd("{pip} install {quiet} -U -e {app} {no_cache}".format(pip=pip_path, quiet=quiet_flag, app=app_path, no_cache=cache_flag)) + exec_cmd("{pip} install {quiet} -U -e {app} {no_cache}".format(pip=pip_path, + quiet=quiet_flag, app=app_path, no_cache=cache_flag)) add_to_appstxt(app, bench_path=bench_path) + if postprocess: + if not skip_assets: + build_assets(bench_path=bench_path, app=app) + conf = get_config(bench_path=bench_path) + + if conf.get('restart_supervisor_on_update'): + restart_supervisor_processes(bench_path=bench_path) + if conf.get('restart_systemd_on_update'): + restart_systemd_processes(bench_path=bench_path) + + def remove_app(app, bench_path='.'): - if not app in get_apps(bench_path): + if app not in get_apps(bench_path): print("No app named {0}".format(app)) sys.exit(1) @@ -188,7 +212,7 @@ def remove_app(app, bench_path='.'): print("Cannot remove, app is installed on site: {0}".format(site)) sys.exit(1) - exec_cmd(["{0} uninstall -y {1}".format(pip, app)]) + exec_cmd("{0} uninstall -y {1}".format(pip, app), cwd=bench_path) remove_from_appstxt(app, bench_path) shutil.rmtree(app_path) run_frappe_cmd("build", bench_path=bench_path) @@ -281,8 +305,7 @@ def get_current_branch(app, bench_path='.'): def get_remote(app, bench_path='.'): repo_dir = get_repo_dir(app, bench_path=bench_path) - contents = subprocess.check_output(['git', 'remote', '-v'], cwd=repo_dir, - stderr=subprocess.STDOUT) + contents = subprocess.check_output(['git', 'remote', '-v'], cwd=repo_dir, stderr=subprocess.STDOUT) contents = contents.decode('utf-8') if re.findall('upstream[\s]+', contents): return 'upstream' @@ -340,8 +363,7 @@ def get_repo_dir(app, bench_path='.'): return os.path.join(bench_path, 'apps', app) def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrade=True): - from .utils import update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets, pre_upgrade, post_upgrade - from . import utils + from bench.utils import update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets, post_upgrade apps_dir = os.path.join(bench_path, 'apps') version_upgrade = (False,) switched_apps = [] @@ -354,44 +376,46 @@ def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrad for app in apps: app_dir = os.path.join(apps_dir, app) - if os.path.exists(app_dir): - try: - if check_upgrade: - version_upgrade = is_version_upgrade(app=app, bench_path=bench_path, branch=branch) - if version_upgrade[0] and not upgrade: - raise MajorVersionUpgradeException("Switching to {0} will cause upgrade from {1} to {2}. Pass --upgrade to confirm".format(branch, version_upgrade[1], version_upgrade[2]), version_upgrade[1], version_upgrade[2]) - print("Switching for "+app) - unshallow = "--unshallow" if os.path.exists(os.path.join(app_dir, ".git", "shallow")) else "" - exec_cmd("git config --unset-all remote.upstream.fetch", cwd=app_dir) - exec_cmd("git config --add remote.upstream.fetch '+refs/heads/*:refs/remotes/upstream/*'", cwd=app_dir) - exec_cmd("git fetch upstream {unshallow}".format(unshallow=unshallow), cwd=app_dir) - exec_cmd("git checkout {branch}".format(branch=branch), cwd=app_dir) - exec_cmd("git merge upstream/{branch}".format(branch=branch), cwd=app_dir) - switched_apps.append(app) - except CommandFailedError: - print("Error switching to branch {0} for {1}".format(branch, app)) - except InvalidRemoteException: - print("Remote does not exist for app "+app) - except InvalidBranchException: - print("Branch {0} does not exist in Upstream for {1}".format(branch, app)) + + if not os.path.exists(app_dir): + bench.utils.log("{} does not exist!".format(app), level=2) + continue + + repo = git.Repo(app_dir) + unshallow_flag = os.path.exists(os.path.join(app_dir, ".git", "shallow")) + bench.utils.log("Fetching upstream {0}for {1}".format("unshallow " if unshallow_flag else "", app)) + + bench.utils.exec_cmd("git remote set-branches upstream '*'", cwd=app_dir) + bench.utils.exec_cmd("git fetch --all{0}".format(" --unshallow" if unshallow_flag else ""), cwd=app_dir) + + if check_upgrade: + version_upgrade = is_version_upgrade(app=app, bench_path=bench_path, branch=branch) + if version_upgrade[0] and not upgrade: + bench.utils.log("Switching to {0} will cause upgrade from {1} to {2}. Pass --upgrade to confirm".format(branch, version_upgrade[1], version_upgrade[2]), level=2) + sys.exit(1) + + print("Switching for "+app) + bench.utils.exec_cmd("git checkout {0}".format(branch), cwd=app_dir) + + if str(repo.active_branch) == branch: + switched_apps.append(app) + else: + bench.utils.log("Switching branches failed for: {}".format(app), level=2) if switched_apps: - print("Successfully switched branches for:\n" + "\n".join(switched_apps)) + bench.utils.log("Successfully switched branches for: " + ", ".join(switched_apps), level=1) + print('Please run `bench update --patch` to be safe from any differences in database schema') if version_upgrade[0] and upgrade: update_requirements() update_node_packages() - pre_upgrade(version_upgrade[1], version_upgrade[2]) - if sys.version_info >= (3, 4): - import importlib - importlib.reload(utils) - else: - reload(utils) + reload_module(utils) backup_all_sites() patch_sites() build_assets() post_upgrade(version_upgrade[1], version_upgrade[2]) + def switch_to_branch(branch=None, apps=None, bench_path='.', upgrade=False): switch_branch(branch, apps=apps, bench_path=bench_path, upgrade=upgrade) @@ -402,8 +426,7 @@ def switch_to_develop(apps=None, bench_path='.', upgrade=True): switch_branch('develop', apps=apps, bench_path=bench_path, upgrade=upgrade) def get_version_from_string(contents, field='__version__'): - match = re.search(r"^(\s*%s\s*=\s*['\\\"])(.+?)(['\"])(?sm)" % field, - contents) + match = re.search(r"^(\s*%s\s*=\s*['\\\"])(.+?)(['\"])(?sm)" % field, contents) return match.group(2) def get_major_version(version): diff --git a/bench/cli.py b/bench/cli.py index 733a0dd6..ad1c90fa 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -1,6 +1,6 @@ import click import os, sys, logging, json, pwd, subprocess -from bench.utils import is_root, PatchError, drop_privileges, get_env_cmd, get_cmd_output, get_frappe, log, is_bench_directory +from bench.utils import is_root, PatchError, drop_privileges, get_env_cmd, get_cmd_output, get_frappe, log, find_parent_bench from bench.app import get_apps from bench.config.common_site_config import get_config from bench.commands import bench_command @@ -29,7 +29,6 @@ def cli(): elif len(sys.argv) > 1 and sys.argv[1]=="--help": print(click.Context(bench_command).get_help()) - print() print(get_frappe_help()) return @@ -99,7 +98,6 @@ def get_frappe_commands(bench_path='.'): return [] try: output = get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-commands".format(python=python), cwd=sites_path) - # output = output.decode('utf-8') return json.loads(output) except subprocess.CalledProcessError as e: if hasattr(e, "stderr"): @@ -109,27 +107,12 @@ def get_frappe_commands(bench_path='.'): def get_frappe_help(bench_path='.'): python = get_env_cmd('python', bench_path=bench_path) sites_path = os.path.join(bench_path, 'sites') - if not os.path.exists(sites_path): - return [] try: out = get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-help".format(python=python), cwd=sites_path) - return "Framework commands:\n" + out.split('Commands:')[1] - except subprocess.CalledProcessError: + return "\n\nFramework commands:\n" + out.split('Commands:')[1] + except: return "" -def find_parent_bench(path): - """Checks if parent directories are benches""" - if is_bench_directory(directory=path): - return path - - home_path = os.path.expanduser("~") - root_path = os.path.abspath(os.sep) - - if path not in {home_path, root_path}: - # NOTE: the os.path.split assumes that given path is absolute - parent_dir = os.path.split(path)[0] - return find_parent_bench(parent_dir) - def change_working_directory(): """Allows bench commands to be run from anywhere inside a bench directory""" cur_dir = os.path.abspath(".") diff --git a/bench/commands/__init__.py b/bench/commands/__init__.py index 7e514012..090013c2 100755 --- a/bench/commands/__init__.py +++ b/bench/commands/__init__.py @@ -40,7 +40,7 @@ bench_command.add_command(switch_to_develop) from bench.commands.utils import (start, restart, set_nginx_port, set_ssl_certificate, set_ssl_certificate_key, set_url_root, - set_mariadb_host, set_default_site, download_translations, shell, backup_site, backup_all_sites, release, renew_lets_encrypt, + set_mariadb_host, set_default_site, download_translations, backup_site, backup_all_sites, release, renew_lets_encrypt, disable_production, bench_src, prepare_beta_release, set_redis_cache_host, set_redis_queue_host, set_redis_socketio_host, find_benches, migrate_env) bench_command.add_command(start) bench_command.add_command(restart) @@ -54,7 +54,6 @@ bench_command.add_command(set_redis_queue_host) bench_command.add_command(set_redis_socketio_host) bench_command.add_command(set_default_site) bench_command.add_command(download_translations) -bench_command.add_command(shell) bench_command.add_command(backup_site) bench_command.add_command(backup_all_sites) bench_command.add_command(release) diff --git a/bench/commands/config.py b/bench/commands/config.py index a7a0f3ec..1dc33a39 100644 --- a/bench/commands/config.py +++ b/bench/commands/config.py @@ -1,112 +1,80 @@ -import click, json -from bench.config.common_site_config import update_config +# imports - standard imports +import ast -## Config -## Not DRY -@click.group() +# imports - module imports +from bench.config.common_site_config import update_config, get_config, put_config + +# imports - third party imports +import click + + +@click.group(help='Change bench configuration') def config(): - "change bench configuration" pass -@click.command('auto_update') -@click.argument('state', type=click.Choice(['on', 'off'])) -def config_auto_update(state): - "Enable/Disable auto update for bench" - state = True if state == 'on' else False - update_config({'auto_update': state}) - -@click.command('restart_supervisor_on_update') +@click.command('restart_supervisor_on_update', help='Enable/Disable auto restart of supervisor processes') @click.argument('state', type=click.Choice(['on', 'off'])) def config_restart_supervisor_on_update(state): - "Enable/Disable auto restart of supervisor processes" - state = True if state == 'on' else False - update_config({'restart_supervisor_on_update': state}) + update_config({'restart_supervisor_on_update': state == 'on'}) -@click.command('restart_systemd_on_update') + +@click.command('restart_systemd_on_update', help='Enable/Disable auto restart of systemd units') @click.argument('state', type=click.Choice(['on', 'off'])) def config_restart_systemd_on_update(state): - "Enable/Disable auto restart of systemd units" - state = True if state == 'on' else False - update_config({'restart_systemd_on_update': state}) + update_config({'restart_systemd_on_update': state == 'on'}) -@click.command('update_bench_on_update') + +@click.command('update_bench_on_update', help='Enable/Disable bench updates on running bench update') @click.argument('state', type=click.Choice(['on', 'off'])) def config_update_bench_on_update(state): - "Enable/Disable bench updates on running bench update" - state = True if state == 'on' else False - update_config({'update_bench_on_update': state}) + update_config({'update_bench_on_update': state == 'on'}) -@click.command('dns_multitenant') +@click.command('dns_multitenant', help='Enable/Disable bench multitenancy on running bench update') @click.argument('state', type=click.Choice(['on', 'off'])) def config_dns_multitenant(state): - "Enable/Disable bench updates on running bench update" - state = True if state == 'on' else False - update_config({'dns_multitenant': state}) + update_config({'dns_multitenant': state == 'on'}) -@click.command('serve_default_site') +@click.command('serve_default_site', help='Configure nginx to serve the default site on port 80') @click.argument('state', type=click.Choice(['on', 'off'])) def config_serve_default_site(state): - "Configure nginx to serve the default site on port 80" - state = True if state == 'on' else False - update_config({'serve_default_site': state}) + update_config({'serve_default_site': state == 'on'}) -@click.command('rebase_on_pull') +@click.command('rebase_on_pull', help='Rebase repositories on pulling') @click.argument('state', type=click.Choice(['on', 'off'])) def config_rebase_on_pull(state): - "Rebase repositories on pulling" - state = True if state == 'on' else False - update_config({'rebase_on_pull': state}) + update_config({'rebase_on_pull': state == 'on'}) -@click.command('http_timeout') +@click.command('http_timeout', help='Set HTTP timeout') @click.argument('seconds', type=int) def config_http_timeout(seconds): - "set http timeout" update_config({'http_timeout': seconds}) -@click.command('set-common-config') +@click.command('set-common-config', help='Set value in common config') @click.option('configs', '-c', '--config', multiple=True, type=(str, str)) def set_common_config(configs): - import ast - from bench.config.common_site_config import update_config - common_site_config = {} for key, value in configs: - if value in ("False", "True"): + if value in ('true', 'false'): + value = value.title() + try: value = ast.literal_eval(value) - - elif "." in value: - try: - value = float(value) - except ValueError: - pass - - elif "{" in value or "[" in value: - try: - value = json.loads(value) - except ValueError: - pass - - else: - try: - value = int(value) - except ValueError: - pass + except ValueError: + pass common_site_config[key] = value update_config(common_site_config, bench_path='.') -@click.command('remove-common-config') +@click.command('remove-common-config', help='Remove specific keys from current bench\'s common config') @click.argument('keys', nargs=-1) def remove_common_config(keys): - from bench.config.common_site_config import get_config, put_config common_site_config = get_config('.') for key in keys: if key in common_site_config: @@ -115,7 +83,6 @@ def remove_common_config(keys): put_config(common_site_config) -config.add_command(config_auto_update) config.add_command(config_update_bench_on_update) config.add_command(config_restart_supervisor_on_update) config.add_command(config_restart_systemd_on_update) diff --git a/bench/commands/git.py b/bench/commands/git.py index ffaa4d5b..8b52661b 100644 --- a/bench/commands/git.py +++ b/bench/commands/git.py @@ -1,28 +1,30 @@ -import click -import os, subprocess, re +# imports - standard imports +import os +import subprocess +# imports - module imports from bench.app import get_repo_dir, get_apps, get_remote from bench.utils import set_git_remote_url +# imports - third party imports +import click -@click.command('remote-set-url') + +@click.command('remote-set-url', help="Set app remote url") @click.argument('git-url') def remote_set_url(git_url): - "Set app remote url" set_git_remote_url(git_url) -@click.command('remote-reset-url') +@click.command('remote-reset-url', help="Reset app remote url to frappe official") @click.argument('app') def remote_reset_url(app): - "Reset app remote url to frappe official" git_url = "https://github.com/frappe/{}.git".format(app) set_git_remote_url(git_url) -@click.command('remote-urls') +@click.command('remote-urls', help="Show apps remote url") def remote_urls(): - "Show apps remote url" for app in get_apps(): repo_dir = get_repo_dir(app) diff --git a/bench/commands/install.py b/bench/commands/install.py index e1fa75f1..e1fc93bb 100644 --- a/bench/commands/install.py +++ b/bench/commands/install.py @@ -1,21 +1,29 @@ -import os, sys, json, click -from bench.utils import run_playbook, setup_sudoers, is_root +# imports - module imports +from bench.utils import run_playbook, setup_sudoers -extra_vars = {"production": True} +# imports - third party imports +import click -@click.group() + +extra_vars = { + "production": True +} + + +@click.group(help="Install system dependencies for setting up Frappe environment") def install(): - "Install system dependancies" pass -@click.command('prerequisites') + +@click.command('prerequisites', help="Installs pre-requisite libraries, essential tools like b2zip, htop, screen, vim, x11-fonts, python libs, cups and Redis") def install_prerequisites(): run_playbook('site.yml', tag='common, redis') -@click.command('mariadb') -@click.option('--mysql_root_password') + +@click.command('mariadb', help="Install and setup MariaDB of specified version and root password") +@click.option('--mysql_root_password', '--mysql-root-password', default="") @click.option('--version', default="10.3") -def install_maridb(mysql_root_password='', version=''): +def install_maridb(mysql_root_password, version): if mysql_root_password: extra_vars.update({ "mysql_root_password": mysql_root_password, @@ -27,41 +35,49 @@ def install_maridb(mysql_root_password='', version=''): run_playbook('site.yml', extra_vars=extra_vars, tag='mariadb') -@click.command('wkhtmltopdf') + +@click.command('wkhtmltopdf', help="Installs wkhtmltopdf v0.12.3 for linux") def install_wkhtmltopdf(): run_playbook('site.yml', extra_vars=extra_vars, tag='wkhtmltopdf') -@click.command('nodejs') + +@click.command('nodejs', help="Installs Node.js v8") def install_nodejs(): run_playbook('site.yml', extra_vars=extra_vars, tag='nodejs') -@click.command('psutil') + +@click.command('psutil', help="Installs psutil via pip") def install_psutil(): run_playbook('site.yml', extra_vars=extra_vars, tag='psutil') -@click.command('supervisor') + +@click.command('supervisor', help="Installs supervisor. If user is specified, sudoers is setup for that user") @click.option('--user') def install_supervisor(user=None): run_playbook('site.yml', extra_vars=extra_vars, tag='supervisor') if user: setup_sudoers(user) -@click.command('nginx') + +@click.command('nginx', help="Installs NGINX. If user is specified, sudoers is setup for that user") @click.option('--user') def install_nginx(user=None): run_playbook('site.yml', extra_vars=extra_vars, tag='nginx') if user: setup_sudoers(user) -@click.command('virtualbox') + +@click.command('virtualbox', help="Installs supervisor") def install_virtualbox(): run_playbook('vm_build.yml', tag='virtualbox') -@click.command('packer') + +@click.command('packer', help="Installs Oracle virtualbox and packer 1.2.1") def install_packer(): run_playbook('vm_build.yml', tag='packer') -@click.command('fail2ban') + +@click.command("fail2ban", help="Install fail2ban, an intrusion prevention software framework that protects computer servers from brute-force attacks") @click.option('--maxretry', default=6, help="Number of matches (i.e. value of the counter) which triggers ban action on the IP.") @click.option('--bantime', default=600, help="The counter is set to zero if no match is found within 'findtime' seconds.") @click.option('--findtime', default=600, help='Duration (in seconds) for IP to be banned for. Negative number for "permanent" ban.') @@ -69,6 +85,7 @@ def install_failtoban(**kwargs): extra_vars.update(kwargs) run_playbook('site.yml', extra_vars=extra_vars, tag='fail2ban') + install.add_command(install_prerequisites) install.add_command(install_maridb) install.add_command(install_wkhtmltopdf) diff --git a/bench/commands/make.py b/bench/commands/make.py index b9b01b3e..7fc799e6 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -1,6 +1,8 @@ +# imports - third party imports import click -@click.command() + +@click.command('init', help='Initialize a new bench instance in the specified path') @click.argument('path') @click.option('--python', type = str, default = 'python3', help = 'Path to Python Executable.') @click.option('--ignore-exist', is_flag = True, default = False, help = "Ignore if Bench instance exists.") @@ -11,14 +13,10 @@ import click @click.option('--clone-without-update', is_flag=True, help="copy repos from path without update") @click.option('--no-procfile', is_flag=True, help="Pull changes in all the apps in bench") @click.option('--no-backups',is_flag=True, help="Run migrations for all sites in the bench") -@click.option('--no-auto-update',is_flag=True, help="Build JS and CSS artifacts for the bench") @click.option('--skip-redis-config-generation', is_flag=True, help="Skip redis config generation if already specifying the common-site-config file") @click.option('--skip-assets',is_flag=True, default=False, help="Do not build assets") @click.option('--verbose',is_flag=True, help="Verbose output during install") -def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, no_auto_update, clone_from, verbose, skip_redis_config_generation, clone_without_update, ignore_exist=False, skip_assets=False, python='python3'): - ''' - Create a New Bench Instance. - ''' +def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, clone_from, verbose, skip_redis_config_generation, clone_without_update, ignore_exist=False, skip_assets=False, python='python3'): from bench.utils import init, log try: @@ -27,7 +25,6 @@ def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, n apps_path=apps_path, no_procfile=no_procfile, no_backups=no_backups, - no_auto_update=no_auto_update, frappe_path=frappe_path, frappe_branch=frappe_branch, verbose=verbose, @@ -41,10 +38,11 @@ def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, n log('Bench {} initialized'.format(path), level=1) except SystemExit: pass - except: + except Exception as e: import os, shutil, time, six # add a sleep here so that the traceback of other processes doesnt overlap with the prompts time.sleep(1) + print(e) log("There was a problem while creating {}".format(path), level=2) if six.moves.input("Do you want to rollback these changes? [Y/n]: ").lower() == "y": print('Rolling back Bench "{}"'.format(path)) @@ -52,42 +50,40 @@ def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, n shutil.rmtree(path) -@click.command('get-app') +@click.command('get-app', help='Clone an app from the internet or filesystem and set it up in your bench') @click.argument('name', nargs=-1) # Dummy argument for backward compatibility @click.argument('git-url') @click.option('--branch', default=None, help="branch to checkout") +@click.option('--overwrite', is_flag=True, default=False) @click.option('--skip-assets', is_flag=True, default=False, help="Do not build assets") -def get_app(git_url, branch, name=None, skip_assets=False): +def get_app(git_url, branch, name=None, overwrite=False, skip_assets=False): "clone an app from the internet and set it up in your bench" from bench.app import get_app - get_app(git_url, branch=branch, skip_assets=skip_assets) + get_app(git_url, branch=branch, skip_assets=skip_assets, overwrite=overwrite) -@click.command('new-app') +@click.command('new-app', help='Create a new Frappe application under apps folder') @click.argument('app-name') def new_app(app_name): - "start a new app" from bench.app import new_app new_app(app_name) -@click.command('remove-app') +@click.command('remove-app', help='Completely remove app from bench and re-build assets if not installed on any site') @click.argument('app-name') def remove_app(app_name): - "completely remove app from bench" from bench.app import remove_app remove_app(app_name) -@click.command('exclude-app') +@click.command('exclude-app', help='Exclude app from updating') @click.argument('app_name') def exclude_app_for_update(app_name): - "Exclude app from updating" from bench.app import add_to_excluded_apps_txt add_to_excluded_apps_txt(app_name) -@click.command('include-app') +@click.command('include-app', help='Include app for updating') @click.argument('app_name') def include_app_for_update(app_name): "Include app from updating" diff --git a/bench/commands/setup.py b/bench/commands/setup.py index 7f93f4cf..352a781b 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -1,233 +1,216 @@ -from bench.utils import exec_cmd -from six import PY3 -import click, sys, json +# imports - standard imports import os +import sys -@click.group() +# imports - module imports +from bench.utils import exec_cmd + +# imports - third party imports +from six import PY3 +import click + + +@click.group(help="Setup command group for enabling setting up a Frappe environment") def setup(): - "Setup bench" pass -@click.command('sudoers') -@click.argument('user') + +@click.command("sudoers", help="Add commands to sudoers list for execution without password") +@click.argument("user") def setup_sudoers(user): - "Add commands to sudoers list for execution without password" from bench.utils import setup_sudoers setup_sudoers(user) -@click.command('nginx') -@click.option('--yes', help='Yes to regeneration of nginx config file', default=False, is_flag=True) + +@click.command("nginx", help="Generate configuration files for NGINX") +@click.option("--yes", help="Yes to regeneration of nginx config file", default=False, is_flag=True) def setup_nginx(yes=False): - "generate config for nginx" from bench.config.nginx import make_nginx_conf make_nginx_conf(bench_path=".", yes=yes) -@click.command('reload-nginx') + +@click.command("reload-nginx", help="Checks NGINX config file and reloads service") def reload_nginx(): from bench.config.production_setup import reload_nginx reload_nginx() -@click.command('supervisor') -@click.option('--user') -@click.option('--yes', help='Yes to regeneration of supervisor config', is_flag=True, default=False) + +@click.command("supervisor", help="Generate configuration for supervisor") +@click.option("--user", help="optional user argument") +@click.option("--yes", help="Yes to regeneration of supervisor config", is_flag=True, default=False) def setup_supervisor(user=None, yes=False): - "generate config for supervisor with an optional user argument" from bench.config.supervisor import generate_supervisor_config generate_supervisor_config(bench_path=".", user=user, yes=yes) -@click.command('redis') + +@click.command("redis", help="Generates configuration for Redis") def setup_redis(): - "generate config for redis cache" from bench.config.redis import generate_config - generate_config('.') + generate_config(".") -@click.command('fonts') +@click.command("fonts", help="Add Frappe fonts to system") def setup_fonts(): - "Add frappe fonts to system" from bench.utils import setup_fonts setup_fonts() -@click.command('production') -@click.argument('user') -@click.option('--yes', help='Yes to regeneration config', is_flag=True, default=False) + +@click.command("production", help="Setup Frappe production environment for specific user") +@click.argument("user") +@click.option("--yes", help="Yes to regeneration config", is_flag=True, default=False) def setup_production(user, yes=False): - "setup bench for production" from bench.config.production_setup import setup_production - from bench.utils import run_playbook # Install prereqs for production from distutils.spawn import find_executable - if not find_executable('ansible'): - exec_cmd("sudo {0} install ansible".format("pip3" if PY3 else "pip2")) - if not find_executable('fail2ban-client'): + if not find_executable("ansible"): + exec_cmd("sudo -H {0} -m pip install ansible".format(sys.executable)) + if not find_executable("fail2ban-client"): exec_cmd("bench setup role fail2ban") - if not find_executable('nginx'): + if not find_executable("nginx"): exec_cmd("bench setup role nginx") - if not find_executable('supervisord'): + if not find_executable("supervisord"): exec_cmd("bench setup role supervisor") setup_production(user=user, yes=yes) -@click.command('auto-update') -def setup_auto_update(): - "Add cronjob for bench auto update" - from bench.utils import setup_auto_update - setup_auto_update() - - -@click.command('backups') +@click.command("backups", help="Add cronjob for bench backups") def setup_backups(): - "Add cronjob for bench backups" from bench.utils import setup_backups setup_backups() -@click.command('env') -@click.option('--python', type = str, default = 'python3', help = 'Path to Python Executable.') -def setup_env(python='python3'): - "Setup virtualenv for bench" + +@click.command("env", help="Setup virtualenv for bench") +@click.option("--python", type = str, default = "python3", help = "Path to Python Executable.") +def setup_env(python="python3"): from bench.utils import setup_env setup_env(python=python) -@click.command('firewall') -@click.option('--ssh_port') -@click.option('--force') + +@click.command("firewall", help="Setup firewall for system") +@click.option("--ssh_port") +@click.option("--force") def setup_firewall(ssh_port=None, force=False): - "Setup firewall" from bench.utils import run_playbook if not force: - click.confirm('Setting up the firewall will block all ports except 80, 443 and 22\n' - 'Do you want to continue?', - abort=True) + click.confirm("Setting up the firewall will block all ports except 80, 443 and {0}\nDo you want to continue?".format(ssh_port), abort=True) if not ssh_port: ssh_port = 22 - run_playbook('roles/bench/tasks/setup_firewall.yml', {"ssh_port": ssh_port}) + run_playbook("roles/bench/tasks/setup_firewall.yml", {"ssh_port": ssh_port}) -@click.command('ssh-port') -@click.argument('port') -@click.option('--force') + +@click.command("ssh-port", help="Set SSH Port for system") +@click.argument("port") +@click.option("--force") def set_ssh_port(port, force=False): - "Set SSH Port" from bench.utils import run_playbook if not force: - click.confirm('This will change your SSH Port to {}\n' - 'Do you want to continue?'.format(port), - abort=True) + click.confirm("This will change your SSH Port to {}\nDo you want to continue?".format(port), abort=True) - run_playbook('roles/bench/tasks/change_ssh_port.yml', {"ssh_port": port}) + run_playbook("roles/bench/tasks/change_ssh_port.yml", {"ssh_port": port}) -@click.command('lets-encrypt') -@click.argument('site') -@click.option('--custom-domain') + +@click.command("lets-encrypt", help="Setup lets-encrypt SSL for site") +@click.argument("site") +@click.option("--custom-domain") @click.option('-n', '--non-interactive', default=False, is_flag=True, help="Run command non-interactively. This flag restarts nginx and runs certbot non interactively. Shouldn't be used on 1'st attempt") def setup_letsencrypt(site, custom_domain, non_interactive): - "Setup lets-encrypt for site" from bench.config.lets_encrypt import setup_letsencrypt - setup_letsencrypt(site, custom_domain, bench_path='.', interactive=not non_interactive) + setup_letsencrypt(site, custom_domain, bench_path=".", interactive=not non_interactive) -@click.command('wildcard-ssl') -@click.argument('domain') -@click.option('--email') -@click.option('--exclude-base-domain', default=False, is_flag=True, help="SSL Certificate not applicable for base domain") +@click.command("wildcard-ssl", help="Setup wildcard SSL certificate for multi-tenant bench") +@click.argument("domain") +@click.option("--email") +@click.option("--exclude-base-domain", default=False, is_flag=True, help="SSL Certificate not applicable for base domain") def setup_wildcard_ssl(domain, email, exclude_base_domain): - ''' Setup wildcard ssl certificate ''' from bench.config.lets_encrypt import setup_wildcard_ssl - setup_wildcard_ssl(domain, email, bench_path='.', exclude_base_domain=exclude_base_domain) + setup_wildcard_ssl(domain, email, bench_path=".", exclude_base_domain=exclude_base_domain) -@click.command('procfile') +@click.command("procfile", help="Generate Procfile for bench start") def setup_procfile(): - "Setup Procfile for bench start" from bench.config.procfile import setup_procfile - setup_procfile('.') + setup_procfile(".") -@click.command('socketio') +@click.command("socketio", help="Setup node dependencies for socketio server") def setup_socketio(): - "Setup node deps for socketio server" from bench.utils import setup_socketio setup_socketio() -@click.command('requirements', help="Update Python and Node packages") -@click.option('--node', help="Update only Node packages", default=False, is_flag=True) -@click.option('--python', help="Update only Python packages", default=False, is_flag=True) + +@click.command("requirements", help="Setup Python and Node dependencies") +@click.option("--node", help="Update only Node packages", default=False, is_flag=True) +@click.option("--python", help="Update only Python packages", default=False, is_flag=True) def setup_requirements(node=False, python=False): - "Setup python and node requirements" - if not node: - setup_python_requirements() + from bench.utils import update_requirements as setup_python_packages + setup_python_packages() + if not python: - setup_node_requirements() - -def setup_python_requirements(): - from bench.utils import update_requirements - update_requirements() - -def setup_node_requirements(): - from bench.utils import update_node_packages - update_node_packages() + from bench.utils import update_node_packages as setup_node_packages + setup_node_packages() -@click.command('manager') -@click.option('--yes', help='Yes to regeneration of nginx config file', default=False, is_flag=True) -@click.option('--port', help='Port on which you want to run bench manager', default=23624) -@click.option('--domain', help='Domain on which you want to run bench manager') +@click.command("manager", help="Setup bench-manager.local site with the bench_manager app installed on it") +@click.option("--yes", help="Yes to regeneration of nginx config file", default=False, is_flag=True) +@click.option("--port", help="Port on which you want to run bench manager", default=23624) +@click.option("--domain", help="Domain on which you want to run bench manager") def setup_manager(yes=False, port=23624, domain=None): - "Setup bench-manager.local site with the bench_manager app installed on it" from six.moves import input + from bench.utils import get_sites + from bench.config.common_site_config import get_config + from bench.config.nginx import make_bench_manager_nginx_conf + create_new_site = True - if 'bench-manager.local' in os.listdir('sites'): - ans = input('Site already exists. Overwrite existing site? [Y/n]: ').lower() - while ans not in ('y', 'n', ''): - ans = input( - 'Please enter "y" or "n". Site already exists. Overwrite existing site? [Y/n]: ').lower() - if ans == 'n': + + if "bench-manager.local" in os.listdir("sites"): + ans = input("Site already exists. Overwrite existing site? [Y/n]: ").lower() + while ans not in ("y", "n", ""): + ans = input("Please enter 'y' or 'n'. Site already exists. Overwrite existing site? [Y/n]: ").lower() + if ans == "n": create_new_site = False + if create_new_site: exec_cmd("bench new-site --force bench-manager.local") - if 'bench_manager' in os.listdir('apps'): - print('App already exists. Skipping app download.') + if "bench_manager" in os.listdir("apps"): + print("App already exists. Skipping app download.") else: exec_cmd("bench get-app bench_manager") exec_cmd("bench --site bench-manager.local install-app bench_manager") - from bench.config.common_site_config import get_config - bench_path = '.' + bench_path = "." conf = get_config(bench_path) - if conf.get('restart_supervisor_on_update') or conf.get('restart_systemd_on_update'): + + if conf.get("restart_supervisor_on_update") or conf.get("restart_systemd_on_update"): # implicates a production setup or so I presume if not domain: print("Please specify the site name on which you want to host bench-manager using the 'domain' flag") sys.exit(1) - from bench.utils import get_sites, get_bench_name - bench_name = get_bench_name(bench_path) - if domain not in get_sites(bench_path): raise Exception("No such site") - from bench.config.nginx import make_bench_manager_nginx_conf make_bench_manager_nginx_conf(bench_path, yes=yes, port=port, domain=domain) -@click.command('config') +@click.command("config", help="Generate or over-write sites/common_site_config.json") def setup_config(): - "overwrite or make config.json" from bench.config.common_site_config import make_config - make_config('.') + make_config(".") -@click.command('add-domain') -@click.argument('domain') -@click.option('--site', prompt=True) -@click.option('--ssl-certificate', help="Absolute path to SSL Certificate") -@click.option('--ssl-certificate-key', help="Absolute path to SSL Certificate Key") +@click.command("add-domain", help="Add a custom domain to a particular site") +@click.argument("domain") +@click.option("--site", prompt=True) +@click.option("--ssl-certificate", help="Absolute path to SSL Certificate") +@click.option("--ssl-certificate-key", help="Absolute path to SSL Certificate Key") def add_domain(domain, site=None, ssl_certificate=None, ssl_certificate_key=None): """Add custom domain to site""" from bench.config.site_config import add_domain @@ -236,24 +219,25 @@ def add_domain(domain, site=None, ssl_certificate=None, ssl_certificate_key=None print("Please specify site") sys.exit(1) - add_domain(site, domain, ssl_certificate, ssl_certificate_key, bench_path='.') + add_domain(site, domain, ssl_certificate, ssl_certificate_key, bench_path=".") -@click.command('remove-domain') -@click.argument('domain') -@click.option('--site', prompt=True) + +@click.command("remove-domain", help="Remove custom domain from a site") +@click.argument("domain") +@click.option("--site", prompt=True) def remove_domain(domain, site=None): - """Remove custom domain from a site""" from bench.config.site_config import remove_domain if not site: print("Please specify site") sys.exit(1) - remove_domain(site, domain, bench_path='.') + remove_domain(site, domain, bench_path=".") -@click.command('sync-domains') -@click.option('--domain', multiple=True) -@click.option('--site', prompt=True) + +@click.command("sync-domains", help="Check if there is a change in domains. If yes, updates the domains list.") +@click.option("--domain", multiple=True) +@click.option("--site", prompt=True) def sync_domains(domain=None, site=None): from bench.config.site_config import sync_domains @@ -262,53 +246,55 @@ def sync_domains(domain=None, site=None): sys.exit(1) try: - domains = list(map(str,domain)) + domains = list(map(str, domain)) except Exception: print("Domains should be a json list of strings or dictionaries") sys.exit(1) - changed = sync_domains(site, domains, bench_path='.') + changed = sync_domains(site, domains, bench_path=".") # if changed, success, else failure sys.exit(0 if changed else 1) -@click.command('role') -@click.argument('role') -@click.option('--admin_emails', default='') -@click.option('--mysql_root_password') -@click.option('--container', is_flag=True, default=False) + +@click.command("role", help="Install dependencies via ansible roles") +@click.argument("role") +@click.option("--admin_emails", default="") +@click.option("--mysql_root_password") +@click.option("--container", is_flag=True, default=False) def setup_roles(role, **kwargs): - "Install dependancies via roles" from bench.utils import run_playbook extra_vars = {"production": True} extra_vars.update(kwargs) if role: - run_playbook('site.yml', extra_vars=extra_vars, tag=role) + run_playbook("site.yml", extra_vars=extra_vars, tag=role) else: - run_playbook('site.yml', extra_vars=extra_vars) + run_playbook("site.yml", extra_vars=extra_vars) -@click.command('fail2ban') -@click.option('--maxretry', default=6, help="Number of matches (i.e. value of the counter) which triggers ban action on the IP. Default is 6 seconds" ) -@click.option('--bantime', default=600, help="The counter is set to zero if no match is found within 'findtime' seconds. Default is 600 seconds") -@click.option('--findtime', default=600, help='Duration (in seconds) for IP to be banned for. Negative number for "permanent" ban. Default is 600 seconds') + +@click.command("fail2ban", help="Setup fail2ban, an intrusion prevention software framework that protects computer servers from brute-force attacks") +@click.option("--maxretry", default=6, help="Number of matches (i.e. value of the counter) which triggers ban action on the IP. Default is 6 seconds" ) +@click.option("--bantime", default=600, help="The counter is set to zero if no match is found within 'findtime' seconds. Default is 600 seconds") +@click.option("--findtime", default=600, help="Duration (in seconds) for IP to be banned for. Negative number for 'permanent' ban. Default is 600 seconds") def setup_nginx_proxy_jail(**kwargs): from bench.utils import run_playbook - run_playbook('roles/fail2ban/tasks/configure_nginx_jail.yml', extra_vars=kwargs) + run_playbook("roles/fail2ban/tasks/configure_nginx_jail.yml", extra_vars=kwargs) -@click.command('systemd') -@click.option('--user') -@click.option('--yes', help='Yes to regeneration of systemd config files', is_flag=True, default=False) -@click.option('--stop', help='Stop bench services', is_flag=True, default=False) -@click.option('--create-symlinks', help='Create Symlinks', is_flag=True, default=False) -@click.option('--delete-symlinks', help='Delete Symlinks', is_flag=True, default=False) + +@click.command("systemd", help="Generate configuration for systemd") +@click.option("--user", help="Optional user argument") +@click.option("--yes", help="Yes to regeneration of systemd config files", is_flag=True, default=False) +@click.option("--stop", help="Stop bench services", is_flag=True, default=False) +@click.option("--create-symlinks", help="Create Symlinks", is_flag=True, default=False) +@click.option("--delete-symlinks", help="Delete Symlinks", is_flag=True, default=False) def setup_systemd(user=None, yes=False, stop=False, create_symlinks=False, delete_symlinks=False): - "generate configs for systemd with an optional user argument" from bench.config.systemd import generate_systemd_config generate_systemd_config(bench_path=".", user=user, yes=yes, stop=stop, create_symlinks=create_symlinks, delete_symlinks=delete_symlinks) + setup.add_command(setup_sudoers) setup.add_command(setup_nginx) setup.add_command(reload_nginx) @@ -317,7 +303,6 @@ setup.add_command(setup_redis) setup.add_command(setup_letsencrypt) setup.add_command(setup_wildcard_ssl) setup.add_command(setup_production) -setup.add_command(setup_auto_update) setup.add_command(setup_backups) setup.add_command(setup_env) setup.add_command(setup_procfile) diff --git a/bench/commands/update.py b/bench/commands/update.py index 82f857ba..8461da2c 100755 --- a/bench/commands/update.py +++ b/bench/commands/update.py @@ -1,124 +1,32 @@ -import click -import sys +# imports - standard imports import os -from bench.config.common_site_config import get_config, update_config -from bench.app import pull_all_apps, is_version_upgrade, validate_branch -from bench.utils import (update_bench, validate_upgrade, pre_upgrade, post_upgrade, before_update, - update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets, - restart_supervisor_processes, restart_systemd_processes, is_bench_directory) -from bench import patches + +# imports - third party imports +import click from six.moves import reload_module +# imports - module imports +from bench.app import pull_all_apps +from bench.utils import post_upgrade, patch_sites, build_assets -@click.command('update') -@click.option('--pull', is_flag=True, help="Pull changes in all the apps in bench") + +@click.command('update', help="Updates bench tool and if executed in a bench directory, without any flags will backup, pull, setup requirements, build, run patches and restart bench. Using specific flags will only do certain tasks instead of all") +@click.option('--pull', is_flag=True, help="Pull updates for all the apps in bench") @click.option('--patch', is_flag=True, help="Run migrations for all sites in the bench") -@click.option('--build', is_flag=True, help="Build JS and CSS artifacts for the bench") -@click.option('--bench', is_flag=True, help="Update bench") -@click.option('--requirements', is_flag=True, help="Update requirements") -@click.option('--restart-supervisor', is_flag=True, help="restart supervisor processes after update") -@click.option('--restart-systemd', is_flag=True, help="restart systemd units after update") -@click.option('--auto', is_flag=True) -@click.option('--no-backup', is_flag=True) -@click.option('--force', is_flag=True) +@click.option('--build', is_flag=True, help="Build JS and CSS assets for the bench") +@click.option('--bench', is_flag=True, help="Update bench CLI tool") +@click.option('--requirements', is_flag=True, help="Update requirements. If run alone, equivalent to `bench setup requirements`") +@click.option('--restart-supervisor', is_flag=True, help="Restart supervisor processes after update") +@click.option('--restart-systemd', is_flag=True, help="Restart systemd units after update") +@click.option('--no-backup', is_flag=True, help="If this flag is set, sites won't be backed up prior to updates. Note: This is not recommended in production.") +@click.option('--force', is_flag=True, help="Forces major version upgrades") @click.option('--reset', is_flag=True, help="Hard resets git branch's to their new states overriding any changes and overriding rebase on pull") -def update(pull=False, patch=False, build=False, bench=False, auto=False, restart_supervisor=False, restart_systemd=False, requirements=False, no_backup=False, force=False, reset=False): - "Update bench" +def update(pull, patch, build, bench, requirements, restart_supervisor, restart_systemd, no_backup, force, reset): + from bench.utils import update + update(pull=pull, patch=patch, build=build, bench=bench, requirements=requirements, restart_supervisor=restart_supervisor, restart_systemd=restart_systemd, backup= not no_backup, force=force, reset=reset) - if not is_bench_directory(): - """Update only bench if bench update called from outside a bench""" - update_bench(bench_repo=True, requirements=True) - sys.exit() - if not (pull or patch or build or bench or requirements): - pull, patch, build, bench, requirements = True, True, True, True, True - - if auto: - sys.exit(1) - - patches.run(bench_path='.') - conf = get_config(".") - - if bench and conf.get('update_bench_on_update'): - update_bench(bench_repo=True, requirements=False) - restart_update({ - 'pull': pull, - 'patch': patch, - 'build': build, - 'requirements': requirements, - 'no-backup': no_backup, - 'restart-supervisor': restart_supervisor, - 'reset': reset - }) - - if conf.get('release_bench'): - print('Release bench, cannot update') - sys.exit(1) - - validate_branch() - - version_upgrade = is_version_upgrade() - if version_upgrade[0]: - print() - print() - print("This update will cause a major version change in Frappe/ERPNext from {0} to {1}.".format(*version_upgrade[1:])) - print("This would take significant time to migrate and might break custom apps.") - click.confirm('Do you want to continue?', abort=True) - - _update(pull, patch, build, bench, auto, restart_supervisor, restart_systemd, requirements, no_backup, force=force, reset=reset) - -def _update(pull=False, patch=False, build=False, update_bench=False, auto=False, restart_supervisor=False, - restart_systemd=False, requirements=False, no_backup=False, bench_path='.', force=False, reset=False): - conf = get_config(bench_path=bench_path) - version_upgrade = is_version_upgrade(bench_path=bench_path) - - if version_upgrade[0] or (not version_upgrade[0] and force): - validate_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) - - before_update(bench_path=bench_path, requirements=requirements) - - conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 }) - update_config(conf, bench_path=bench_path) - - if not no_backup: - print('Backing up sites...') - backup_all_sites(bench_path=bench_path) - - if pull: - pull_all_apps(bench_path=bench_path, reset=reset) - - if requirements: - update_requirements(bench_path=bench_path) - update_node_packages(bench_path=bench_path) - - if version_upgrade[0] or (not version_upgrade[0] and force): - pre_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) - import bench.utils, bench.app - print('Reloading bench...') - reload_module(bench.utils) - reload_module(bench.app) - - if patch: - print('Patching sites...') - patch_sites(bench_path=bench_path) - if build: - build_assets(bench_path=bench_path) - if version_upgrade[0] or (not version_upgrade[0] and force): - post_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) - if restart_supervisor or conf.get('restart_supervisor_on_update'): - restart_supervisor_processes(bench_path=bench_path) - if restart_systemd or conf.get('restart_systemd_on_update'): - restart_systemd_processes(bench_path=bench_path) - - conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 }) - update_config(conf, bench_path=bench_path) - - print("_"*80) - print("Bench: Deployment tool for Frappe and ERPNext (https://erpnext.org).") - print("Open source depends on your contributions, so please contribute bug reports, patches, fixes or cash and be a part of the community") - print() - -@click.command('retry-upgrade') +@click.command('retry-upgrade', help="Retry a failed upgrade") @click.option('--version', default=5) def retry_upgrade(version): pull_all_apps() @@ -126,35 +34,24 @@ def retry_upgrade(version): build_assets() post_upgrade(version-1, version) -def restart_update(kwargs): - args = ['--'+k for k, v in list(kwargs.items()) if v] - os.execv(sys.argv[0], sys.argv[:2] + args) -@click.command('switch-to-branch') +@click.command('switch-to-branch', help="Switch all apps to specified branch, or specify apps separated by space") @click.argument('branch') @click.argument('apps', nargs=-1) @click.option('--upgrade',is_flag=True) def switch_to_branch(branch, apps, upgrade=False): - "Switch all apps to specified branch, or specify apps separated by space" from bench.app import switch_to_branch switch_to_branch(branch=branch, apps=list(apps), upgrade=upgrade) - print('Switched to ' + branch) - print('Please run `bench update --patch` to be safe from any differences in database schema') -@click.command('switch-to-master') + +@click.command('switch-to-master', help="[DEPRECATED]: Switch frappe and erpnext to master branch") def switch_to_master(): - "Switch frappe and erpnext to master branch" - from bench.app import switch_to_master - switch_to_master(apps=['frappe', 'erpnext']) - print() - print('Switched to master') - print('Please run `bench update --patch` to be safe from any differences in database schema') + from bench.utils import log + log("`switch-to-master` has been deprecated as master branches were renamed to version-11") + @click.command('switch-to-develop') def switch_to_develop(upgrade=False): "Switch frappe and erpnext to develop branch" from bench.app import switch_to_develop switch_to_develop(apps=['frappe', 'erpnext']) - print() - print('Switched to develop') - print('Please run `bench update --patch` to be safe from any differences in database schema') diff --git a/bench/commands/utils.py b/bench/commands/utils.py index f2bac0af..822086a7 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -1,23 +1,25 @@ +# imports - standard imports +import os +import sys + +# imports - third party imports import click -import sys, os, copy -@click.command('start') +@click.command('start', help="Start Frappe development processes") @click.option('--no-dev', is_flag=True, default=False) @click.option('--concurrency', '-c', type=str) @click.option('--procfile', '-p', type=str) def start(no_dev, concurrency, procfile): - "Start Frappe development processes" from bench.utils import start start(no_dev=no_dev, concurrency=concurrency, procfile=procfile) -@click.command('restart') +@click.command('restart', help="Restart supervisor processes or systemd units") @click.option('--web', is_flag=True, default=False) @click.option('--supervisor', is_flag=True, default=False) @click.option('--systemd', is_flag=True, default=False) def restart(web, supervisor, systemd): - "Restart supervisor processes or systemd units" from bench.utils import restart_supervisor_processes, restart_systemd_processes from bench.config.common_site_config import get_config if get_config('.').get('restart_supervisor_on_update') or supervisor: @@ -25,134 +27,112 @@ def restart(web, supervisor, systemd): if get_config('.').get('restart_systemd_on_update') or systemd: restart_systemd_processes(bench_path='.', web_workers=web) -@click.command('set-nginx-port') + +@click.command('set-nginx-port', help="Set NGINX port for site") @click.argument('site') @click.argument('port', type=int) def set_nginx_port(site, port): - "Set nginx port for site" from bench.config.site_config import set_nginx_port set_nginx_port(site, port) -@click.command('set-ssl-certificate') +@click.command('set-ssl-certificate', help="Set SSL certificate path for site") @click.argument('site') @click.argument('ssl-certificate-path') def set_ssl_certificate(site, ssl_certificate_path): - "Set ssl certificate path for site" from bench.config.site_config import set_ssl_certificate set_ssl_certificate(site, ssl_certificate_path) -@click.command('set-ssl-key') +@click.command('set-ssl-key', help="Set SSL certificate private key path for site") @click.argument('site') @click.argument('ssl-certificate-key-path') def set_ssl_certificate_key(site, ssl_certificate_key_path): - "Set ssl certificate private key path for site" from bench.config.site_config import set_ssl_certificate_key set_ssl_certificate_key(site, ssl_certificate_key_path) -@click.command('set-url-root') +@click.command('set-url-root', help="Set URL root for site") @click.argument('site') @click.argument('url-root') def set_url_root(site, url_root): - "Set url root for site" from bench.config.site_config import set_url_root set_url_root(site, url_root) -@click.command('set-mariadb-host') +@click.command('set-mariadb-host', help="Set MariaDB host for bench") @click.argument('host') def set_mariadb_host(host): - "Set MariaDB host for bench" from bench.utils import set_mariadb_host set_mariadb_host(host) -@click.command('set-redis-cache-host') + +@click.command('set-redis-cache-host', help="Set Redis cache host for bench") @click.argument('host') def set_redis_cache_host(host): """ - Set Redis cache host for bench - Eg: bench set-redis-cache-host localhost:6379/1 + Usage: bench set-redis-cache-host localhost:6379/1 """ from bench.utils import set_redis_cache_host set_redis_cache_host(host) -@click.command('set-redis-queue-host') + +@click.command('set-redis-queue-host', help="Set Redis queue host for bench") @click.argument('host') def set_redis_queue_host(host): """ - Set Redis queue host for bench - Eg: bench set-redis-queue-host localhost:6379/2 + Usage: bench set-redis-queue-host localhost:6379/2 """ from bench.utils import set_redis_queue_host set_redis_queue_host(host) -@click.command('set-redis-socketio-host') + +@click.command('set-redis-socketio-host', help="Set Redis socketio host for bench") @click.argument('host') def set_redis_socketio_host(host): """ - Set Redis socketio host for bench - Eg: bench set-redis-socketio-host localhost:6379/3 + Usage: bench set-redis-socketio-host localhost:6379/3 """ from bench.utils import set_redis_socketio_host set_redis_socketio_host(host) -@click.command('set-default-site') +@click.command('set-default-site', help="Set default site for bench") @click.argument('site') def set_default_site(site): - "Set default site for bench" from bench.utils import set_default_site set_default_site(site) -@click.command('download-translations') +@click.command('download-translations', help="Download latest translations") def download_translations(): - "Download latest translations" from bench.utils import download_translations_p download_translations_p() -@click.command('renew-lets-encrypt') + +@click.command('renew-lets-encrypt', help="Renew Let's Encrypt certificate") def renew_lets_encrypt(): - "Renew Let's Encrypt certificate" from bench.config.lets_encrypt import renew_certs renew_certs() -@click.command() -def shell(bench_path='.'): - if not os.environ.get('SHELL'): - print("Cannot get shell") - sys.exit(1) - if not os.path.exists('sites'): - print("sites dir doesn't exist") - sys.exit(1) - env = copy.copy(os.environ) - env['PS1'] = '(' + os.path.basename(os.path.dirname(os.path.abspath(__file__))) + ')' + env.get('PS1', '') - env['PATH'] = os.path.dirname(os.path.abspath(os.path.join('env','bin')) + ':' + env['PATH']) - os.chdir('sites') - os.execve(env['SHELL'], [env['SHELL']], env) - -@click.command('backup') +@click.command('backup', help="Backup single site") @click.argument('site') def backup_site(site): - "backup site" from bench.utils import get_sites, backup_site if site not in get_sites(bench_path='.'): - print('site not found') + print('Site `{0}` not found'.format(site)) sys.exit(1) backup_site(site, bench_path='.') -@click.command('backup-all-sites') +@click.command('backup-all-sites', help="Backup all sites in current bench") def backup_all_sites(): - "backup all sites" from bench.utils import backup_all_sites backup_all_sites(bench_path='.') -@click.command('release') +@click.command('release', help="Release a Frappe app (internal to the Frappe team)") @click.argument('app') @click.argument('bump-type', type=click.Choice(['major', 'minor', 'patch', 'stable', 'prerelease'])) @click.option('--from-branch', default='develop') @@ -162,48 +142,41 @@ def backup_all_sites(): @click.option('--repo-name') @click.option('--dont-frontport', is_flag=True, default=False, help='Front port fixes to new branches, example merging hotfix(v10) into staging-fixes(v11)') def release(app, bump_type, from_branch, to_branch, owner, repo_name, remote, dont_frontport): - "Release app (internal to the Frappe team)" from bench.release import release frontport = not dont_frontport - release(bench_path='.', app=app, bump_type=bump_type, from_branch=from_branch, to_branch=to_branch, - remote=remote, owner=owner, repo_name=repo_name, frontport=frontport) + release(bench_path='.', app=app, bump_type=bump_type, from_branch=from_branch, to_branch=to_branch, remote=remote, owner=owner, repo_name=repo_name, frontport=frontport) -@click.command('prepare-beta-release') +@click.command('prepare-beta-release', help="Prepare major beta release from develop branch") @click.argument('app') @click.option('--owner', default='frappe') def prepare_beta_release(app, owner): - """Prepare major beta release from develop branch""" from bench.prepare_beta_release import prepare_beta_release prepare_beta_release(bench_path='.', app=app, owner=owner) -@click.command('disable-production') +@click.command('disable-production', help="Disables production environment for the bench.") def disable_production(): - """Disables production environment for the bench.""" from bench.config.production_setup import disable_production disable_production(bench_path='.') -@click.command('src') +@click.command('src', help="Prints bench source folder path, which can be used as: cd `bench src`") def bench_src(): - """Prints bench source folder path, which can be used as: cd `bench src` """ import bench print(os.path.dirname(bench.__path__[0])) -@click.command('find') +@click.command('find', help="Finds benches recursively from location") @click.argument('location', default='') def find_benches(location): - """Finds benches recursively from location""" from bench.utils import find_benches find_benches(directory=location) -@click.command('migrate-env') +@click.command('migrate-env', help="Migrate Virtual Environment to desired Python Version") @click.argument('python', type=str) @click.option('--no-backup', 'backup', is_flag=True, default=True) def migrate_env(python, backup=True): - """Migrate Virtual Environment to desired Python Version""" from bench.utils import migrate_env migrate_env(python=python, backup=backup) diff --git a/bench/config/nginx.py b/bench/config/nginx.py index 7e584c3a..9f10c6f7 100644 --- a/bench/config/nginx.py +++ b/bench/config/nginx.py @@ -1,8 +1,24 @@ -import os, json, click, random, string, hashlib -from bench.utils import get_sites, get_bench_name, exec_cmd +# imports - standard imports +import hashlib +import os +import random +import string + +# imports - third party imports +import click from six import string_types +# imports - module imports +from bench.utils import get_bench_name, get_sites + + def make_nginx_conf(bench_path, yes=False): + conf_path = os.path.join(bench_path, "config", "nginx.conf") + + if not yes and os.path.exists(conf_path): + if not click.confirm('nginx.conf already exists and this will overwrite it. Do you want to continue?'): + return + from bench import env from bench.config.common_site_config import get_config @@ -37,10 +53,6 @@ def make_nginx_conf(bench_path, yes=False): nginx_conf = template.render(**template_vars) - conf_path = os.path.join(bench_path, "config", "nginx.conf") - if not yes and os.path.exists(conf_path): - click.confirm('nginx.conf already exists and this will overwrite it. Do you want to continue?', - abort=True) with open(conf_path, "w") as f: f.write(nginx_conf) diff --git a/bench/config/procfile.py b/bench/config/procfile.py index 3dd7cece..a6982c83 100755 --- a/bench/config/procfile.py +++ b/bench/config/procfile.py @@ -3,7 +3,7 @@ from bench.utils import find_executable from bench.app import use_rq from bench.config.common_site_config import get_config -def setup_procfile(bench_path, yes=False): +def setup_procfile(bench_path, yes=False, skip_redis=False): config = get_config(bench_path=bench_path) procfile_path = os.path.join(bench_path, 'Procfile') if not yes and os.path.exists(procfile_path): @@ -14,7 +14,8 @@ def setup_procfile(bench_path, yes=False): node=find_executable("node") or find_executable("nodejs"), use_rq=use_rq(bench_path), webserver_port=config.get('webserver_port'), - CI=os.environ.get('CI')) + CI=os.environ.get('CI'), + skip_redis=skip_redis) with open(procfile_path, 'w') as f: f.write(procfile) diff --git a/bench/config/templates/Procfile b/bench/config/templates/Procfile index b9c81118..e557f561 100644 --- a/bench/config/templates/Procfile +++ b/bench/config/templates/Procfile @@ -1,6 +1,8 @@ +{% if not skip_redis %} redis_cache: redis-server config/redis_cache.conf redis_socketio: redis-server config/redis_socketio.conf redis_queue: redis-server config/redis_queue.conf +{% endif %} web: bench serve {% if webserver_port -%} --port {{ webserver_port }} {%- endif %} socketio: {{ node }} apps/frappe/socketio.js diff --git a/bench/prepare_staging.py b/bench/prepare_staging.py index baa2a7c3..9d004684 100755 --- a/bench/prepare_staging.py +++ b/bench/prepare_staging.py @@ -19,9 +19,7 @@ def prepare_staging(bench_path, app, remote='upstream'): print('No commits to release') return - print() print(message) - print() click.confirm('Do you want to continue?', abort=True) @@ -52,13 +50,13 @@ def create_staging(repo_path, from_branch='develop'): g.merge(from_branch, '--no-ff') except git.exc.GitCommandError as e: handle_merge_error(e, source=from_branch, target='staging') - + g.checkout(from_branch) try: g.merge('staging') except git.exc.GitCommandError as e: handle_merge_error(e, source='staging', target=from_branch) - + def push_commits(repo_path, remote='upstream'): print('pushing staging branch of', repo_path) diff --git a/bench/release.py b/bench/release.py index ebf60f6a..f026b379 100755 --- a/bench/release.py +++ b/bench/release.py @@ -83,9 +83,7 @@ def bump(bench_path, app, bump_type, from_branch, to_branch, remote, owner, repo print('No commits to release') return - print() print(message) - print() click.confirm('Do you want to continue?', abort=True) diff --git a/bench/tests/test_base.py b/bench/tests/test_base.py new file mode 100644 index 00000000..b5022398 --- /dev/null +++ b/bench/tests/test_base.py @@ -0,0 +1,96 @@ +# imports - standard imports +import json +import os +import shutil +import subprocess +import sys +import unittest +import getpass + +# imports - module imports +import bench +import bench.utils + + +class TestBenchBase(unittest.TestCase): + def setUp(self): + self.benches_path = "." + self.benches = [] + + def tearDown(self): + for bench_name in self.benches: + bench_path = os.path.join(self.benches_path, bench_name) + mariadb_password = "travis" if os.environ.get("CI") else getpass.getpass(prompt="Enter MariaDB root Password: ") + if os.path.exists(bench_path): + sites = bench.utils.get_sites(bench_path=bench_path) + for site in sites: + subprocess.call(["bench", "drop-site", site, "--force", "--no-backup", "--root-password", mariadb_password], cwd=bench_path) + shutil.rmtree(bench_path, ignore_errors=True) + + def assert_folders(self, bench_name): + for folder in bench.utils.folders_in_bench: + self.assert_exists(bench_name, folder) + self.assert_exists(bench_name, "apps", "frappe") + + def assert_virtual_env(self, bench_name): + bench_path = os.path.abspath(bench_name) + python_path = os.path.abspath(os.path.join(bench_path, "env", "bin", "python")) + self.assertTrue(python_path.startswith(bench_path)) + for subdir in ("bin", "include", "lib", "share"): + self.assert_exists(bench_name, "env", subdir) + + def assert_config(self, bench_name): + for config, search_key in ( + ("redis_queue.conf", "redis_queue.rdb"), + ("redis_socketio.conf", "redis_socketio.rdb"), + ("redis_cache.conf", "redis_cache.rdb")): + + self.assert_exists(bench_name, "config", config) + + with open(os.path.join(bench_name, "config", config), "r") as f: + self.assertTrue(search_key in f.read()) + + def assert_common_site_config(self, bench_name, expected_config): + common_site_config_path = os.path.join(self.benches_path, bench_name, 'sites', 'common_site_config.json') + self.assertTrue(os.path.exists(common_site_config_path)) + + with open(common_site_config_path, "r") as f: + config = json.load(f) + + for key, value in list(expected_config.items()): + self.assertEqual(config.get(key), value) + + def assert_exists(self, *args): + self.assertTrue(os.path.exists(os.path.join(*args))) + + def new_site(self, site_name, bench_name): + new_site_cmd = ["bench", "new-site", site_name, "--admin-password", "admin"] + + if os.environ.get('CI'): + new_site_cmd.extend(["--mariadb-root-password", "travis"]) + + subprocess.call(new_site_cmd, cwd=os.path.join(self.benches_path, bench_name)) + + def init_bench(self, bench_name, **kwargs): + self.benches.append(bench_name) + frappe_tmp_path = "/tmp/frappe" + + if not os.path.exists(frappe_tmp_path): + bench.utils.exec_cmd("git clone https://github.com/frappe/frappe --depth 1 --origin upstream {location}".format(location=frappe_tmp_path)) + + kwargs.update(dict( + python=sys.executable, + no_procfile=True, + no_backups=True, + skip_assets=True, + frappe_path=frappe_tmp_path + )) + + if not os.path.exists(os.path.join(self.benches_path, bench_name)): + bench.utils.init(bench_name, **kwargs) + bench.utils.exec_cmd("git remote set-url upstream https://github.com/frappe/frappe", cwd=os.path.join(self.benches_path, bench_name, "apps", "frappe")) + + def file_exists(self, path): + if os.environ.get("CI"): + return not subprocess.call(["sudo", "test", "-f", path]) + return os.path.isfile(path) diff --git a/bench/tests/test_init.py b/bench/tests/test_init.py index 170ccf17..195f90dc 100755 --- a/bench/tests/test_init.py +++ b/bench/tests/test_init.py @@ -1,26 +1,20 @@ - +# imports - standard imports +import json +import os +import subprocess import unittest -import json, os, shutil, subprocess + +# imports - third paty imports +import git + +# imports - module imports import bench import bench.utils -import bench.app -import bench.config.common_site_config -import bench.cli from bench.release import get_bumped_version +from bench.tests.test_base import TestBenchBase -bench.cli.from_command_line = True - -class TestBenchInit(unittest.TestCase): - def setUp(self): - self.benches_path = "." - self.benches = [] - - def tearDown(self): - for bench_name in self.benches: - bench_path = os.path.join(self.benches_path, bench_name) - if os.path.exists(bench_path): - shutil.rmtree(bench_path, ignore_errors=True) +class TestBenchInit(TestBenchBase): def test_semantic_version(self): self.assertEqual( get_bumped_version('11.0.4', 'major'), '12.0.0' ) self.assertEqual( get_bumped_version('11.0.4', 'minor'), '11.1.0' ) @@ -35,20 +29,14 @@ class TestBenchInit(unittest.TestCase): def test_init(self, bench_name="test-bench", **kwargs): self.init_bench(bench_name, **kwargs) - self.assert_folders(bench_name) - self.assert_virtual_env(bench_name) - - self.assert_common_site_config(bench_name, bench.config.common_site_config.default_config) - self.assert_config(bench_name) - self.assert_socketio(bench_name) def test_multiple_benches(self): - # 1st bench - self.test_init("test-bench-1") + for bench_name in ("test-bench-1", "test-bench-2"): + self.init_bench(bench_name) self.assert_common_site_config("test-bench-1", { "webserver_port": 8000, @@ -59,9 +47,6 @@ class TestBenchInit(unittest.TestCase): "redis_cache": "redis://localhost:13000" }) - # 2nd bench - self.test_init("test-bench-2") - self.assert_common_site_config("test-bench-2", { "webserver_port": 8001, "socketio_port": 9001, @@ -71,196 +56,92 @@ class TestBenchInit(unittest.TestCase): "redis_cache": "redis://localhost:13001" }) + + def test_new_site(self): - self.init_bench('test-bench') - self.new_site("test-site-1.dev") + bench_name = "test-bench" + site_name = "test-site.local" + bench_path = os.path.join(self.benches_path, bench_name) + site_path = os.path.join(bench_path, "sites", site_name) + site_config_path = os.path.join(site_path, "site_config.json") - def new_site(self, site_name): - new_site_cmd = ["bench", "new-site", site_name, "--admin-password", "admin"] - - # set in CI - if os.environ.get('CI'): - new_site_cmd.extend(["--mariadb-root-password", "travis"]) - - subprocess.check_output(new_site_cmd, cwd=os.path.join(self.benches_path, "test-bench")) - - site_path = os.path.join(self.benches_path, "test-bench", "sites", site_name) + self.init_bench(bench_name) + bench.utils.exec_cmd("bench setup requirements --node", cwd=bench_path) + self.new_site(site_name, bench_name) self.assertTrue(os.path.exists(site_path)) self.assertTrue(os.path.exists(os.path.join(site_path, "private", "backups"))) self.assertTrue(os.path.exists(os.path.join(site_path, "private", "files"))) self.assertTrue(os.path.exists(os.path.join(site_path, "public", "files"))) - - site_config_path = os.path.join(site_path, "site_config.json") self.assertTrue(os.path.exists(site_config_path)) + with open(site_config_path, "r") as f: site_config = json.loads(f.read()) - for key in ("db_name", "db_password"): - self.assertTrue(key in site_config) - self.assertTrue(site_config[key]) + for key in ("db_name", "db_password"): + self.assertTrue(key in site_config) + self.assertTrue(site_config[key]) def test_get_app(self): - site_name = "test-site-2.dev" - self.init_bench('test-bench') - - self.new_site(site_name) + self.init_bench("test-bench") bench_path = os.path.join(self.benches_path, "test-bench") + bench.utils.exec_cmd("bench get-app frappe_theme --skip-assets", cwd=bench_path) + self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "frappe_theme"))) + app_installed_in_env = "frappe_theme" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8') + self.assertTrue(app_installed_in_env) - bench.app.get_app("https://github.com/frappe/frappe-client", bench_path=bench_path) - self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "frappeclient"))) def test_install_app(self): - site_name = "test-site-3.dev" - self.init_bench('test-bench') - - self.new_site(site_name) + bench_name = "test-bench" + site_name = "install-app.test" bench_path = os.path.join(self.benches_path, "test-bench") - # get app - bench.app.get_app("https://github.com/frappe/erpnext", "develop", bench_path=bench_path) + self.init_bench(bench_name) + bench.utils.exec_cmd("bench setup requirements --node", cwd=bench_path) + bench.utils.exec_cmd("bench build", cwd=bench_path) + bench.utils.exec_cmd("bench get-app erpnext", cwd=bench_path) self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "erpnext"))) - # install app - bench.app.install_app("erpnext", bench_path=bench_path) + # check if app is installed + app_installed_in_env = "erpnext" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8') + self.assertTrue(app_installed_in_env) - # install it to site - subprocess.check_output(["bench", "--site", site_name, "install-app", "erpnext"], cwd=bench_path) + # create and install app on site + self.new_site(site_name, bench_name) + bench.utils.exec_cmd("bench --site {0} install-app erpnext".format(site_name), cwd=bench_path) - out = subprocess.check_output(["bench", "--site", site_name, "list-apps"], cwd=bench_path) - self.assertTrue("erpnext" in out) + app_installed_on_site = subprocess.check_output(["bench", "--site", site_name, "list-apps"], cwd=bench_path).decode('utf8') + self.assertTrue("erpnext" in app_installed_on_site) def test_remove_app(self): - self.init_bench('test-bench') - + self.init_bench("test-bench") bench_path = os.path.join(self.benches_path, "test-bench") - # get app - bench.app.get_app("https://github.com/frappe/erpnext", "develop", bench_path=bench_path) - - self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "erpnext"))) - - # remove it - bench.app.remove_app("erpnext", bench_path=bench_path) + bench.utils.exec_cmd("bench setup requirements --node", cwd=bench_path) + bench.utils.exec_cmd("bench get-app erpnext --branch version-12 --skip-assets --overwrite", cwd=bench_path) + bench.utils.exec_cmd("bench remove-app erpnext", cwd=bench_path) + with open(os.path.join(bench_path, "sites", "apps.txt")) as f: + self.assertFalse("erpnext" in f.read()) + self.assertFalse("erpnext" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8')) self.assertFalse(os.path.exists(os.path.join(bench_path, "apps", "erpnext"))) def test_switch_to_branch(self): - self.init_bench('test-bench') - + self.init_bench("test-bench") bench_path = os.path.join(self.benches_path, "test-bench") app_path = os.path.join(bench_path, "apps", "frappe") - bench.app.switch_branch(branch="master", apps=["frappe"], bench_path=bench_path, check_upgrade=False) - out = subprocess.check_output(['git', 'status'], cwd=app_path) - self.assertTrue("master" in out) + bench.utils.exec_cmd("bench switch-to-branch version-12 frappe", cwd=bench_path) + app_branch_after_switch = str(git.Repo(path=app_path).active_branch) + self.assertEqual("version-12", app_branch_after_switch) - # bring it back to develop! - bench.app.switch_branch(branch="develop", apps=["frappe"], bench_path=bench_path, check_upgrade=False) - out = subprocess.check_output(['git', 'status'], cwd=app_path) - self.assertTrue("develop" in out) + bench.utils.exec_cmd("bench switch-to-branch develop frappe", cwd=bench_path) + app_branch_after_second_switch = str(git.Repo(path=app_path).active_branch) + self.assertEqual("develop", app_branch_after_second_switch) - def init_bench(self, bench_name, **kwargs): - self.benches.append(bench_name) - bench.utils.init(bench_name, **kwargs) - def test_drop_site(self): - self.init_bench('test-bench') - # Check without archive_path given to drop-site command - self.drop_site("test-drop-without-archive-path") - - # Check with archive_path given to drop-site command - home = os.path.abspath(os.path.expanduser('~')) - archived_sites_path = os.path.join(home, 'archived_sites') - - self.drop_site("test-drop-with-archive-path", archived_sites_path=archived_sites_path) - - def drop_site(self, site_name, archived_sites_path=None): - self.new_site(site_name) - - drop_site_cmd = ['bench', 'drop-site', site_name] - - if archived_sites_path: - drop_site_cmd.extend(['--archived-sites-path', archived_sites_path]) - - if os.environ.get('CI'): - drop_site_cmd.extend(['--root-password', 'travis']) - - bench_path = os.path.join(self.benches_path, 'test-bench') - try: - subprocess.check_output(drop_site_cmd, cwd=bench_path) - except subprocess.CalledProcessError as err: - print(err.output) - - if not archived_sites_path: - archived_sites_path = os.path.join(bench_path, 'archived_sites') - self.assertTrue(os.path.exists(archived_sites_path)) - self.assertTrue(os.path.exists(os.path.join(archived_sites_path, site_name))) - - else: - self.assertTrue(os.path.exists(archived_sites_path)) - self.assertTrue(os.path.exists(os.path.join(archived_sites_path, site_name))) - - def assert_folders(self, bench_name): - for folder in bench.utils.folders_in_bench: - self.assert_exists(bench_name, folder) - - self.assert_exists(bench_name, "sites", "assets") - self.assert_exists(bench_name, "apps", "frappe") - self.assert_exists(bench_name, "apps", "frappe", "setup.py") - - def assert_virtual_env(self, bench_name): - bench_path = os.path.abspath(bench_name) - python = os.path.join(bench_path, "env", "bin", "python") - python_path = bench.utils.get_cmd_output('{python} -c "import os; print os.path.dirname(os.__file__)"'.format(python=python)) - - # part of bench's virtualenv - self.assertTrue(python_path.startswith(bench_path)) - self.assert_exists(python_path) - self.assert_exists(python_path, "site-packages") - self.assert_exists(python_path, "site-packages", "IPython") - self.assert_exists(python_path, "site-packages", "pip") - - site_packages = os.listdir(os.path.join(python_path, "site-packages")) - # removing test case temporarily - # as develop and master branch havin differnt version of mysqlclient - #self.assertTrue(any(package.startswith("mysqlclient-1.3.12") for package in site_packages)) - - def assert_config(self, bench_name): - for config, search_key in ( - ("redis_queue.conf", "redis_queue.rdb"), - ("redis_socketio.conf", "redis_socketio.rdb"), - ("redis_cache.conf", "redis_cache.rdb")): - - self.assert_exists(bench_name, "config", config) - - with open(os.path.join(bench_name, "config", config), "r") as f: - f = f.read().decode("utf-8") - self.assertTrue(search_key in f) - - def assert_socketio(self, bench_name): - try: # for v10 and under - self.assert_exists(bench_name, "node_modules") - self.assert_exists(bench_name, "node_modules", "socket.io") - except: # for v11 and above - self.assert_exists(bench_name, "apps", "frappe", "node_modules") - self.assert_exists(bench_name, "apps", "frappe", "node_modules", "socket.io") - - def assert_common_site_config(self, bench_name, expected_config): - common_site_config_path = os.path.join(bench_name, 'sites', 'common_site_config.json') - self.assertTrue(os.path.exists(common_site_config_path)) - - config = self.load_json(common_site_config_path) - - for key, value in list(expected_config.items()): - self.assertEqual(config.get(key), value) - - def assert_exists(self, *args): - self.assertTrue(os.path.exists(os.path.join(*args))) - - def load_json(self, path): - with open(path, "r") as f: - return json.loads(f.read().decode("utf-8")) +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/bench/tests/test_setup_production.py b/bench/tests/test_setup_production.py index e22a5858..ca5b4abc 100644 --- a/bench/tests/test_setup_production.py +++ b/bench/tests/test_setup_production.py @@ -1,69 +1,51 @@ - -from bench.tests import test_init -from bench.config.production_setup import setup_production, get_supervisor_confdir, disable_production -import bench.utils -import os +# imports - standard imports import getpass +import os import re -import unittest +import subprocess import time +import unittest -class TestSetupProduction(test_init.TestBenchInit): - # setUp, tearDown and other tests are defiend in TestBenchInit +# imports - module imports +import bench.utils +from bench.config.production_setup import get_supervisor_confdir +from bench.tests.test_base import TestBenchBase + +class TestSetupProduction(TestBenchBase): def test_setup_production(self): - self.test_multiple_benches() - user = getpass.getuser() for bench_name in ("test-bench-1", "test-bench-2"): bench_path = os.path.join(os.path.abspath(self.benches_path), bench_name) - setup_production(user, bench_path) + self.init_bench(bench_name) + bench.utils.exec_cmd("sudo bench setup production {0}".format(user), cwd=bench_path) self.assert_nginx_config(bench_name) self.assert_supervisor_config(bench_name) - - # test after start of both benches - for bench_name in ("test-bench-1", "test-bench-2"): self.assert_supervisor_process(bench_name) self.assert_nginx_process() - - # sudoers - bench.utils.setup_sudoers(user) + bench.utils.exec_cmd("sudo bench setup sudoers {0}".format(user)) self.assert_sudoers(user) - for bench_name in ("test-bench-1", "test-bench-2"): + for bench_name in self.benches: bench_path = os.path.join(os.path.abspath(self.benches_path), bench_name) - disable_production(bench_path) + bench.utils.exec_cmd("sudo bench disable-production", cwd=bench_path) - def test_disable_production(self): - bench_name = 'test-disable-prod' - self.test_init(bench_name, frappe_branch='master') - - user = getpass.getuser() - - bench_path = os.path.join(os.path.abspath(self.benches_path), bench_name) - setup_production(user, bench_path) - - disable_production(bench_path) - - self.assert_nginx_link(bench_name) - self.assert_supervisor_link(bench_name) - self.assert_supervisor_process(bench_name=bench_name, disable_production=True) def assert_nginx_config(self, bench_name): conf_src = os.path.join(os.path.abspath(self.benches_path), bench_name, 'config', 'nginx.conf') conf_dest = "/etc/nginx/conf.d/{bench_name}.conf".format(bench_name=bench_name) - self.assertTrue(os.path.exists(conf_src)) - self.assertTrue(os.path.exists(conf_dest)) + self.assertTrue(self.file_exists(conf_src)) + self.assertTrue(self.file_exists(conf_dest)) # symlink matches self.assertEqual(os.path.realpath(conf_dest), conf_src) # file content with open(conf_src, "r") as f: - f = f.read().decode("utf-8") + f = f.read() for key in ( "upstream {bench_name}-frappe", @@ -71,48 +53,56 @@ class TestSetupProduction(test_init.TestBenchInit): ): self.assertTrue(key.format(bench_name=bench_name) in f) + def assert_nginx_process(self): out = bench.utils.get_cmd_output("sudo nginx -t 2>&1") self.assertTrue("nginx: configuration file /etc/nginx/nginx.conf test is successful" in out) + def assert_sudoers(self, user): sudoers_file = '/etc/sudoers.d/frappe' - self.assertTrue(os.path.exists(sudoers_file)) + self.assertTrue(self.file_exists(sudoers_file)) - with open(sudoers_file, 'r') as f: - sudoers = f.read().decode('utf-8') + if os.environ.get("CI"): + sudoers = subprocess.check_output(["sudo", "cat", sudoers_file]).decode("utf-8") + else: + with open(sudoers_file, 'r') as f: + sudoers = f.read() self.assertTrue('{user} ALL = (root) NOPASSWD: /usr/sbin/service nginx *'.format(user=user) in sudoers) self.assertTrue('{user} ALL = (root) NOPASSWD: /usr/bin/supervisorctl'.format(user=user) in sudoers) self.assertTrue('{user} ALL = (root) NOPASSWD: /usr/sbin/nginx'.format(user=user) in sudoers) + def assert_supervisor_config(self, bench_name, use_rq=True): conf_src = os.path.join(os.path.abspath(self.benches_path), bench_name, 'config', 'supervisor.conf') supervisor_conf_dir = get_supervisor_confdir() conf_dest = "{supervisor_conf_dir}/{bench_name}.conf".format(supervisor_conf_dir=supervisor_conf_dir, bench_name=bench_name) - self.assertTrue(os.path.exists(conf_src)) - self.assertTrue(os.path.exists(conf_dest)) + self.assertTrue(self.file_exists(conf_src)) + self.assertTrue(self.file_exists(conf_dest)) # symlink matches self.assertEqual(os.path.realpath(conf_dest), conf_src) # file content with open(conf_src, "r") as f: - f = f.read().decode("utf-8") + f = f.read() tests = [ "program:{bench_name}-frappe-web", "program:{bench_name}-redis-cache", "program:{bench_name}-redis-queue", "program:{bench_name}-redis-socketio", - "program:{bench_name}-node-socketio", "group:{bench_name}-web", "group:{bench_name}-workers", "group:{bench_name}-redis" ] + if not os.environ.get("CI"): + tests.append("program:{bench_name}-node-socketio") + if use_rq: tests.extend([ "program:{bench_name}-frappe-schedule", @@ -130,8 +120,11 @@ class TestSetupProduction(test_init.TestBenchInit): ]) for key in tests: + if key.format(bench_name=bench_name) not in f: + print(key.format(bench_name=bench_name)) self.assertTrue(key.format(bench_name=bench_name) in f) + def assert_supervisor_process(self, bench_name, use_rq=True, disable_production=False): out = bench.utils.get_cmd_output("sudo supervisorctl status") @@ -172,15 +165,6 @@ class TestSetupProduction(test_init.TestBenchInit): else: self.assertTrue(re.search(key.format(bench_name=bench_name), out)) - def assert_nginx_link(self, bench_name): - nginx_conf_name = '{bench_name}.conf'.format(bench_name=bench_name) - nginx_conf_path = os.path.join('/etc/nginx/conf.d', nginx_conf_name) - self.assertFalse(os.path.islink(nginx_conf_path)) - - def assert_supervisor_link(self, bench_name): - supervisor_conf_dir = get_supervisor_confdir() - supervisor_conf_name = '{bench_name}.conf'.format(bench_name=bench_name) - supervisor_conf_path = os.path.join(supervisor_conf_dir, supervisor_conf_name) - - self.assertFalse(os.path.islink(supervisor_conf_path)) +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/bench/utils.py b/bench/utils.py index 10a4f801..b0d7543b 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -1,14 +1,31 @@ -import errno, glob, grp, itertools, json, logging, multiprocessing, os, platform, pwd, re, select, shutil, site, subprocess, sys +# imports - standard imports +import errno +import glob +import grp +import itertools +import json +import logging +import multiprocessing +import os +import platform +import pwd +import re +import select +import shutil +import site +import subprocess +import sys from datetime import datetime from distutils.spawn import find_executable +# imports - third party imports +import click import requests -import semantic_version from six import iteritems from six.moves.urllib.parse import urlparse +# imports - module imports import bench -from bench import env class PatchError(Exception): @@ -28,6 +45,7 @@ class color: green = '\033[92m' yellow = '\033[93m' red = '\033[91m' + silver = '\033[90m' def is_bench_directory(directory=os.path.curdir): @@ -60,6 +78,7 @@ def safe_decode(string, encoding = 'utf-8'): pass return string + def get_frappe(bench_path='.'): frappe = get_env_cmd('frappe', bench_path=bench_path) if not os.path.exists(frappe): @@ -67,12 +86,16 @@ def get_frappe(bench_path='.'): print('bench get-app https://github.com/frappe/frappe.git') return frappe + def get_env_cmd(cmd, bench_path='.'): return os.path.abspath(os.path.join(bench_path, 'env', 'bin', cmd)) -def init(path, apps_path=None, no_procfile=False, no_backups=False, no_auto_update=False, - frappe_path=None, frappe_branch=None, wheel_cache_dir=None, verbose=False, clone_from=None, - skip_redis_config_generation=False, clone_without_update=False, ignore_exist = False, skip_assets=False, python='python3'): + +def init(path, apps_path=None, no_procfile=False, no_backups=False, + frappe_path=None, frappe_branch=None, verbose=False, clone_from=None, + skip_redis_config_generation=False, clone_without_update=False, ignore_exist=False, skip_assets=False, + python='python3'): + """Initialize a new bench directory""" from bench.app import get_app, install_apps_from_path from bench.config import redis from bench.config.common_site_config import make_config @@ -124,17 +147,110 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, no_auto_upda redis.generate_config(path) if not no_procfile: - setup_procfile(path) + setup_procfile(path, skip_redis=skip_redis_config_generation) if not no_backups: setup_backups(bench_path=path) - if not no_auto_update: - setup_auto_update(bench_path=path) + copy_patches_txt(path) + +def restart_update(kwargs): + args = ['--'+k for k, v in list(kwargs.items()) if v] + os.execv(sys.argv[0], sys.argv[:2] + args) + + +def update(pull=False, patch=False, build=False, bench=False, restart_supervisor=False, + restart_systemd=False, requirements=False, backup=True, force=False, reset=False): + """command: bench update""" + + if not is_bench_directory(): + """Update only bench CLI if bench update called from outside a bench""" + update_bench(bench_repo=True, requirements=True) + sys.exit(0) + + from bench import patches + from bench.app import is_version_upgrade, pull_all_apps, validate_branch + from bench.config.common_site_config import get_config, update_config + + bench_path = os.path.abspath(".") + patches.run(bench_path=bench_path) + conf = get_config(bench_path) + + if conf.get('release_bench'): + print('Release bench detected, cannot update!') + sys.exit(1) + + if not (pull or patch or build or bench or requirements): + pull, patch, build, bench, requirements = True, True, True, True, True + + if bench and conf.get('update_bench_on_update'): + update_bench(bench_repo=True, requirements=False) + restart_update({ + 'pull': pull, + 'patch': patch, + 'build': build, + 'requirements': requirements, + 'no-backup': backup, + 'restart-supervisor': restart_supervisor, + 'reset': reset + }) + + validate_branch() + version_upgrade = is_version_upgrade() + + if version_upgrade[0]: + if force: + print("Force flag has been used for a major version change in Frappe and it's apps. \nThis will take significant time to migrate and might break custom apps.") + else: + print("This update will cause a major version change in Frappe/ERPNext from {0} to {1}. \nThis would take significant time to migrate and might break custom apps.".format(*version_upgrade[1:])) + click.confirm('Do you want to continue?', abort=True) + + if version_upgrade[0] or (not version_upgrade[0] and force): + validate_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) + + before_update(bench_path=bench_path, requirements=requirements) + + conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 }) + update_config(conf, bench_path=bench_path) + + if backup: + print('Backing up sites...') + backup_all_sites(bench_path=bench_path) + + if pull: + pull_all_apps(bench_path=bench_path, reset=reset) + + if requirements: + update_requirements(bench_path=bench_path) + update_node_packages(bench_path=bench_path) + + if patch: + print('Patching sites...') + patch_sites(bench_path=bench_path) + + if build: + build_assets(bench_path=bench_path) + + if version_upgrade[0] or (not version_upgrade[0] and force): + post_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) + + if restart_supervisor or conf.get('restart_supervisor_on_update'): + restart_supervisor_processes(bench_path=bench_path) + + if restart_systemd or conf.get('restart_systemd_on_update'): + restart_systemd_processes(bench_path=bench_path) + + conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 }) + update_config(conf, bench_path=bench_path) + + print("_" * 80 + "\nBench: Deployment tool for Frappe and Frappe Applications (https://frappe.io/bench).\nOpen source depends on your contributions, so please contribute bug reports, patches, fixes or cash and be a part of the community") + + def copy_patches_txt(bench_path): shutil.copy(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'patches', 'patches.txt'), os.path.join(bench_path, 'patches.txt')) + def clone_apps_from(bench_path, clone_from, update_app=True): from .app import install_app print('Copying apps from {0}...'.format(clone_from)) @@ -171,30 +287,15 @@ def clone_apps_from(bench_path, clone_from, update_app=True): for app in apps: setup_app(app) + def exec_cmd(cmd, cwd='.'): - from .cli import from_command_line + import shlex + print("{0}$ {1}{2}".format(color.silver, cmd, color.nc)) + cmd = shlex.split(cmd) + subprocess.call(cmd, cwd=cwd, universal_newlines=True) - is_async = False if from_command_line else True - if is_async: - stderr = stdout = subprocess.PIPE - else: - stderr = stdout = None - - logger.info(cmd) - - p = subprocess.Popen(cmd, cwd=cwd, shell=True, stdout=stdout, stderr=stderr, - universal_newlines=True) - - if is_async: - return_code = print_output(p) - else: - return_code = p.wait() - - if return_code > 0: - raise CommandFailedError(cmd) def which(executable, raise_err = False): - from distutils.spawn import find_executable exec_ = find_executable(executable) if not exec_ and raise_err: @@ -204,6 +305,7 @@ def which(executable, raise_err = False): return exec_ + def setup_env(bench_path='.', python = 'python3'): python = which(python, raise_err = True) pip = os.path.join('env', 'bin', 'pip') @@ -212,10 +314,12 @@ def setup_env(bench_path='.', python = 'python3'): exec_cmd('{} -q install -U pip wheel six'.format(pip), cwd=bench_path) exec_cmd('{} -q install -e git+https://github.com/frappe/python-pdfkit.git#egg=pdfkit'.format(pip), cwd=bench_path) + def setup_socketio(bench_path='.'): exec_cmd("npm install socket.io redis express superagent cookie babel-core less chokidar \ babel-cli babel-preset-es2015 babel-preset-es2016 babel-preset-es2017 babel-preset-babili", cwd=bench_path) + def patch_sites(bench_path='.'): bench.set_frappe_version(bench_path=bench_path) @@ -227,6 +331,7 @@ def patch_sites(bench_path='.'): except subprocess.CalledProcessError: raise PatchError + def build_assets(bench_path='.', app=None): bench.set_frappe_version(bench_path=bench_path) @@ -238,19 +343,16 @@ def build_assets(bench_path='.', app=None): command += ' --app {}'.format(app) exec_cmd(command, cwd=bench_path) + def get_sites(bench_path='.'): sites_path = os.path.join(bench_path, 'sites') sites = (site for site in os.listdir(sites_path) if os.path.exists(os.path.join(sites_path, site, 'site_config.json'))) return sites + def get_bench_dir(bench_path='.'): return os.path.abspath(bench_path) -def setup_auto_update(bench_path='.'): - logger.info('setting up auto update') - add_to_crontab('0 10 * * * cd {bench_dir} && {bench} update --auto >> {logfile} 2>&1'.format(bench_dir=get_bench_dir(bench_path=bench_path), - bench=os.path.join(get_bench_dir(bench_path=bench_path), 'env', 'bin', 'bench'), - logfile=os.path.join(get_bench_dir(bench_path=bench_path), 'logs', 'auto_update_log.log'))) def setup_backups(bench_path='.'): logger.info('setting up backups') @@ -265,24 +367,27 @@ def setup_backups(bench_path='.'): add_to_crontab('0 */6 * * * {backup_command} >> {logfile} 2>&1'.format(backup_command=backup_command, logfile=os.path.join(get_bench_dir(bench_path=bench_path), 'logs', 'backup.log'))) + def add_to_crontab(line): current_crontab = read_crontab() line = str.encode(line) if not line in current_crontab: cmd = ["crontab"] - if platform.system() == 'FreeBSD' or platform.linux_distribution()[0]=="arch": + if platform.system() == 'FreeBSD': cmd = ["crontab", "-"] s = subprocess.Popen(cmd, stdin=subprocess.PIPE) s.stdin.write(current_crontab) s.stdin.write(line + b'\n') s.stdin.close() + def read_crontab(): s = subprocess.Popen(["crontab", "-l"], stdin=subprocess.PIPE, stdout=subprocess.PIPE) out = s.stdout.read() s.stdout.close() return out + def update_bench(bench_repo=True, requirements=True): logger.info("Updating bench") @@ -301,7 +406,10 @@ def update_bench(bench_repo=True, requirements=True): logger.info("Bench Updated!") + def setup_sudoers(user): + from bench import env + if not os.path.exists('/etc/sudoers.d'): os.makedirs('/etc/sudoers.d') @@ -333,6 +441,7 @@ def setup_sudoers(user): os.chmod(sudoers_file, 0o440) + def setup_logging(bench_path='.'): if os.path.exists(os.path.join(bench_path, 'logs')): logger = logging.getLogger('bench') @@ -343,6 +452,7 @@ def setup_logging(bench_path='.'): logger.addHandler(hdlr) logger.setLevel(logging.DEBUG) + def get_program(programs): program = None for p in programs: @@ -351,9 +461,11 @@ def get_program(programs): break return program + def get_process_manager(): return get_program(['foreman', 'forego', 'honcho']) + def start(no_dev=False, concurrency=None, procfile=None): program = get_process_manager() if not program: @@ -371,6 +483,7 @@ def start(no_dev=False, concurrency=None, procfile=None): os.execv(program, command) + def check_cmd(cmd, cwd='.'): try: subprocess.check_call(cmd, cwd=cwd, shell=True) @@ -378,6 +491,7 @@ def check_cmd(cmd, cwd='.'): except subprocess.CalledProcessError: return False + def get_git_version(): '''returns git version from `git --version` extracts version number from string `get version 1.9.1` etc''' @@ -387,6 +501,7 @@ def get_git_version(): version = '.'.join(version.split('.')[0:2]) return float(version) + def check_git_for_shallow_clone(): from .config.common_site_config import get_config config = get_config('.') @@ -402,6 +517,7 @@ def check_git_for_shallow_clone(): if git_version > 1.9: return True + def get_cmd_output(cmd, cwd='.'): try: output = subprocess.check_output(cmd, cwd=cwd, shell=True, stderr=subprocess.PIPE).strip() @@ -412,6 +528,7 @@ def get_cmd_output(cmd, cwd='.'): print(e.output) raise + def safe_encode(what, encoding = 'utf-8'): try: what = what.encode(encoding) @@ -420,6 +537,7 @@ def safe_encode(what, encoding = 'utf-8'): return what + def restart_supervisor_processes(bench_path='.', web_workers=False): from .config.common_site_config import get_config conf = get_config(bench_path=bench_path) @@ -449,38 +567,44 @@ def restart_supervisor_processes(bench_path='.', web_workers=False): exec_cmd('sudo supervisorctl restart {group}'.format(group=group), cwd=bench_path) + def restart_systemd_processes(bench_path='.', web_workers=False): from .config.common_site_config import get_config bench_name = get_bench_name(bench_path) exec_cmd('sudo systemctl stop -- $(systemctl show -p Requires {bench_name}.target | cut -d= -f2)'.format(bench_name=bench_name)) exec_cmd('sudo systemctl start -- $(systemctl show -p Requires {bench_name}.target | cut -d= -f2)'.format(bench_name=bench_name)) + def set_default_site(site, bench_path='.'): if site not in get_sites(bench_path=bench_path): raise Exception("Site not in bench") exec_cmd("{frappe} --use {site}".format(frappe=get_frappe(bench_path=bench_path), site=site), cwd=os.path.join(bench_path, 'sites')) + def update_bench_requirements(): bench_req_file = os.path.join(os.path.dirname(bench.__path__[0]), 'requirements.txt') install_requirements(bench_req_file, user=True) + def update_env_pip(bench_path): env_pip = os.path.join(bench_path, 'env', 'bin', 'pip') exec_cmd("{pip} install -q -U pip".format(pip=env_pip)) + def update_requirements(bench_path='.'): from bench.app import get_apps, install_app print('Updating Python libraries...') - # update env pip - update_env_pip(bench_path) - # Update bench requirements (at user level) update_bench_requirements() + # update env pip + update_env_pip(bench_path) + for app in get_apps(): - install_app(app, bench_path=bench_path) + install_app(app, bench_path=bench_path, skip_assets=True) + def update_node_packages(bench_path='.'): print('Updating node packages...') @@ -488,7 +612,6 @@ def update_node_packages(bench_path='.'): from distutils.version import LooseVersion v = LooseVersion(get_develop_version('frappe', bench_path = bench_path)) - # After rollup was merged, frappe_version = 10.1 # if develop_verion is 11 and up, only then install yarn if v < LooseVersion('11.x.x-develop'): @@ -496,6 +619,7 @@ def update_node_packages(bench_path='.'): else: update_yarn_packages(bench_path) + def update_yarn_packages(bench_path='.'): apps_dir = os.path.join(bench_path, 'apps') @@ -556,6 +680,7 @@ def install_requirements(req_file, user=False): exec_cmd("{python} -m pip install {user_flag} -q -U -r {req_file}".format(python=python, user_flag=user_flag, req_file=req_file)) + def backup_site(site, bench_path='.'): bench.set_frappe_version(bench_path=bench_path) @@ -565,30 +690,38 @@ def backup_site(site, bench_path='.'): else: run_frappe_cmd('--site', site, 'backup', bench_path=bench_path) + def backup_all_sites(bench_path='.'): for site in get_sites(bench_path=bench_path): backup_site(site, bench_path=bench_path) + def is_root(): if os.getuid() == 0: return True return False + def set_mariadb_host(host, bench_path='.'): update_common_site_config({'db_host': host}, bench_path=bench_path) + def set_redis_cache_host(host, bench_path='.'): update_common_site_config({'redis_cache': "redis://{}".format(host)}, bench_path=bench_path) + def set_redis_queue_host(host, bench_path='.'): update_common_site_config({'redis_queue': "redis://{}".format(host)}, bench_path=bench_path) + def set_redis_socketio_host(host, bench_path='.'): update_common_site_config({'redis_socketio': "redis://{}".format(host)}, bench_path=bench_path) + def update_common_site_config(ddict, bench_path='.'): update_json_file(os.path.join(bench_path, 'sites', 'common_site_config.json'), ddict) + def update_json_file(filename, ddict): if os.path.exists(filename): with open(filename, 'r') as f: @@ -601,6 +734,7 @@ def update_json_file(filename, ddict): with open(filename, 'w') as f: json.dump(content, f, indent=1, sort_keys=True) + def drop_privileges(uid_name='nobody', gid_name='nogroup'): # from http://stackoverflow.com/a/2699996 if os.getuid() != 0: @@ -621,6 +755,7 @@ def drop_privileges(uid_name='nobody', gid_name='nogroup'): # Ensure a very conservative umask os.umask(0o22) + def fix_prod_setup_perms(bench_path='.', frappe_user=None): from .config.common_site_config import get_config @@ -638,6 +773,7 @@ def fix_prod_setup_perms(bench_path='.', frappe_user=None): gid = grp.getgrnam(frappe_user).gr_gid os.chown(path, uid, gid) + def fix_file_perms(): for dir_path, dirs, files in os.walk('.'): for _dir in dirs: @@ -650,10 +786,12 @@ def fix_file_perms(): if not _file.startswith('activate'): os.chmod(os.path.join(bin_dir, _file), 0o755) + def get_current_frappe_version(bench_path='.'): from .app import get_current_frappe_version as fv return fv(bench_path=bench_path) + def run_frappe_cmd(*args, **kwargs): from .cli import from_command_line @@ -677,7 +815,7 @@ def run_frappe_cmd(*args, **kwargs): if return_code > 0: sys.exit(return_code) - #raise CommandFailedError(args) + def get_frappe_cmd_output(*args, **kwargs): bench_path = kwargs.get('bench_path', '.') @@ -685,24 +823,12 @@ def get_frappe_cmd_output(*args, **kwargs): sites_dir = os.path.join(bench_path, 'sites') return subprocess.check_output((f, '-m', 'frappe.utils.bench_helper', 'frappe') + args, cwd=sites_dir) + def validate_upgrade(from_ver, to_ver, bench_path='.'): if to_ver >= 6: if not find_executable('npm') and not (find_executable('node') or find_executable('nodejs')): raise Exception("Please install nodejs and npm") -def pre_upgrade(from_ver, to_ver, bench_path='.'): - pip = os.path.join(bench_path, 'env', 'bin', 'pip') - - if from_ver <= 4 and to_ver >= 5: - from .migrate_to_v5 import remove_shopping_cart - apps = ('frappe', 'erpnext') - remove_shopping_cart(bench_path=bench_path) - - for app in apps: - cwd = os.path.abspath(os.path.join(bench_path, 'apps', app)) - if os.path.exists(cwd): - exec_cmd("git clean -dxf", cwd=cwd) - exec_cmd("{pip} install --upgrade -e {app}".format(pip=pip, app=cwd)) def post_upgrade(from_ver, to_ver, bench_path='.'): from .config.common_site_config import get_config @@ -710,8 +836,7 @@ def post_upgrade(from_ver, to_ver, bench_path='.'): from .config.supervisor import generate_supervisor_config from .config.nginx import make_nginx_conf conf = get_config(bench_path=bench_path) - print("-"*80) - print("Your bench was upgraded to version {0}".format(to_ver)) + print("-" * 80 + "Your bench was upgraded to version {0}".format(to_ver)) if conf.get('restart_supervisor_on_update'): redis.generate_config(bench_path=bench_path) @@ -987,6 +1112,7 @@ def in_virtual_env(): return False + def migrate_env(python, backup=False): from bench.config.common_site_config import get_config from bench.app import get_apps @@ -1042,4 +1168,18 @@ def migrate_env(python, backup=False): log.debug('Migration Successful to {}'.format(python)) except: log.debug('Migration Error') - raise \ No newline at end of file + raise + + +def find_parent_bench(path): + """Checks if parent directories are benches""" + if is_bench_directory(directory=path): + return path + + home_path = os.path.expanduser("~") + root_path = os.path.abspath(os.sep) + + if path not in {home_path, root_path}: + # NOTE: the os.path.split assumes that given path is absolute + parent_dir = os.path.split(path)[0] + return find_parent_bench(parent_dir) diff --git a/playbooks/install.py b/playbooks/install.py index 0da571b5..9867d185 100644 --- a/playbooks/install.py +++ b/playbooks/install.py @@ -1,5 +1,16 @@ #!/usr/bin/env python3 -import os, sys, subprocess, getpass, json, multiprocessing, shutil, platform, warnings, datetime +from __future__ import print_function +import os +import sys +import subprocess +import getpass +import json +import multiprocessing +import shutil +import platform +import warnings +import datetime + tmp_bench_repo = os.path.join('/', 'tmp', '.bench') tmp_log_folder = os.path.join('/', 'tmp', 'logs') @@ -395,8 +406,20 @@ def parse_commandline_args(): if __name__ == '__main__': if sys.version[0] == '2': - if not raw_input("It is recommended to run this script with Python 3\nDo you still wish to continue? [Y/n]: ").lower() == "y": - sys.exit() + if not os.environ.get('CI'): + if not raw_input("It is recommended to run this script with Python 3\nDo you still wish to continue? [Y/n]: ").lower() == "y": + sys.exit() + + try: + from distutils.spawn import find_executable + except ImportError: + try: + subprocess.check_call('pip install --upgrade setuptools') + except subprocess.CalledProcessError: + print("Install distutils or use Python3 to run the script") + sys.exit(1) + + shutil.which = find_executable if not is_sudo_user(): log("Please run this script as a non-root user with sudo privileges", level=3) diff --git a/requirements.txt b/requirements.txt index 1922bb98..e00b90ea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,18 +1,10 @@ Click==7.0 -GitPython==2.1.11 +GitPython==2.1.15 honcho==1.0.1 Jinja2==2.10.3 python-crontab==2.4.0 requests==2.22.0 -semantic_version==2.8.2 +semantic-version==2.8.2 setuptools==40.8.0 six==1.12.0 virtualenv==16.6.0 -gitdb2==2.0.6;python_version<'3.4' -MarkupSafe==1.1.1 -python-dateutil==2.8.1 -idna==2.8 -certifi==2019.9.11 -urllib3==1.25.7 -chardet==3.0.4 -smmap2==2.0.5