diff --git a/.github/semantic.yml b/.github/semantic.yml new file mode 100644 index 00000000..e1e53bc1 --- /dev/null +++ b/.github/semantic.yml @@ -0,0 +1,13 @@ +# Always validate the PR title AND all the commits +titleAndCommits: true + +# Allow use of Merge commits (eg on github: "Merge branch 'master' into feature/ride-unicorns") +# this is only relevant when using commitsOnly: true (or titleAndCommits: true) +allowMergeCommits: true + +# Allow use of Revert commits (eg on github: "Revert "feat: ride unicorns"") +# this is only relevant when using commitsOnly: true (or titleAndCommits: true) +allowRevertCommits: true + +# For allowed PR types: https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json +# Tool Reference: https://github.com/zeke/semantic-pull-requests diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..fbb96207 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,28 @@ +name: Generate Semantic Release and publish on PyPI +on: + push: + branches: + - v5.x +jobs: + release: + name: Release + runs-on: ubuntu-18.04 + steps: + - name: Checkout Entire Repository + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Setup Node.js v12 + uses: actions/setup-node@v1 + with: + node-version: 12 + - name: Setup dependencies + run: | + npm install @semantic-release/git @semantic-release/exec --no-save + pip install wheel twine + - name: Create Release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }} + run: npx semantic-release diff --git a/.releaserc b/.releaserc new file mode 100644 index 00000000..43cc0109 --- /dev/null +++ b/.releaserc @@ -0,0 +1,35 @@ +{ + "branches": ["v5.x"], + "plugins": [ + "@semantic-release/commit-analyzer", + "@semantic-release/release-notes-generator", + [ + "@semantic-release/exec", { + "prepareCmd": 'sed -ir "s/[0-9]*\.[0-9]*\.[0-9]*/${nextRelease.version}/" bench/__init__.py' + } + ], + [ + "@semantic-release/exec", { + "prepareCmd": "python setup.py bdist_wheel --universal" + } + ], + [ + "@semantic-release/git", { + "assets": ["bench/__init__.py"], + "message": "chore(release): Bumped to Version ${nextRelease.version}\n\n${nextRelease.notes}" + } + ], + [ + "@semantic-release/github", { + "assets": [ + {"path": "dist/*"}, + ] + } + ], + [ + "@semantic-release/exec", { + "publishCmd": "python -m twine upload dist/* -u $PYPI_USERNAME -p $PYPI_PASSWORD" + } + ] + ] +} diff --git a/README.md b/README.md index abbbb7f4..f11b7879 100755 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Bench is a command-line utility that helps you to install, update, and manage mu - [Production Setup](#docker-installation-for-production) - [Easy Install Script](#easy-install-script) - [Manual Installation](#manual-installation) - - [Usage](#usage) + - [Usage](#basic-usage) - [Custom Bench commands](#custom-bench-commands) - [Bench Manager](#bench-manager) - [Guides](#guides) diff --git a/bench/__init__.py b/bench/__init__.py index eec75ea4..19d73b97 100644 --- a/bench/__init__.py +++ b/bench/__init__.py @@ -1,4 +1,4 @@ -VERSION = "5.2.1" +VERSION = "5.0.0-dev" PROJECT_NAME = "frappe-bench" FRAPPE_VERSION = None diff --git a/bench/app.py b/bench/app.py index e55498e0..02925c9b 100755 --- a/bench/app.py +++ b/bench/app.py @@ -6,21 +6,15 @@ import json import logging import os import re -import shutil import subprocess import sys # imports - third party imports import click -import git -import requests -import semantic_version -from six.moves import reload_module from setuptools.config import read_configuration # imports - module imports import bench -from bench.config.common_site_config import get_config from bench.utils import color, CommandFailedError, build_assets, check_git_for_shallow_clone, exec_cmd, get_cmd_output, get_frappe, restart_supervisor_processes, restart_systemd_processes, run_frappe_cmd @@ -92,6 +86,9 @@ def remove_from_excluded_apps_txt(app, bench_path='.'): return write_excluded_apps_txt(apps, bench_path=bench_path) def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=False, restart_bench=True, overwrite=False): + import requests + import shutil + if not os.path.exists(git_url): if not is_git_url(git_url): orgs = ['frappe', 'erpnext'] @@ -166,17 +163,13 @@ def new_app(app, bench_path='.'): app = app.lower().replace(" ", "_").replace("-", "_") logger.log('creating new app {}'.format(app)) apps = os.path.abspath(os.path.join(bench_path, 'apps')) - bench.set_frappe_version(bench_path=bench_path) - - if bench.FRAPPE_VERSION == 4: - exec_cmd("{frappe} --make_app {apps} {app}".format(frappe=get_frappe(bench_path=bench_path), - apps=apps, app=app)) - else: - run_frappe_cmd('make-app', apps, app, bench_path=bench_path) + run_frappe_cmd('make-app', apps, app, bench_path=bench_path) install_app(app, bench_path=bench_path) def install_app(app, bench_path=".", verbose=False, no_cache=False, restart_bench=True, skip_assets=False): + from bench.config.common_site_config import get_config + print('\n{0}Installing {1}{2}'.format(color.yellow, app, color.nc)) logger.log("installing {}".format(app)) @@ -205,6 +198,9 @@ def install_app(app, bench_path=".", verbose=False, no_cache=False, restart_benc def remove_app(app, bench_path='.'): + import shutil + from bench.config.common_site_config import get_config + if app not in get_apps(bench_path): print("No app named {0}".format(app)) sys.exit(1) @@ -232,6 +228,8 @@ def remove_app(app, bench_path='.'): def pull_apps(apps=None, bench_path='.', reset=False): '''Check all apps if there no local changes, pull''' + from bench.config.common_site_config import get_config + rebase = '--rebase' if get_config(bench_path).get('rebase_on_pull') else '' apps = apps or get_apps(bench_path=bench_path) @@ -380,6 +378,8 @@ def get_repo_dir(app, bench_path='.'): return os.path.join(bench_path, 'apps', app) def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrade=True): + import git + from six.moves import reload_module from bench.utils import update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets, post_upgrade apps_dir = os.path.join(bench_path, 'apps') version_upgrade = (False,) @@ -447,6 +447,8 @@ def get_version_from_string(contents, field='__version__'): return match.group(2) def get_major_version(version): + import semantic_version + return semantic_version.Version(version).major def install_apps_from_path(path, bench_path='.'): @@ -455,6 +457,8 @@ def install_apps_from_path(path, bench_path='.'): get_app(app['url'], branch=app.get('branch'), bench_path=bench_path, skip_assets=True) def get_apps_json(path): + import requests + if path.startswith('http'): r = requests.get(path) return r.json() diff --git a/bench/cli.py b/bench/cli.py index 4ed7149d..05c74d86 100755 --- a/bench/cli.py +++ b/bench/cli.py @@ -30,11 +30,10 @@ def cli(): logger = setup_logging() logger.info(command) - if sys.argv[1] not in ("src", ): + if len(sys.argv) > 1 and sys.argv[1] not in ("src", ): check_uid() change_uid() - - change_dir() + change_dir() if is_dist_editable(bench.PROJECT_NAME) and len(sys.argv) > 1 and sys.argv[1] != "src" and not get_config(".").get("developer_mode"): log("bench is installed in editable mode!\n\nThis is not the recommended mode of installation for production. Instead, install the package from PyPI with: `pip install frappe-bench`\n", level=3) diff --git a/bench/commands/config.py b/bench/commands/config.py index 4d4bec59..b3691941 100644 --- a/bench/commands/config.py +++ b/bench/commands/config.py @@ -1,6 +1,3 @@ -# imports - standard imports -import ast - # imports - module imports from bench.config.common_site_config import update_config, get_config, put_config @@ -52,6 +49,8 @@ def config_http_timeout(seconds): @click.command('set-common-config', help='Set value in common config') @click.option('configs', '-c', '--config', multiple=True, type=(str, str)) def set_common_config(configs): + import ast + common_site_config = {} for key, value in configs: if value in ('true', 'false'): diff --git a/bench/commands/make.py b/bench/commands/make.py index 7fc799e6..c20b25b3 100755 --- a/bench/commands/make.py +++ b/bench/commands/make.py @@ -8,11 +8,11 @@ import click @click.option('--ignore-exist', is_flag = True, default = False, help = "Ignore if Bench instance exists.") @click.option('--apps_path', default=None, help="path to json files with apps to install after init") @click.option('--frappe-path', default=None, help="path to frappe repo") -@click.option('--frappe-branch', default=None, help="path to frappe repo") +@click.option('--frappe-branch', default=None, help="Clone a particular branch of frappe") @click.option('--clone-from', default=None, help="copy repos from path") @click.option('--clone-without-update', is_flag=True, help="copy repos from path without update") -@click.option('--no-procfile', is_flag=True, help="Pull changes in all the apps in bench") -@click.option('--no-backups',is_flag=True, help="Run migrations for all sites in the bench") +@click.option('--no-procfile', is_flag=True, help="Do not create a Procfile") +@click.option('--no-backups',is_flag=True, help="Do not set up automatic periodic backups for all sites on this bench") @click.option('--skip-redis-config-generation', is_flag=True, help="Skip redis config generation if already specifying the common-site-config file") @click.option('--skip-assets',is_flag=True, default=False, help="Do not build assets") @click.option('--verbose',is_flag=True, help="Verbose output during install") diff --git a/bench/commands/setup.py b/bench/commands/setup.py index 08e2d4f2..8021f835 100755 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -6,10 +6,7 @@ import sys import click # imports - module imports -import bench.config.lets_encrypt -import bench.config.nginx import bench.config.procfile -import bench.config.production_setup import bench.config.redis import bench.config.site_config import bench.config.supervisor @@ -31,20 +28,25 @@ def setup_sudoers(user): @click.command("nginx", help="Generate configuration files for NGINX") @click.option("--yes", help="Yes to regeneration of nginx config file", default=False, is_flag=True) def setup_nginx(yes=False): + import bench.config.nginx + bench.config.nginx.make_nginx_conf(bench_path=".", yes=yes) @click.command("reload-nginx", help="Checks NGINX config file and reloads service") def reload_nginx(): + import bench.config.production_setup + bench.config.production_setup.reload_nginx() @click.command("supervisor", help="Generate configuration for supervisor") @click.option("--user", help="optional user argument") @click.option("--yes", help="Yes to regeneration of supervisor config", is_flag=True, default=False) -def setup_supervisor(user=None, yes=False): +@click.option("--skip-redis", help="Skip redis configuration", is_flag=True, default=False) +def setup_supervisor(user=None, yes=False, skip_redis=False): bench.config.supervisor.update_supervisord_config(user=user, yes=yes) - bench.config.supervisor.generate_supervisor_config(bench_path=".", user=user, yes=yes) + bench.config.supervisor.generate_supervisor_config(bench_path=".", user=user, yes=yes, skip_redis=skip_redis) @click.command("redis", help="Generates configuration for Redis") @@ -61,6 +63,8 @@ def setup_fonts(): @click.argument("user") @click.option("--yes", help="Yes to regeneration config", is_flag=True, default=False) def setup_production(user, yes=False): + import bench.config.production_setup + bench.config.production_setup.setup_production(user=user, yes=yes) @@ -103,6 +107,8 @@ def set_ssh_port(port, force=False): @click.option("--custom-domain") @click.option('-n', '--non-interactive', default=False, is_flag=True, help="Run command non-interactively. This flag restarts nginx and runs certbot non interactively. Shouldn't be used on 1'st attempt") def setup_letsencrypt(site, custom_domain, non_interactive): + import bench.config.lets_encrypt + bench.config.lets_encrypt.setup_letsencrypt(site, custom_domain, bench_path=".", interactive=not non_interactive) @@ -111,6 +117,8 @@ def setup_letsencrypt(site, custom_domain, non_interactive): @click.option("--email") @click.option("--exclude-base-domain", default=False, is_flag=True, help="SSL Certificate not applicable for base domain") def setup_wildcard_ssl(domain, email, exclude_base_domain): + import bench.config.lets_encrypt + bench.config.lets_encrypt.setup_wildcard_ssl(domain, email, bench_path=".", exclude_base_domain=exclude_base_domain) @@ -253,8 +261,8 @@ def setup_roles(role, **kwargs): @click.command("fail2ban", help="Setup fail2ban, an intrusion prevention software framework that protects computer servers from brute-force attacks") @click.option("--maxretry", default=6, help="Number of matches (i.e. value of the counter) which triggers ban action on the IP. Default is 6 seconds" ) -@click.option("--bantime", default=600, help="The counter is set to zero if no match is found within 'findtime' seconds. Default is 600 seconds") -@click.option("--findtime", default=600, help="Duration (in seconds) for IP to be banned for. Negative number for 'permanent' ban. Default is 600 seconds") +@click.option("--bantime", default=600, help="Duration (in seconds) for IP to be banned for. Negative number for 'permanent' ban. Default is 600 seconds") +@click.option("--findtime", default=600, help="The counter is set to zero if match found within 'findtime' seconds doesn't exceed 'maxretry'. Default is 600 seconds") def setup_nginx_proxy_jail(**kwargs): run_playbook("roles/fail2ban/tasks/configure_nginx_jail.yml", extra_vars=kwargs) diff --git a/bench/commands/update.py b/bench/commands/update.py index dee57d28..e1113b40 100755 --- a/bench/commands/update.py +++ b/bench/commands/update.py @@ -15,11 +15,12 @@ from bench.utils import post_upgrade, patch_sites, build_assets @click.option('--restart-supervisor', is_flag=True, help="Restart supervisor processes after update") @click.option('--restart-systemd', is_flag=True, help="Restart systemd units after update") @click.option('--no-backup', is_flag=True, help="If this flag is set, sites won't be backed up prior to updates. Note: This is not recommended in production.") +@click.option('--no-compile', is_flag=True, help="If set, Python bytecode won't be compiled before restarting the processes") @click.option('--force', is_flag=True, help="Forces major version upgrades") @click.option('--reset', is_flag=True, help="Hard resets git branch's to their new states overriding any changes and overriding rebase on pull") -def update(pull, apps, patch, build, requirements, restart_supervisor, restart_systemd, no_backup, force, reset): +def update(pull, apps, patch, build, requirements, restart_supervisor, restart_systemd, no_backup, no_compile, force, reset): from bench.utils import update - update(pull=pull, apps=apps, patch=patch, build=build, requirements=requirements, restart_supervisor=restart_supervisor, restart_systemd=restart_systemd, backup=not no_backup, force=force, reset=reset) + update(pull=pull, apps=apps, patch=patch, build=build, requirements=requirements, restart_supervisor=restart_supervisor, restart_systemd=restart_systemd, backup=not no_backup, compile=not no_compile, force=force, reset=reset) @click.command('retry-upgrade', help="Retry a failed upgrade") diff --git a/bench/config/__init__.py b/bench/config/__init__.py index 301b515b..f79be532 100644 --- a/bench/config/__init__.py +++ b/bench/config/__init__.py @@ -1,6 +1,6 @@ """Module for setting up system and respective bench configurations""" -# imports - third party imports -from jinja2 import Environment, PackageLoader -env = Environment(loader=PackageLoader('bench.config')) +def env(): + from jinja2 import Environment, PackageLoader + return Environment(loader=PackageLoader('bench.config')) diff --git a/bench/config/common_site_config.py b/bench/config/common_site_config.py index 02c6778a..41d9cc9c 100644 --- a/bench/config/common_site_config.py +++ b/bench/config/common_site_config.py @@ -1,11 +1,8 @@ # imports - standard imports import getpass import json -import multiprocessing import os -# imports - third party imports -from six.moves.urllib.parse import urlparse default_config = { @@ -54,8 +51,10 @@ def get_config_path(bench_path): def get_gunicorn_workers(): '''This function will return the maximum workers that can be started depending upon number of cpu's present on the machine''' + import multiprocessing + return { - "gunicorn_workers": multiprocessing.cpu_count() + "gunicorn_workers": multiprocessing.cpu_count() * 2 + 1 } def update_config_for_frappe(config, bench_path): @@ -73,6 +72,8 @@ def update_config_for_frappe(config, bench_path): # TODO Optionally we need to add the host or domain name in case dns_multitenant is false def make_ports(bench_path): + from six.moves.urllib.parse import urlparse + benches_path = os.path.dirname(os.path.abspath(bench_path)) default_ports = { diff --git a/bench/config/lets_encrypt.py b/bench/config/lets_encrypt.py index 32ca3380..752d361c 100755 --- a/bench/config/lets_encrypt.py +++ b/bench/config/lets_encrypt.py @@ -3,8 +3,6 @@ import os # imports - third party imports import click -from crontab import CronTab -from six.moves.urllib.request import urlretrieve # imports - module imports import bench @@ -48,7 +46,7 @@ def setup_letsencrypt(site, custom_domain, bench_path, interactive): def create_config(site, custom_domain): - config = bench.config.env.get_template('letsencrypt.cfg').render(domain=custom_domain or site) + config = bench.config.env().get_template('letsencrypt.cfg').render(domain=custom_domain or site) config_path = '/etc/letsencrypt/configs/{site}.cfg'.format(site=custom_domain or site) create_dir_if_missing(config_path) @@ -86,6 +84,8 @@ def run_certbot_and_setup_ssl(site, custom_domain, bench_path, interactive=True) def setup_crontab(): + from crontab import CronTab + job_command = '/opt/certbot-auto renew -a nginx --post-hook "systemctl reload nginx"' job_comment = 'Renew lets-encrypt every month' print("Setting Up cron job to {0}".format(job_comment)) @@ -106,6 +106,8 @@ def create_dir_if_missing(path): def get_certbot(): + from six.moves.urllib.request import urlretrieve + certbot_path = get_certbot_path() create_dir_if_missing(certbot_path) diff --git a/bench/config/nginx.py b/bench/config/nginx.py index fbb73680..4ed4233d 100644 --- a/bench/config/nginx.py +++ b/bench/config/nginx.py @@ -20,7 +20,7 @@ def make_nginx_conf(bench_path, yes=False): if not click.confirm('nginx.conf already exists and this will overwrite it. Do you want to continue?'): return - template = bench.config.env.get_template('nginx.conf') + template = bench.config.env().get_template('nginx.conf') bench_path = os.path.abspath(bench_path) sites_path = os.path.join(bench_path, "sites") @@ -59,7 +59,7 @@ def make_bench_manager_nginx_conf(bench_path, yes=False, port=23624, domain=None from bench.config.site_config import get_site_config from bench.config.common_site_config import get_config - template = bench.config.env.get_template('bench_manager_nginx.conf') + template = bench.config.env().get_template('bench_manager_nginx.conf') bench_path = os.path.abspath(bench_path) sites_path = os.path.join(bench_path, "sites") diff --git a/bench/config/procfile.py b/bench/config/procfile.py index 1b3f20ed..e57e579d 100755 --- a/bench/config/procfile.py +++ b/bench/config/procfile.py @@ -18,7 +18,7 @@ def setup_procfile(bench_path, yes=False, skip_redis=False): click.confirm('A Procfile already exists and this will overwrite it. Do you want to continue?', abort=True) - procfile = bench.config.env.get_template('Procfile').render( + procfile = bench.config.env().get_template('Procfile').render( node=find_executable("node") or find_executable("nodejs"), use_rq=use_rq(bench_path), webserver_port=config.get('webserver_port'), diff --git a/bench/config/redis.py b/bench/config/redis.py index 42005b07..7af1bbf7 100644 --- a/bench/config/redis.py +++ b/bench/config/redis.py @@ -3,16 +3,14 @@ import os import re import subprocess -# imports - third party imports -import semantic_version -from six.moves.urllib.parse import urlparse - # imports - module imports import bench from bench.config.common_site_config import get_config def generate_config(bench_path): + from six.moves.urllib.parse import urlparse + config = get_config(bench_path) ports = {} @@ -52,7 +50,7 @@ def generate_config(bench_path): os.makedirs(pid_path) def write_redis_config(template_name, context, bench_path): - template = bench.config.env.get_template(template_name) + template = bench.config.env().get_template(template_name) if "pid_path" not in context: context["pid_path"] = os.path.abspath(os.path.join(bench_path, "config", "pids")) @@ -61,6 +59,8 @@ def write_redis_config(template_name, context, bench_path): f.write(template.render(**context)) def get_redis_version(): + import semantic_version + version_string = subprocess.check_output('redis-server --version', shell=True) version_string = version_string.decode('utf-8').strip() # extract version number from string diff --git a/bench/config/site_config.py b/bench/config/site_config.py index 696185c8..9ea67a86 100644 --- a/bench/config/site_config.py +++ b/bench/config/site_config.py @@ -4,7 +4,6 @@ import os from collections import defaultdict # imports - module imports -from bench.config.nginx import make_nginx_conf from bench.utils import get_sites @@ -35,6 +34,8 @@ def set_ssl_certificate_key(site, ssl_certificate_key, bench_path='.', gen_confi set_site_config_nginx_property(site, {"ssl_certificate_key": ssl_certificate_key}, bench_path=bench_path, gen_config=gen_config) def set_site_config_nginx_property(site, config, bench_path='.', gen_config=True): + from bench.config.nginx import make_nginx_conf + if site not in get_sites(bench_path=bench_path): raise Exception("No such site") update_site_config(site, config, bench_path=bench_path) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 8788757f..76ffd877 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -5,24 +5,23 @@ import os # imports - module imports import bench -from bench.app import get_current_frappe_version, use_rq +from bench.app import use_rq from bench.utils import get_bench_name, find_executable from bench.config.common_site_config import get_config, update_config, get_gunicorn_workers # imports - third party imports import click -from six.moves import configparser logger = logging.getLogger(bench.PROJECT_NAME) -def generate_supervisor_config(bench_path, user=None, yes=False): +def generate_supervisor_config(bench_path, user=None, yes=False, skip_redis=False): """Generate supervisor config for respective bench path""" if not user: user = getpass.getuser() - template = bench.config.env.get_template('supervisor.conf') + template = bench.config.env().get_template('supervisor.conf') config = get_config(bench_path=bench_path) bench_dir = os.path.abspath(bench_path) @@ -30,7 +29,6 @@ def generate_supervisor_config(bench_path, user=None, yes=False): "bench_dir": bench_dir, "sites_dir": os.path.join(bench_dir, 'sites'), "user": user, - "frappe_version": get_current_frappe_version(bench_path), "use_rq": use_rq(bench_path), "http_timeout": config.get("http_timeout", 120), "redis_server": find_executable('redis-server'), @@ -42,7 +40,8 @@ def generate_supervisor_config(bench_path, user=None, yes=False): "gunicorn_workers": config.get('gunicorn_workers', get_gunicorn_workers()["gunicorn_workers"]), "bench_name": get_bench_name(bench_path), "background_workers": config.get('background_workers') or 1, - "bench_cmd": find_executable('bench') + "bench_cmd": find_executable('bench'), + "skip_redis": skip_redis, }) conf_path = os.path.join(bench_path, 'config', 'supervisor.conf') @@ -68,6 +67,8 @@ def get_supervisord_conf(): def update_supervisord_config(user=None, yes=False): """From bench v5.x, we're moving to supervisor running as user""" + from six.moves import configparser + from bench.config.production_setup import service if not user: diff --git a/bench/config/systemd.py b/bench/config/systemd.py index 4a414825..c0a0053c 100644 --- a/bench/config/systemd.py +++ b/bench/config/systemd.py @@ -7,7 +7,7 @@ import click # imports - module imports import bench -from bench.app import get_current_frappe_version, use_rq +from bench.app import use_rq from bench.config.common_site_config import get_config, get_gunicorn_workers, update_config from bench.utils import exec_cmd, find_executable, get_bench_name @@ -51,7 +51,6 @@ def generate_systemd_config(bench_path, user=None, yes=False, "bench_dir": bench_dir, "sites_dir": os.path.join(bench_dir, 'sites'), "user": user, - "frappe_version": get_current_frappe_version(bench_path), "use_rq": use_rq(bench_path), "http_timeout": config.get("http_timeout", 120), "redis_server": find_executable('redis-server'), @@ -85,7 +84,7 @@ def setup_systemd_directory(bench_path): def setup_main_config(bench_info, bench_path): # Main config - bench_template = bench.config.env.get_template('systemd/frappe-bench.target') + bench_template = bench.config.env().get_template('systemd/frappe-bench.target') bench_config = bench_template.render(**bench_info) bench_config_path = os.path.join(bench_path, 'config', 'systemd' , bench_info.get("bench_name") + '.target') @@ -94,11 +93,11 @@ def setup_main_config(bench_info, bench_path): def setup_workers_config(bench_info, bench_path): # Worker Group - bench_workers_target_template = bench.config.env.get_template('systemd/frappe-bench-workers.target') - bench_default_worker_template = bench.config.env.get_template('systemd/frappe-bench-frappe-default-worker.service') - bench_short_worker_template = bench.config.env.get_template('systemd/frappe-bench-frappe-short-worker.service') - bench_long_worker_template = bench.config.env.get_template('systemd/frappe-bench-frappe-long-worker.service') - bench_schedule_worker_template = bench.config.env.get_template('systemd/frappe-bench-frappe-schedule.service') + bench_workers_target_template = bench.config.env().get_template('systemd/frappe-bench-workers.target') + bench_default_worker_template = bench.config.env().get_template('systemd/frappe-bench-frappe-default-worker.service') + bench_short_worker_template = bench.config.env().get_template('systemd/frappe-bench-frappe-short-worker.service') + bench_long_worker_template = bench.config.env().get_template('systemd/frappe-bench-frappe-long-worker.service') + bench_schedule_worker_template = bench.config.env().get_template('systemd/frappe-bench-frappe-schedule.service') bench_workers_target_config = bench_workers_target_template.render(**bench_info) bench_default_worker_config = bench_default_worker_template.render(**bench_info) @@ -129,9 +128,9 @@ def setup_workers_config(bench_info, bench_path): def setup_web_config(bench_info, bench_path): # Web Group - bench_web_target_template = bench.config.env.get_template('systemd/frappe-bench-web.target') - bench_web_service_template = bench.config.env.get_template('systemd/frappe-bench-frappe-web.service') - bench_node_socketio_template = bench.config.env.get_template('systemd/frappe-bench-node-socketio.service') + bench_web_target_template = bench.config.env().get_template('systemd/frappe-bench-web.target') + bench_web_service_template = bench.config.env().get_template('systemd/frappe-bench-frappe-web.service') + bench_node_socketio_template = bench.config.env().get_template('systemd/frappe-bench-node-socketio.service') bench_web_target_config = bench_web_target_template.render(**bench_info) bench_web_service_config = bench_web_service_template.render(**bench_info) @@ -152,10 +151,10 @@ def setup_web_config(bench_info, bench_path): def setup_redis_config(bench_info, bench_path): # Redis Group - bench_redis_target_template = bench.config.env.get_template('systemd/frappe-bench-redis.target') - bench_redis_cache_template = bench.config.env.get_template('systemd/frappe-bench-redis-cache.service') - bench_redis_queue_template = bench.config.env.get_template('systemd/frappe-bench-redis-queue.service') - bench_redis_socketio_template = bench.config.env.get_template('systemd/frappe-bench-redis-socketio.service') + bench_redis_target_template = bench.config.env().get_template('systemd/frappe-bench-redis.target') + bench_redis_cache_template = bench.config.env().get_template('systemd/frappe-bench-redis-cache.service') + bench_redis_queue_template = bench.config.env().get_template('systemd/frappe-bench-redis-queue.service') + bench_redis_socketio_template = bench.config.env().get_template('systemd/frappe-bench-redis-socketio.service') bench_redis_target_config = bench_redis_target_template.render(**bench_info) bench_redis_cache_config = bench_redis_cache_template.render(**bench_info) diff --git a/bench/config/templates/nginx.conf b/bench/config/templates/nginx.conf index 06fc0b2b..82f16d46 100644 --- a/bench/config/templates/nginx.conf +++ b/bench/config/templates/nginx.conf @@ -29,6 +29,10 @@ server { {% if allow_rate_limiting %} limit_conn per_host_{{ bench_name_hash }} 8; {% endif %} + + proxy_buffer_size 128k; + proxy_buffers 4 256k; + proxy_busy_buffers_size 256k; {% if ssl_certificate and ssl_certificate_key %} ssl on; diff --git a/bench/config/templates/supervisor.conf b/bench/config/templates/supervisor.conf index a9d06331..d24b1cb6 100644 --- a/bench/config/templates/supervisor.conf +++ b/bench/config/templates/supervisor.conf @@ -114,6 +114,7 @@ killasgroup=true {% endif %} +{% if not skip_redis %} [program:{{ bench_name }}-redis-cache] command={{ redis_server }} {{ redis_cache_config }} priority=1 @@ -133,8 +134,9 @@ stdout_logfile={{ bench_dir }}/logs/redis-queue.log stderr_logfile={{ bench_dir }}/logs/redis-queue.error.log user={{ user }} directory={{ sites_dir }} +{% endif %} -{% if frappe_version > 5 %} +{% if not skip_redis %} [program:{{ bench_name }}-redis-socketio] command={{ redis_server }} {{ redis_socketio_config }} priority=1 @@ -144,6 +146,7 @@ stdout_logfile={{ bench_dir }}/logs/redis-socketio.log stderr_logfile={{ bench_dir }}/logs/redis-socketio.error.log user={{ user }} directory={{ sites_dir }} +{% endif %} {% if node %} [program:{{ bench_name }}-node-socketio] @@ -157,8 +160,6 @@ user={{ user }} directory={{ bench_dir }} {% endif %} -{% endif %} - [group:{{ bench_name }}-web] programs={{ bench_name }}-frappe-web {%- if node -%} ,{{ bench_name }}-node-socketio {%- endif%} @@ -174,5 +175,7 @@ programs={{ bench_name }}-frappe-workerbeat,{{ bench_name }}-frappe-worker,{{ be {% endif %} +{% if not skip_redis %} [group:{{ bench_name }}-redis] -programs={{ bench_name }}-redis-cache,{{ bench_name }}-redis-queue {%- if frappe_version > 5 -%} ,{{ bench_name }}-redis-socketio {%- endif %} +programs={{ bench_name }}-redis-cache,{{ bench_name }}-redis-queue,{{ bench_name }}-redis-socketio +{% endif %} diff --git a/bench/playbooks/create_user.yml b/bench/playbooks/create_user.yml index 2991afb3..10695833 100644 --- a/bench/playbooks/create_user.yml +++ b/bench/playbooks/create_user.yml @@ -11,7 +11,7 @@ - name: Set home folder perms file: - path: '/home/{{ frappe_user }}' + path: '{{ user_directory }}' mode: 'o+rx' owner: '{{ frappe_user }}' group: '{{ frappe_user }}' diff --git a/bench/playbooks/roles/bench/tasks/main.yml b/bench/playbooks/roles/bench/tasks/main.yml index e01d18b7..164a216e 100644 --- a/bench/playbooks/roles/bench/tasks/main.yml +++ b/bench/playbooks/roles/bench/tasks/main.yml @@ -33,7 +33,7 @@ - name: Fix permissions become_user: root - command: chown {{ frappe_user }} -R /home/{{ frappe_user }} + command: chown {{ frappe_user }} -R {{ user_directory }} - name: python3 bench init for develop command: bench init {{ bench_path }} --frappe-path {{ frappe_repo_url }} --frappe-branch {{ frappe_branch }} --python {{ python }} @@ -77,6 +77,6 @@ # Setup Bench for production environment - include_tasks: setup_bench_production.yml vars: - bench_path: "/home/{{ frappe_user }}/{{ bench_name }}" + bench_path: "{{ user_directory }}/{{ bench_name }}" when: not run_travis and production ... diff --git a/bench/playbooks/roles/bench/tasks/setup_erpnext.yml b/bench/playbooks/roles/bench/tasks/setup_erpnext.yml index 32e498d1..9db153ab 100644 --- a/bench/playbooks/roles/bench/tasks/setup_erpnext.yml +++ b/bench/playbooks/roles/bench/tasks/setup_erpnext.yml @@ -13,16 +13,17 @@ - name: Check whether the site already exists stat: path="{{ bench_path }}/sites/{{ site }}" register: site_folder + when: not without_site - name: Create a new site command: "bench new-site {{ site }} --admin-password '{{ admin_password }}' --mariadb-root-password '{{ mysql_root_password }}'" args: chdir: "{{ bench_path }}" - when: not site_folder.stat.exists + when: not without_site and not site_folder.stat.exists - name: Install ERPNext to default site command: "bench --site {{ site }} install-app erpnext" args: chdir: "{{ bench_path }}" - when: not without_erpnext + when: not without_site and not without_erpnext ... \ No newline at end of file diff --git a/bench/playbooks/roles/bench/tasks/setup_inputrc.yml b/bench/playbooks/roles/bench/tasks/setup_inputrc.yml index 9c88b933..14b47a34 100644 --- a/bench/playbooks/roles/bench/tasks/setup_inputrc.yml +++ b/bench/playbooks/roles/bench/tasks/setup_inputrc.yml @@ -1,11 +1,11 @@ --- - name: insert/update inputrc for history blockinfile: - dest: "/home/{{ frappe_user }}/.inputrc" + dest: "{{ user_directory }}/.inputrc" create: yes block: | ## arrow up "\e[A":history-search-backward ## arrow down "\e[B":history-search-forward -... \ No newline at end of file +... diff --git a/bench/playbooks/roles/mariadb/tasks/main.yml b/bench/playbooks/roles/mariadb/tasks/main.yml index 1f119005..8079583d 100644 --- a/bench/playbooks/roles/mariadb/tasks/main.yml +++ b/bench/playbooks/roles/mariadb/tasks/main.yml @@ -48,6 +48,10 @@ [mysqld] pid-file = /var/run/mysqld/mysqld.pid socket = /var/run/mysqld/mysqld.sock + + # setting appeared inside mysql but overwritten by mariadb inside mariadb.conf.d/xx-server.cnf valued as utf8mb4_general_ci + + collation-server = utf8mb4_unicode_ci create: yes become: yes become_user: root diff --git a/bench/playbooks/roles/wkhtmltopdf/tasks/main.yml b/bench/playbooks/roles/wkhtmltopdf/tasks/main.yml index aea2f55f..2a6a89ea 100644 --- a/bench/playbooks/roles/wkhtmltopdf/tasks/main.yml +++ b/bench/playbooks/roles/wkhtmltopdf/tasks/main.yml @@ -24,7 +24,14 @@ get_url: url: https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.focal_amd64.deb dest: /tmp/wkhtmltox.deb - when: ansible_distribution == 'Ubuntu' and ansible_distribution_major_version == '20' + when: ansible_distribution == 'Ubuntu' and ansible_distribution_major_version == '20' and ansible_architecture != 'aarch64' + +- name: download wkthmltox Ubuntu 20 arm64 + get_url: + # wkhtmltox supports arm64 starting from 0.12.6 + url: https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_arm64.deb + dest: /tmp/wkhtmltox.deb + when: ansible_distribution == 'Ubuntu' and ansible_distribution_major_version == '20' and ansible_architecture == 'aarch64' - name: download wkthmltox Ubuntu 18 get_url: diff --git a/bench/playbooks/site.yml b/bench/playbooks/site.yml index 328cfcba..7a3e106c 100644 --- a/bench/playbooks/site.yml +++ b/bench/playbooks/site.yml @@ -40,8 +40,8 @@ - name: setup bench and dev environment hosts: localhost vars: - bench_repo_path: "/home/{{ frappe_user }}/.bench" - bench_path: "/home/{{ frappe_user }}/{{ bench_name }}" + bench_repo_path: "{{ user_directory }}/.bench" + bench_path: "{{ user_directory }}/{{ bench_name }}" roles: # setup frappe-bench - { role: bench, tags: "bench", when: not run_travis and not without_bench_setup } diff --git a/bench/release.py b/bench/release.py index f026b379..81efdc02 100755 --- a/bench/release.py +++ b/bench/release.py @@ -4,11 +4,8 @@ import os import sys import semantic_version import git -import requests import getpass import re -from requests.auth import HTTPBasicAuth -import requests.exceptions from time import sleep from .config.common_site_config import get_config import click @@ -47,6 +44,9 @@ def release(bench_path, app, bump_type, from_branch, to_branch, repo_name=repo_name, remote=remote, frontport=frontport) def validate(bench_path, config): + import requests + from requests.auth import HTTPBasicAuth + global github_username, github_password github_username = config.get('github_username') @@ -306,6 +306,9 @@ def push_release(repo_path, from_branch, to_branch, remote='upstream'): def create_github_release(repo_path, tag_name, message, remote='upstream', owner='frappe', repo_name=None, gh_username=None, gh_password=None, prerelease=False): + import requests + import requests.exceptions + from requests.auth import HTTPBasicAuth print('creating release on github') diff --git a/bench/tests/test_init.py b/bench/tests/test_init.py index 69d49a45..2f409899 100755 --- a/bench/tests/test_init.py +++ b/bench/tests/test_init.py @@ -9,6 +9,7 @@ import git # imports - module imports import bench +import bench.cli import bench.utils from bench.release import get_bumped_version from bench.tests.test_base import FRAPPE_BRANCH, TestBenchBase @@ -27,6 +28,10 @@ class TestBenchInit(TestBenchBase): self.assertEqual( get_bumped_version('11.0.5-beta.22', 'prerelease'), '11.0.5-beta.23' ) + def test_utils(self): + self.assertEqual(subprocess.call("bench"), 0) + + def test_init(self, bench_name="test-bench", **kwargs): self.init_bench(bench_name, **kwargs) self.assert_folders(bench_name) diff --git a/bench/utils.py b/bench/utils.py index 8e76903f..54350ba9 100755 --- a/bench/utils.py +++ b/bench/utils.py @@ -2,18 +2,17 @@ # -*- coding: utf-8 -*- # imports - standard imports +import compileall import errno import glob import grp import itertools import json import logging -import multiprocessing import os import pwd import re import select -import shutil import site import subprocess import sys @@ -22,11 +21,7 @@ from distutils.spawn import find_executable # imports - third party imports import click -from crontab import CronTab -import requests -from semantic_version import Version from six import iteritems -from six.moves.urllib.parse import urlparse # imports - module imports import bench @@ -92,6 +87,12 @@ def safe_decode(string, encoding = 'utf-8'): def check_latest_version(): + if bench.VERSION.endswith("dev"): + return + + import requests + from semantic_version import Version + try: pypi_request = requests.get("https://pypi.org/pypi/frappe-bench/json") except Exception: @@ -162,11 +163,8 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, if apps_path: install_apps_from_path(apps_path, bench_path=path) - - bench.set_frappe_version(bench_path=path) - if bench.FRAPPE_VERSION > 5: - if not skip_assets: - update_node_packages(bench_path=path) + if not skip_assets: + update_node_packages(bench_path=path) set_all_patches_executed(bench_path=path) if not skip_assets: @@ -183,8 +181,8 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, copy_patches_txt(path) -def update(pull=False, apps=None, patch=False, build=False, requirements=False, backup=True, force=False, reset=False, - restart_supervisor=False, restart_systemd=False): +def update(pull=False, apps=None, patch=False, build=False, requirements=False, backup=True, compile=True, + force=False, reset=False, restart_supervisor=False, restart_systemd=False): """command: bench update""" from bench import patches from bench.app import is_version_upgrade, pull_apps, validate_branch @@ -218,7 +216,6 @@ def update(pull=False, apps=None, patch=False, build=False, requirements=False, if version_upgrade[0] or (not version_upgrade[0] and force): validate_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) - conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 }) update_config(conf, bench_path=bench_path) @@ -246,6 +243,10 @@ def update(pull=False, apps=None, patch=False, build=False, requirements=False, if version_upgrade[0] or (not version_upgrade[0] and force): post_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) + if pull and compile: + print("Compiling Python files...") + compileall.compile_dir('../apps', quiet=1, rx=re.compile('.*node_modules.*')) + if restart_supervisor or conf.get('restart_supervisor_on_update'): restart_supervisor_processes(bench_path=bench_path) @@ -259,6 +260,8 @@ def update(pull=False, apps=None, patch=False, build=False, requirements=False, def copy_patches_txt(bench_path): + import shutil + shutil.copy(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'patches', 'patches.txt'), os.path.join(bench_path, 'patches.txt')) @@ -352,27 +355,17 @@ def setup_socketio(bench_path='.'): def patch_sites(bench_path='.'): - bench.set_frappe_version(bench_path=bench_path) - try: - if bench.FRAPPE_VERSION == 4: - exec_cmd("{frappe} --latest all".format(frappe=get_frappe(bench_path=bench_path)), cwd=os.path.join(bench_path, 'sites')) - else: - run_frappe_cmd('--site', 'all', 'migrate', bench_path=bench_path) + run_frappe_cmd('--site', 'all', 'migrate', bench_path=bench_path) except subprocess.CalledProcessError: raise PatchError def build_assets(bench_path='.', app=None): - bench.set_frappe_version(bench_path=bench_path) - - if bench.FRAPPE_VERSION == 4: - exec_cmd("{frappe} --build".format(frappe=get_frappe(bench_path=bench_path)), cwd=os.path.join(bench_path, 'sites')) - else: - command = 'bench build' - if app: - command += ' --app {}'.format(app) - exec_cmd(command, cwd=bench_path) + command = 'bench build' + if app: + command += ' --app {}'.format(app) + exec_cmd(command, cwd=bench_path) def get_sites(bench_path='.'): @@ -382,20 +375,15 @@ def get_sites(bench_path='.'): def setup_backups(bench_path='.'): + from crontab import CronTab from bench.config.common_site_config import get_config logger.log('setting up backups') bench_dir = os.path.abspath(bench_path) user = get_config(bench_path=bench_dir).get('frappe_user') logfile = os.path.join(bench_dir, 'logs', 'backup.log') - bench.set_frappe_version(bench_path=bench_path) system_crontab = CronTab(user=user) - - if bench.FRAPPE_VERSION == 4: - backup_command = "cd {sites_dir} && {frappe} --backup all".format(frappe=get_frappe(bench_path=bench_path),) - else: - backup_command = "cd {bench_dir} && {bench} --verbose --site all backup".format(bench_dir=bench_dir, bench=sys.argv[0]) - + backup_command = "cd {bench_dir} && {bench} --verbose --site all backup".format(bench_dir=bench_dir, bench=sys.argv[0]) job_command = "{backup_command} >> {logfile} 2>&1".format(backup_command=backup_command, logfile=logfile) if job_command not in str(system_crontab): @@ -418,7 +406,7 @@ def setup_sudoers(user): if set_permissions: os.chmod('/etc/sudoers', 0o440) - template = bench.config.env.get_template('frappe_sudoers') + template = bench.config.env().get_template('frappe_sudoers') frappe_sudoers = template.render(**{ 'user': user, 'service': find_executable('service'), @@ -481,7 +469,7 @@ def start(no_dev=False, concurrency=None, procfile=None, no_prefix=False): if no_prefix: command.extend(['--no-prefix']) - + os.execv(program, command) @@ -655,13 +643,7 @@ def update_npm_packages(bench_path='.'): def backup_site(site, bench_path='.'): - bench.set_frappe_version(bench_path=bench_path) - - if bench.FRAPPE_VERSION == 4: - exec_cmd("{frappe} --backup {site}".format(frappe=get_frappe(bench_path=bench_path), site=site), - cwd=os.path.join(bench_path, 'sites')) - else: - run_frappe_cmd('--site', site, 'backup', bench_path=bench_path) + run_frappe_cmd('--site', site, 'backup', bench_path=bench_path) def backup_all_sites(bench_path='.'): @@ -747,11 +729,6 @@ def fix_prod_setup_perms(bench_path='.', frappe_user=None): os.chown(path, uid, gid) -def get_current_frappe_version(bench_path='.'): - from .app import get_current_frappe_version as fv - return fv(bench_path=bench_path) - - def run_frappe_cmd(*args, **kwargs): from .cli import from_command_line @@ -811,6 +788,8 @@ sudo supervisorctl reload def update_translations_p(args): + import requests + try: update_translations(*args) except requests.exceptions.HTTPError: @@ -818,6 +797,8 @@ def update_translations_p(args): def download_translations_p(): + import multiprocessing + pool = multiprocessing.Pool(multiprocessing.cpu_count()) langs = get_langs() @@ -842,6 +823,8 @@ def get_langs(): def update_translations(app, lang): + import requests + translations_dir = os.path.join('apps', app, app, 'translations') csv_file = os.path.join(translations_dir, lang + '.csv') url = "https://translate.erpnext.com/files/{}-{}.csv".format(app, lang) @@ -891,6 +874,8 @@ def get_bench_name(bench_path): def setup_fonts(): + import shutil + fonts_path = os.path.join('/tmp', 'fonts') if os.path.exists('/etc/fonts_backup'): @@ -963,6 +948,8 @@ def find_benches(directory=None): def migrate_env(python, backup=False): + import shutil + from six.moves.urllib.parse import urlparse from bench.config.common_site_config import get_config from bench.app import get_apps diff --git a/docs/easy_install.md b/docs/easy_install.md index 3c59f308..91ee5566 100644 --- a/docs/easy_install.md +++ b/docs/easy_install.md @@ -34,6 +34,8 @@ If you are on a fresh server and logged in as root, at first create a dedicated *(it is very common to use "frappe" as frappe-username, but this comes with the security flaw of ["frappe" ranking very high](https://www.reddit.com/r/dataisbeautiful/comments/b3sirt/i_deployed_over_a_dozen_cyber_honeypots_all_over/?st=JTJ0SC0Q&sh=76e05240) in as a username challenged in hacking attempts. So, for production sites it is highly recommended to use a custom username harder to guess)* +*(you can specify the flag --home to specify a directory for your [frappe-user]. Bench will follow the home directory specified by the user's home directory e.g. /data/[frappe-user]/frappe-bench)* + Switch to `[frappe-user]` (using `su [frappe-user]`) and start the setup wget https://raw.githubusercontent.com/frappe/bench/develop/install.py @@ -71,7 +73,7 @@ use --python flag to specify virtual environments python version, by default scr ## How do I start ERPNext -1. For development: Go to your bench folder (`frappe-bench` by default) and start the bench with `bench start` +1. For development: Go to your bench folder (`~[frappe-user]/frappe-bench` by default) and start the bench with `bench start` 2. For production: Your process will be setup and managed by `nginx` and `supervisor`. Checkout [Setup Production](https://frappe.io/docs/user/en/bench/guides/setup-production.html) for more information. --- diff --git a/install.py b/install.py index 46ce4247..44d53892 100644 --- a/install.py +++ b/install.py @@ -157,13 +157,24 @@ def install_prerequisites(): ] }) + # until psycopg2-binary is available for aarch64 (Arm 64-bit), we'll need libpq and libssl dev packages to build psycopg2 from source + if platform.machine() == 'aarch64': + log("Installing libpq and libssl dev packages to build psycopg2 for aarch64...") + run_os_command({ + 'apt-get': ['sudo apt-get install -y libpq-dev libssl-dev'], + 'yum': ['sudo yum install -y libpq-devel openssl-devel'] + }) + install_package('curl') install_package('wget') install_package('git') install_package('pip3', 'python3-pip') + run_os_command({ + 'python3': "sudo -H python3 -m pip install --upgrade pip setuptools-rust" + }) success = run_os_command({ - 'python3': "sudo -H python3 -m pip install --upgrade setuptools wheel cryptography ansible==2.8.5 pip" + 'python3': "sudo -H python3 -m pip install --upgrade setuptools wheel cryptography ansible~=2.8.15" }) if not (success or shutil.which('ansible')): @@ -223,10 +234,11 @@ def install_bench(args): # create user if not exists extra_vars = vars(args) extra_vars.update(frappe_user=args.user) + + extra_vars.update(user_directory=get_user_home_directory(args.user)) if os.path.exists(tmp_bench_repo): repo_path = tmp_bench_repo - else: repo_path = os.path.join(os.path.expanduser('~'), 'bench') @@ -247,12 +259,11 @@ def install_bench(args): else: frappe_branch = "version-{0}".format(args.version) erpnext_branch = "version-{0}".format(args.version) - else: - if args.frappe_branch: - frappe_branch = args.frappe_branch - - if args.erpnext_branch: - erpnext_branch = args.erpnext_branch + # Allow override of frappe_branch and erpnext_branch, regardless of args.version (which always has a default set) + if args.frappe_branch: + frappe_branch = args.frappe_branch + if args.erpnext_branch: + erpnext_branch = args.erpnext_branch extra_vars.update(frappe_branch=frappe_branch) extra_vars.update(erpnext_branch=erpnext_branch) @@ -261,6 +272,10 @@ def install_bench(args): extra_vars.update(bench_name=bench_name) # Will install ERPNext production setup by default + if args.without_erpnext: + log("Initializing bench {bench_name}:\n\tFrappe Branch: {frappe_branch}\n\tERPNext will not be installed due to --without-erpnext".format(bench_name=bench_name, frappe_branch=frappe_branch)) + else: + log("Initializing bench {bench_name}:\n\tFrappe Branch: {frappe_branch}\n\tERPNext Branch: {erpnext_branch}".format(bench_name=bench_name, frappe_branch=frappe_branch, erpnext_branch=erpnext_branch)) run_playbook('site.yml', sudo=True, extra_vars=extra_vars) if os.path.exists(tmp_bench_repo): @@ -273,11 +288,15 @@ def clone_bench_repo(args): repo_url = args.repo_url or 'https://github.com/frappe/bench' if os.path.exists(tmp_bench_repo): + log('Not cloning already existing Bench repository at {tmp_bench_repo}'.format(tmp_bench_repo=tmp_bench_repo)) return 0 elif args.without_bench_setup: clone_path = os.path.join(os.path.expanduser('~'), 'bench') + log('--without-bench-setup specified, clone path is: {clone_path}'.format(clone_path=clone_path)) else: clone_path = tmp_bench_repo + # Not logging repo_url to avoid accidental credential leak in case credential is embedded in URL + log('Cloning bench repository branch {branch} into {clone_path}'.format(branch=branch, clone_path=clone_path)) success = run_os_command( {'git': 'git clone --quiet {repo_url} {bench_repo} --depth 1 --branch {branch}'.format( @@ -327,8 +346,8 @@ def get_passwords(args): mysql_root_password = '' continue - # admin password - if not admin_password: + # admin password, only needed if we're also creating a site + if not admin_password and not args.without_site: admin_password = getpass.unix_getpass(prompt='Please enter the default Administrator user password: ') conf_admin_passswd = getpass.unix_getpass(prompt='Re-enter Administrator password: ') @@ -336,6 +355,8 @@ def get_passwords(args): passwords_didnt_match("Administrator") admin_password = '' continue + elif args.without_site: + log("Not creating a new site due to --without-site") pass_set = False else: @@ -366,6 +387,11 @@ def get_extra_vars_json(extra_args): return ('@' + json_path) +def get_user_home_directory(user): + # Return home directory /home/USERNAME or anything else defined as home directory in + # passwd for user. + return os.path.expanduser('~'+user) + def run_playbook(playbook_name, sudo=False, extra_vars=None): args = ['ansible-playbook', '-c', 'local', playbook_name , '-vvvv'] @@ -405,8 +431,8 @@ def parse_commandline_args(): args_group.add_argument('--develop', dest='develop', action='store_true', default=False, help='Install developer setup') args_group.add_argument('--production', dest='production', action='store_true', default=False, help='Setup Production environment for bench') - parser.add_argument('--site', dest='site', action='store', default='site1.local', help='Specifiy name for your first ERPNext site') - parser.add_argument('--without-site', dest='without_site', action='store_true', default=False) + parser.add_argument('--site', dest='site', action='store', default='site1.local', help='Specify name for your first ERPNext site') + parser.add_argument('--without-site', dest='without_site', action='store_true', default=False, help='Do not create a new site') parser.add_argument('--verbose', dest='verbose', action='store_true', default=False, help='Run the script in verbose mode') parser.add_argument('--user', dest='user', help='Install frappe-bench for this user') parser.add_argument('--bench-branch', dest='bench_branch', help='Clone a particular branch of bench repository') diff --git a/requirements.txt b/requirements.txt index ea1bdd08..92df9331 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ Click==7.0 GitPython==2.1.15 honcho==1.0.1 -Jinja2==2.10.3 +Jinja2==2.11.3 python-crontab==2.4.0 requests==2.22.0 semantic-version==2.8.2