accounting/bench
2
0
Fork 0
mirror of https://github.com/frappe/bench.git synced 2024-11-12 00:06:36 +00:00
Code Issues Releases Activity

Merge branch 'develop' of github.com:frappe/bench into netchampfaris/minor-message

Browse Source
This commit is contained in:
Gavin D'souza 2020-03-25 12:34:15 +05:30
commit 2e23133a56
32 changed files with 1394 additions and 1130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.circleci/config.yml
View File

@ -7,7 +7,7 @@ jobs:
- checkout
- run:
name: Setup
command: |
command: |
sudo pip install --ignore-installed setuptools
sudo pip install urllib3 pyOpenSSL ndg-httpsclient pyasn1
sudo cp -r ~/.ssh/* /root/.ssh
@ -24,7 +24,7 @@ jobs:
name: Setup Tests
command: |
cd ~
sudo pip install --upgrade pip==19.3.1
sudo pip install --upgrade pip
sudo pip install -e ~/.bench
- run:

16
.deepsource.toml Normal file
View File

@ -0,0 +1,16 @@
version = 1
exclude_patterns = [
".*"
]
test_patterns = [
"bench/tests/**"
]
[[analyzers]]
name = "python"
enabled = true
dependency_file_paths = [
"requirements.txt"
]

110
.travis.yml
View File

@ -1,26 +1,96 @@
language: python
dist: xenial
dist: bionic
sudo: true
python:
- "2.7"
git:
depth: 1
cache:
- pip
- npm
- yarn
addons:
mariadb: '10.3'
matrix:
include:
- name: "Python 2.7 Basic Setup"
python: 2.7
env: TEST=bench
script: python -m unittest -v bench.tests.test_init
- name: "Python 3.6 Basic Setup"
python: 3.6
env: TEST=bench
script: python -m unittest -v bench.tests.test_init
- name: "Python 3.7 Basic Setup"
python: 3.7
env: TEST=bench
script: python -m unittest -v bench.tests.test_init
- name: "Python 3.8 Production Setup"
python: 3.8
env: TEST=bench
script: python -m unittest -v bench.tests.test_setup_production
- name: "Python 2.7 Production Setup"
python: 2.7
env: TEST=bench
script: python -m unittest -v bench.tests.test_setup_production
- name: "Python 3.6 Production Setup"
python: 3.6
env: TEST=bench
script: python -m unittest -v bench.tests.test_setup_production
- name: "Python 3.7 Production Setup"
python: 3.7
env: TEST=bench
script: python -m unittest -v bench.tests.test_setup_production
- name: "Python 3.8 Production Setup"
python: 3.8
env: TEST=bench
script: python -m unittest -v bench.tests.test_setup_production
- name: "Python 3.6 Easy Install"
python: 3.6
env: TEST=easy_install
script: sudo python $TRAVIS_BUILD_DIR/playbooks/install.py --user travis --run-travis --production --verbose
- name: "Python 3.7 Easy Install"
python: 3.7
env: TEST=easy_install
script: sudo python $TRAVIS_BUILD_DIR/playbooks/install.py --user travis --run-travis --production --verbose
- name: "Python 3.8 Easy Install"
python: 3.8
env: TEST=easy_install
script: sudo python $TRAVIS_BUILD_DIR/playbooks/install.py --user travis --run-travis --production --verbose
install:
- sudo pip install urllib3 pyOpenSSL ndg-httpsclient pyasn1
- sudo apt-get purge -y mysql-common mysql-server mysql-client
- sudo apt-get install --only-upgrade -y git
- sudo apt-get install hhvm && rm -rf /home/travis/.kiex/
- mkdir -p ~/.bench
- mkdir -p /tmp/.bench
- cp -r $TRAVIS_BUILD_DIR/* ~/.bench
- cp -r $TRAVIS_BUILD_DIR/* /tmp/.bench
- pip install urllib3 pyOpenSSL ndg-httpsclient pyasn1
- sudo python $TRAVIS_BUILD_DIR/playbooks/install.py --user travis --run-travis --production --verbose
# - sudo bash $TRAVIS_BUILD_DIR/install_scripts/setup_frappe.sh --skip-install-bench --mysql-root-password travis
# - cd ~ && sudo python bench-repo/installer/install.py --only-dependencies
- if [ $TEST == "bench" ];then
wget -q -O /tmp/wkhtmltox.tar.xz https://github.com/frappe/wkhtmltopdf/raw/master/wkhtmltox-0.12.3_linux-generic-amd64.tar.xz;
tar -xf /tmp/wkhtmltox.tar.xz -C /tmp;
sudo mv /tmp/wkhtmltox/bin/wkhtmltopdf /usr/local/bin/wkhtmltopdf;
sudo chmod o+x /usr/local/bin/wkhtmltopdf;
script:
- cd ~
- sudo pip install --upgrade pip==19.3.1
- sudo pip install -e ~/.bench
# - sudo python -m unittest bench.tests.test_setup_production.TestSetupProduction.test_setup_production_v6
- sudo python -m unittest -v bench.tests.test_setup_production
mkdir -p ~/.bench;
cp -r $TRAVIS_BUILD_DIR/* ~/.bench;
pip install -q -U -e ~/.bench;
sudo pip install -q -U -e ~/.bench;
mysql -u root -e "SET GLOBAL character_set_server = 'utf8mb4'";
mysql -u root -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'";
mysql -u root -e "UPDATE mysql.user SET Password=PASSWORD('travis') WHERE User='root'";
mysql -u root -e "FLUSH PRIVILEGES";
fi
- if [ $TEST == "easy_install" ];then
mkdir -p /tmp/.bench;
cp -r $TRAVIS_BUILD_DIR/* /tmp/.bench;
fi

24
README.md
View File

@ -15,6 +15,7 @@ bench is a command-line utility that helps you to install apps, manage multiple
- [bench CLI](#bench-cli)
- [Usage](#usage)
- [Installation](#installation)
- [Custom Bench commands](#custom-bench-commands)
- [Easy Install Script](#easy-install-script)
- [Release Bench](#release-bench)
- [Guides](#guides)
@ -49,7 +50,7 @@ Bench is a command line tool that helps you install, setup, manage multiple site
* Install apps on a particular site
bench --site [site-name] install-app [app-name]
* Start bench (only for development)
bench start
@ -60,7 +61,7 @@ Bench is a command line tool that helps you install, setup, manage multiple site
_Note:_ Apart from `bench init`, all other bench commands have to be run having the respective bench directory as the working directory. _(`bench update` may also be run, but it's behaviour is covered in depth in the docs)_
For more in depth information on commands and usage follow [here](https://github.com/frappe/bench/blob/master/docs/commands_and_usage.md).
For more in depth information on commands and usage follow [here](https://github.com/frappe/bench/blob/master/docs/commands_and_usage.md). As for a consolidated list of bench commands, go through [this page](https://github.com/frappe/bench/blob/master/docs/bench_usage.md).
---
@ -68,8 +69,8 @@ For more in depth information on commands and usage follow [here](https://github
To do this install, you must have basic information on how Linux works and should be able to use the command-line. bench will also create nginx and supervisor config files, setup backups and much more. If you are using on a VPS make sure it has >= 1Gb of RAM or has swap setup properly.
git clone https://github.com/frappe/bench ~/.bench
pip3 install --user -e ~/.bench
git clone https://github.com/frappe/bench ~/.bench
pip3 install --user -e ~/.bench
As bench is a python application, its installation really depends on `python` + `pip` + `git`. The Frappe Framework, however has various other system dependencies like `nodejs`, `yarn`, `redis` and a database system like `mariadb` or `postgres`. Go through the [installation requirements](https://github.com/frappe/bench/blob/master/docs/installation.md) for an updated list.
@ -77,6 +78,12 @@ If you have questions, please ask them on the [forum](https://discuss.erpnext.co
---
## Custom Bench Commands
Want to utilize a bench command you've added in your custom Frappe application? [This](https://github.com/frappe/bench/blob/master/docs/bench_custom_cmd.md) guide might be of some help.
---
# Easy Install Script
- This is an opinionated setup so it is best to setup on a blank server.
@ -110,12 +117,11 @@ Releases can be made for [Frappe](https://github.com/frappe/frappe) apps using b
---
# Docker Install
# Docker
1. For developer setup, you can also use the official [Frappe Docker](https://github.com/frappe/frappe_docker/).
2. The app, mariadb and redis run on individual containers.
3. This setup supports multi-tenancy and exposes the frappe-bench volume as a external storage.
4. For more details, [ead the instructions on the [Frappe Docker README](https://github.com/frappe/frappe_docker/)
- For official images and resources [Frappe Docker](https://github.com/frappe/frappe_docker)
- Production Installation [README](https://github.com/frappe/frappe_docker/blob/develop/README.md)
- Developer Setup [README](https://github.com/frappe/frappe_docker/blob/develop/development/README.md)
---

214
bench/app.py
View File

@ -1,23 +1,32 @@
# imports - compatibility imports
from __future__ import print_function
import os
from .utils import (exec_cmd, get_frappe, check_git_for_shallow_clone, build_assets,
restart_supervisor_processes, get_cmd_output, run_frappe_cmd, CommandFailedError,
restart_systemd_processes)
from .config.common_site_config import get_config
# imports - standard imports
import json
import logging
import os
import re
import shutil
import subprocess
import sys
# imports - third party imports
import click
import git
import requests
import semantic_version
import json
import re
import subprocess
import bench
import sys
import shutil
from six.moves import reload_module
logging.basicConfig(level="DEBUG")
# imports - module imports
import bench
from bench.config.common_site_config import get_config
from bench.utils import CommandFailedError, build_assets, check_git_for_shallow_clone, exec_cmd, get_cmd_output, get_frappe, restart_supervisor_processes, restart_systemd_processes, run_frappe_cmd
logging.basicConfig(level="INFO")
logger = logging.getLogger(__name__)
class InvalidBranchException(Exception): pass
class InvalidRemoteException(Exception): pass
@ -50,16 +59,13 @@ def write_appstxt(apps, bench_path='.'):
with open(os.path.join(bench_path, 'sites', 'apps.txt'), 'w') as f:
return f.write('\n'.join(apps))
def check_url(url, raise_err = True):
try:
from urlparse import urlparse
except ImportError:
from urllib.parse import urlparse
def check_url(url, raise_err=True):
from six.moves.urllib.parse import urlparse
parsed = urlparse(url)
if not parsed.scheme:
if raise_err:
raise TypeError('{url} Not a valid URL'.format(url = url))
raise TypeError('{url} Not a valid URL'.format(url=url))
else:
return False
@ -92,59 +98,61 @@ def remove_from_excluded_apps_txt(app, bench_path='.'):
apps.remove(app)
return write_excluded_apps_txt(apps, bench_path=bench_path)
def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=False,
postprocess = True):
# from bench.utils import check_url
try:
from urlparse import urljoin
except ImportError:
from urllib.parse import urljoin
def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=False, postprocess=True, overwrite=False):
if not os.path.exists(git_url):
if not check_url(git_url, raise_err=False):
orgs = ['frappe', 'erpnext']
for org in orgs:
url = 'https://api.github.com/repos/{org}/{app}'.format(org=org, app=git_url)
res = requests.get(url)
if res.ok:
data = res.json()
if 'name' in data:
if git_url == data['name']:
git_url = 'https://github.com/{org}/{app}'.format(org=org, app=git_url)
break
if not check_url(git_url, raise_err = False):
orgs = ['frappe', 'erpnext']
for org in orgs:
url = 'https://api.github.com/repos/{org}/{app}'.format(org = org, app = git_url)
res = requests.get(url)
if res.ok:
data = res.json()
if 'name' in data:
if git_url == data['name']:
git_url = 'https://github.com/{org}/{app}'.format(org = org, app = git_url)
break
# Gets repo name from URL
repo_name = git_url.rsplit('/', 1)[1].rsplit('.', 1)[0]
shallow_clone = '--depth 1' if check_git_for_shallow_clone() else ''
branch = '--branch {branch}'.format(branch=branch) if branch else ''
else:
repo_name = git_url.split(os.sep)[-1]
shallow_clone = ''
branch = '--branch {branch}'.format(branch=branch) if branch else ''
#Gets repo name from URL
repo_name = git_url.rsplit('/', 1)[1].rsplit('.', 1)[0]
logger.info('getting app {}'.format(repo_name))
shallow_clone = '--depth 1' if check_git_for_shallow_clone() else ''
branch = '--branch {branch}'.format(branch=branch) if branch else ''
if os.path.isdir(os.path.join(bench_path, 'apps', repo_name)):
# application directory already exists
# prompt user to overwrite it
if overwrite or click.confirm('''A directory for the application "{0}" already exists.
Do you want to continue and overwrite it?'''.format(repo_name)):
shutil.rmtree(os.path.join(bench_path, 'apps', repo_name))
elif click.confirm('''Do you want to reinstall the existing application?''', abort=True):
app_name = get_app_name(bench_path, repo_name)
install_app(app=app_name, bench_path=bench_path, verbose=verbose, skip_assets=skip_assets)
sys.exit()
exec_cmd("git clone -q {git_url} {branch} {shallow_clone} --origin upstream".format(
git_url=git_url,
shallow_clone=shallow_clone,
branch=branch),
cwd=os.path.join(bench_path, 'apps'))
logger.info('Getting app {0}'.format(repo_name))
exec_cmd("git clone {git_url} {branch} {shallow_clone} --origin upstream".format(
git_url=git_url,
shallow_clone=shallow_clone,
branch=branch),
cwd=os.path.join(bench_path, 'apps'))
#Retrieves app name from setup.py
app_name = get_app_name(bench_path, repo_name)
install_app(app=app_name, bench_path=bench_path, verbose=verbose, skip_assets=skip_assets)
def get_app_name(bench_path, repo_name):
# retrieves app name from setup.py
app_path = os.path.join(bench_path, 'apps', repo_name, 'setup.py')
with open(app_path, 'rb') as f:
app_name = re.search(r'name\s*=\s*[\'"](.*)[\'"]', f.read().decode('utf-8')).group(1)
if repo_name != app_name:
apps_path = os.path.join(os.path.abspath(bench_path), 'apps')
os.rename(os.path.join(apps_path, repo_name), os.path.join(apps_path, app_name))
return app_name
print('installing', app_name)
install_app(app=app_name, bench_path=bench_path, verbose=verbose)
if postprocess:
if not skip_assets:
build_assets(bench_path=bench_path, app=app_name)
conf = get_config(bench_path=bench_path)
if conf.get('restart_supervisor_on_update'):
restart_supervisor_processes(bench_path=bench_path)
if conf.get('restart_systemd_on_update'):
restart_systemd_processes(bench_path=bench_path)
def new_app(app, bench_path='.'):
# For backwards compatibility
@ -160,7 +168,8 @@ def new_app(app, bench_path='.'):
run_frappe_cmd('make-app', apps, app, bench_path=bench_path)
install_app(app, bench_path=bench_path)
def install_app(app, bench_path=".", verbose=False, no_cache=False):
def install_app(app, bench_path=".", verbose=False, no_cache=False, postprocess=True, skip_assets=False):
logger.info("installing {}".format(app))
pip_path = os.path.join(bench_path, "env", "bin", "pip")
@ -168,11 +177,23 @@ def install_app(app, bench_path=".", verbose=False, no_cache=False):
app_path = os.path.join(bench_path, "apps", app)
cache_flag = "--no-cache-dir" if no_cache else ""
exec_cmd("{pip} install {quiet} -U -e {app} {no_cache}".format(pip=pip_path, quiet=quiet_flag, app=app_path, no_cache=cache_flag))
exec_cmd("{pip} install {quiet} -U -e {app} {no_cache}".format(pip=pip_path,
quiet=quiet_flag, app=app_path, no_cache=cache_flag))
add_to_appstxt(app, bench_path=bench_path)
if postprocess:
if not skip_assets:
build_assets(bench_path=bench_path, app=app)
conf = get_config(bench_path=bench_path)
if conf.get('restart_supervisor_on_update'):
restart_supervisor_processes(bench_path=bench_path)
if conf.get('restart_systemd_on_update'):
restart_systemd_processes(bench_path=bench_path)
def remove_app(app, bench_path='.'):
if not app in get_apps(bench_path):
if app not in get_apps(bench_path):
print("No app named {0}".format(app))
sys.exit(1)
@ -188,7 +209,7 @@ def remove_app(app, bench_path='.'):
print("Cannot remove, app is installed on site: {0}".format(site))
sys.exit(1)
exec_cmd(["{0} uninstall -y {1}".format(pip, app)])
exec_cmd("{0} uninstall -y {1}".format(pip, app), cwd=bench_path)
remove_from_appstxt(app, bench_path)
shutil.rmtree(app_path)
run_frappe_cmd("build", bench_path=bench_path)
@ -281,8 +302,7 @@ def get_current_branch(app, bench_path='.'):
def get_remote(app, bench_path='.'):
repo_dir = get_repo_dir(app, bench_path=bench_path)
contents = subprocess.check_output(['git', 'remote', '-v'], cwd=repo_dir,
stderr=subprocess.STDOUT)
contents = subprocess.check_output(['git', 'remote', '-v'], cwd=repo_dir, stderr=subprocess.STDOUT)
contents = contents.decode('utf-8')
if re.findall('upstream[\s]+', contents):
return 'upstream'
@ -340,8 +360,7 @@ def get_repo_dir(app, bench_path='.'):
return os.path.join(bench_path, 'apps', app)
def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrade=True):
from .utils import update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets, pre_upgrade, post_upgrade
from . import utils
from bench.utils import update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets, post_upgrade
apps_dir = os.path.join(bench_path, 'apps')
version_upgrade = (False,)
switched_apps = []
@ -354,44 +373,46 @@ def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrad
for app in apps:
app_dir = os.path.join(apps_dir, app)
if os.path.exists(app_dir):
try:
if check_upgrade:
version_upgrade = is_version_upgrade(app=app, bench_path=bench_path, branch=branch)
if version_upgrade[0] and not upgrade:
raise MajorVersionUpgradeException("Switching to {0} will cause upgrade from {1} to {2}. Pass --upgrade to confirm".format(branch, version_upgrade[1], version_upgrade[2]), version_upgrade[1], version_upgrade[2])
print("Switching for "+app)
unshallow = "--unshallow" if os.path.exists(os.path.join(app_dir, ".git", "shallow")) else ""
exec_cmd("git config --unset-all remote.upstream.fetch", cwd=app_dir)
exec_cmd("git config --add remote.upstream.fetch '+refs/heads/*:refs/remotes/upstream/*'", cwd=app_dir)
exec_cmd("git fetch upstream {unshallow}".format(unshallow=unshallow), cwd=app_dir)
exec_cmd("git checkout {branch}".format(branch=branch), cwd=app_dir)
exec_cmd("git merge upstream/{branch}".format(branch=branch), cwd=app_dir)
switched_apps.append(app)
except CommandFailedError:
print("Error switching to branch {0} for {1}".format(branch, app))
except InvalidRemoteException:
print("Remote does not exist for app "+app)
except InvalidBranchException:
print("Branch {0} does not exist in Upstream for {1}".format(branch, app))
if not os.path.exists(app_dir):
bench.utils.log("{} does not exist!".format(app), level=2)
continue
repo = git.Repo(app_dir)
unshallow_flag = os.path.exists(os.path.join(app_dir, ".git", "shallow"))
bench.utils.log("Fetching upstream {0}for {1}".format("unshallow " if unshallow_flag else "", app))
bench.utils.exec_cmd("git remote set-branches upstream '*'", cwd=app_dir)
bench.utils.exec_cmd("git fetch --all{0}".format(" --unshallow" if unshallow_flag else ""), cwd=app_dir)
if check_upgrade:
version_upgrade = is_version_upgrade(app=app, bench_path=bench_path, branch=branch)
if version_upgrade[0] and not upgrade:
bench.utils.log("Switching to {0} will cause upgrade from {1} to {2}. Pass --upgrade to confirm".format(branch, version_upgrade[1], version_upgrade[2]), level=2)
sys.exit(1)
print("Switching for "+app)
bench.utils.exec_cmd("git checkout {0}".format(branch), cwd=app_dir)
if str(repo.active_branch) == branch:
switched_apps.append(app)
else:
bench.utils.log("Switching branches failed for: {}".format(app), level=2)
if switched_apps:
print("Successfully switched branches for:\n" + "\n".join(switched_apps))
bench.utils.log("Successfully switched branches for: " + ", ".join(switched_apps), level=1)
print('Please run `bench update --patch` to be safe from any differences in database schema')
if version_upgrade[0] and upgrade:
update_requirements()
update_node_packages()
pre_upgrade(version_upgrade[1], version_upgrade[2])
if sys.version_info >= (3, 4):
import importlib
importlib.reload(utils)
else:
reload(utils)
reload_module(utils)
backup_all_sites()
patch_sites()
build_assets()
post_upgrade(version_upgrade[1], version_upgrade[2])
def switch_to_branch(branch=None, apps=None, bench_path='.', upgrade=False):
switch_branch(branch, apps=apps, bench_path=bench_path, upgrade=upgrade)
@ -402,8 +423,7 @@ def switch_to_develop(apps=None, bench_path='.', upgrade=True):
switch_branch('develop', apps=apps, bench_path=bench_path, upgrade=upgrade)
def get_version_from_string(contents, field='__version__'):
match = re.search(r"^(\s*%s\s*=\s*['\\\"])(.+?)(['\"])(?sm)" % field,
contents)
match = re.search(r"^(\s*%s\s*=\s*['\\\"])(.+?)(['\"])(?sm)" % field, contents)
return match.group(2)
def get_major_version(version):

23
bench/cli.py
View File

@ -1,6 +1,6 @@
import click
import os, sys, logging, json, pwd, subprocess
from bench.utils import is_root, PatchError, drop_privileges, get_env_cmd, get_cmd_output, get_frappe, log, is_bench_directory
from bench.utils import is_root, PatchError, drop_privileges, get_env_cmd, get_cmd_output, get_frappe, log, find_parent_bench
from bench.app import get_apps
from bench.config.common_site_config import get_config
from bench.commands import bench_command
@ -29,7 +29,6 @@ def cli():
elif len(sys.argv) > 1 and sys.argv[1]=="--help":
print(click.Context(bench_command).get_help())
print()
print(get_frappe_help())
return
@ -99,7 +98,6 @@ def get_frappe_commands(bench_path='.'):
return []
try:
output = get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-commands".format(python=python), cwd=sites_path)
# output = output.decode('utf-8')
return json.loads(output)
except subprocess.CalledProcessError as e:
if hasattr(e, "stderr"):
@ -109,27 +107,12 @@ def get_frappe_commands(bench_path='.'):
def get_frappe_help(bench_path='.'):
python = get_env_cmd('python', bench_path=bench_path)
sites_path = os.path.join(bench_path, 'sites')
if not os.path.exists(sites_path):
return []
try:
out = get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-help".format(python=python), cwd=sites_path)
return "Framework commands:\n" + out.split('Commands:')[1]
except subprocess.CalledProcessError:
return "\n\nFramework commands:\n" + out.split('Commands:')[1]
except:
return ""
def find_parent_bench(path):
"""Checks if parent directories are benches"""
if is_bench_directory(directory=path):
return path
home_path = os.path.expanduser("~")
root_path = os.path.abspath(os.sep)
if path not in {home_path, root_path}:
# NOTE: the os.path.split assumes that given path is absolute
parent_dir = os.path.split(path)[0]
return find_parent_bench(parent_dir)
def change_working_directory():
"""Allows bench commands to be run from anywhere inside a bench directory"""
cur_dir = os.path.abspath(".")

128
bench/commands/__init__.py
View File

@ -1,15 +1,5 @@
import click
import os, shutil
import os.path as osp
import logging
from datetime import datetime
from bench.utils import which, exec_cmd
log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
def print_bench_version(ctx, param, value):
"""Prints current bench version"""
@ -31,11 +21,14 @@ def bench_command(bench_path='.'):
setup_logging(bench_path=bench_path)
from bench.commands.make import init, get_app, new_app, remove_app
from bench.commands.make import init, get_app, new_app, remove_app, exclude_app_for_update, include_app_for_update, pip
bench_command.add_command(init)
bench_command.add_command(get_app)
bench_command.add_command(new_app)
bench_command.add_command(remove_app)
bench_command.add_command(exclude_app_for_update)
bench_command.add_command(include_app_for_update)
bench_command.add_command(pip)
from bench.commands.update import update, retry_upgrade, switch_to_branch, switch_to_master, switch_to_develop
@ -45,9 +38,10 @@ bench_command.add_command(switch_to_branch)
bench_command.add_command(switch_to_master)
bench_command.add_command(switch_to_develop)
from bench.commands.utils import (start, restart, set_nginx_port, set_ssl_certificate, set_ssl_certificate_key, set_url_root,
set_mariadb_host, set_default_site, download_translations, shell, backup_site, backup_all_sites, release, renew_lets_encrypt,
disable_production, bench_src, prepare_beta_release, set_redis_cache_host, set_redis_queue_host, set_redis_socketio_host, find_benches)
set_mariadb_host, set_default_site, download_translations, backup_site, backup_all_sites, release, renew_lets_encrypt,
disable_production, bench_src, prepare_beta_release, set_redis_cache_host, set_redis_queue_host, set_redis_socketio_host, find_benches, migrate_env)
bench_command.add_command(start)
bench_command.add_command(restart)
bench_command.add_command(set_nginx_port)
@ -60,7 +54,6 @@ bench_command.add_command(set_redis_queue_host)
bench_command.add_command(set_redis_socketio_host)
bench_command.add_command(set_default_site)
bench_command.add_command(download_translations)
bench_command.add_command(shell)
bench_command.add_command(backup_site)
bench_command.add_command(backup_all_sites)
bench_command.add_command(release)
@ -69,6 +62,8 @@ bench_command.add_command(disable_production)
bench_command.add_command(bench_src)
bench_command.add_command(prepare_beta_release)
bench_command.add_command(find_benches)
bench_command.add_command(migrate_env)
from bench.commands.setup import setup
bench_command.add_command(setup)
@ -84,108 +79,3 @@ bench_command.add_command(remote_urls)
from bench.commands.install import install
bench_command.add_command(install)
from bench.config.common_site_config import get_config
try:
from urlparse import urlparse
except ImportError:
from urllib.parse import urlparse
@click.command('migrate-env')
@click.argument('python', type = str)
@click.option('--no-backup', is_flag=True)
def migrate_env(python, no_backup = False):
"""
Migrate Virtual Environment to desired Python Version.
"""
try:
# Clear Cache before Bench Dies.
config = get_config(bench_path = os.getcwd())
rredis = urlparse(config['redis_cache'])
redis = '{redis} -p {port}'.format(
redis = which('redis-cli'),
port = rredis.port
)
log.debug('Clearing Redis Cache...')
exec_cmd('{redis} FLUSHALL'.format(redis = redis))
log.debug('Clearing Redis DataBase...')
exec_cmd('{redis} FLUSHDB'.format(redis = redis))
except Exception:
log.warn('Please ensure Redis Connections are running or Daemonized.')
try:
# This is with the assumption that a bench is set-up within path.
path = os.getcwd()
# I know, bad name for a flag. Thanks, Ameya! :| - <achilles@frappe.io>
if not no_backup:
# Back, the f*ck up.
parch = osp.join(path, 'archived_envs')
if not osp.exists(parch):
os.mkdir(parch)
# Simply moving. Thanks, Ameya.
# I'm keen to zip.
source = osp.join(path, 'env')
target = parch
log.debug('Backing up Virtual Environment')
stamp = datetime.now().strftime('%Y%m%d_%H%M%S')
dest = osp.join(path, str(stamp))
# WARNING: This is an archive, you might have to use virtualenv --relocate
# That's because virtualenv creates symlinks with shebangs pointing to executables.
# shebangs, shebangs - ricky martin.
# ...and shutil.copytree is a f*cking mess.
os.rename(source, dest)
shutil.move(dest, target)
log.debug('Setting up a New Virtual {python} Environment'.format(
python = python
))
# Path to Python Executable (Basically $PYTHONPTH)
python = which(python)
virtualenv = which('virtualenv')
nvenv = 'env'
pvenv = osp.join(path, nvenv)
exec_cmd('{virtualenv} --python {python} {pvenv}'.format(
virtualenv = virtualenv,
python = python,
pvenv = pvenv
), cwd = path)
pip = osp.join(pvenv, 'bin', 'pip')
exec_cmd('{pip} install --upgrade pip'.format(pip=pip))
exec_cmd('{pip} install --upgrade setuptools'.format(pip=pip))
# TODO: Options
papps = osp.join(path, 'apps')
apps = ['frappe', 'erpnext'] + [app for app in os.listdir(papps) if app not in ['frappe', 'erpnext']]
for app in apps:
papp = osp.join(papps, app)
if osp.isdir(papp) and osp.exists(osp.join(papp, 'setup.py')):
exec_cmd('{pip} install -e {app}'.format(
pip = pip, app = papp
))
log.debug('Migration Successful to {python}'.format(
python = python
))
except:
log.debug('Migration Error')
raise
bench_command.add_command(migrate_env)
from bench.commands.make import exclude_app_for_update, include_app_for_update
bench_command.add_command(exclude_app_for_update)
bench_command.add_command(include_app_for_update)

98
bench/commands/config.py
View File

@ -1,112 +1,74 @@
import click, json
from bench.config.common_site_config import update_config
# imports - standard imports
import ast
## Config
## Not DRY
@click.group()
# imports - module imports
from bench.config.common_site_config import update_config, get_config, put_config
# imports - third party imports
import click
@click.group(help='Change bench configuration')
def config():
"change bench configuration"
pass
@click.command('auto_update')
@click.argument('state', type=click.Choice(['on', 'off']))
def config_auto_update(state):
"Enable/Disable auto update for bench"
state = True if state == 'on' else False
update_config({'auto_update': state})
@click.command('restart_supervisor_on_update')
@click.command('restart_supervisor_on_update', help='Enable/Disable auto restart of supervisor processes')
@click.argument('state', type=click.Choice(['on', 'off']))
def config_restart_supervisor_on_update(state):
"Enable/Disable auto restart of supervisor processes"
state = True if state == 'on' else False
update_config({'restart_supervisor_on_update': state})
update_config({'restart_supervisor_on_update': state == 'on'})
@click.command('restart_systemd_on_update')
@click.command('restart_systemd_on_update', help='Enable/Disable auto restart of systemd units')
@click.argument('state', type=click.Choice(['on', 'off']))
def config_restart_systemd_on_update(state):
"Enable/Disable auto restart of systemd units"
state = True if state == 'on' else False
update_config({'restart_systemd_on_update': state})
@click.command('update_bench_on_update')
@click.argument('state', type=click.Choice(['on', 'off']))
def config_update_bench_on_update(state):
"Enable/Disable bench updates on running bench update"
state = True if state == 'on' else False
update_config({'update_bench_on_update': state})
update_config({'restart_systemd_on_update': state == 'on'})
@click.command('dns_multitenant')
@click.command('dns_multitenant', help='Enable/Disable bench multitenancy on running bench update')
@click.argument('state', type=click.Choice(['on', 'off']))
def config_dns_multitenant(state):
"Enable/Disable bench updates on running bench update"
state = True if state == 'on' else False
update_config({'dns_multitenant': state})
update_config({'dns_multitenant': state == 'on'})
@click.command('serve_default_site')
@click.command('serve_default_site', help='Configure nginx to serve the default site on port 80')
@click.argument('state', type=click.Choice(['on', 'off']))
def config_serve_default_site(state):
"Configure nginx to serve the default site on port 80"
state = True if state == 'on' else False
update_config({'serve_default_site': state})
update_config({'serve_default_site': state == 'on'})
@click.command('rebase_on_pull')
@click.command('rebase_on_pull', help='Rebase repositories on pulling')
@click.argument('state', type=click.Choice(['on', 'off']))
def config_rebase_on_pull(state):
"Rebase repositories on pulling"
state = True if state == 'on' else False
update_config({'rebase_on_pull': state})
update_config({'rebase_on_pull': state == 'on'})
@click.command('http_timeout')
@click.command('http_timeout', help='Set HTTP timeout')
@click.argument('seconds', type=int)
def config_http_timeout(seconds):
"set http timeout"
update_config({'http_timeout': seconds})
@click.command('set-common-config')
@click.command('set-common-config', help='Set value in common config')
@click.option('configs', '-c', '--config', multiple=True, type=(str, str))
def set_common_config(configs):
import ast
from bench.config.common_site_config import update_config
common_site_config = {}
for key, value in configs:
if value in ("False", "True"):
if value in ('true', 'false'):
value = value.title()
try:
value = ast.literal_eval(value)
elif "." in value:
try:
value = float(value)
except ValueError:
pass
elif "{" in value or "[" in value:
try:
value = json.loads(value)
except ValueError:
pass
else:
try:
value = int(value)
except ValueError:
pass
except ValueError:
pass
common_site_config[key] = value
update_config(common_site_config, bench_path='.')
@click.command('remove-common-config')
@click.command('remove-common-config', help='Remove specific keys from current bench\'s common config')
@click.argument('keys', nargs=-1)
def remove_common_config(keys):
from bench.config.common_site_config import get_config, put_config
common_site_config = get_config('.')
for key in keys:
if key in common_site_config:
@ -115,8 +77,6 @@ def remove_common_config(keys):
put_config(common_site_config)
config.add_command(config_auto_update)
config.add_command(config_update_bench_on_update)
config.add_command(config_restart_supervisor_on_update)
config.add_command(config_restart_systemd_on_update)
config.add_command(config_dns_multitenant)

18
bench/commands/git.py
View File

@ -1,28 +1,30 @@
import click
import os, subprocess, re
# imports - standard imports
import os
import subprocess
# imports - module imports
from bench.app import get_repo_dir, get_apps, get_remote
from bench.utils import set_git_remote_url
# imports - third party imports
import click
@click.command('remote-set-url')
@click.command('remote-set-url', help="Set app remote url")
@click.argument('git-url')
def remote_set_url(git_url):
"Set app remote url"
set_git_remote_url(git_url)
@click.command('remote-reset-url')
@click.command('remote-reset-url', help="Reset app remote url to frappe official")
@click.argument('app')
def remote_reset_url(app):
"Reset app remote url to frappe official"
git_url = "https://github.com/frappe/{}.git".format(app)
set_git_remote_url(git_url)
@click.command('remote-urls')
@click.command('remote-urls', help="Show apps remote url")
def remote_urls():
"Show apps remote url"
for app in get_apps():
repo_dir = get_repo_dir(app)

51
bench/commands/install.py
View File

@ -1,21 +1,29 @@
import os, sys, json, click
from bench.utils import run_playbook, setup_sudoers, is_root
# imports - module imports
from bench.utils import run_playbook, setup_sudoers
extra_vars = {"production": True}
# imports - third party imports
import click
@click.group()
extra_vars = {
"production": True
}
@click.group(help="Install system dependencies for setting up Frappe environment")
def install():
"Install system dependancies"
pass
@click.command('prerequisites')
@click.command('prerequisites', help="Installs pre-requisite libraries, essential tools like b2zip, htop, screen, vim, x11-fonts, python libs, cups and Redis")
def install_prerequisites():
run_playbook('site.yml', tag='common, redis')
@click.command('mariadb')
@click.option('--mysql_root_password')
@click.command('mariadb', help="Install and setup MariaDB of specified version and root password")
@click.option('--mysql_root_password', '--mysql-root-password', default="")
@click.option('--version', default="10.3")
def install_maridb(mysql_root_password='', version=''):
def install_maridb(mysql_root_password, version):
if mysql_root_password:
extra_vars.update({
"mysql_root_password": mysql_root_password,
@ -27,41 +35,49 @@ def install_maridb(mysql_root_password='', version=''):
run_playbook('site.yml', extra_vars=extra_vars, tag='mariadb')
@click.command('wkhtmltopdf')
@click.command('wkhtmltopdf', help="Installs wkhtmltopdf v0.12.3 for linux")
def install_wkhtmltopdf():
run_playbook('site.yml', extra_vars=extra_vars, tag='wkhtmltopdf')
@click.command('nodejs')
@click.command('nodejs', help="Installs Node.js v8")
def install_nodejs():
run_playbook('site.yml', extra_vars=extra_vars, tag='nodejs')
@click.command('psutil')
@click.command('psutil', help="Installs psutil via pip")
def install_psutil():
run_playbook('site.yml', extra_vars=extra_vars, tag='psutil')
@click.command('supervisor')
@click.command('supervisor', help="Installs supervisor. If user is specified, sudoers is setup for that user")
@click.option('--user')
def install_supervisor(user=None):
run_playbook('site.yml', extra_vars=extra_vars, tag='supervisor')
if user:
setup_sudoers(user)
@click.command('nginx')
@click.command('nginx', help="Installs NGINX. If user is specified, sudoers is setup for that user")
@click.option('--user')
def install_nginx(user=None):
run_playbook('site.yml', extra_vars=extra_vars, tag='nginx')
if user:
setup_sudoers(user)
@click.command('virtualbox')
@click.command('virtualbox', help="Installs supervisor")
def install_virtualbox():
run_playbook('vm_build.yml', tag='virtualbox')
@click.command('packer')
@click.command('packer', help="Installs Oracle virtualbox and packer 1.2.1")
def install_packer():
run_playbook('vm_build.yml', tag='packer')
@click.command('fail2ban')
@click.command("fail2ban", help="Install fail2ban, an intrusion prevention software framework that protects computer servers from brute-force attacks")
@click.option('--maxretry', default=6, help="Number of matches (i.e. value of the counter) which triggers ban action on the IP.")
@click.option('--bantime', default=600, help="The counter is set to zero if no match is found within 'findtime' seconds.")
@click.option('--findtime', default=600, help='Duration (in seconds) for IP to be banned for. Negative number for "permanent" ban.')
@ -69,6 +85,7 @@ def install_failtoban(**kwargs):
extra_vars.update(kwargs)
run_playbook('site.yml', extra_vars=extra_vars, tag='fail2ban')
install.add_command(install_prerequisites)
install.add_command(install_maridb)
install.add_command(install_wkhtmltopdf)

43
bench/commands/make.py
View File

@ -1,6 +1,8 @@
# imports - third party imports
import click
@click.command()
@click.command('init', help='Initialize a new bench instance in the specified path')
@click.argument('path')
@click.option('--python', type = str, default = 'python3', help = 'Path to Python Executable.')
@click.option('--ignore-exist', is_flag = True, default = False, help = "Ignore if Bench instance exists.")
@ -11,14 +13,10 @@ import click
@click.option('--clone-without-update', is_flag=True, help="copy repos from path without update")
@click.option('--no-procfile', is_flag=True, help="Pull changes in all the apps in bench")
@click.option('--no-backups',is_flag=True, help="Run migrations for all sites in the bench")
@click.option('--no-auto-update',is_flag=True, help="Build JS and CSS artifacts for the bench")
@click.option('--skip-redis-config-generation', is_flag=True, help="Skip redis config generation if already specifying the common-site-config file")
@click.option('--skip-assets',is_flag=True, default=False, help="Do not build assets")
@click.option('--verbose',is_flag=True, help="Verbose output during install")
def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, no_auto_update, clone_from, verbose, skip_redis_config_generation, clone_without_update, ignore_exist=False, skip_assets=False, python='python3'):
'''
Create a New Bench Instance.
'''
def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, clone_from, verbose, skip_redis_config_generation, clone_without_update, ignore_exist=False, skip_assets=False, python='python3'):
from bench.utils import init, log
try:
@ -27,7 +25,6 @@ def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, n
apps_path=apps_path,
no_procfile=no_procfile,
no_backups=no_backups,
no_auto_update=no_auto_update,
frappe_path=frappe_path,
frappe_branch=frappe_branch,
verbose=verbose,
@ -41,10 +38,11 @@ def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, n
log('Bench {} initialized'.format(path), level=1)
except SystemExit:
pass
except:
except Exception as e:
import os, shutil, time, six
# add a sleep here so that the traceback of other processes doesnt overlap with the prompts
time.sleep(1)
print(e)
log("There was a problem while creating {}".format(path), level=2)
if six.moves.input("Do you want to rollback these changes? [Y/n]: ").lower() == "y":
print('Rolling back Bench "{}"'.format(path))
@ -52,44 +50,53 @@ def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, n
shutil.rmtree(path)
@click.command('get-app')
@click.command('get-app', help='Clone an app from the internet or filesystem and set it up in your bench')
@click.argument('name', nargs=-1) # Dummy argument for backward compatibility
@click.argument('git-url')
@click.option('--branch', default=None, help="branch to checkout")
@click.option('--overwrite', is_flag=True, default=False)
@click.option('--skip-assets', is_flag=True, default=False, help="Do not build assets")
def get_app(git_url, branch, name=None, skip_assets=False):
def get_app(git_url, branch, name=None, overwrite=False, skip_assets=False):
"clone an app from the internet and set it up in your bench"
from bench.app import get_app
get_app(git_url, branch=branch, skip_assets=skip_assets)
get_app(git_url, branch=branch, skip_assets=skip_assets, overwrite=overwrite)
@click.command('new-app')
@click.command('new-app', help='Create a new Frappe application under apps folder')
@click.argument('app-name')
def new_app(app_name):
"start a new app"
from bench.app import new_app
new_app(app_name)
@click.command('remove-app')
@click.command('remove-app', help='Completely remove app from bench and re-build assets if not installed on any site')
@click.argument('app-name')
def remove_app(app_name):
"completely remove app from bench"
from bench.app import remove_app
remove_app(app_name)
@click.command('exclude-app')
@click.command('exclude-app', help='Exclude app from updating')
@click.argument('app_name')
def exclude_app_for_update(app_name):
"Exclude app from updating"
from bench.app import add_to_excluded_apps_txt
add_to_excluded_apps_txt(app_name)
@click.command('include-app')
@click.command('include-app', help='Include app for updating')
@click.argument('app_name')
def include_app_for_update(app_name):
"Include app from updating"
from bench.app import remove_from_excluded_apps_txt
remove_from_excluded_apps_txt(app_name)
@click.command('pip', context_settings={"ignore_unknown_options": True}, help="For pip help use `bench pip help [COMMAND]` or `bench pip [COMMAND] -h`")
@click.argument('args', nargs=-1)
@click.pass_context
def pip(ctx, args):
"Run pip commands in bench env"
import os
from bench.utils import get_env_cmd
env_pip = get_env_cmd('pip')
os.execv(env_pip, (env_pip,) + args)

289
bench/commands/setup.py
View File

@ -1,233 +1,216 @@
from bench.utils import exec_cmd
from six import PY3
import click, sys, json
# imports - standard imports
import os
import sys
@click.group()
# imports - module imports
from bench.utils import exec_cmd
# imports - third party imports
from six import PY3
import click
@click.group(help="Setup command group for enabling setting up a Frappe environment")
def setup():
"Setup bench"
pass
@click.command('sudoers')
@click.argument('user')
@click.command("sudoers", help="Add commands to sudoers list for execution without password")
@click.argument("user")
def setup_sudoers(user):
"Add commands to sudoers list for execution without password"
from bench.utils import setup_sudoers
setup_sudoers(user)
@click.command('nginx')
@click.option('--yes', help='Yes to regeneration of nginx config file', default=False, is_flag=True)
@click.command("nginx", help="Generate configuration files for NGINX")
@click.option("--yes", help="Yes to regeneration of nginx config file", default=False, is_flag=True)
def setup_nginx(yes=False):
"generate config for nginx"
from bench.config.nginx import make_nginx_conf
make_nginx_conf(bench_path=".", yes=yes)
@click.command('reload-nginx')
@click.command("reload-nginx", help="Checks NGINX config file and reloads service")
def reload_nginx():
from bench.config.production_setup import reload_nginx
reload_nginx()
@click.command('supervisor')
@click.option('--user')
@click.option('--yes', help='Yes to regeneration of supervisor config', is_flag=True, default=False)
@click.command("supervisor", help="Generate configuration for supervisor")
@click.option("--user", help="optional user argument")
@click.option("--yes", help="Yes to regeneration of supervisor config", is_flag=True, default=False)
def setup_supervisor(user=None, yes=False):
"generate config for supervisor with an optional user argument"
from bench.config.supervisor import generate_supervisor_config
generate_supervisor_config(bench_path=".", user=user, yes=yes)
@click.command('redis')
@click.command("redis", help="Generates configuration for Redis")
def setup_redis():
"generate config for redis cache"
from bench.config.redis import generate_config
generate_config('.')
generate_config(".")
@click.command('fonts')
@click.command("fonts", help="Add Frappe fonts to system")
def setup_fonts():
"Add frappe fonts to system"
from bench.utils import setup_fonts
setup_fonts()
@click.command('production')
@click.argument('user')
@click.option('--yes', help='Yes to regeneration config', is_flag=True, default=False)
@click.command("production", help="Setup Frappe production environment for specific user")
@click.argument("user")
@click.option("--yes", help="Yes to regeneration config", is_flag=True, default=False)
def setup_production(user, yes=False):
"setup bench for production"
from bench.config.production_setup import setup_production
from bench.utils import run_playbook
# Install prereqs for production
from distutils.spawn import find_executable
if not find_executable('ansible'):
exec_cmd("sudo {0} install ansible".format("pip3" if PY3 else "pip2"))
if not find_executable('fail2ban-client'):
if not find_executable("ansible"):
exec_cmd("sudo -H {0} -m pip install ansible".format(sys.executable))
if not find_executable("fail2ban-client"):
exec_cmd("bench setup role fail2ban")
if not find_executable('nginx'):
if not find_executable("nginx"):
exec_cmd("bench setup role nginx")
if not find_executable('supervisord'):
if not find_executable("supervisord"):
exec_cmd("bench setup role supervisor")
setup_production(user=user, yes=yes)
@click.command('auto-update')
def setup_auto_update():
"Add cronjob for bench auto update"
from bench.utils import setup_auto_update
setup_auto_update()
@click.command('backups')
@click.command("backups", help="Add cronjob for bench backups")
def setup_backups():
"Add cronjob for bench backups"
from bench.utils import setup_backups
setup_backups()
@click.command('env')
@click.option('--python', type = str, default = 'python3', help = 'Path to Python Executable.')
def setup_env(python='python3'):
"Setup virtualenv for bench"
@click.command("env", help="Setup virtualenv for bench")
@click.option("--python", type = str, default = "python3", help = "Path to Python Executable.")
def setup_env(python="python3"):
from bench.utils import setup_env
setup_env(python=python)
@click.command('firewall')
@click.option('--ssh_port')
@click.option('--force')
@click.command("firewall", help="Setup firewall for system")
@click.option("--ssh_port")
@click.option("--force")
def setup_firewall(ssh_port=None, force=False):
"Setup firewall"
from bench.utils import run_playbook
if not force:
click.confirm('Setting up the firewall will block all ports except 80, 443 and 22\n'
'Do you want to continue?',
abort=True)
click.confirm("Setting up the firewall will block all ports except 80, 443 and {0}\nDo you want to continue?".format(ssh_port), abort=True)
if not ssh_port:
ssh_port = 22
run_playbook('roles/bench/tasks/setup_firewall.yml', {"ssh_port": ssh_port})
run_playbook("roles/bench/tasks/setup_firewall.yml", {"ssh_port": ssh_port})
@click.command('ssh-port')
@click.argument('port')
@click.option('--force')
@click.command("ssh-port", help="Set SSH Port for system")
@click.argument("port")
@click.option("--force")
def set_ssh_port(port, force=False):
"Set SSH Port"
from bench.utils import run_playbook
if not force:
click.confirm('This will change your SSH Port to {}\n'
'Do you want to continue?'.format(port),
abort=True)
click.confirm("This will change your SSH Port to {}\nDo you want to continue?".format(port), abort=True)
run_playbook('roles/bench/tasks/change_ssh_port.yml', {"ssh_port": port})
run_playbook("roles/bench/tasks/change_ssh_port.yml", {"ssh_port": port})
@click.command('lets-encrypt')
@click.argument('site')
@click.option('--custom-domain')
@click.option('-n', '--non-interactive', default=False, is_flag=True, help="Run certbot non-interactively. Shouldn't be used on 1'st attempt")
@click.command("lets-encrypt", help="Setup lets-encrypt SSL for site")
@click.argument("site")
@click.option("--custom-domain")
@click.option('-n', '--non-interactive', default=False, is_flag=True, help="Run command non-interactively. This flag restarts nginx and runs certbot non interactively. Shouldn't be used on 1'st attempt")
def setup_letsencrypt(site, custom_domain, non_interactive):
"Setup lets-encrypt for site"
from bench.config.lets_encrypt import setup_letsencrypt
setup_letsencrypt(site, custom_domain, bench_path='.', interactive=not non_interactive)
setup_letsencrypt(site, custom_domain, bench_path=".", interactive=not non_interactive)
@click.command('wildcard-ssl')
@click.argument('domain')
@click.option('--email')
@click.option('--exclude-base-domain', default=False, is_flag=True, help="SSL Certificate not applicable for base domain")
@click.command("wildcard-ssl", help="Setup wildcard SSL certificate for multi-tenant bench")
@click.argument("domain")
@click.option("--email")
@click.option("--exclude-base-domain", default=False, is_flag=True, help="SSL Certificate not applicable for base domain")
def setup_wildcard_ssl(domain, email, exclude_base_domain):
''' Setup wildcard ssl certificate '''
from bench.config.lets_encrypt import setup_wildcard_ssl
setup_wildcard_ssl(domain, email, bench_path='.', exclude_base_domain=exclude_base_domain)
setup_wildcard_ssl(domain, email, bench_path=".", exclude_base_domain=exclude_base_domain)
@click.command('procfile')
@click.command("procfile", help="Generate Procfile for bench start")
def setup_procfile():
"Setup Procfile for bench start"
from bench.config.procfile import setup_procfile
setup_procfile('.')
setup_procfile(".")
@click.command('socketio')
@click.command("socketio", help="Setup node dependencies for socketio server")
def setup_socketio():
"Setup node deps for socketio server"
from bench.utils import setup_socketio
setup_socketio()
@click.command('requirements', help="Update Python and Node packages")
@click.option('--node', help="Update only Node packages", default=False, is_flag=True)
@click.option('--python', help="Update only Python packages", default=False, is_flag=True)
@click.command("requirements", help="Setup Python and Node dependencies")
@click.option("--node", help="Update only Node packages", default=False, is_flag=True)
@click.option("--python", help="Update only Python packages", default=False, is_flag=True)
def setup_requirements(node=False, python=False):
"Setup python and node requirements"
if not node:
setup_python_requirements()
from bench.utils import update_requirements as setup_python_packages
setup_python_packages()
if not python:
setup_node_requirements()
def setup_python_requirements():
from bench.utils import update_requirements
update_requirements()
def setup_node_requirements():
from bench.utils import update_node_packages
update_node_packages()
from bench.utils import update_node_packages as setup_node_packages
setup_node_packages()
@click.command('manager')
@click.option('--yes', help='Yes to regeneration of nginx config file', default=False, is_flag=True)
@click.option('--port', help='Port on which you want to run bench manager', default=23624)
@click.option('--domain', help='Domain on which you want to run bench manager')
@click.command("manager", help="Setup bench-manager.local site with the bench_manager app installed on it")
@click.option("--yes", help="Yes to regeneration of nginx config file", default=False, is_flag=True)
@click.option("--port", help="Port on which you want to run bench manager", default=23624)
@click.option("--domain", help="Domain on which you want to run bench manager")
def setup_manager(yes=False, port=23624, domain=None):
"Setup bench-manager.local site with the bench_manager app installed on it"
from six.moves import input
from bench.utils import get_sites
from bench.config.common_site_config import get_config
from bench.config.nginx import make_bench_manager_nginx_conf
create_new_site = True
if 'bench-manager.local' in os.listdir('sites'):
ans = input('Site already exists. Overwrite existing site? [Y/n]: ').lower()
while ans not in ('y', 'n', ''):
ans = input(
'Please enter "y" or "n". Site already exists. Overwrite existing site? [Y/n]: ').lower()
if ans == 'n':
if "bench-manager.local" in os.listdir("sites"):
ans = input("Site already exists. Overwrite existing site? [Y/n]: ").lower()
while ans not in ("y", "n", ""):
ans = input("Please enter 'y' or 'n'. Site already exists. Overwrite existing site? [Y/n]: ").lower()
if ans == "n":
create_new_site = False
if create_new_site:
exec_cmd("bench new-site --force bench-manager.local")
if 'bench_manager' in os.listdir('apps'):
print('App already exists. Skipping app download.')
if "bench_manager" in os.listdir("apps"):
print("App already exists. Skipping app download.")
else:
exec_cmd("bench get-app bench_manager")
exec_cmd("bench --site bench-manager.local install-app bench_manager")
from bench.config.common_site_config import get_config
bench_path = '.'
bench_path = "."
conf = get_config(bench_path)
if conf.get('restart_supervisor_on_update') or conf.get('restart_systemd_on_update'):
if conf.get("restart_supervisor_on_update") or conf.get("restart_systemd_on_update"):
# implicates a production setup or so I presume
if not domain:
print("Please specify the site name on which you want to host bench-manager using the 'domain' flag")
sys.exit(1)
from bench.utils import get_sites, get_bench_name
bench_name = get_bench_name(bench_path)
if domain not in get_sites(bench_path):
raise Exception("No such site")
from bench.config.nginx import make_bench_manager_nginx_conf
make_bench_manager_nginx_conf(bench_path, yes=yes, port=port, domain=domain)
@click.command('config')
@click.command("config", help="Generate or over-write sites/common_site_config.json")
def setup_config():
"overwrite or make config.json"
from bench.config.common_site_config import make_config
make_config('.')
make_config(".")
@click.command('add-domain')
@click.argument('domain')
@click.option('--site', prompt=True)
@click.option('--ssl-certificate', help="Absolute path to SSL Certificate")
@click.option('--ssl-certificate-key', help="Absolute path to SSL Certificate Key")
@click.command("add-domain", help="Add a custom domain to a particular site")
@click.argument("domain")
@click.option("--site", prompt=True)
@click.option("--ssl-certificate", help="Absolute path to SSL Certificate")
@click.option("--ssl-certificate-key", help="Absolute path to SSL Certificate Key")
def add_domain(domain, site=None, ssl_certificate=None, ssl_certificate_key=None):
"""Add custom domain to site"""
from bench.config.site_config import add_domain
@ -236,24 +219,25 @@ def add_domain(domain, site=None, ssl_certificate=None, ssl_certificate_key=None
print("Please specify site")
sys.exit(1)
add_domain(site, domain, ssl_certificate, ssl_certificate_key, bench_path='.')
add_domain(site, domain, ssl_certificate, ssl_certificate_key, bench_path=".")
@click.command('remove-domain')
@click.argument('domain')
@click.option('--site', prompt=True)
@click.command("remove-domain", help="Remove custom domain from a site")
@click.argument("domain")
@click.option("--site", prompt=True)
def remove_domain(domain, site=None):
"""Remove custom domain from a site"""
from bench.config.site_config import remove_domain
if not site:
print("Please specify site")
sys.exit(1)
remove_domain(site, domain, bench_path='.')
remove_domain(site, domain, bench_path=".")
@click.command('sync-domains')
@click.option('--domain', multiple=True)
@click.option('--site', prompt=True)
@click.command("sync-domains", help="Check if there is a change in domains. If yes, updates the domains list.")
@click.option("--domain", multiple=True)
@click.option("--site", prompt=True)
def sync_domains(domain=None, site=None):
from bench.config.site_config import sync_domains
@ -262,53 +246,55 @@ def sync_domains(domain=None, site=None):
sys.exit(1)
try:
domains = list(map(str,domain))
domains = list(map(str, domain))
except Exception:
print("Domains should be a json list of strings or dictionaries")
sys.exit(1)
changed = sync_domains(site, domains, bench_path='.')
changed = sync_domains(site, domains, bench_path=".")
# if changed, success, else failure
sys.exit(0 if changed else 1)
@click.command('role')
@click.argument('role')
@click.option('--admin_emails', default='')
@click.option('--mysql_root_password')
@click.option('--container', is_flag=True, default=False)
@click.command("role", help="Install dependencies via ansible roles")
@click.argument("role")
@click.option("--admin_emails", default="")
@click.option("--mysql_root_password")
@click.option("--container", is_flag=True, default=False)
def setup_roles(role, **kwargs):
"Install dependancies via roles"
from bench.utils import run_playbook
extra_vars = {"production": True}
extra_vars.update(kwargs)
if role:
run_playbook('site.yml', extra_vars=extra_vars, tag=role)
run_playbook("site.yml", extra_vars=extra_vars, tag=role)
else:
run_playbook('site.yml', extra_vars=extra_vars)
run_playbook("site.yml", extra_vars=extra_vars)
@click.command('fail2ban')
@click.option('--maxretry', default=6, help="Number of matches (i.e. value of the counter) which triggers ban action on the IP. Default is 6 seconds" )
@click.option('--bantime', default=600, help="The counter is set to zero if no match is found within 'findtime' seconds. Default is 600 seconds")
@click.option('--findtime', default=600, help='Duration (in seconds) for IP to be banned for. Negative number for "permanent" ban. Default is 600 seconds')
@click.command("fail2ban", help="Setup fail2ban, an intrusion prevention software framework that protects computer servers from brute-force attacks")
@click.option("--maxretry", default=6, help="Number of matches (i.e. value of the counter) which triggers ban action on the IP. Default is 6 seconds" )
@click.option("--bantime", default=600, help="The counter is set to zero if no match is found within 'findtime' seconds. Default is 600 seconds")
@click.option("--findtime", default=600, help="Duration (in seconds) for IP to be banned for. Negative number for 'permanent' ban. Default is 600 seconds")
def setup_nginx_proxy_jail(**kwargs):
from bench.utils import run_playbook
run_playbook('roles/fail2ban/tasks/configure_nginx_jail.yml', extra_vars=kwargs)
run_playbook("roles/fail2ban/tasks/configure_nginx_jail.yml", extra_vars=kwargs)
@click.command('systemd')
@click.option('--user')
@click.option('--yes', help='Yes to regeneration of systemd config files', is_flag=True, default=False)
@click.option('--stop', help='Stop bench services', is_flag=True, default=False)
@click.option('--create-symlinks', help='Create Symlinks', is_flag=True, default=False)
@click.option('--delete-symlinks', help='Delete Symlinks', is_flag=True, default=False)
@click.command("systemd", help="Generate configuration for systemd")
@click.option("--user", help="Optional user argument")
@click.option("--yes", help="Yes to regeneration of systemd config files", is_flag=True, default=False)
@click.option("--stop", help="Stop bench services", is_flag=True, default=False)
@click.option("--create-symlinks", help="Create Symlinks", is_flag=True, default=False)
@click.option("--delete-symlinks", help="Delete Symlinks", is_flag=True, default=False)
def setup_systemd(user=None, yes=False, stop=False, create_symlinks=False, delete_symlinks=False):
"generate configs for systemd with an optional user argument"
from bench.config.systemd import generate_systemd_config
generate_systemd_config(bench_path=".", user=user, yes=yes,
stop=stop, create_symlinks=create_symlinks, delete_symlinks=delete_symlinks)
setup.add_command(setup_sudoers)
setup.add_command(setup_nginx)
setup.add_command(reload_nginx)
@ -317,7 +303,6 @@ setup.add_command(setup_redis)
setup.add_command(setup_letsencrypt)
setup.add_command(setup_wildcard_ssl)
setup.add_command(setup_production)
setup.add_command(setup_auto_update)
setup.add_command(setup_backups)
setup.add_command(setup_env)
setup.add_command(setup_procfile)

156
bench/commands/update.py
View File

@ -1,124 +1,31 @@
import click
import sys
# imports - standard imports
import os
from bench.config.common_site_config import get_config, update_config
from bench.app import pull_all_apps, is_version_upgrade, validate_branch
from bench.utils import (update_bench, validate_upgrade, pre_upgrade, post_upgrade, before_update,
update_requirements, update_node_packages, backup_all_sites, patch_sites, build_assets,
restart_supervisor_processes, restart_systemd_processes, is_bench_directory)
from bench import patches
# imports - third party imports
import click
from six.moves import reload_module
# imports - module imports
from bench.app import pull_all_apps
from bench.utils import post_upgrade, patch_sites, build_assets
@click.command('update')
@click.option('--pull', is_flag=True, help="Pull changes in all the apps in bench")
@click.command('update', help="Updates bench tool and if executed in a bench directory, without any flags will backup, pull, setup requirements, build, run patches and restart bench. Using specific flags will only do certain tasks instead of all")
@click.option('--pull', is_flag=True, help="Pull updates for all the apps in bench")
@click.option('--patch', is_flag=True, help="Run migrations for all sites in the bench")
@click.option('--build', is_flag=True, help="Build JS and CSS artifacts for the bench")
@click.option('--bench', is_flag=True, help="Update bench")
@click.option('--requirements', is_flag=True, help="Update requirements")
@click.option('--restart-supervisor', is_flag=True, help="restart supervisor processes after update")
@click.option('--restart-systemd', is_flag=True, help="restart systemd units after update")
@click.option('--auto', is_flag=True)
@click.option('--no-backup', is_flag=True)
@click.option('--force', is_flag=True)
@click.option('--build', is_flag=True, help="Build JS and CSS assets for the bench")
@click.option('--requirements', is_flag=True, help="Update requirements. If run alone, equivalent to `bench setup requirements`")
@click.option('--restart-supervisor', is_flag=True, help="Restart supervisor processes after update")
@click.option('--restart-systemd', is_flag=True, help="Restart systemd units after update")
@click.option('--no-backup', is_flag=True, help="If this flag is set, sites won't be backed up prior to updates. Note: This is not recommended in production.")
@click.option('--force', is_flag=True, help="Forces major version upgrades")
@click.option('--reset', is_flag=True, help="Hard resets git branch's to their new states overriding any changes and overriding rebase on pull")
def update(pull=False, patch=False, build=False, bench=False, auto=False, restart_supervisor=False, restart_systemd=False, requirements=False, no_backup=False, force=False, reset=False):
"Update bench"
def update(pull, patch, build, requirements, restart_supervisor, restart_systemd, no_backup, force, reset):
from bench.utils import update
update(pull=pull, patch=patch, build=build, requirements=requirements, restart_supervisor=restart_supervisor, restart_systemd=restart_systemd, backup=not no_backup, force=force, reset=reset)
if not is_bench_directory():
"""Update only bench if bench update called from outside a bench"""
update_bench(bench_repo=True, requirements=True)
sys.exit()
if not (pull or patch or build or bench or requirements):
pull, patch, build, bench, requirements = True, True, True, True, True
if auto:
sys.exit(1)
patches.run(bench_path='.')
conf = get_config(".")
if bench and conf.get('update_bench_on_update'):
update_bench(bench_repo=True, requirements=False)
restart_update({
'pull': pull,
'patch': patch,
'build': build,
'requirements': requirements,
'no-backup': no_backup,
'restart-supervisor': restart_supervisor,
'reset': reset
})
if conf.get('release_bench'):
print('Release bench, cannot update')
sys.exit(1)
validate_branch()
version_upgrade = is_version_upgrade()
if version_upgrade[0]:
print()
print()
print("This update will cause a major version change in Frappe/ERPNext from {0} to {1}.".format(*version_upgrade[1:]))
print("This would take significant time to migrate and might break custom apps.")
click.confirm('Do you want to continue?', abort=True)
_update(pull, patch, build, bench, auto, restart_supervisor, restart_systemd, requirements, no_backup, force=force, reset=reset)
def _update(pull=False, patch=False, build=False, update_bench=False, auto=False, restart_supervisor=False,
restart_systemd=False, requirements=False, no_backup=False, bench_path='.', force=False, reset=False):
conf = get_config(bench_path=bench_path)
version_upgrade = is_version_upgrade(bench_path=bench_path)
if version_upgrade[0] or (not version_upgrade[0] and force):
validate_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path)
before_update(bench_path=bench_path, requirements=requirements)
conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 })
update_config(conf, bench_path=bench_path)
if not no_backup:
print('Backing up sites...')
backup_all_sites(bench_path=bench_path)
if pull:
pull_all_apps(bench_path=bench_path, reset=reset)
if requirements:
update_requirements(bench_path=bench_path)
update_node_packages(bench_path=bench_path)
if version_upgrade[0] or (not version_upgrade[0] and force):
pre_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path)
import bench.utils, bench.app
print('Reloading bench...')
reload_module(bench.utils)
reload_module(bench.app)
if patch:
print('Patching sites...')
patch_sites(bench_path=bench_path)
if build:
build_assets(bench_path=bench_path)
if version_upgrade[0] or (not version_upgrade[0] and force):
post_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path)
if restart_supervisor or conf.get('restart_supervisor_on_update'):
restart_supervisor_processes(bench_path=bench_path)
if restart_systemd or conf.get('restart_systemd_on_update'):
restart_systemd_processes(bench_path=bench_path)
conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 })
update_config(conf, bench_path=bench_path)
print("_"*80)
print("Bench: Deployment tool for Frappe and ERPNext (https://erpnext.org).")
print("Open source depends on your contributions, so please contribute bug reports, patches, fixes or cash and be a part of the community")
print()
@click.command('retry-upgrade')
@click.command('retry-upgrade', help="Retry a failed upgrade")
@click.option('--version', default=5)
def retry_upgrade(version):
pull_all_apps()
@ -126,35 +33,24 @@ def retry_upgrade(version):
build_assets()
post_upgrade(version-1, version)
def restart_update(kwargs):
args = ['--'+k for k, v in list(kwargs.items()) if v]
os.execv(sys.argv[0], sys.argv[:2] + args)
@click.command('switch-to-branch')
@click.command('switch-to-branch', help="Switch all apps to specified branch, or specify apps separated by space")
@click.argument('branch')
@click.argument('apps', nargs=-1)
@click.option('--upgrade',is_flag=True)
def switch_to_branch(branch, apps, upgrade=False):
"Switch all apps to specified branch, or specify apps separated by space"
from bench.app import switch_to_branch
switch_to_branch(branch=branch, apps=list(apps), upgrade=upgrade)
print('Switched to ' + branch)
print('Please run `bench update --patch` to be safe from any differences in database schema')
@click.command('switch-to-master')
@click.command('switch-to-master', help="[DEPRECATED]: Switch frappe and erpnext to master branch")
def switch_to_master():
"Switch frappe and erpnext to master branch"
from bench.app import switch_to_master
switch_to_master(apps=['frappe', 'erpnext'])
print()
print('Switched to master')
print('Please run `bench update --patch` to be safe from any differences in database schema')
from bench.utils import log
log("`switch-to-master` has been deprecated as master branches were renamed to version-11")
@click.command('switch-to-develop')
def switch_to_develop(upgrade=False):
"Switch frappe and erpnext to develop branch"
from bench.app import switch_to_develop
switch_to_develop(apps=['frappe', 'erpnext'])
print()
print('Switched to develop')
print('Please run `bench update --patch` to be safe from any differences in database schema')

104
bench/commands/utils.py
View File

@ -1,23 +1,25 @@
# imports - standard imports
import os
import sys
# imports - third party imports
import click
import sys, os, copy
@click.command('start')
@click.command('start', help="Start Frappe development processes")
@click.option('--no-dev', is_flag=True, default=False)
@click.option('--concurrency', '-c', type=str)
@click.option('--procfile', '-p', type=str)
def start(no_dev, concurrency, procfile):
"Start Frappe development processes"
from bench.utils import start
start(no_dev=no_dev, concurrency=concurrency, procfile=procfile)
@click.command('restart')
@click.command('restart', help="Restart supervisor processes or systemd units")
@click.option('--web', is_flag=True, default=False)
@click.option('--supervisor', is_flag=True, default=False)
@click.option('--systemd', is_flag=True, default=False)
def restart(web, supervisor, systemd):
"Restart supervisor processes or systemd units"
from bench.utils import restart_supervisor_processes, restart_systemd_processes
from bench.config.common_site_config import get_config
if get_config('.').get('restart_supervisor_on_update') or supervisor:
@ -25,134 +27,112 @@ def restart(web, supervisor, systemd):
if get_config('.').get('restart_systemd_on_update') or systemd:
restart_systemd_processes(bench_path='.', web_workers=web)
@click.command('set-nginx-port')
@click.command('set-nginx-port', help="Set NGINX port for site")
@click.argument('site')
@click.argument('port', type=int)
def set_nginx_port(site, port):
"Set nginx port for site"
from bench.config.site_config import set_nginx_port
set_nginx_port(site, port)
@click.command('set-ssl-certificate')
@click.command('set-ssl-certificate', help="Set SSL certificate path for site")
@click.argument('site')
@click.argument('ssl-certificate-path')
def set_ssl_certificate(site, ssl_certificate_path):
"Set ssl certificate path for site"
from bench.config.site_config import set_ssl_certificate
set_ssl_certificate(site, ssl_certificate_path)
@click.command('set-ssl-key')
@click.command('set-ssl-key', help="Set SSL certificate private key path for site")
@click.argument('site')
@click.argument('ssl-certificate-key-path')
def set_ssl_certificate_key(site, ssl_certificate_key_path):
"Set ssl certificate private key path for site"
from bench.config.site_config import set_ssl_certificate_key
set_ssl_certificate_key(site, ssl_certificate_key_path)
@click.command('set-url-root')
@click.command('set-url-root', help="Set URL root for site")
@click.argument('site')
@click.argument('url-root')
def set_url_root(site, url_root):
"Set url root for site"
from bench.config.site_config import set_url_root
set_url_root(site, url_root)
@click.command('set-mariadb-host')
@click.command('set-mariadb-host', help="Set MariaDB host for bench")
@click.argument('host')
def set_mariadb_host(host):
"Set MariaDB host for bench"
from bench.utils import set_mariadb_host
set_mariadb_host(host)
@click.command('set-redis-cache-host')
@click.command('set-redis-cache-host', help="Set Redis cache host for bench")
@click.argument('host')
def set_redis_cache_host(host):
"""
Set Redis cache host for bench
Eg: bench set-redis-cache-host localhost:6379/1
Usage: bench set-redis-cache-host localhost:6379/1
"""
from bench.utils import set_redis_cache_host
set_redis_cache_host(host)
@click.command('set-redis-queue-host')
@click.command('set-redis-queue-host', help="Set Redis queue host for bench")
@click.argument('host')
def set_redis_queue_host(host):
"""
Set Redis queue host for bench
Eg: bench set-redis-queue-host localhost:6379/2
Usage: bench set-redis-queue-host localhost:6379/2
"""
from bench.utils import set_redis_queue_host
set_redis_queue_host(host)
@click.command('set-redis-socketio-host')
@click.command('set-redis-socketio-host', help="Set Redis socketio host for bench")
@click.argument('host')
def set_redis_socketio_host(host):
"""
Set Redis socketio host for bench
Eg: bench set-redis-socketio-host localhost:6379/3
Usage: bench set-redis-socketio-host localhost:6379/3
"""
from bench.utils import set_redis_socketio_host
set_redis_socketio_host(host)
@click.command('set-default-site')
@click.command('set-default-site', help="Set default site for bench")
@click.argument('site')
def set_default_site(site):
"Set default site for bench"
from bench.utils import set_default_site
set_default_site(site)
@click.command('download-translations')
@click.command('download-translations', help="Download latest translations")
def download_translations():
"Download latest translations"
from bench.utils import download_translations_p
download_translations_p()
@click.command('renew-lets-encrypt')
@click.command('renew-lets-encrypt', help="Renew Let's Encrypt certificate")
def renew_lets_encrypt():
"Renew Let's Encrypt certificate"
from bench.config.lets_encrypt import renew_certs
renew_certs()
@click.command()
def shell(bench_path='.'):
if not os.environ.get('SHELL'):
print("Cannot get shell")
sys.exit(1)
if not os.path.exists('sites'):
print("sites dir doesn't exist")
sys.exit(1)
env = copy.copy(os.environ)
env['PS1'] = '(' + os.path.basename(os.path.dirname(os.path.abspath(__file__))) + ')' + env.get('PS1', '')
env['PATH'] = os.path.dirname(os.path.abspath(os.path.join('env','bin')) + ':' + env['PATH'])
os.chdir('sites')
os.execve(env['SHELL'], [env['SHELL']], env)
@click.command('backup')
@click.command('backup', help="Backup single site")
@click.argument('site')
def backup_site(site):
"backup site"
from bench.utils import get_sites, backup_site
if site not in get_sites(bench_path='.'):
print('site not found')
print('Site `{0}` not found'.format(site))
sys.exit(1)
backup_site(site, bench_path='.')
@click.command('backup-all-sites')
@click.command('backup-all-sites', help="Backup all sites in current bench")
def backup_all_sites():
"backup all sites"
from bench.utils import backup_all_sites
backup_all_sites(bench_path='.')
@click.command('release')
@click.command('release', help="Release a Frappe app (internal to the Frappe team)")
@click.argument('app')
@click.argument('bump-type', type=click.Choice(['major', 'minor', 'patch', 'stable', 'prerelease']))
@click.option('--from-branch', default='develop')
@ -162,39 +142,41 @@ def backup_all_sites():
@click.option('--repo-name')
@click.option('--dont-frontport', is_flag=True, default=False, help='Front port fixes to new branches, example merging hotfix(v10) into staging-fixes(v11)')
def release(app, bump_type, from_branch, to_branch, owner, repo_name, remote, dont_frontport):
"Release app (internal to the Frappe team)"
from bench.release import release
frontport = not dont_frontport
release(bench_path='.', app=app, bump_type=bump_type, from_branch=from_branch, to_branch=to_branch,
remote=remote, owner=owner, repo_name=repo_name, frontport=frontport)
release(bench_path='.', app=app, bump_type=bump_type, from_branch=from_branch, to_branch=to_branch, remote=remote, owner=owner, repo_name=repo_name, frontport=frontport)
@click.command('prepare-beta-release')
@click.command('prepare-beta-release', help="Prepare major beta release from develop branch")
@click.argument('app')
@click.option('--owner', default='frappe')
def prepare_beta_release(app, owner):
"""Prepare major beta release from develop branch"""
from bench.prepare_beta_release import prepare_beta_release
prepare_beta_release(bench_path='.', app=app, owner=owner)
@click.command('disable-production')
@click.command('disable-production', help="Disables production environment for the bench.")
def disable_production():
"""Disables production environment for the bench."""
from bench.config.production_setup import disable_production
disable_production(bench_path='.')
@click.command('src')
@click.command('src', help="Prints bench source folder path, which can be used as: cd `bench src`")
def bench_src():
"""Prints bench source folder path, which can be used as: cd `bench src` """
import bench
print(os.path.dirname(bench.__path__[0]))
@click.command('find')
@click.command('find', help="Finds benches recursively from location")
@click.argument('location', default='')
def find_benches(location):
"""Finds benches recursively from location"""
from bench.utils import find_benches
find_benches(directory=location)
@click.command('migrate-env', help="Migrate Virtual Environment to desired Python Version")
@click.argument('python', type=str)
@click.option('--no-backup', 'backup', is_flag=True, default=True)
def migrate_env(python, backup=True):
from bench.utils import migrate_env
migrate_env(python=python, backup=backup)

14
bench/config/common_site_config.py
View File

@ -1,9 +1,12 @@
import os, multiprocessing, getpass, json
# imports - standard imports
import getpass
import json
import multiprocessing
import os
# imports - third party imports
from six.moves.urllib.parse import urlparse
try:
from urllib.parse import urlparse
except ImportError:
from urlparse import urlparse
default_config = {
'restart_supervisor_on_update': False,
@ -11,7 +14,6 @@ default_config = {
'auto_update': False,
'serve_default_site': True,
'rebase_on_pull': False,
'update_bench_on_update': True,
'frappe_user': getpass.getuser(),
'shallow_clone': True,
'background_workers': 1

14
bench/config/lets_encrypt.py
View File

@ -29,9 +29,10 @@ def setup_letsencrypt(site, custom_domain, bench_path, interactive):
print("No custom domain named {0} set for site".format(custom_domain))
return
click.confirm('Running this will stop the nginx service temporarily causing your sites to go offline\n'
'Do you want to continue?',
abort=True)
if interactive:
click.confirm('Running this will stop the nginx service temporarily causing your sites to go offline\n'
'Do you want to continue?',
abort=True)
if not get_config(bench_path).get("dns_multitenant"):
print("You cannot setup SSL without DNS Multitenancy")
@ -82,11 +83,10 @@ def run_certbot_and_setup_ssl(site, custom_domain, bench_path, interactive=True)
def setup_crontab():
job_command = '/opt/certbot-auto renew -a nginx --post-hook "systemctl reload nginx"'
system_crontab = CronTab(tabfile='/etc/crontab', user=True)
system_crontab = CronTab(user='root')
if job_command not in str(system_crontab):
job = system_crontab.new(command=job_command, comment="Renew lets-encrypt every month")
job.every().month()
job.enable()
job = system_crontab.new(command=job_command, comment="Renew lets-encrypt every month")
job.day.on(1)
system_crontab.write()

24
bench/config/nginx.py
View File

@ -1,8 +1,24 @@
import os, json, click, random, string, hashlib
from bench.utils import get_sites, get_bench_name, exec_cmd
# imports - standard imports
import hashlib
import os
import random
import string
# imports - third party imports
import click
from six import string_types
# imports - module imports
from bench.utils import get_bench_name, get_sites
def make_nginx_conf(bench_path, yes=False):
conf_path = os.path.join(bench_path, "config", "nginx.conf")
if not yes and os.path.exists(conf_path):
if not click.confirm('nginx.conf already exists and this will overwrite it. Do you want to continue?'):
return
from bench import env
from bench.config.common_site_config import get_config
@ -37,10 +53,6 @@ def make_nginx_conf(bench_path, yes=False):
nginx_conf = template.render(**template_vars)
conf_path = os.path.join(bench_path, "config", "nginx.conf")
if not yes and os.path.exists(conf_path):
click.confirm('nginx.conf already exists and this will overwrite it. Do you want to continue?',
abort=True)
with open(conf_path, "w") as f:
f.write(nginx_conf)

5
bench/config/procfile.py
View File

@ -3,7 +3,7 @@ from bench.utils import find_executable
from bench.app import use_rq
from bench.config.common_site_config import get_config
def setup_procfile(bench_path, yes=False):
def setup_procfile(bench_path, yes=False, skip_redis=False):
config = get_config(bench_path=bench_path)
procfile_path = os.path.join(bench_path, 'Procfile')
if not yes and os.path.exists(procfile_path):
@ -14,7 +14,8 @@ def setup_procfile(bench_path, yes=False):
node=find_executable("node") or find_executable("nodejs"),
use_rq=use_rq(bench_path),
webserver_port=config.get('webserver_port'),
CI=os.environ.get('CI'))
CI=os.environ.get('CI'),
skip_redis=skip_redis)
with open(procfile_path, 'w') as f:
f.write(procfile)

19
bench/config/redis.py
View File

@ -1,11 +1,16 @@
from .common_site_config import get_config
import re, os, subprocess, semantic_version
import bench
# imports - standard imports
import os
import re
import subprocess
# imports - third party imports
import semantic_version
from six.moves.urllib.parse import urlparse
# imports - module imports
import bench
from bench.config.common_site_config import get_config
try:
from urllib.parse import urlparse
except ImportError:
from urlparse import urlparse
def generate_config(bench_path):
config = get_config(bench_path)

2
bench/config/templates/Procfile
View File

@ -1,6 +1,8 @@
{% if not skip_redis %}
redis_cache: redis-server config/redis_cache.conf
redis_socketio: redis-server config/redis_socketio.conf
redis_queue: redis-server config/redis_queue.conf
{% endif %}
web: bench serve {% if webserver_port -%} --port {{ webserver_port }} {%- endif %}
socketio: {{ node }} apps/frappe/socketio.js

6
bench/prepare_staging.py
View File

@ -19,9 +19,7 @@ def prepare_staging(bench_path, app, remote='upstream'):
print('No commits to release')
return
print()
print(message)
print()
click.confirm('Do you want to continue?', abort=True)
@ -52,13 +50,13 @@ def create_staging(repo_path, from_branch='develop'):
g.merge(from_branch, '--no-ff')
except git.exc.GitCommandError as e:
handle_merge_error(e, source=from_branch, target='staging')
g.checkout(from_branch)
try:
g.merge('staging')
except git.exc.GitCommandError as e:
handle_merge_error(e, source='staging', target=from_branch)
def push_commits(repo_path, remote='upstream'):
print('pushing staging branch of', repo_path)

2
bench/release.py
View File

@ -83,9 +83,7 @@ def bump(bench_path, app, bump_type, from_branch, to_branch, remote, owner, repo
print('No commits to release')
return
print()
print(message)
print()
click.confirm('Do you want to continue?', abort=True)

96
bench/tests/test_base.py Normal file
View File

@ -0,0 +1,96 @@
# imports - standard imports
import json
import os
import shutil
import subprocess
import sys
import unittest
import getpass
# imports - module imports
import bench
import bench.utils
class TestBenchBase(unittest.TestCase):
def setUp(self):
self.benches_path = "."
self.benches = []
def tearDown(self):
for bench_name in self.benches:
bench_path = os.path.join(self.benches_path, bench_name)
mariadb_password = "travis" if os.environ.get("CI") else getpass.getpass(prompt="Enter MariaDB root Password: ")
if os.path.exists(bench_path):
sites = bench.utils.get_sites(bench_path=bench_path)
for site in sites:
subprocess.call(["bench", "drop-site", site, "--force", "--no-backup", "--root-password", mariadb_password], cwd=bench_path)
shutil.rmtree(bench_path, ignore_errors=True)
def assert_folders(self, bench_name):
for folder in bench.utils.folders_in_bench:
self.assert_exists(bench_name, folder)
self.assert_exists(bench_name, "apps", "frappe")
def assert_virtual_env(self, bench_name):
bench_path = os.path.abspath(bench_name)
python_path = os.path.abspath(os.path.join(bench_path, "env", "bin", "python"))
self.assertTrue(python_path.startswith(bench_path))
for subdir in ("bin", "include", "lib", "share"):
self.assert_exists(bench_name, "env", subdir)
def assert_config(self, bench_name):
for config, search_key in (
("redis_queue.conf", "redis_queue.rdb"),
("redis_socketio.conf", "redis_socketio.rdb"),
("redis_cache.conf", "redis_cache.rdb")):
self.assert_exists(bench_name, "config", config)
with open(os.path.join(bench_name, "config", config), "r") as f:
self.assertTrue(search_key in f.read())
def assert_common_site_config(self, bench_name, expected_config):
common_site_config_path = os.path.join(self.benches_path, bench_name, 'sites', 'common_site_config.json')
self.assertTrue(os.path.exists(common_site_config_path))
with open(common_site_config_path, "r") as f:
config = json.load(f)
for key, value in list(expected_config.items()):
self.assertEqual(config.get(key), value)
def assert_exists(self, *args):
self.assertTrue(os.path.exists(os.path.join(*args)))
def new_site(self, site_name, bench_name):
new_site_cmd = ["bench", "new-site", site_name, "--admin-password", "admin"]
if os.environ.get('CI'):
new_site_cmd.extend(["--mariadb-root-password", "travis"])
subprocess.call(new_site_cmd, cwd=os.path.join(self.benches_path, bench_name))
def init_bench(self, bench_name, **kwargs):
self.benches.append(bench_name)
frappe_tmp_path = "/tmp/frappe"
if not os.path.exists(frappe_tmp_path):
bench.utils.exec_cmd("git clone https://github.com/frappe/frappe --depth 1 --origin upstream {location}".format(location=frappe_tmp_path))
kwargs.update(dict(
python=sys.executable,
no_procfile=True,
no_backups=True,
skip_assets=True,
frappe_path=frappe_tmp_path
))
if not os.path.exists(os.path.join(self.benches_path, bench_name)):
bench.utils.init(bench_name, **kwargs)
bench.utils.exec_cmd("git remote set-url upstream https://github.com/frappe/frappe", cwd=os.path.join(self.benches_path, bench_name, "apps", "frappe"))
def file_exists(self, path):
if os.environ.get("CI"):
return not subprocess.call(["sudo", "test", "-f", path])
return os.path.isfile(path)

243
bench/tests/test_init.py
View File

@ -1,26 +1,20 @@
# imports - standard imports
import json
import os
import subprocess
import unittest
import json, os, shutil, subprocess
# imports - third paty imports
import git
# imports - module imports
import bench
import bench.utils
import bench.app
import bench.config.common_site_config
import bench.cli
from bench.release import get_bumped_version
from bench.tests.test_base import TestBenchBase
bench.cli.from_command_line = True
class TestBenchInit(unittest.TestCase):
def setUp(self):
self.benches_path = "."
self.benches = []
def tearDown(self):
for bench_name in self.benches:
bench_path = os.path.join(self.benches_path, bench_name)
if os.path.exists(bench_path):
shutil.rmtree(bench_path, ignore_errors=True)
class TestBenchInit(TestBenchBase):
def test_semantic_version(self):
self.assertEqual( get_bumped_version('11.0.4', 'major'), '12.0.0' )
self.assertEqual( get_bumped_version('11.0.4', 'minor'), '11.1.0' )
@ -35,20 +29,14 @@ class TestBenchInit(unittest.TestCase):
def test_init(self, bench_name="test-bench", **kwargs):
self.init_bench(bench_name, **kwargs)
self.assert_folders(bench_name)
self.assert_virtual_env(bench_name)
self.assert_common_site_config(bench_name, bench.config.common_site_config.default_config)
self.assert_config(bench_name)
self.assert_socketio(bench_name)
def test_multiple_benches(self):
# 1st bench
self.test_init("test-bench-1")
for bench_name in ("test-bench-1", "test-bench-2"):
self.init_bench(bench_name)
self.assert_common_site_config("test-bench-1", {
"webserver_port": 8000,
@ -59,9 +47,6 @@ class TestBenchInit(unittest.TestCase):
"redis_cache": "redis://localhost:13000"
})
# 2nd bench
self.test_init("test-bench-2")
self.assert_common_site_config("test-bench-2", {
"webserver_port": 8001,
"socketio_port": 9001,
@ -71,196 +56,92 @@ class TestBenchInit(unittest.TestCase):
"redis_cache": "redis://localhost:13001"
})
def test_new_site(self):
self.init_bench('test-bench')
self.new_site("test-site-1.dev")
bench_name = "test-bench"
site_name = "test-site.local"
bench_path = os.path.join(self.benches_path, bench_name)
site_path = os.path.join(bench_path, "sites", site_name)
site_config_path = os.path.join(site_path, "site_config.json")
def new_site(self, site_name):
new_site_cmd = ["bench", "new-site", site_name, "--admin-password", "admin"]
# set in CI
if os.environ.get('CI'):
new_site_cmd.extend(["--mariadb-root-password", "travis"])
subprocess.check_output(new_site_cmd, cwd=os.path.join(self.benches_path, "test-bench"))
site_path = os.path.join(self.benches_path, "test-bench", "sites", site_name)
self.init_bench(bench_name)
bench.utils.exec_cmd("bench setup requirements --node", cwd=bench_path)
self.new_site(site_name, bench_name)
self.assertTrue(os.path.exists(site_path))
self.assertTrue(os.path.exists(os.path.join(site_path, "private", "backups")))
self.assertTrue(os.path.exists(os.path.join(site_path, "private", "files")))
self.assertTrue(os.path.exists(os.path.join(site_path, "public", "files")))
site_config_path = os.path.join(site_path, "site_config.json")
self.assertTrue(os.path.exists(site_config_path))
with open(site_config_path, "r") as f:
site_config = json.loads(f.read())
for key in ("db_name", "db_password"):
self.assertTrue(key in site_config)
self.assertTrue(site_config[key])
for key in ("db_name", "db_password"):
self.assertTrue(key in site_config)
self.assertTrue(site_config[key])
def test_get_app(self):
site_name = "test-site-2.dev"
self.init_bench('test-bench')
self.new_site(site_name)
self.init_bench("test-bench")
bench_path = os.path.join(self.benches_path, "test-bench")
bench.utils.exec_cmd("bench get-app frappe_theme --skip-assets", cwd=bench_path)
self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "frappe_theme")))
app_installed_in_env = "frappe_theme" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8')
self.assertTrue(app_installed_in_env)
bench.app.get_app("https://github.com/frappe/frappe-client", bench_path=bench_path)
self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "frappeclient")))
def test_install_app(self):
site_name = "test-site-3.dev"
self.init_bench('test-bench')
self.new_site(site_name)
bench_name = "test-bench"
site_name = "install-app.test"
bench_path = os.path.join(self.benches_path, "test-bench")
# get app
bench.app.get_app("https://github.com/frappe/erpnext", "develop", bench_path=bench_path)
self.init_bench(bench_name)
bench.utils.exec_cmd("bench setup requirements --node", cwd=bench_path)
bench.utils.exec_cmd("bench build", cwd=bench_path)
bench.utils.exec_cmd("bench get-app erpnext", cwd=bench_path)
self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "erpnext")))
# install app
bench.app.install_app("erpnext", bench_path=bench_path)
# check if app is installed
app_installed_in_env = "erpnext" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8')
self.assertTrue(app_installed_in_env)
# install it to site
subprocess.check_output(["bench", "--site", site_name, "install-app", "erpnext"], cwd=bench_path)
# create and install app on site
self.new_site(site_name, bench_name)
bench.utils.exec_cmd("bench --site {0} install-app erpnext".format(site_name), cwd=bench_path)
out = subprocess.check_output(["bench", "--site", site_name, "list-apps"], cwd=bench_path)
self.assertTrue("erpnext" in out)
app_installed_on_site = subprocess.check_output(["bench", "--site", site_name, "list-apps"], cwd=bench_path).decode('utf8')
self.assertTrue("erpnext" in app_installed_on_site)
def test_remove_app(self):
self.init_bench('test-bench')
self.init_bench("test-bench")
bench_path = os.path.join(self.benches_path, "test-bench")
# get app
bench.app.get_app("https://github.com/frappe/erpnext", "develop", bench_path=bench_path)
self.assertTrue(os.path.exists(os.path.join(bench_path, "apps", "erpnext")))
# remove it
bench.app.remove_app("erpnext", bench_path=bench_path)
bench.utils.exec_cmd("bench setup requirements --node", cwd=bench_path)
bench.utils.exec_cmd("bench get-app erpnext --branch version-12 --skip-assets --overwrite", cwd=bench_path)
bench.utils.exec_cmd("bench remove-app erpnext", cwd=bench_path)
with open(os.path.join(bench_path, "sites", "apps.txt")) as f:
self.assertFalse("erpnext" in f.read())
self.assertFalse("erpnext" in subprocess.check_output(["bench", "pip", "freeze"], cwd=bench_path).decode('utf8'))
self.assertFalse(os.path.exists(os.path.join(bench_path, "apps", "erpnext")))
def test_switch_to_branch(self):
self.init_bench('test-bench')
self.init_bench("test-bench")
bench_path = os.path.join(self.benches_path, "test-bench")
app_path = os.path.join(bench_path, "apps", "frappe")
bench.app.switch_branch(branch="master", apps=["frappe"], bench_path=bench_path, check_upgrade=False)
out = subprocess.check_output(['git', 'status'], cwd=app_path)
self.assertTrue("master" in out)
bench.utils.exec_cmd("bench switch-to-branch version-12 frappe", cwd=bench_path)
app_branch_after_switch = str(git.Repo(path=app_path).active_branch)
self.assertEqual("version-12", app_branch_after_switch)
# bring it back to develop!
bench.app.switch_branch(branch="develop", apps=["frappe"], bench_path=bench_path, check_upgrade=False)
out = subprocess.check_output(['git', 'status'], cwd=app_path)
self.assertTrue("develop" in out)
bench.utils.exec_cmd("bench switch-to-branch develop frappe", cwd=bench_path)
app_branch_after_second_switch = str(git.Repo(path=app_path).active_branch)
self.assertEqual("develop", app_branch_after_second_switch)
def init_bench(self, bench_name, **kwargs):
self.benches.append(bench_name)
bench.utils.init(bench_name, **kwargs)
def test_drop_site(self):
self.init_bench('test-bench')
# Check without archive_path given to drop-site command
self.drop_site("test-drop-without-archive-path")
# Check with archive_path given to drop-site command
home = os.path.abspath(os.path.expanduser('~'))
archived_sites_path = os.path.join(home, 'archived_sites')
self.drop_site("test-drop-with-archive-path", archived_sites_path=archived_sites_path)
def drop_site(self, site_name, archived_sites_path=None):
self.new_site(site_name)
drop_site_cmd = ['bench', 'drop-site', site_name]
if archived_sites_path:
drop_site_cmd.extend(['--archived-sites-path', archived_sites_path])
if os.environ.get('CI'):
drop_site_cmd.extend(['--root-password', 'travis'])
bench_path = os.path.join(self.benches_path, 'test-bench')
try:
subprocess.check_output(drop_site_cmd, cwd=bench_path)
except subprocess.CalledProcessError as err:
print(err.output)
if not archived_sites_path:
archived_sites_path = os.path.join(bench_path, 'archived_sites')
self.assertTrue(os.path.exists(archived_sites_path))
self.assertTrue(os.path.exists(os.path.join(archived_sites_path, site_name)))
else:
self.assertTrue(os.path.exists(archived_sites_path))
self.assertTrue(os.path.exists(os.path.join(archived_sites_path, site_name)))
def assert_folders(self, bench_name):
for folder in bench.utils.folders_in_bench:
self.assert_exists(bench_name, folder)
self.assert_exists(bench_name, "sites", "assets")
self.assert_exists(bench_name, "apps", "frappe")
self.assert_exists(bench_name, "apps", "frappe", "setup.py")
def assert_virtual_env(self, bench_name):
bench_path = os.path.abspath(bench_name)
python = os.path.join(bench_path, "env", "bin", "python")
python_path = bench.utils.get_cmd_output('{python} -c "import os; print os.path.dirname(os.__file__)"'.format(python=python))
# part of bench's virtualenv
self.assertTrue(python_path.startswith(bench_path))
self.assert_exists(python_path)
self.assert_exists(python_path, "site-packages")
self.assert_exists(python_path, "site-packages", "IPython")
self.assert_exists(python_path, "site-packages", "pip")
site_packages = os.listdir(os.path.join(python_path, "site-packages"))
# removing test case temporarily
# as develop and master branch havin differnt version of mysqlclient
#self.assertTrue(any(package.startswith("mysqlclient-1.3.12") for package in site_packages))
def assert_config(self, bench_name):
for config, search_key in (
("redis_queue.conf", "redis_queue.rdb"),
("redis_socketio.conf", "redis_socketio.rdb"),
("redis_cache.conf", "redis_cache.rdb")):
self.assert_exists(bench_name, "config", config)
with open(os.path.join(bench_name, "config", config), "r") as f:
f = f.read().decode("utf-8")
self.assertTrue(search_key in f)
def assert_socketio(self, bench_name):
try: # for v10 and under
self.assert_exists(bench_name, "node_modules")
self.assert_exists(bench_name, "node_modules", "socket.io")
except: # for v11 and above
self.assert_exists(bench_name, "apps", "frappe", "node_modules")
self.assert_exists(bench_name, "apps", "frappe", "node_modules", "socket.io")
def assert_common_site_config(self, bench_name, expected_config):
common_site_config_path = os.path.join(bench_name, 'sites', 'common_site_config.json')
self.assertTrue(os.path.exists(common_site_config_path))
config = self.load_json(common_site_config_path)
for key, value in list(expected_config.items()):
self.assertEqual(config.get(key), value)
def assert_exists(self, *args):
self.assertTrue(os.path.exists(os.path.join(*args)))
def load_json(self, path):
with open(path, "r") as f:
return json.loads(f.read().decode("utf-8"))
if __name__ == '__main__':
unittest.main()

92
bench/tests/test_setup_production.py
View File

@ -1,69 +1,51 @@
from bench.tests import test_init
from bench.config.production_setup import setup_production, get_supervisor_confdir, disable_production
import bench.utils
import os
# imports - standard imports
import getpass
import os
import re
import unittest
import subprocess
import time
import unittest
class TestSetupProduction(test_init.TestBenchInit):
# setUp, tearDown and other tests are defiend in TestBenchInit
# imports - module imports
import bench.utils
from bench.config.production_setup import get_supervisor_confdir
from bench.tests.test_base import TestBenchBase
class TestSetupProduction(TestBenchBase):
def test_setup_production(self):
self.test_multiple_benches()
user = getpass.getuser()
for bench_name in ("test-bench-1", "test-bench-2"):
bench_path = os.path.join(os.path.abspath(self.benches_path), bench_name)
setup_production(user, bench_path)
self.init_bench(bench_name)
bench.utils.exec_cmd("sudo bench setup production {0}".format(user), cwd=bench_path)
self.assert_nginx_config(bench_name)
self.assert_supervisor_config(bench_name)
# test after start of both benches
for bench_name in ("test-bench-1", "test-bench-2"):
self.assert_supervisor_process(bench_name)
self.assert_nginx_process()
# sudoers
bench.utils.setup_sudoers(user)
bench.utils.exec_cmd("sudo bench setup sudoers {0}".format(user))
self.assert_sudoers(user)
for bench_name in ("test-bench-1", "test-bench-2"):
for bench_name in self.benches:
bench_path = os.path.join(os.path.abspath(self.benches_path), bench_name)
disable_production(bench_path)
bench.utils.exec_cmd("sudo bench disable-production", cwd=bench_path)
def test_disable_production(self):
bench_name = 'test-disable-prod'
self.test_init(bench_name, frappe_branch='master')
user = getpass.getuser()
bench_path = os.path.join(os.path.abspath(self.benches_path), bench_name)
setup_production(user, bench_path)
disable_production(bench_path)
self.assert_nginx_link(bench_name)
self.assert_supervisor_link(bench_name)
self.assert_supervisor_process(bench_name=bench_name, disable_production=True)
def assert_nginx_config(self, bench_name):
conf_src = os.path.join(os.path.abspath(self.benches_path), bench_name, 'config', 'nginx.conf')
conf_dest = "/etc/nginx/conf.d/{bench_name}.conf".format(bench_name=bench_name)
self.assertTrue(os.path.exists(conf_src))
self.assertTrue(os.path.exists(conf_dest))
self.assertTrue(self.file_exists(conf_src))
self.assertTrue(self.file_exists(conf_dest))
# symlink matches
self.assertEqual(os.path.realpath(conf_dest), conf_src)
# file content
with open(conf_src, "r") as f:
f = f.read().decode("utf-8")
f = f.read()
for key in (
"upstream {bench_name}-frappe",
@ -71,48 +53,56 @@ class TestSetupProduction(test_init.TestBenchInit):
):
self.assertTrue(key.format(bench_name=bench_name) in f)
def assert_nginx_process(self):
out = bench.utils.get_cmd_output("sudo nginx -t 2>&1")
self.assertTrue("nginx: configuration file /etc/nginx/nginx.conf test is successful" in out)
def assert_sudoers(self, user):
sudoers_file = '/etc/sudoers.d/frappe'
self.assertTrue(os.path.exists(sudoers_file))
self.assertTrue(self.file_exists(sudoers_file))
with open(sudoers_file, 'r') as f:
sudoers = f.read().decode('utf-8')
if os.environ.get("CI"):
sudoers = subprocess.check_output(["sudo", "cat", sudoers_file]).decode("utf-8")
else:
with open(sudoers_file, 'r') as f:
sudoers = f.read()
self.assertTrue('{user} ALL = (root) NOPASSWD: /usr/sbin/service nginx *'.format(user=user) in sudoers)
self.assertTrue('{user} ALL = (root) NOPASSWD: /usr/bin/supervisorctl'.format(user=user) in sudoers)
self.assertTrue('{user} ALL = (root) NOPASSWD: /usr/sbin/nginx'.format(user=user) in sudoers)
def assert_supervisor_config(self, bench_name, use_rq=True):
conf_src = os.path.join(os.path.abspath(self.benches_path), bench_name, 'config', 'supervisor.conf')
supervisor_conf_dir = get_supervisor_confdir()
conf_dest = "{supervisor_conf_dir}/{bench_name}.conf".format(supervisor_conf_dir=supervisor_conf_dir, bench_name=bench_name)
self.assertTrue(os.path.exists(conf_src))
self.assertTrue(os.path.exists(conf_dest))
self.assertTrue(self.file_exists(conf_src))
self.assertTrue(self.file_exists(conf_dest))
# symlink matches
self.assertEqual(os.path.realpath(conf_dest), conf_src)
# file content
with open(conf_src, "r") as f:
f = f.read().decode("utf-8")
f = f.read()
tests = [
"program:{bench_name}-frappe-web",
"program:{bench_name}-redis-cache",
"program:{bench_name}-redis-queue",
"program:{bench_name}-redis-socketio",
"program:{bench_name}-node-socketio",
"group:{bench_name}-web",
"group:{bench_name}-workers",
"group:{bench_name}-redis"
]
if not os.environ.get("CI"):
tests.append("program:{bench_name}-node-socketio")
if use_rq:
tests.extend([
"program:{bench_name}-frappe-schedule",
@ -130,8 +120,11 @@ class TestSetupProduction(test_init.TestBenchInit):
])
for key in tests:
if key.format(bench_name=bench_name) not in f:
print(key.format(bench_name=bench_name))
self.assertTrue(key.format(bench_name=bench_name) in f)
def assert_supervisor_process(self, bench_name, use_rq=True, disable_production=False):
out = bench.utils.get_cmd_output("sudo supervisorctl status")
@ -172,15 +165,6 @@ class TestSetupProduction(test_init.TestBenchInit):
else:
self.assertTrue(re.search(key.format(bench_name=bench_name), out))
def assert_nginx_link(self, bench_name):
nginx_conf_name = '{bench_name}.conf'.format(bench_name=bench_name)
nginx_conf_path = os.path.join('/etc/nginx/conf.d', nginx_conf_name)
self.assertFalse(os.path.islink(nginx_conf_path))
def assert_supervisor_link(self, bench_name):
supervisor_conf_dir = get_supervisor_confdir()
supervisor_conf_name = '{bench_name}.conf'.format(bench_name=bench_name)
supervisor_conf_path = os.path.join(supervisor_conf_dir, supervisor_conf_name)
self.assertFalse(os.path.islink(supervisor_conf_path))
if __name__ == '__main__':
unittest.main()

355
bench/utils.py
View File

@ -1,12 +1,31 @@
import errno, glob, grp, itertools, json, logging, multiprocessing, os, platform, pwd, re, select, shutil, site, subprocess, sys
# imports - standard imports
import errno
import glob
import grp
import itertools
import json
import logging
import multiprocessing
import os
import platform
import pwd
import re
import select
import shutil
import site
import subprocess
import sys
from datetime import datetime
from distutils.spawn import find_executable
# imports - third party imports
import click
import requests
import semantic_version
from six import iteritems
from six.moves.urllib.parse import urlparse
# imports - module imports
import bench
from bench import env
class PatchError(Exception):
@ -26,6 +45,7 @@ class color:
green = '\033[92m'
yellow = '\033[93m'
red = '\033[91m'
silver = '\033[90m'
def is_bench_directory(directory=os.path.curdir):
@ -58,6 +78,7 @@ def safe_decode(string, encoding = 'utf-8'):
pass
return string
def get_frappe(bench_path='.'):
frappe = get_env_cmd('frappe', bench_path=bench_path)
if not os.path.exists(frappe):
@ -65,12 +86,16 @@ def get_frappe(bench_path='.'):
print('bench get-app https://github.com/frappe/frappe.git')
return frappe
def get_env_cmd(cmd, bench_path='.'):
return os.path.abspath(os.path.join(bench_path, 'env', 'bin', cmd))
def init(path, apps_path=None, no_procfile=False, no_backups=False, no_auto_update=False,
frappe_path=None, frappe_branch=None, wheel_cache_dir=None, verbose=False, clone_from=None,
skip_redis_config_generation=False, clone_without_update=False, ignore_exist = False, skip_assets=False, python='python3'):
def init(path, apps_path=None, no_procfile=False, no_backups=False,
frappe_path=None, frappe_branch=None, verbose=False, clone_from=None,
skip_redis_config_generation=False, clone_without_update=False, ignore_exist=False, skip_assets=False,
python='python3'):
"""Initialize a new bench directory"""
from bench.app import get_app, install_apps_from_path
from bench.config import redis
from bench.config.common_site_config import make_config
@ -122,17 +147,85 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, no_auto_upda
redis.generate_config(path)
if not no_procfile:
setup_procfile(path)
setup_procfile(path, skip_redis=skip_redis_config_generation)
if not no_backups:
setup_backups(bench_path=path)
if not no_auto_update:
setup_auto_update(bench_path=path)
copy_patches_txt(path)
def update(pull=False, patch=False, build=False, requirements=False, backup=True, force=False, reset=False,
restart_supervisor=False, restart_systemd=False):
"""command: bench update"""
from bench import patches
from bench.app import is_version_upgrade, pull_all_apps, validate_branch
from bench.config.common_site_config import get_config, update_config
bench_path = os.path.abspath(".")
patches.run(bench_path=bench_path)
conf = get_config(bench_path)
if conf.get('release_bench'):
print('Release bench detected, cannot update!')
sys.exit(1)
if not (pull or patch or build or requirements):
pull, patch, build, requirements = True, True, True, True
validate_branch()
version_upgrade = is_version_upgrade()
if version_upgrade[0]:
if force:
print("Force flag has been used for a major version change in Frappe and it's apps. \nThis will take significant time to migrate and might break custom apps.")
else:
print("This update will cause a major version change in Frappe/ERPNext from {0} to {1}. \nThis would take significant time to migrate and might break custom apps.".format(*version_upgrade[1:]))
click.confirm('Do you want to continue?', abort=True)
if version_upgrade[0] or (not version_upgrade[0] and force):
validate_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path)
conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 })
update_config(conf, bench_path=bench_path)
if backup:
print('Backing up sites...')
backup_all_sites(bench_path=bench_path)
if pull:
pull_all_apps(bench_path=bench_path, reset=reset)
if requirements:
update_requirements(bench_path=bench_path)
update_node_packages(bench_path=bench_path)
if patch:
print('Patching sites...')
patch_sites(bench_path=bench_path)
if build:
build_assets(bench_path=bench_path)
if version_upgrade[0] or (not version_upgrade[0] and force):
post_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path)
if restart_supervisor or conf.get('restart_supervisor_on_update'):
restart_supervisor_processes(bench_path=bench_path)
if restart_systemd or conf.get('restart_systemd_on_update'):
restart_systemd_processes(bench_path=bench_path)
conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 })
update_config(conf, bench_path=bench_path)
print("_" * 80 + "\nBench: Deployment tool for Frappe and Frappe Applications (https://frappe.io/bench).\nOpen source depends on your contributions, so please contribute bug reports, patches, fixes or cash and be a part of the community")
def copy_patches_txt(bench_path):
shutil.copy(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'patches', 'patches.txt'),
os.path.join(bench_path, 'patches.txt'))
def clone_apps_from(bench_path, clone_from, update_app=True):
from .app import install_app
print('Copying apps from {0}...'.format(clone_from))
@ -169,30 +262,15 @@ def clone_apps_from(bench_path, clone_from, update_app=True):
for app in apps:
setup_app(app)
def exec_cmd(cmd, cwd='.'):
from .cli import from_command_line
import shlex
print("{0}$ {1}{2}".format(color.silver, cmd, color.nc))
cmd = shlex.split(cmd)
subprocess.call(cmd, cwd=cwd, universal_newlines=True)
is_async = False if from_command_line else True
if is_async:
stderr = stdout = subprocess.PIPE
else:
stderr = stdout = None
logger.info(cmd)
p = subprocess.Popen(cmd, cwd=cwd, shell=True, stdout=stdout, stderr=stderr,
universal_newlines=True)
if is_async:
return_code = print_output(p)
else:
return_code = p.wait()
if return_code > 0:
raise CommandFailedError(cmd)
def which(executable, raise_err = False):
from distutils.spawn import find_executable
exec_ = find_executable(executable)
if not exec_ and raise_err:
@ -202,18 +280,33 @@ def which(executable, raise_err = False):
return exec_
def setup_env(bench_path='.', python = 'python3'):
python = which(python, raise_err = True)
pip = os.path.join('env', 'bin', 'pip')
exec_cmd('virtualenv -q {} -p {}'.format('env', python), cwd=bench_path)
exec_cmd('{} -q install -U pip==19.3.1 wheel six'.format(pip), cwd=bench_path)
exec_cmd('{} -q install -e git+https://github.com/frappe/python-pdfkit.git#egg=pdfkit'.format(pip), cwd=bench_path)
def get_venv_path():
venv = which('virtualenv')
if not venv:
current_python = sys.executable
with open(os.devnull, "wb") as devnull:
is_venv_installed = not subprocess.call([current_python, "-m", "venv", "--help"], stdout=devnull)
if is_venv_installed:
venv = "{} -m venv".format(current_python)
return venv or log("virtualenv cannot be found", level=2)
def setup_env(bench_path='.', python='python3'):
frappe = os.path.join(bench_path, "apps", "frappe")
pip = os.path.join(".", "env", "bin", "pip")
virtualenv = get_venv_path()
exec_cmd('{} -q env -p {}'.format(virtualenv, python), cwd=bench_path)
exec_cmd('{} install -q -U -e {}'.format(pip, frappe), cwd=bench_path)
def setup_socketio(bench_path='.'):
exec_cmd("npm install socket.io redis express superagent cookie babel-core less chokidar \
babel-cli babel-preset-es2015 babel-preset-es2016 babel-preset-es2017 babel-preset-babili", cwd=bench_path)
def patch_sites(bench_path='.'):
bench.set_frappe_version(bench_path=bench_path)
@ -225,6 +318,7 @@ def patch_sites(bench_path='.'):
except subprocess.CalledProcessError:
raise PatchError
def build_assets(bench_path='.', app=None):
bench.set_frappe_version(bench_path=bench_path)
@ -236,19 +330,16 @@ def build_assets(bench_path='.', app=None):
command += ' --app {}'.format(app)
exec_cmd(command, cwd=bench_path)
def get_sites(bench_path='.'):
sites_path = os.path.join(bench_path, 'sites')
sites = (site for site in os.listdir(sites_path) if os.path.exists(os.path.join(sites_path, site, 'site_config.json')))
return sites
def get_bench_dir(bench_path='.'):
return os.path.abspath(bench_path)
def setup_auto_update(bench_path='.'):
logger.info('setting up auto update')
add_to_crontab('0 10 * * * cd {bench_dir} && {bench} update --auto >> {logfile} 2>&1'.format(bench_dir=get_bench_dir(bench_path=bench_path),
bench=os.path.join(get_bench_dir(bench_path=bench_path), 'env', 'bin', 'bench'),
logfile=os.path.join(get_bench_dir(bench_path=bench_path), 'logs', 'auto_update_log.log')))
def setup_backups(bench_path='.'):
logger.info('setting up backups')
@ -263,43 +354,30 @@ def setup_backups(bench_path='.'):
add_to_crontab('0 */6 * * * {backup_command} >> {logfile} 2>&1'.format(backup_command=backup_command,
logfile=os.path.join(get_bench_dir(bench_path=bench_path), 'logs', 'backup.log')))
def add_to_crontab(line):
current_crontab = read_crontab()
line = str.encode(line)
if not line in current_crontab:
cmd = ["crontab"]
if platform.system() == 'FreeBSD' or platform.linux_distribution()[0]=="arch":
if platform.system() == 'FreeBSD':
cmd = ["crontab", "-"]
s = subprocess.Popen(cmd, stdin=subprocess.PIPE)
s.stdin.write(current_crontab)
s.stdin.write(line + b'\n')
s.stdin.close()
def read_crontab():
s = subprocess.Popen(["crontab", "-l"], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
out = s.stdout.read()
s.stdout.close()
return out
def update_bench(bench_repo=True, requirements=True):
logger.info("Updating bench")
# bench-repo folder
cwd = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if bench_repo:
try:
exec_cmd("git pull", cwd=cwd)
except bench.utils.CommandFailedError:
exec_cmd("git -c user.name=bench -c user.email=developers@frappe.io stash", cwd=cwd)
logger.info("Stashing changes made at {}\nUse git stash apply to recover changes after the successful update!".format(cwd))
if requirements:
update_bench_requirements()
logger.info("Bench Updated!")
def setup_sudoers(user):
from bench import env
if not os.path.exists('/etc/sudoers.d'):
os.makedirs('/etc/sudoers.d')
@ -331,6 +409,7 @@ def setup_sudoers(user):
os.chmod(sudoers_file, 0o440)
def setup_logging(bench_path='.'):
if os.path.exists(os.path.join(bench_path, 'logs')):
logger = logging.getLogger('bench')
@ -341,6 +420,7 @@ def setup_logging(bench_path='.'):
logger.addHandler(hdlr)
logger.setLevel(logging.DEBUG)
def get_program(programs):
program = None
for p in programs:
@ -349,9 +429,11 @@ def get_program(programs):
break
return program
def get_process_manager():
return get_program(['foreman', 'forego', 'honcho'])
def start(no_dev=False, concurrency=None, procfile=None):
program = get_process_manager()
if not program:
@ -369,6 +451,7 @@ def start(no_dev=False, concurrency=None, procfile=None):
os.execv(program, command)
def check_cmd(cmd, cwd='.'):
try:
subprocess.check_call(cmd, cwd=cwd, shell=True)
@ -376,6 +459,7 @@ def check_cmd(cmd, cwd='.'):
except subprocess.CalledProcessError:
return False
def get_git_version():
'''returns git version from `git --version`
extracts version number from string `get version 1.9.1` etc'''
@ -385,6 +469,7 @@ def get_git_version():
version = '.'.join(version.split('.')[0:2])
return float(version)
def check_git_for_shallow_clone():
from .config.common_site_config import get_config
config = get_config('.')
@ -400,6 +485,7 @@ def check_git_for_shallow_clone():
if git_version > 1.9:
return True
def get_cmd_output(cmd, cwd='.'):
try:
output = subprocess.check_output(cmd, cwd=cwd, shell=True, stderr=subprocess.PIPE).strip()
@ -410,6 +496,7 @@ def get_cmd_output(cmd, cwd='.'):
print(e.output)
raise
def safe_encode(what, encoding = 'utf-8'):
try:
what = what.encode(encoding)
@ -418,6 +505,7 @@ def safe_encode(what, encoding = 'utf-8'):
return what
def restart_supervisor_processes(bench_path='.', web_workers=False):
from .config.common_site_config import get_config
conf = get_config(bench_path=bench_path)
@ -447,26 +535,26 @@ def restart_supervisor_processes(bench_path='.', web_workers=False):
exec_cmd('sudo supervisorctl restart {group}'.format(group=group), cwd=bench_path)
def restart_systemd_processes(bench_path='.', web_workers=False):
from .config.common_site_config import get_config
bench_name = get_bench_name(bench_path)
exec_cmd('sudo systemctl stop -- $(systemctl show -p Requires {bench_name}.target | cut -d= -f2)'.format(bench_name=bench_name))
exec_cmd('sudo systemctl start -- $(systemctl show -p Requires {bench_name}.target | cut -d= -f2)'.format(bench_name=bench_name))
def set_default_site(site, bench_path='.'):
if site not in get_sites(bench_path=bench_path):
raise Exception("Site not in bench")
exec_cmd("{frappe} --use {site}".format(frappe=get_frappe(bench_path=bench_path), site=site),
cwd=os.path.join(bench_path, 'sites'))
def update_bench_requirements():
bench_req_file = os.path.join(os.path.dirname(bench.__path__[0]), 'requirements.txt')
install_requirements(bench_req_file, user=True)
def update_env_pip(bench_path):
env_pip = os.path.join(bench_path, 'env', 'bin', 'pip')
exec_cmd("{pip} install -q -U pip".format(pip=env_pip))
def update_requirements(bench_path='.'):
from bench.app import get_apps, install_app
print('Updating Python libraries...')
@ -474,11 +562,9 @@ def update_requirements(bench_path='.'):
# update env pip
update_env_pip(bench_path)
# Update bench requirements (at user level)
update_bench_requirements()
for app in get_apps():
install_app(app, bench_path=bench_path)
install_app(app, bench_path=bench_path, skip_assets=True)
def update_node_packages(bench_path='.'):
print('Updating node packages...')
@ -486,7 +572,6 @@ def update_node_packages(bench_path='.'):
from distutils.version import LooseVersion
v = LooseVersion(get_develop_version('frappe', bench_path = bench_path))
# After rollup was merged, frappe_version = 10.1
# if develop_verion is 11 and up, only then install yarn
if v < LooseVersion('11.x.x-develop'):
@ -494,6 +579,7 @@ def update_node_packages(bench_path='.'):
else:
update_yarn_packages(bench_path)
def update_yarn_packages(bench_path='.'):
apps_dir = os.path.join(bench_path, 'apps')
@ -555,6 +641,7 @@ def install_requirements(req_file, user=False):
exec_cmd("{python} -m pip install {user_flag} -q -U -r {req_file}".format(python=python, user_flag=user_flag, req_file=req_file))
def backup_site(site, bench_path='.'):
bench.set_frappe_version(bench_path=bench_path)
@ -564,30 +651,38 @@ def backup_site(site, bench_path='.'):
else:
run_frappe_cmd('--site', site, 'backup', bench_path=bench_path)
def backup_all_sites(bench_path='.'):
for site in get_sites(bench_path=bench_path):
backup_site(site, bench_path=bench_path)
def is_root():
if os.getuid() == 0:
return True
return False
def set_mariadb_host(host, bench_path='.'):
update_common_site_config({'db_host': host}, bench_path=bench_path)
def set_redis_cache_host(host, bench_path='.'):
update_common_site_config({'redis_cache': "redis://{}".format(host)}, bench_path=bench_path)
def set_redis_queue_host(host, bench_path='.'):
update_common_site_config({'redis_queue': "redis://{}".format(host)}, bench_path=bench_path)
def set_redis_socketio_host(host, bench_path='.'):
update_common_site_config({'redis_socketio': "redis://{}".format(host)}, bench_path=bench_path)
def update_common_site_config(ddict, bench_path='.'):
update_json_file(os.path.join(bench_path, 'sites', 'common_site_config.json'), ddict)
def update_json_file(filename, ddict):
if os.path.exists(filename):
with open(filename, 'r') as f:
@ -600,6 +695,7 @@ def update_json_file(filename, ddict):
with open(filename, 'w') as f:
json.dump(content, f, indent=1, sort_keys=True)
def drop_privileges(uid_name='nobody', gid_name='nogroup'):
# from http://stackoverflow.com/a/2699996
if os.getuid() != 0:
@ -620,6 +716,7 @@ def drop_privileges(uid_name='nobody', gid_name='nogroup'):
# Ensure a very conservative umask
os.umask(0o22)
def fix_prod_setup_perms(bench_path='.', frappe_user=None):
from .config.common_site_config import get_config
@ -637,6 +734,7 @@ def fix_prod_setup_perms(bench_path='.', frappe_user=None):
gid = grp.getgrnam(frappe_user).gr_gid
os.chown(path, uid, gid)
def fix_file_perms():
for dir_path, dirs, files in os.walk('.'):
for _dir in dirs:
@ -649,10 +747,12 @@ def fix_file_perms():
if not _file.startswith('activate'):
os.chmod(os.path.join(bin_dir, _file), 0o755)
def get_current_frappe_version(bench_path='.'):
from .app import get_current_frappe_version as fv
return fv(bench_path=bench_path)
def run_frappe_cmd(*args, **kwargs):
from .cli import from_command_line
@ -676,7 +776,7 @@ def run_frappe_cmd(*args, **kwargs):
if return_code > 0:
sys.exit(return_code)
#raise CommandFailedError(args)
def get_frappe_cmd_output(*args, **kwargs):
bench_path = kwargs.get('bench_path', '.')
@ -684,24 +784,12 @@ def get_frappe_cmd_output(*args, **kwargs):
sites_dir = os.path.join(bench_path, 'sites')
return subprocess.check_output((f, '-m', 'frappe.utils.bench_helper', 'frappe') + args, cwd=sites_dir)
def validate_upgrade(from_ver, to_ver, bench_path='.'):
if to_ver >= 6:
if not find_executable('npm') and not (find_executable('node') or find_executable('nodejs')):
raise Exception("Please install nodejs and npm")
def pre_upgrade(from_ver, to_ver, bench_path='.'):
pip = os.path.join(bench_path, 'env', 'bin', 'pip')
if from_ver <= 4 and to_ver >= 5:
from .migrate_to_v5 import remove_shopping_cart
apps = ('frappe', 'erpnext')
remove_shopping_cart(bench_path=bench_path)
for app in apps:
cwd = os.path.abspath(os.path.join(bench_path, 'apps', app))
if os.path.exists(cwd):
exec_cmd("git clean -dxf", cwd=cwd)
exec_cmd("{pip} install --upgrade -e {app}".format(pip=pip, app=cwd))
def post_upgrade(from_ver, to_ver, bench_path='.'):
from .config.common_site_config import get_config
@ -709,8 +797,7 @@ def post_upgrade(from_ver, to_ver, bench_path='.'):
from .config.supervisor import generate_supervisor_config
from .config.nginx import make_nginx_conf
conf = get_config(bench_path=bench_path)
print("-"*80)
print("Your bench was upgraded to version {0}".format(to_ver))
print("-" * 80 + "Your bench was upgraded to version {0}".format(to_ver))
if conf.get('restart_supervisor_on_update'):
redis.generate_config(bench_path=bench_path)
@ -814,38 +901,6 @@ def get_output(*cmd):
return out
def before_update(bench_path, requirements):
validate_pillow_dependencies(bench_path, requirements)
def validate_pillow_dependencies(bench_path, requirements):
if not requirements:
return
try:
pip = os.path.join(bench_path, 'env', 'bin', 'pip')
exec_cmd("{pip} install Pillow".format(pip=pip))
except CommandFailedError:
distro = platform.linux_distribution()
distro_name = distro[0].lower()
if "centos" in distro_name or "fedora" in distro_name:
print("Please install these dependencies using the command:")
print("sudo yum install libtiff-devel libjpeg-devel libzip-devel freetype-devel lcms2-devel libwebp-devel tcl-devel tk-devel")
raise
elif "ubuntu" in distro_name or "elementary os" in distro_name or "debian" in distro_name:
print("Please install these dependencies using the command:")
if "ubuntu" in distro_name and distro[1]=="12.04":
print("sudo apt-get install -y libtiff4-dev libjpeg8-dev zlib1g-dev libfreetype6-dev liblcms2-dev libwebp-dev tcl8.5-dev tk8.5-dev python-tk")
else:
print("sudo apt-get install -y libtiff5-dev libjpeg8-dev zlib1g-dev libfreetype6-dev liblcms2-dev libwebp-dev tcl8.6-dev tk8.6-dev python-tk")
raise
def get_bench_name(bench_path):
return os.path.basename(os.path.abspath(bench_path))
@ -985,3 +1040,75 @@ def in_virtual_env():
return _no_global_under_venv()
return False
def migrate_env(python, backup=False):
from bench.config.common_site_config import get_config
from bench.app import get_apps
log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
nvenv = 'env'
path = os.getcwd()
python = which(python)
virtualenv = which('virtualenv')
pvenv = os.path.join(path, nvenv)
pip = os.path.join(pvenv, 'bin', 'pip')
# Clear Cache before Bench Dies.
try:
config = get_config(bench_path=os.getcwd())
rredis = urlparse(config['redis_cache'])
redis = '{redis} -p {port}'.format(redis=which('redis-cli'), port=rredis.port)
log.debug('Clearing Redis Cache...')
exec_cmd('{redis} FLUSHALL'.format(redis = redis))
log.debug('Clearing Redis DataBase...')
exec_cmd('{redis} FLUSHDB'.format(redis = redis))
except:
log.warn('Please ensure Redis Connections are running or Daemonized.')
# Backup venv: restore using `virtualenv --relocatable` if needed
if backup:
parch = os.path.join(path, 'archived_envs')
if not os.path.exists(parch):
os.mkdir(parch)
source = os.path.join(path, 'env')
target = parch
log.debug('Backing up Virtual Environment')
stamp = datetime.now().strftime('%Y%m%d_%H%M%S')
dest = os.path.join(path, str(stamp))
os.rename(source, dest)
shutil.move(dest, target)
# Create virtualenv using specified python
try:
log.debug('Setting up a New Virtual {} Environment'.format(python))
exec_cmd('{virtualenv} --python {python} {pvenv}'.format(virtualenv=virtualenv, python=python, pvenv=pvenv))
apps = ' '.join(["-e {}".format(os.path.join("apps", app)) for app in get_apps()])
exec_cmd('{0} install -q -U {1}'.format(pip, apps))
log.debug('Migration Successful to {}'.format(python))
except:
log.debug('Migration Error')
raise
def find_parent_bench(path):
"""Checks if parent directories are benches"""
if is_bench_directory(directory=path):
return path
home_path = os.path.expanduser("~")
root_path = os.path.abspath(os.sep)
if path not in {home_path, root_path}:
# NOTE: the os.path.split assumes that given path is absolute
parent_dir = os.path.split(path)[0]
return find_parent_bench(parent_dir)

48
docs/bench_custom_cmd.md Normal file
View File

@ -0,0 +1,48 @@
## How are Frappe Framework commands available via bench?
bench utilizes `frappe.utils.bench_manager` to get the framework's as well as those of any custom commands written in application installed in the Frappe environment. Currently, with *version 12* there are commands related to the scheduler, sites, translations and other utils in Frappe inherited by bench.
## Can I add CLI commands in my custom app and call them via bench?
Along with the framework commands, Frappe's `bench_manager` module also searches for any commands in your custom applications. Thereby, bench communicates with the respective bench's Frappe which in turn checks for available commands in all of the applications.
To make your custom command available to bench, just create a `commands` module under your parent module and write the command with a click wrapper and a variable commands which contains a list of click functions, which are your own commands. The directory strcuture may be visualized as:
```
frappe-bench
|──apps
|── frappe
├── custom_app
│   ├── README.md
│   ├── custom_app
│   │   ├── commands <------ commands module
│   ├── license.txt
│   ├── requirements.txt
│   └── setup.py
```
The commands module maybe a single file such as `commands.py` or a directory with an `__init__.py` file. For a custom application of name 'flags', example may be given as
```python
# file_path: frappe-bench/apps/flags/flags/commands.py
import click
@click.command('set-flags')
@click.argument('state', type=click.Choice(['on', 'off']))
def set_flags(state):
from flags.utils import set_flags
set_flags(state=state)
commands = [
set_flags
]
```
and with context of the current bench, this command maybe executed simply as
```zsh
➜ bench set-flags
Flags are set to state: 'on'
```

201
docs/bench_usage.md Normal file
View File

@ -0,0 +1,201 @@
# bench CLI Usage
This may not be known to a lot of people but half the bench commands we're used to, exist in the Frappe Framework and not in bench directly. Those commands generally are the `--site` commands. This page is concerned only with the commands in the bench project. Any framework commands won't be a part of this consolidation.
# bench CLI Commands
Under Click's structure, `bench` is the main command group, under which there are three main groups of commands in bench currently, namely
- **install**: The install command group deals with commands used to install system dependencies for setting up Frappe environment
- **setup**: This command group for consists of commands used to maipulate the requirements and environments required by your Frappe environment
- **config**: The config command group deals with making changes in the current bench (not the CLI tool) configuration
## Using the bench command line
```zsh
➜ bench
Usage: bench [OPTIONS] COMMAND [ARGS]...
Bench manager for Frappe
Options:
--version
--help Show this message and exit.
Commands:
backup Backup single site
backup-all-sites Backup all sites in current bench
config Change bench configuration
disable-production Disables production environment for the bench.
download-translations Download latest translations
exclude-app Exclude app from updating
find Finds benches recursively from location
get-app Clone an app from the internet or filesystem and...
```
Similarly, all available flags and options can be checked for commands individually by executing them with the `--help` flag. The `init` command for instance:
```zsh
➜ bench init --help
Usage: bench init [OPTIONS] PATH
Initialize a new bench instance in the specified path
Options:
--python TEXT Path to Python Executable.
--ignore-exist Ignore if Bench instance exists.
--apps_path TEXT path to json files with apps to install
after init
```
## bench and sudo
Some bench commands may require sudo, such as some `setup` commands and everything else under the `install` commands group. For these commands, you may not be asked for your root password if sudoers setup has been done. The security implications, well we'll talk about those soon.
## General Commands
These commands belong directly to the bench group so they can be invoked directly prefixing each with `bench` in your shell. Therefore, the usage for these commands is as
```zsh
bench COMMAND [ARGS]...
```
### The usual commands
- **init**: Initialize a new bench instance in the specified path. This sets up a complete bench folder with an `apps` folder which contains all the Frappe apps available in the current bench, `sites` folder that stores all site data seperated by individual site folders, `config` folder that contains your redis, NGINX and supervisor configuration files. The `env` folder consists of all python dependencies the current bench and installed Frappe applications have.
- **restart**: Restart web, supervisor, systemd processes units. Used in production setup.
- **update**: Updates bench tool and if executed in a bench directory, without any flags will backup, pull, setup requirements, build, run patches and restart bench. Using specific flags will only do certain tasks instead of all.
- **migrate-env**: Migrate Virtual Environment to desired Python version. This regenerates the `env` folder with the specified Python version.
- **retry-upgrade**: Retry a failed upgrade
- **disable-production**: Disables production environment for the bench.
- **renew-lets-encrypt**: Renew Let's Encrypt certificate for site SSL.
- **backup**: Backup single site data. Can be used to backup files as well.
- **backup-all-sites**: Backup all sites in current bench.
- **get-app**: Download an app from the internet or filesystem and set it up in your bench. This clones the git repo of the Frappe project and installs it in the bench environment.
- **remove-app**: Completely remove app from bench and re-build assets if not installed on any site.
- **exclude-app**: Exclude app from updating during a `bench update`
- **include-app**: Include app for updating. All Frappe applications are included by default when installed.
- **remote-set-url**: Set app remote url
- **remote-reset-url**: Reset app remote url to frappe official
- **remote-urls**: Show apps remote url
- **switch-to-branch**: Switch all apps to specified branch, or specify apps separated by space
- **switch-to-develop**: Switch Frappe and ERPNext to develop branch
### A little advanced
- **set-nginx-port**: Set NGINX port for site
- **set-ssl-certificate**: Set SSL certificate path for site
- **set-ssl-key**: Set SSL certificate private key path for site
- **set-url-root**: Set URL root for site
- **set-mariadb-host**: Set MariaDB host for bench
- **set-redis-cache-host**: Set Redis cache host for bench
- **set-redis-queue-host**: Set Redis queue host for bench
- **set-redis-socketio-host**: Set Redis socketio host for bench
- **set-default-site**: Set default site for bench
- **download-translations**: Download latest translations
### Developer's commands
- **start**: Start Frappe development processes. Uses the Procfile to start the Frappe development environment.
- **src**: Prints bench source folder path, which can be used to cd into the bench installation repository by `cd $(bench src)`.
- **find**: Finds benches recursively from location or specified path.
- **pip**: Use the current bench's pip to manage Python packages. For help about pip usage: `bench pip help [COMMAND]` or `bench pip [COMMAND] -h`.
- **new-app**: Create a new Frappe application under apps folder.
### Release bench
- **release**: Create a release of a Frappe application
- **prepare-beta-release**: Prepare major beta release from develop branch
## Setup commands
The setup commands used for setting up the Frappe environment in context of the current bench need to be executed using `bench setup` as the prefix. So, the general usage of these commands is as
```zsh
bench setup COMMAND [ARGS]...
```
- **sudoers**: Add commands to sudoers list for allowing bench commands execution without root password
- **env**: Setup virtualenv for bench. This sets up a `env` folder under the root of the bench directory.
- **redis**: Generates configuration for Redis
- **fonts**: Add Frappe fonts to system
- **config**: Generate or over-write sites/common_site_config.json
- **backups**: Add cronjob for bench backups
- **socketio**: Setup node dependencies for socketio server
- **requirements**: Setup Python and Node dependencies
- **manager**: Setup `bench-manager.local` site with the [Bench Manager](https://github.com/frappe/bench_manager) app, a GUI for bench installed on it.
- **procfile**: Generate Procfile for bench start
- **production**: Setup Frappe production environment for specific user. This installs ansible, NGINX, supervisor, fail2ban and generates the respective configuration files.
- **nginx**: Generate configuration files for NGINX
- **fail2ban**: Setup fail2ban, an intrusion prevention software framework that protects computer servers from brute-force attacks
- **systemd**: Generate configuration for systemd
- **firewall**: Setup firewall for system
- **ssh-port**: Set SSH Port for system
- **reload-nginx**: Checks NGINX config file and reloads service
- **supervisor**: Generate configuration for supervisor
- **lets-encrypt**: Setup lets-encrypt SSL for site
- **wildcard-ssl**: Setup wildcard SSL certificate for multi-tenant bench
- **add-domain**: Add a custom domain to a particular site
- **remove-domain**: Remove custom domain from a site
- **sync-domains**: Check if there is a change in domains. If yes, updates the domains list.
- **role**: Install dependencies via ansible roles
## Config commands
The config group commands are used for manipulating configurations in the current bench context. The usage for these commands is as
```zsh
bench config COMMAND [ARGS]...
```
- **set-common-config**: Set value in common config
- **remove-common-config**: Remove specific keys from current bench's common config
- **update_bench_on_update**: Enable/Disable bench updates on running bench update
- **restart_supervisor_on_update**: Enable/Disable auto restart of supervisor processes
- **restart_systemd_on_update**: Enable/Disable auto restart of systemd units
- **dns_multitenant**: Enable/Disable bench multitenancy on running bench update
- **serve_default_site**: Configure nginx to serve the default site on port 80
- **http_timeout**: Set HTTP timeout
## Install commands
The install group commands are used for manipulating system level dependencies. The usage for these commands is as
```zsh
bench install COMMAND [ARGS]...
```
- **prerequisites**: Installs pre-requisite libraries, essential tools like b2zip, htop, screen, vim, x11-fonts, python libs, cups and Redis
- **nodejs**: Installs Node.js v8
- **nginx**: Installs NGINX. If user is specified, sudoers is setup for that user
- **packer**: Installs Oracle virtualbox and packer 1.2.1
- **psutil**: Installs psutil via pip
- **mariadb**: Install and setup MariaDB of specified version and root password
- **wkhtmltopdf**: Installs wkhtmltopdf v0.12.3 for linux
- **supervisor**: Installs supervisor. If user is specified, sudoers is setup for that user
- **fail2ban**: Install fail2ban, an intrusion prevention software framework that protects computer servers from brute-force attacks
- **virtualbox**: Installs supervisor

6
docs/easy_install.md
View File

@ -1,7 +1,7 @@
# Easy Install Script
- This script will install the pre-requisites, install bench and setup an ERPNext site `(site1.local under frappe-bench)`
- Passwords for Frappe Administrator and MariaDB (root) will be asked and saved under `~/passwoords.txt`
- Passwords for Frappe Administrator and MariaDB (root) will be asked and saved under `~/passwords.txt`
- MariaDB (root) password may be `password` on a fresh server
- You can then login as **Administrator** with the Administrator password
- The log file is saved under `/tmp/logs/install_bench.log` in case you run into any issues during the install.
@ -45,7 +45,7 @@ Switch to `[frappe-user]` (using `su [frappe-user]`) and start the setup
*Note: `user` flag to create a user and install using that user (By default, the script will create a user with the username `frappe` if the --user flag is not used)*
For production or development, append teh `--production` or `--develop` flag to the command respectively.
For production or development, append the `--production` or `--develop` flag to the command respectively.
sudo python3 install.py --production --user [frappe-user]
@ -88,4 +88,4 @@ TLDR; Save the logs!
3. A lot of things can go wrong in setting up the environment due to prior settings, company protocols or even breaking changes in system packages and their dependencies.
4. Sharing your logfile in any issues opened related to this can help us find solutions to it faster and make the sript better!
4. Sharing your logfile in any issues opened related to this can help us find solutions to it faster and make the sript better!

29
playbooks/install.py
View File

@ -1,5 +1,16 @@
#!/usr/bin/env python3
import os, sys, subprocess, getpass, json, multiprocessing, shutil, platform, warnings, datetime
from __future__ import print_function
import os
import sys
import subprocess
import getpass
import json
import multiprocessing
import shutil
import platform
import warnings
import datetime
tmp_bench_repo = os.path.join('/', 'tmp', '.bench')
tmp_log_folder = os.path.join('/', 'tmp', 'logs')
@ -395,8 +406,20 @@ def parse_commandline_args():
if __name__ == '__main__':
if sys.version[0] == '2':
if not raw_input("It is recommended to run this script with Python 3\nDo you still wish to continue? [Y/n]: ").lower() == "y":
sys.exit()
if not os.environ.get('CI'):
if not raw_input("It is recommended to run this script with Python 3\nDo you still wish to continue? [Y/n]: ").lower() == "y":
sys.exit()
try:
from distutils.spawn import find_executable
except ImportError:
try:
subprocess.check_call('pip install --upgrade setuptools')
except subprocess.CalledProcessError:
print("Install distutils or use Python3 to run the script")
sys.exit(1)
shutil.which = find_executable
if not is_sudo_user():
log("Please run this script as a non-root user with sudo privileges", level=3)

84
playbooks/roles/wkhtmltopdf/tasks/main.yml
View File

@ -20,24 +20,76 @@
force: yes
when: ansible_os_family == 'Debian'
# wkhtmltopdf has been locked down to 0.12.3 intentionally since 0.12.4 has problems.
# I you want to upgrade try it on multiple platforms and then decide
- name: download wkthmltox linux
get_url: url=https://github.com/frappe/wkhtmltopdf/raw/master/wkhtmltox-0.12.3_linux-generic-{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.tar.xz dest=/tmp/wkhtmltox.tar.xz
- name: download wkthmltox Ubuntu 18
get_url:
url: https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.deb
dest: /tmp/wkhtmltox.deb
checksum: "sha256:{{ 'db48fa1a043309c4bfe8c8e0e38dc06c183f821599dd88d4e3cea47c5a5d4cd3' if ansible_architecture == 'x86_64' else '1f5ac84c1cb25e385b49b94a04807d60bf73da217bc6c9fe2cbd1f0a61d33f63' }}"
when: ansible_distribution == 'Ubuntu' and ansible_distribution_major_version == '18'
- name: Creates directory
file: path=/tmp/wkhtmltox state=directory
- name: download wkthmltox Ubuntu 16
get_url:
url: https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.xenial_{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.deb
dest: /tmp/wkhtmltox.deb
checksum: "sha256:{{ 'df203cee4dc9b3efb8d0cd6fc25fa819883224f50c75b76bd9c856903711dc14' if ansible_architecture == 'x86_64' else '27b6edafee099b87b2911cc68b780e79cffed3948bb5a074e8ea1cf8820da156' }}"
when: ansible_distribution == 'Ubuntu' and ansible_distribution_major_version == '16'
- name: unarchive wkhtmltopdf
unarchive: src=/tmp/wkhtmltox.tar.xz dest=/tmp
- name: download wkthmltox Ubuntu 14
get_url:
url: https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.trusty_{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.deb
dest: /tmp/wkhtmltox.deb
checksum: "sha256:{{ '2a3d1fe80da0dbc69da56cf90a3d0ec2786d1b919be29527630d609fea4a6b7c' if ansible_architecture == 'x86_64' else '582e02881e4bc6be9aaa634da1fe8c02d3233fb57f6daab9efa137edb812dd3b' }}"
when: ansible_distribution == 'Ubuntu' and ansible_distribution_major_version == '14'
- name: copy to /usr/local/bin
copy: src="/tmp/wkhtmltox/bin/wkhtmltopdf" dest="/usr/local/bin/wkhtmltopdf"
become: true
become_user: root
- name: download wkthmltox CentOS 6
get_url:
url: https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox-0.12.5-1.centos6.{{ "x86_64" if ansible_architecture == "x86_64" else "i686"}}.rpm
dest: /tmp/wkhtmltox.rpm
checksum: "sha256:{{ '17bff4966143d240a126b6cc414c6f79aa2106c0c97c772228e84d685221c25f' if ansible_architecture == 'x86_64' else 'c60e75fef5bfa1e79983919ffb47b40dcfbb49d121a510f11ca4b2a2603c00f1' }}"
when: ansible_distribution == 'CentOS' and ansible_distribution_major_version == '6'
- name: make wkhtmltopdf executable
file: path=/usr/local/bin/wkhtmltopdf mode="o+x"
become: true
become_user: root
- name: download wkthmltox CentOS 7
get_url:
url: https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox-0.12.5-1.centos7.{{ "x86_64" if ansible_architecture == "x86_64" else "i686"}}.rpm
dest: /tmp/wkhtmltox.rpm
checksum: "sha256:{{ 'ac4f909b836fa1fc0188d19a1ab844910f91612e9ccefcb5298aa955a058ffe4' if ansible_architecture == 'x86_64' else '1030279ac4b5b15dda04de2587b2a1942bde1c78aa1837dfec4ddcbea426721f' }}"
when: ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7'
- name: download wkthmltox CentOS 8
get_url:
url: https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox-0.12.5-1.centos8.x86_64.rpm
dest: /tmp/wkhtmltox.rpm
when: ansible_distribution == 'CentOS' and ansible_distribution_major_version == '8'
- name: download wkthmltox Debian 8
get_url:
url: https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.jessie_{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.deb
dest: /tmp/wkhtmltox.deb
checksum: "sha256:{{ '2583399a865d7604726da166ee7cec656b87ae0a6016e6bce7571dcd3045f98b' if ansible_architecture == 'x86_64' else '3a6969f3ed207a805092e05794644eb9e152aaa6518e9204c819fa318947a8a8' }}"
when: ansible_distribution == 'Debian' and ansible_distribution_major_version == '8'
- name: download wkthmltox Debian 9
get_url:
url: https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.stretch_{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.deb
dest: /tmp/wkhtmltox.deb
checksum: '{{ 1140b0ab02aa6e17346af2f14ed0de807376de475ba90e1db3975f112fbd20bb if ansible_architecture == "x86_64" else 5b2d15e738ac479e7a8ca6fd765f406c3684a48091813520f87878278d6dd22a }}'
when: ansible_distribution == 'Debian' and ansible_distribution_major_version == '9'
- name: download wkthmltox Debian 10
get_url:
url: https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.buster_{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.deb
dest: /tmp/wkhtmltox.deb
when: ansible_distribution == 'Debian' and ansible_distribution_major_version == '10'
- name: Install wkhtmltox rpm
yum:
name: /tmp/wkhtmltox.rpm
state: present
when: ansible_os_family == 'RedHat'
- name: Install wkhtmltox deb
apt:
deb: /tmp/wkhtmltox.deb
state: present
when: ansible_os_family == 'Debian'
...

6
requirements.txt
View File

@ -1,10 +1,10 @@
Click==7.0
GitPython==2.1.11
GitPython==2.1.15
honcho==1.0.1
Jinja2==2.10.3
python_crontab==2.4.0
python-crontab==2.4.0
requests==2.22.0
semantic_version==2.8.2
semantic-version==2.8.2
setuptools==40.8.0
six==1.12.0
virtualenv==16.6.0