Merge branch 'master' into bench-cmd-help

README.md

@@ -15,6 +15,7 @@ bench is a command-line utility that helps you to install apps, manage multiple
  - [bench CLI](#bench-cli)
 	- [Usage](#usage)
 	- [Installation](#installation)
+	- [Custom Bench commands](#custom-bench-commands)
  - [Easy Install Script](#easy-install-script)
  - [Release Bench](#release-bench)
  - [Guides](#guides)
@@ -60,7 +61,7 @@ Bench is a command line tool that helps you install, setup, manage multiple site
 
 _Note:_ Apart from `bench init`, all other bench commands have to be run having the respective bench directory as the working directory. _(`bench update` may also be run, but it's behaviour is covered in depth in the docs)_
 
-For more in depth information on commands and usage follow [here](https://github.com/frappe/bench/blob/master/docs/commands_and_usage.md).
+For more in depth information on commands and usage follow [here](https://github.com/frappe/bench/blob/master/docs/commands_and_usage.md). As for a consolidated list of bench commands, go through [this page](https://github.com/frappe/bench/blob/master/docs/bench_usage.md).
 
 ---
 
@@ -77,6 +78,12 @@ If you have questions, please ask them on the [forum](https://discuss.erpnext.co
 
 ---
 
+## Custom Bench Commands
+
+Want to utilize a bench command you've added in your custom Frappe application? [This](https://github.com/frappe/bench/blob/master/docs/bench_custom_cmd.md) guide might be of some help.
+
+---
+
 # Easy Install Script
 
 - This is an opinionated setup so it is best to setup on a blank server.

bench/app.py

@@ -4,6 +4,7 @@ from .utils import (exec_cmd, get_frappe, check_git_for_shallow_clone, build_ass
 	restart_supervisor_processes, get_cmd_output, run_frappe_cmd, CommandFailedError,
 	restart_systemd_processes)
 from .config.common_site_config import get_config
+from six.moves import reload_module
 
 import logging
 import requests
@@ -14,6 +15,7 @@ import subprocess
 import bench
 import sys
 import shutil
+import click
 
 logging.basicConfig(level="DEBUG")
 logger = logging.getLogger(__name__)
@@ -93,7 +95,7 @@ def remove_from_excluded_apps_txt(app, bench_path='.'):
 		return write_excluded_apps_txt(apps, bench_path=bench_path)
 
 def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=False,
-	postprocess = True):
+		postprocess=True, overwrite=False):
 	# from bench.utils import check_url
 	try:
 		from urlparse import urljoin
@@ -114,37 +116,41 @@ def get_app(git_url, branch=None, bench_path='.', skip_assets=False, verbose=Fal
 
 	# Gets repo name from URL
 	repo_name = git_url.rsplit('/', 1)[1].rsplit('.', 1)[0]
-	logger.info('getting app {}'.format(repo_name))
 	shallow_clone = '--depth 1' if check_git_for_shallow_clone() else ''
 	branch = '--branch {branch}'.format(branch=branch) if branch else ''
 
-	exec_cmd("git clone -q {git_url} {branch} {shallow_clone} --origin upstream".format(
+	if os.path.isdir(os.path.join(bench_path, 'apps', repo_name)):
+		# application directory already exists
+		# prompt user to overwrite it
+		if overwrite or click.confirm('''A directory for the application "{0}" already exists.
+Do you want to continue and overwrite it?'''.format(repo_name)):
+			shutil.rmtree(os.path.join(bench_path, 'apps', repo_name))
+		elif click.confirm('''Do you want to reinstall the existing application?''', abort=True):
+			app_name = get_app_name(bench_path, repo_name)
+			install_app(app=app_name, bench_path=bench_path, verbose=verbose, skip_assets=skip_assets)
+			sys.exit()
+
+	logger.info('Getting app {0}'.format(repo_name))
+	exec_cmd("git clone {git_url} {branch} {shallow_clone} --origin upstream".format(
 		git_url=git_url,
 		shallow_clone=shallow_clone,
 		branch=branch),
 		cwd=os.path.join(bench_path, 'apps'))
 
-	#Retrieves app name from setup.py
+	app_name = get_app_name(bench_path, repo_name)
+	install_app(app=app_name, bench_path=bench_path, verbose=verbose, skip_assets=skip_assets)
+
+
+def get_app_name(bench_path, repo_name):
+	# retrieves app name from setup.py
 	app_path = os.path.join(bench_path, 'apps', repo_name, 'setup.py')
 	with open(app_path, 'rb') as f:
 		app_name = re.search(r'name\s*=\s*[\'"](.*)[\'"]', f.read().decode('utf-8')).group(1)
 		if repo_name != app_name:
 			apps_path = os.path.join(os.path.abspath(bench_path), 'apps')
 			os.rename(os.path.join(apps_path, repo_name), os.path.join(apps_path, app_name))
+		return app_name
 
-	print('installing', app_name)
-	install_app(app=app_name, bench_path=bench_path, verbose=verbose)
-
-	if postprocess:
-
-		if not skip_assets:
-			build_assets(bench_path=bench_path, app=app_name)
-		conf = get_config(bench_path=bench_path)
-
-		if conf.get('restart_supervisor_on_update'):
-			restart_supervisor_processes(bench_path=bench_path)
-		if conf.get('restart_systemd_on_update'):
-			restart_systemd_processes(bench_path=bench_path)
 
 def new_app(app, bench_path='.'):
 	# For backwards compatibility
@@ -160,7 +166,8 @@ def new_app(app, bench_path='.'):
 		run_frappe_cmd('make-app', apps, app, bench_path=bench_path)
 	install_app(app, bench_path=bench_path)
 
-def install_app(app, bench_path=".", verbose=False, no_cache=False):
+
+def install_app(app, bench_path=".", verbose=False, no_cache=False, postprocess=True, skip_assets=False):
 	logger.info("installing {}".format(app))
 
 	pip_path = os.path.join(bench_path, "env", "bin", "pip")
@@ -168,11 +175,23 @@ def install_app(app, bench_path=".", verbose=False, no_cache=False):
 	app_path = os.path.join(bench_path, "apps", app)
 	cache_flag = "--no-cache-dir" if no_cache else ""
 
-	exec_cmd("{pip} install {quiet} -U -e {app} {no_cache}".format(pip=pip_path, quiet=quiet_flag, app=app_path, no_cache=cache_flag))
+	exec_cmd("{pip} install {quiet} -U -e {app} {no_cache}".format(pip=pip_path,
+									quiet=quiet_flag, app=app_path, no_cache=cache_flag))
 	add_to_appstxt(app, bench_path=bench_path)
 
+	if postprocess:
+		if not skip_assets:
+			build_assets(bench_path=bench_path, app=app)
+		conf = get_config(bench_path=bench_path)
+
+		if conf.get('restart_supervisor_on_update'):
+			restart_supervisor_processes(bench_path=bench_path)
+		if conf.get('restart_systemd_on_update'):
+			restart_systemd_processes(bench_path=bench_path)
+
+
 def remove_app(app, bench_path='.'):
-	if not app in get_apps(bench_path):
+	if app not in get_apps(bench_path):
 		print("No app named {0}".format(app))
 		sys.exit(1)
 
@@ -281,8 +300,7 @@ def get_current_branch(app, bench_path='.'):
 
 def get_remote(app, bench_path='.'):
 	repo_dir = get_repo_dir(app, bench_path=bench_path)
-	contents = subprocess.check_output(['git', 'remote', '-v'], cwd=repo_dir,
+	contents = subprocess.check_output(['git', 'remote', '-v'], cwd=repo_dir, stderr=subprocess.STDOUT)
-									   stderr=subprocess.STDOUT)
 	contents = contents.decode('utf-8')
 	if re.findall('upstream[\s]+', contents):
 		return 'upstream'
@@ -371,7 +389,7 @@ def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrad
 			except CommandFailedError:
 				print("Error switching to branch {0} for {1}".format(branch, app))
 			except InvalidRemoteException:
-				print("Remote does not exist for app "+app)
+				print("Remote does not exist for app {0}".format(app))
 			except InvalidBranchException:
 				print("Branch {0} does not exist in Upstream for {1}".format(branch, app))
 
@@ -382,11 +400,7 @@ def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrad
 		update_requirements()
 		update_node_packages()
 		pre_upgrade(version_upgrade[1], version_upgrade[2])
-		if sys.version_info >= (3, 4):
+		reload_module(utils)
-			import importlib
-			importlib.reload(utils)
-		else:
-			reload(utils)
 		backup_all_sites()
 		patch_sites()
 		build_assets()
@@ -402,8 +416,7 @@ def switch_to_develop(apps=None, bench_path='.', upgrade=True):
 	switch_branch('develop', apps=apps, bench_path=bench_path, upgrade=upgrade)
 
 def get_version_from_string(contents, field='__version__'):
-	match = re.search(r"^(\s*%s\s*=\s*['\\\"])(.+?)(['\"])(?sm)" % field,
+	match = re.search(r"^(\s*%s\s*=\s*['\\\"])(.+?)(['\"])(?sm)" % field, contents)
-			contents)
 	return match.group(2)
 
 def get_major_version(version):

bench/commands/make.py

@@ -53,10 +53,12 @@ def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups, c
 @click.argument('name', nargs=-1) # Dummy argument for backward compatibility
 @click.argument('git-url')
 @click.option('--branch', default=None, help="branch to checkout")
+@click.option('--overwrite', is_flag=True, default=False)
 @click.option('--skip-assets', is_flag=True, default=False, help="Do not build assets")
-def get_app(git_url, branch, name=None, skip_assets=False):
+def get_app(git_url, branch, name=None, overwrite=False, skip_assets=False):
+	"clone an app from the internet and set it up in your bench"
 	from bench.app import get_app
-	get_app(git_url, branch=branch, skip_assets=skip_assets)
+	get_app(git_url, branch=branch, skip_assets=skip_assets, overwrite=overwrite)
 
 
 @click.command('new-app', help='Create a new Frappe application under apps folder')

bench/config/lets_encrypt.py

@@ -83,11 +83,10 @@ def run_certbot_and_setup_ssl(site, custom_domain, bench_path, interactive=True)
 
 def setup_crontab():
 	job_command = '/opt/certbot-auto renew -a nginx --post-hook "systemctl reload nginx"'
-	system_crontab = CronTab(tabfile='/etc/crontab', user=True)
+	system_crontab = CronTab(user='root')
 	if job_command not in str(system_crontab):
 		job = system_crontab.new(command=job_command, comment="Renew lets-encrypt every month")
-		job.every().month()
+		job.day.on(1)
-		job.enable()
 		system_crontab.write()
 
 

bench/config/nginx.py

@@ -1,8 +1,24 @@
-import os, json, click, random, string, hashlib
+# imports - standard imports
-from bench.utils import get_sites, get_bench_name, exec_cmd
+import hashlib
+import os
+import random
+import string
+
+# imports - third party imports
+import click
 from six import string_types
 
+# imports - module imports
+from bench.utils import get_bench_name, get_sites
+
+
 def make_nginx_conf(bench_path, yes=False):
+	conf_path = os.path.join(bench_path, "config", "nginx.conf")
+
+	if not yes and os.path.exists(conf_path):
+		if not click.confirm('nginx.conf already exists and this will overwrite it. Do you want to continue?'):
+			return
+
 	from bench import env
 	from bench.config.common_site_config import get_config
 
@@ -37,10 +53,6 @@ def make_nginx_conf(bench_path, yes=False):
 
 	nginx_conf = template.render(**template_vars)
 
-	conf_path = os.path.join(bench_path, "config", "nginx.conf")
-	if not yes and os.path.exists(conf_path):
-		click.confirm('nginx.conf already exists and this will overwrite it. Do you want to continue?',
-			abort=True)
 
 	with open(conf_path, "w") as f:
 		f.write(nginx_conf)

docs/bench_custom_cmd.md

@@ -0,0 +1,48 @@
+## How are Frappe Framework commands available via bench?
+
+bench utilizes `frappe.utils.bench_manager` to get the framework's as well as those of any custom commands written in application installed in the Frappe environment. Currently, with *version 12* there are commands related to the scheduler, sites, translations and other utils in Frappe inherited by bench.
+
+
+## Can I add CLI commands in my custom app and call them via bench?
+
+Along with the framework commands, Frappe's `bench_manager` module also searches for any commands in your custom applications. Thereby, bench communicates with the respective bench's Frappe which in turn checks for available commands in all of the applications.
+
+To make your custom command available to bench, just create a `commands` module under your parent module and write the command with a click wrapper and a variable commands which contains a list of click functions, which are your own commands. The directory strcuture may be visualized as:
+
+```
+frappe-bench
+|──apps
+    |── frappe
+    ├── custom_app
+    │   ├── README.md
+    │   ├── custom_app
+    │   │   ├── commands    <------ commands module
+    │   ├── license.txt
+    │   ├── requirements.txt
+    │   └── setup.py
+```
+
+The commands module maybe a single file such as `commands.py` or a directory with an `__init__.py` file. For a custom application of name 'flags', example may be given as
+
+```python
+# file_path: frappe-bench/apps/flags/flags/commands.py
+import click
+
+@click.command('set-flags')
+@click.argument('state', type=click.Choice(['on', 'off']))
+def set_flags(state):
+    from flags.utils import set_flags
+    set_flags(state=state)
+
+commands = [
+    set_flags
+]
+```
+
+and with context of the current bench, this command maybe executed simply as
+
+```zsh
+➜ bench set-flags
+Flags are set to state: 'on'
+```
+

docs/bench_usage.md

@@ -0,0 +1,201 @@
+# bench CLI Usage
+
+This may not be known to a lot of people but half the bench commands we're used to, exist in the Frappe Framework and not in bench directly. Those commands generally are the `--site` commands. This page is concerned only with the commands in the bench project. Any framework commands won't be a part of this consolidation.
+
+
+# bench CLI Commands
+
+Under Click's structure, `bench` is the main command group, under which there are three main groups of commands in bench currently, namely
+
+ - **install**: The install command group deals with commands used to install system dependencies for setting up Frappe environment
+
+ - **setup**: This command group for consists of commands used to maipulate the requirements and environments required by your Frappe environment
+
+ - **config**: The config command group deals with making changes in the current bench (not the CLI tool) configuration
+
+
+## Using the bench command line
+
+```zsh
+➜ bench
+Usage: bench [OPTIONS] COMMAND [ARGS]...
+
+  Bench manager for Frappe
+
+Options:
+  --version
+  --help     Show this message and exit.
+
+Commands:
+  backup                   Backup single site
+  backup-all-sites         Backup all sites in current bench
+  config                   Change bench configuration
+  disable-production       Disables production environment for the bench.
+  download-translations    Download latest translations
+  exclude-app              Exclude app from updating
+  find                     Finds benches recursively from location
+  get-app                  Clone an app from the internet or filesystem and...
+```
+
+Similarly, all available flags and options can be checked for commands individually by executing them with the `--help` flag. The `init` command for instance:
+
+```zsh
+➜ bench init --help
+Usage: bench init [OPTIONS] PATH
+
+  Initialize a new bench instance in the specified path
+
+Options:
+  --python TEXT                   Path to Python Executable.
+  --ignore-exist                  Ignore if Bench instance exists.
+  --apps_path TEXT                path to json files with apps to install
+                                  after init
+```
+
+
+
+## bench and sudo
+
+Some bench commands may require sudo, such as some `setup` commands and everything else under the `install` commands group. For these commands, you may not be asked for your root password if sudoers setup has been done. The security implications, well we'll talk about those soon.
+
+
+
+## General Commands
+
+These commands belong directly to the bench group so they can be invoked directly prefixing each with `bench` in your shell. Therefore, the usage for these commands is as
+
+```zsh
+    bench COMMAND [ARGS]...
+```
+
+### The usual commands
+
+ - **init**: Initialize a new bench instance in the specified path. This sets up a complete bench folder with an `apps` folder which contains all the Frappe apps available in the current bench, `sites` folder that stores all site data seperated by individual site folders, `config` folder that contains your redis, NGINX and supervisor configuration files. The `env` folder consists of all python dependencies the current bench and installed Frappe applications have.
+ - **restart**: Restart web, supervisor, systemd processes units. Used in production setup.
+ - **update**: Updates bench tool and if executed in a bench directory, without any flags will backup, pull, setup requirements, build, run patches and restart bench. Using specific flags will only do certain tasks instead of all.
+ - **migrate-env**: Migrate Virtual Environment to desired Python version. This regenerates the `env` folder with the specified Python version.
+ - **retry-upgrade**: Retry a failed upgrade
+ - **disable-production**: Disables production environment for the bench.
+ - **renew-lets-encrypt**: Renew Let's Encrypt certificate for site SSL.
+ - **backup**: Backup single site data. Can be used to backup files as well.
+ - **backup-all-sites**: Backup all sites in current bench.
+
+ - **get-app**: Download an app from the internet or filesystem and set it up in your bench. This clones the git repo of the Frappe project and installs it in the bench environment.
+ - **remove-app**: Completely remove app from bench and re-build assets if not installed on any site.
+ - **exclude-app**: Exclude app from updating during a `bench update`
+ - **include-app**: Include app for updating. All Frappe applications are included by default when installed.
+ - **remote-set-url**: Set app remote url
+ - **remote-reset-url**: Reset app remote url to frappe official
+ - **remote-urls**: Show apps remote url
+ - **switch-to-branch**: Switch all apps to specified branch, or specify apps separated by space
+ - **switch-to-develop**: Switch Frappe and ERPNext to develop branch
+
+
+### A little advanced
+
+ - **set-nginx-port**: Set NGINX port for site
+ - **set-ssl-certificate**: Set SSL certificate path for site
+ - **set-ssl-key**: Set SSL certificate private key path for site
+ - **set-url-root**: Set URL root for site
+ - **set-mariadb-host**: Set MariaDB host for bench
+ - **set-redis-cache-host**: Set Redis cache host for bench
+ - **set-redis-queue-host**: Set Redis queue host for bench
+ - **set-redis-socketio-host**: Set Redis socketio host for bench
+ - **set-default-site**: Set default site for bench
+ - **download-translations**: Download latest translations
+
+
+### Developer's commands
+
+ - **start**: Start Frappe development processes. Uses the Procfile to start the Frappe development environment.
+ - **src**: Prints bench source folder path, which can be used to cd into the bench installation repository by `cd $(bench src)`.
+ - **find**: Finds benches recursively from location or specified path.
+ - **pip**: Use the current bench's pip to manage Python packages. For help about pip usage: `bench pip help [COMMAND]` or `bench pip [COMMAND] -h`.
+ - **new-app**: Create a new Frappe application under apps folder.
+
+
+### Release bench
+ - **release**: Create a release of a Frappe application
+ - **prepare-beta-release**: Prepare major beta release from develop branch
+
+
+
+## Setup commands
+
+The setup commands used for setting up the Frappe environment in context of the current bench need to be executed using `bench setup` as the prefix. So, the general usage of these commands is as
+
+```zsh
+    bench setup COMMAND [ARGS]...
+```
+
+ - **sudoers**: Add commands to sudoers list for allowing bench commands execution without root password
+
+ - **env**: Setup virtualenv for bench. This sets up a `env` folder under the root of the bench directory.
+ - **redis**: Generates configuration for Redis
+ - **fonts**: Add Frappe fonts to system
+ - **config**: Generate or over-write sites/common_site_config.json
+ - **backups**: Add cronjob for bench backups
+ - **socketio**: Setup node dependencies for socketio server
+ - **requirements**: Setup Python and Node dependencies
+
+ - **manager**: Setup `bench-manager.local` site with the [Bench Manager](https://github.com/frappe/bench_manager) app, a GUI for bench installed on it.
+
+ - **procfile**: Generate Procfile for bench start
+
+ - **production**: Setup Frappe production environment for specific user. This installs ansible, NGINX, supervisor, fail2ban and generates the respective configuration files.
+ - **nginx**: Generate configuration files for NGINX
+ - **fail2ban**: Setup fail2ban, an intrusion prevention software framework that protects computer servers from brute-force attacks
+ - **systemd**: Generate configuration for systemd
+ - **firewall**: Setup firewall for system
+ - **ssh-port**: Set SSH Port for system
+ - **reload-nginx**: Checks NGINX config file and reloads service
+ - **supervisor**: Generate configuration for supervisor
+ - **lets-encrypt**: Setup lets-encrypt SSL for site
+ - **wildcard-ssl**: Setup wildcard SSL certificate for multi-tenant bench
+
+ - **add-domain**: Add a custom domain to a particular site
+ - **remove-domain**: Remove custom domain from a site
+ - **sync-domains**: Check if there is a change in domains. If yes, updates the domains list.
+
+ - **role**: Install dependencies via ansible roles
+
+
+
+## Config commands
+
+The config group commands are used for manipulating configurations in the current bench context. The usage for these commands is as
+
+```zsh
+    bench config COMMAND [ARGS]...
+```
+
+ - **set-common-config**: Set value in common config
+ - **remove-common-config**: Remove specific keys from current bench's common config
+
+ - **update_bench_on_update**: Enable/Disable bench updates on running bench update
+ - **restart_supervisor_on_update**: Enable/Disable auto restart of supervisor processes
+ - **restart_systemd_on_update**: Enable/Disable auto restart of systemd units
+ - **dns_multitenant**: Enable/Disable bench multitenancy on running bench update
+ - **serve_default_site**: Configure nginx to serve the default site on port 80
+ - **http_timeout**: Set HTTP timeout
+
+
+
+## Install commands
+
+The install group commands are used for manipulating system level dependencies. The usage for these commands is as
+
+```zsh
+    bench install COMMAND [ARGS]...
+```
+
+ - **prerequisites**: Installs pre-requisite libraries, essential tools like b2zip, htop, screen, vim, x11-fonts, python libs, cups and Redis
+ - **nodejs**: Installs Node.js v8
+ - **nginx**: Installs NGINX. If user is specified, sudoers is setup for that user
+ - **packer**: Installs Oracle virtualbox and packer 1.2.1
+ - **psutil**: Installs psutil via pip
+ - **mariadb**: Install and setup MariaDB of specified version and root password
+ - **wkhtmltopdf**: Installs wkhtmltopdf v0.12.3 for linux
+ - **supervisor**: Installs supervisor. If user is specified, sudoers is setup for that user
+ - **fail2ban**: Install fail2ban, an intrusion prevention software framework that protects computer servers from brute-force attacks
+ - **virtualbox**: Installs supervisor

playbooks/install.py

@@ -1,5 +1,16 @@
 #!/usr/bin/env python3
-import os, sys, subprocess, getpass, json, multiprocessing, shutil, platform, warnings, datetime
+from __future__ import print_function
+import os
+import sys
+import subprocess
+import getpass
+import json
+import multiprocessing
+import shutil
+import platform
+import warnings
+import datetime
+
 
 tmp_bench_repo = os.path.join('/', 'tmp', '.bench')
 tmp_log_folder = os.path.join('/', 'tmp', 'logs')
@@ -395,9 +406,21 @@ def parse_commandline_args():
 
 if __name__ == '__main__':
 	if sys.version[0] == '2':
+		if not os.environ.get('CI'):
 			if not raw_input("It is recommended to run this script with Python 3\nDo you still wish to continue? [Y/n]: ").lower() == "y":
 				sys.exit()
 
+		try:
+			from distutils.spawn import find_executable
+		except ImportError:
+			try:
+				subprocess.check_call('pip install --upgrade setuptools')
+			except subprocess.CalledProcessError:
+				print("Install distutils or use Python3 to run the script")
+				sys.exit(1)
+
+		shutil.which = find_executable
+
 	if not is_sudo_user():
 		log("Please run this script as a non-root user with sudo privileges", level=3)
 		sys.exit()

requirements.txt

@@ -1,10 +1,10 @@
 Click==7.0
-GitPython==2.1.11
+GitPython==2.1.15
 honcho==1.0.1
 Jinja2==2.10.3
-python_crontab==2.4.0
+python-crontab==2.4.0
 requests==2.22.0
-semantic_version==2.8.2
+semantic-version==2.8.2
 setuptools==40.8.0
 six==1.12.0
 virtualenv==16.6.0