Merge branch 'master' into minor

.travis.yml

@@ -1,11 +1,14 @@
 language: python
 dist: trusty
+group: deprecated-2017Q2
 sudo: required
 
 python:
   - "2.7"
 
 install:
+  - sudo rm /etc/apt/sources.list.d/docker.list
+  - sudo pip install urllib3 pyOpenSSL ndg-httpsclient pyasn1
   - sudo apt-get purge -y mysql-common mysql-server mysql-client
   - sudo apt-get install --only-upgrade -y git
   - mkdir -p ~/.bench

README.md

@@ -82,6 +82,7 @@ Note: Please do not remove the bench directory the above commands will create
 - This is an opinionated setup so it is best to setup on a blank server.
 - Works on Ubuntu 14.04 to 16.04, CentOS 7+, Debian 7 to 8 and MacOS X.
 - You may have to install Python 2.7 (eg on Ubuntu 16.04+) by running `apt-get install python-minimal`
+- You may also have to install build-essential and python-setuptools by running `apt-get install build-essential python-setuptools`
 - This script will install the pre-requisites, install bench and setup an ERPNext site
 - Passwords for Frappe Administrator and MariaDB (root) will be asked
 - You can then login as **Administrator** with the Administrator password
@@ -134,6 +135,13 @@ For production:
 
 ---
 
+## Docker Install - For Developers (beta)
+
+1. For developer setup, you can also use the official [Frappé Docker](https://github.com/frappe/frappe_docker/).
+2. The app, mariadb and redis run on individual containers
+3. This setup supports multi-tenancy and exposes the frappe-bench volume as a external storage.
+4. For more details, [read the instructions on the Frappé Docker README](https://github.com/frappe/frappe_docker/)
+
 Help
 ====
 

bench/app.py

@@ -253,7 +253,7 @@ def get_repo_dir(app, bench_path='.'):
 	return os.path.join(bench_path, 'apps', app)
 
 def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrade=True):
-	from .utils import update_requirements, backup_all_sites, patch_sites, build_assets, pre_upgrade, post_upgrade
+	from .utils import update_requirements, update_npm_packages, backup_all_sites, patch_sites, build_assets, pre_upgrade, post_upgrade
 	from . import utils
 	apps_dir = os.path.join(bench_path, 'apps')
 	version_upgrade = (False,)
@@ -293,6 +293,7 @@ def switch_branch(branch, apps=None, bench_path='.', upgrade=False, check_upgrad
 
 	if version_upgrade[0] and upgrade:
 		update_requirements()
+		update_npm_packages()
 		pre_upgrade(version_upgrade[1], version_upgrade[2])
 		reload(utils)
 		backup_all_sites()

bench/commands/__init__.py

@@ -37,7 +37,7 @@ bench_command.add_command(switch_to_develop)
 
 from bench.commands.utils import (start, restart, set_nginx_port, set_ssl_certificate, set_ssl_certificate_key, set_url_root,
 	set_mariadb_host, set_default_site, download_translations, shell, backup_site, backup_all_sites, release, renew_lets_encrypt,
-	disable_production, bench_src)
+	disable_production, bench_src, prepare_staging)
 bench_command.add_command(start)
 bench_command.add_command(restart)
 bench_command.add_command(set_nginx_port)
@@ -51,6 +51,7 @@ bench_command.add_command(shell)
 bench_command.add_command(backup_site)
 bench_command.add_command(backup_all_sites)
 bench_command.add_command(release)
+bench_command.add_command(prepare_staging)
 bench_command.add_command(renew_lets_encrypt)
 bench_command.add_command(disable_production)
 bench_command.add_command(bench_src)

bench/commands/make.py

@@ -10,13 +10,15 @@ import click
 @click.option('--no-backups',is_flag=True, help="Run migrations for all sites in the bench")
 @click.option('--no-auto-update',is_flag=True, help="Build JS and CSS artifacts for the bench")
 @click.option('--verbose',is_flag=True, help="Verbose output during install")
+@click.option('--skip-bench-mkdir', is_flag=True, help="Skip mkdir frappe-bench")
+@click.option('--skip-redis-config-generation', is_flag=True, help="Skip redis config generation if already specifying the common-site-config file")
 def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups,
-		no_auto_update, clone_from, verbose):
+		no_auto_update, clone_from, verbose, skip_bench_mkdir, skip_redis_config_generation):
 	"Create a new bench"
 	from bench.utils import init
 	init(path, apps_path=apps_path, no_procfile=no_procfile, no_backups=no_backups,
 			no_auto_update=no_auto_update, frappe_path=frappe_path, frappe_branch=frappe_branch,
-			verbose=verbose, clone_from=clone_from)
+			verbose=verbose, clone_from=clone_from, skip_bench_mkdir=skip_bench_mkdir, skip_redis_config_generation=skip_redis_config_generation)
 	click.echo('Bench {} initialized'.format(path))
 
 
@@ -44,5 +46,3 @@ def remove_app(app_name):
 	"completely remove app from bench"
 	from bench.app import remove_app
 	remove_app(app_name)
-
-

bench/commands/setup.py

@@ -118,6 +118,12 @@ def setup_socketio():
 	from bench.utils import setup_socketio
 	setup_socketio()
 
+@click.command('requirements')
+def setup_requirements():
+	"Setup python and node requirements"
+	from bench.utils import update_requirements, update_npm_packages
+	update_requirements()
+	update_npm_packages()
 
 @click.command('config')
 def setup_config():
@@ -155,17 +161,18 @@ def remove_domain(domain, site=None):
 	remove_domain(site, domain, bench_path='.')
 
 @click.command('sync-domains')
-@click.argument('domains')
+@click.option('--domain', multiple=True)
 @click.option('--site', prompt=True)
-def sync_domains(domains, site=None):
+def sync_domains(domain=None, site=None):
 	from bench.config.site_config import sync_domains
 
 	if not site:
 		print("Please specify site")
 		sys.exit(1)
 
-	domains = json.loads(domains)
+	try:
-	if not isinstance(domains, list):
+		domains = list(map(str,domain))
+	except Exception:
 		print("Domains should be a json list of strings or dictionaries")
 		sys.exit(1)
 
@@ -186,6 +193,7 @@ setup.add_command(setup_backups)
 setup.add_command(setup_env)
 setup.add_command(setup_procfile)
 setup.add_command(setup_socketio)
+setup.add_command(setup_requirements)
 setup.add_command(setup_config)
 setup.add_command(setup_fonts)
 setup.add_command(add_domain)

bench/commands/update.py

@@ -3,7 +3,7 @@ import sys, os
 from bench.config.common_site_config import get_config
 from bench.app import pull_all_apps, is_version_upgrade
 from bench.utils import (update_bench, validate_upgrade, pre_upgrade, post_upgrade, before_update,
-	update_requirements, backup_all_sites, patch_sites, build_assets, restart_supervisor_processes)
+	update_requirements, update_npm_packages, backup_all_sites, patch_sites, build_assets, restart_supervisor_processes)
 from bench import patches
 
 #TODO: Not DRY
@@ -62,7 +62,8 @@ def update(pull=False, patch=False, build=False, bench=False, auto=False, restar
 	_update(pull, patch, build, bench, auto, restart_supervisor, requirements, no_backup, upgrade, force=force, reset=reset)
 
 
-def _update(pull=False, patch=False, build=False, update_bench=False, auto=False, restart_supervisor=False, requirements=False, no_backup=False, upgrade=False, bench_path='.', force=False, reset=False):
+def _update(pull=False, patch=False, build=False, update_bench=False, auto=False, restart_supervisor=False,
+		requirements=False, no_backup=False, upgrade=False, bench_path='.', force=False, reset=False):
 	conf = get_config(bench_path=bench_path)
 	version_upgrade = is_version_upgrade(bench_path=bench_path)
 
@@ -78,8 +79,8 @@ def _update(pull=False, patch=False, build=False, update_bench=False, auto=False
 		pull_all_apps(bench_path=bench_path, reset=reset)
 
 	if requirements:
-		print('Updating Python libraries...')
 		update_requirements(bench_path=bench_path)
+		update_npm_packages(bench_path=bench_path)
 
 	if upgrade and (version_upgrade[0] or (not version_upgrade[0] and force)):
 		pre_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path)

bench/commands/utils.py

@@ -119,17 +119,24 @@ def backup_all_sites():
 @click.command('release')
 @click.argument('app')
 @click.argument('bump-type', type=click.Choice(['major', 'minor', 'patch', 'stable', 'prerelease']))
-@click.option('--develop', default='develop')
+@click.option('--from-branch', default='develop')
-@click.option('--master', default='master')
+@click.option('--to-branch', default='master')
 @click.option('--remote', default='upstream')
 @click.option('--owner', default='frappe')
 @click.option('--repo-name')
-def release(app, bump_type, develop, master, owner, repo_name, remote):
+def release(app, bump_type, from_branch, to_branch, owner, repo_name, remote):
 	"Release app (internal to the Frappe team)"
 	from bench.release import release
-	release(bench_path='.', app=app, bump_type=bump_type, develop=develop, master=master,
+	release(bench_path='.', app=app, bump_type=bump_type, from_branch=from_branch, to_branch=to_branch,
 		remote=remote, owner=owner, repo_name=repo_name)
 
+@click.command('prepare-staging')
+@click.argument('app')
+def prepare_staging(app):
+	"""Prepare staging branch from develop branch"""
+	from bench.prepare_staging import prepare_staging
+	prepare_staging(bench_path='.', app=app)
+	
 
 @click.command('disable-production')
 def disable_production():

bench/config/common_site_config.py

@@ -62,7 +62,7 @@ def update_config_for_frappe(config, bench_path):
 		if key not in config:
 			config[key] = "redis://localhost:{0}".format(ports[key])
 
-	for key in ('webserver_port', 'socketio_port'):
+	for key in ('webserver_port', 'socketio_port', 'file_watcher_port'):
 		if key not in config:
 			config[key] = ports[key]
 
@@ -75,6 +75,7 @@ def make_ports(bench_path):
 	default_ports = {
 		"webserver_port": 8000,
 		"socketio_port": 9000,
+		"file_watcher_port": 6787,
 		"redis_queue": 11000,
 		"redis_socketio": 12000,
 		"redis_cache": 13000

bench/config/lets_encrypt.py

@@ -81,12 +81,12 @@ def run_certbot_and_setup_ssl(site, custom_domain, bench_path):
 
 def setup_crontab():
 	job_command = 'sudo service nginx stop && /opt/certbot-auto renew && sudo service nginx start'
-	user_crontab = CronTab()
+	system_crontab = CronTab(tabfile='/etc/crontab', user=True)
-	if job_command not in str(user_crontab):
+	if job_command not in str(system_crontab):
-		job  = user_crontab.new(command=job_command, comment="Renew lets-encrypt every month")
+		job  = system_crontab.new(command=job_command, comment="Renew lets-encrypt every month")
 		job.every().month()
 		job.enable()
-		user_crontab.write()
+		system_crontab.write()
 
 
 def create_dir_if_missing(path):

bench/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "frappe",
+  "description": "Default package.json for frappe apps",
+  "dependencies": {
+    "babel-core": "^6.24.1",
+    "babel-preset-babili": "0.0.12",
+    "babel-preset-es2015": "^6.24.1",
+    "babel-preset-es2016": "^6.24.1",
+    "babel-preset-es2017": "^6.24.1",
+    "chokidar": "^1.7.0",
+    "cookie": "^0.3.1",
+    "express": "^4.15.3",
+    "less": "^2.7.2",
+    "redis": "^2.7.1",
+    "socket.io": "^2.0.1",
+    "superagent": "^3.5.2"
+  }
+}

bench/prepare_staging.py

@@ -0,0 +1,73 @@
+#! env python
+import os
+import git
+import click
+from .config.common_site_config import get_config
+
+github_username = None
+github_password = None
+
+def prepare_staging(bench_path, app, remote='upstream'):
+	from .release import get_release_message
+	validate(bench_path)
+
+	repo_path = os.path.join(bench_path, 'apps', app)
+	update_branches(repo_path, remote)
+	message = get_release_message(repo_path, from_branch='develop', to_branch='staging', remote=remote)
+
+	if not message:
+		print('No commits to release')
+		return
+
+	print()
+	print(message)
+	print()
+
+	click.confirm('Do you want to continue?', abort=True)
+
+	create_staging(repo_path)
+	push_commits(repo_path)
+
+def validate(bench_path):
+	from .release import validate
+
+	config = get_config(bench_path)
+	validate(bench_path, config)
+
+def update_branches(repo_path, remote):
+	from .release import update_branch
+	update_branch(repo_path, 'staging', remote)
+	update_branch(repo_path, 'develop', remote)
+
+	git.Repo(repo_path).git.checkout('develop')
+
+def create_staging(repo_path, from_branch='develop'):
+	from .release import handle_merge_error
+
+	print('creating staging from', from_branch)
+	repo = git.Repo(repo_path)
+	g = repo.git
+	g.checkout('staging')
+	try:
+		g.merge(from_branch, '--no-ff')
+	except git.exc.GitCommandError as e:
+		handle_merge_error(e, source=from_branch, target='staging')
+	
+	g.checkout(from_branch)
+	try:
+		g.merge('staging')
+	except git.exc.GitCommandError as e:
+		handle_merge_error(e, source='staging', target=from_branch)
+	
+def push_commits(repo_path, remote='upstream'):
+	print('pushing staging branch of', repo_path)
+
+	repo = git.Repo(repo_path)
+	g = repo.git
+
+	args = [
+		'develop:develop',
+		'staging:staging'
+	]
+
+	print(g.push(remote, *args))

bench/release.py

@@ -4,10 +4,8 @@ import os
 import sys
 import semantic_version
 import git
-import json
 import requests
 import getpass
-import argparse
 import re
 from requests.auth import HTTPBasicAuth
 import requests.exceptions
@@ -15,30 +13,39 @@ from time import sleep
 from .config.common_site_config import get_config
 import click
 
+branches_to_update = {
+	'develop': [],
+	'hotfix': ['develop']
+}
+
 github_username = None
 github_password = None
 
-def release(bench_path, app, bump_type, develop='develop', master='master',
+def release(bench_path, app, bump_type, from_branch='develop', to_branch='master',
 		remote='upstream', owner='frappe', repo_name=None):
 
-	validate(bench_path)
-
-	bump(bench_path, app, bump_type, develop=develop, master=master, owner=owner,
-		repo_name=repo_name, remote=remote)
-
-def validate(bench_path):
 	config = get_config(bench_path)
+
 	if not config.get('release_bench'):
 		print('bench not configured to release')
 		sys.exit(1)
 
+	if config.get('branches_to_update'):
+		branches_to_update.update(config.get('branches_to_update'))
+
+	validate(bench_path, config)
+
+	bump(bench_path, app, bump_type, from_branch=from_branch, to_branch=to_branch, owner=owner,
+		repo_name=repo_name, remote=remote)
+
+def validate(bench_path, config):
 	global github_username, github_password
 
 	github_username = config.get('github_username')
 	github_password = config.get('github_password')
 
 	if not github_username:
-		github_username = input('Username: ')
+		github_username = click.prompt('Username', type=str)
 
 	if not github_password:
 		github_password = getpass.getpass()
@@ -46,12 +53,12 @@ def validate(bench_path):
 	r = requests.get('https://api.github.com/user', auth=HTTPBasicAuth(github_username, github_password))
 	r.raise_for_status()
 
-def bump(bench_path, app, bump_type, develop, master, remote, owner, repo_name=None):
+def bump(bench_path, app, bump_type, from_branch, to_branch, remote, owner, repo_name=None):
 	assert bump_type in ['minor', 'major', 'patch', 'stable', 'prerelease']
 
 	repo_path = os.path.join(bench_path, 'apps', app)
-	update_branches_and_check_for_changelog(repo_path, develop, master, remote=remote)
+	update_branches_and_check_for_changelog(repo_path, from_branch, to_branch, remote=remote)
-	message = get_release_message(repo_path, develop=develop, master=master, remote=remote)
+	message = get_release_message(repo_path, from_branch=from_branch, to_branch=to_branch, remote=remote)
 
 	if not message:
 		print('No commits to release')
@@ -63,21 +70,22 @@ def bump(bench_path, app, bump_type, develop, master, remote, owner, repo_name=N
 
 	click.confirm('Do you want to continue?', abort=True)
 
-	new_version = bump_repo(repo_path, bump_type, develop=develop, master=master)
+	new_version = bump_repo(repo_path, bump_type, from_branch=from_branch, to_branch=to_branch)
 	commit_changes(repo_path, new_version)
-	tag_name = create_release(repo_path, new_version, develop=develop, master=master)
+	tag_name = create_release(repo_path, new_version, from_branch=from_branch, to_branch=to_branch)
-	push_release(repo_path, develop=develop, master=master, remote=remote)
+	push_release(repo_path, from_branch=from_branch, to_branch=to_branch, remote=remote)
 	create_github_release(repo_path, tag_name, message, remote=remote, owner=owner, repo_name=repo_name)
 	print('Released {tag} for {repo_path}'.format(tag=tag_name, repo_path=repo_path))
 
-def update_branches_and_check_for_changelog(repo_path, develop='develop', master='master', remote='upstream'):
+def update_branches_and_check_for_changelog(repo_path, from_branch='develop', to_branch='master', remote='upstream'):
 
-	update_branch(repo_path, master, remote=remote)
+	update_branch(repo_path, to_branch, remote=remote)
-	update_branch(repo_path, develop, remote=remote)
+	update_branch(repo_path, from_branch, remote=remote)
-	if develop != 'develop':
-		update_branch(repo_path, 'develop', remote=remote)
 
-	git.Repo(repo_path).git.checkout(develop)
+	for branch in branches_to_update[from_branch]:
+		update_branch(repo_path, branch, remote=remote)
+
+	git.Repo(repo_path).git.checkout(from_branch)
 	check_for_unmerged_changelog(repo_path)
 
 def update_branch(repo_path, branch, remote):
@@ -94,18 +102,18 @@ def check_for_unmerged_changelog(repo_path):
 	if os.path.exists(current) and [f for f in os.listdir(current) if f != "readme.md"]:
 		raise Exception("Unmerged change log! in " + repo_path)
 
-def get_release_message(repo_path, develop='develop', master='master', remote='upstream'):
+def get_release_message(repo_path, from_branch='develop', to_branch='master', remote='upstream'):
-	print('getting release message for', repo_path, 'comparing', master, '...', develop)
+	print('getting release message for', repo_path, 'comparing', to_branch, '...', from_branch)
 
 	repo = git.Repo(repo_path)
 	g = repo.git
-	log = g.log('{remote}/{master}..{remote}/{develop}'.format(
+	log = g.log('{remote}/{to_branch}..{remote}/{from_branch}'.format(
-		remote=remote, master=master, develop=develop), '--format=format:%s', '--no-merges')
+		remote=remote, to_branch=to_branch, from_branch=from_branch), '--format=format:%s', '--no-merges')
 
 	if log:
 		return "* " + log.replace('\n', '\n* ')
 
-def bump_repo(repo_path, bump_type, develop='develop', master='master'):
+def bump_repo(repo_path, bump_type, from_branch='develop', to_branch='master'):
 	current_version = get_current_version(repo_path)
 	new_version = get_bumped_version(current_version, bump_type)
 
@@ -199,32 +207,32 @@ def commit_changes(repo_path, new_version):
 	repo.index.add([os.path.join(app_name, '__init__.py')])
 	repo.index.commit('bumped to version {}'.format(new_version))
 
-def create_release(repo_path, new_version, develop='develop', master='master'):
+def create_release(repo_path, new_version, from_branch='develop', to_branch='master'):
 	print('creating release for version', new_version)
 	repo = git.Repo(repo_path)
 	g = repo.git
-	g.checkout(master)
+	g.checkout(to_branch)
 	try:
-		g.merge(develop, '--no-ff')
+		g.merge(from_branch, '--no-ff')
 	except git.exc.GitCommandError as e:
-		handle_merge_error(e, source=develop, target=master)
+		handle_merge_error(e, source=from_branch, target=to_branch)
 
 	tag_name = 'v' + new_version
 	repo.create_tag(tag_name, message='Release {}'.format(new_version))
-	g.checkout(develop)
+	g.checkout(from_branch)
 
 	try:
-		g.merge(master)
+		g.merge(to_branch)
 	except git.exc.GitCommandError as e:
-		handle_merge_error(e, source=master, target=develop)
+		handle_merge_error(e, source=to_branch, target=from_branch)
 
-	if develop != 'develop':
+	for branch in branches_to_update[from_branch]:
-		print('merging master into develop')
+		print('merging master into', branch)
-		g.checkout('develop')
+		g.checkout(branch)
 		try:
-			g.merge(master)
+			g.merge(to_branch)
 		except git.exc.GitCommandError as e:
-			handle_merge_error(e, source=master, target='develop')
+			handle_merge_error(e, source=to_branch, target=branch)
 
 	return tag_name
 
@@ -236,18 +244,18 @@ def handle_merge_error(e, source, target):
 	print('-'*80)
 	click.confirm('Have you manually resolved the error?', abort=True)
 
-def push_release(repo_path, develop='develop', master='master', remote='upstream'):
+def push_release(repo_path, from_branch='develop', to_branch='master', remote='upstream'):
-	print('pushing branches', master, develop, 'of', repo_path)
+	print('pushing branches', to_branch, from_branch, 'of', repo_path)
 	repo = git.Repo(repo_path)
 	g = repo.git
 	args = [
-		'{master}:{master}'.format(master=master),
+		'{to_branch}:{to_branch}'.format(to_branch=to_branch),
-		'{develop}:{develop}'.format(develop=develop)
+		'{from_branch}:{from_branch}'.format(from_branch=from_branch)
 	]
 
-	if develop != 'develop':
+	for branch in branches_to_update[from_branch]:
-		print('pushing develop branch of', repo_path)
+		print('pushing {0} branch of'.format(branch), repo_path)
-		args.append('develop:develop')
+		args.append('{branch}:{branch}'.format(branch=branch))
 
 	args.append('--tags')
 

bench/tests/test_init.py

@@ -40,6 +40,7 @@ class TestBenchInit(unittest.TestCase):
 		self.assert_common_site_config("test-bench-1", {
 			"webserver_port": 8000,
 			"socketio_port": 9000,
+			"file_watcher_port": 6787,
 			"redis_queue": "redis://localhost:11000",
 			"redis_socketio": "redis://localhost:12000",
 			"redis_cache": "redis://localhost:13000"
@@ -51,6 +52,7 @@ class TestBenchInit(unittest.TestCase):
 		self.assert_common_site_config("test-bench-2", {
 			"webserver_port": 8001,
 			"socketio_port": 9001,
+			"file_watcher_port": 6788,
 			"redis_queue": "redis://localhost:11001",
 			"redis_socketio": "redis://localhost:12001",
 			"redis_cache": "redis://localhost:13001"
@@ -210,7 +212,7 @@ class TestBenchInit(unittest.TestCase):
 		self.assert_exists(python_path, "site-packages", "pip")
 
 		site_packages = os.listdir(os.path.join(python_path, "site-packages"))
-		self.assertTrue(any(package.startswith("MySQL_python-1.2.5") for package in site_packages))
+		self.assertTrue(any(package.startswith("mysqlclient-1.3.10") for package in site_packages))
 
 	def assert_config(self, bench_name):
 		for config, search_key in (

bench/utils.py

@@ -2,6 +2,8 @@ import os, sys, shutil, subprocess, logging, itertools, requests, json, platform
 from distutils.spawn import find_executable
 import bench
 from bench import env
+from six import iteritems
+
 
 class PatchError(Exception):
 	pass
@@ -25,21 +27,27 @@ def get_env_cmd(cmd, bench_path='.'):
 
 def init(path, apps_path=None, no_procfile=False, no_backups=False,
 		no_auto_update=False, frappe_path=None, frappe_branch=None, wheel_cache_dir=None,
-		verbose=False, clone_from=None):
+		verbose=False, clone_from=None, skip_bench_mkdir=False, skip_redis_config_generation=False):
 	from .app import get_app, install_apps_from_path
 	from .config.common_site_config import make_config
 	from .config import redis
 	from .config.procfile import setup_procfile
 	from bench.patches import set_all_patches_executed
 
-	if os.path.exists(path):
+	if(skip_bench_mkdir):
-		print('Directory {} already exists!'.format(path))
+		pass
-		raise Exception("Site directory already exists")
+	else:
-		# sys.exit(1)
+		if os.path.exists(path):
+			print('Directory {} already exists!'.format(path))
+			raise Exception("Site directory already exists")
+		os.makedirs(path)
 
-	os.makedirs(path)
 	for dirname in folders_in_bench:
-		os.mkdir(os.path.join(path, dirname))
+		try:
+			os.makedirs(os.path.join(path, dirname))
+		except OSError, e:
+			if e.errno != os.errno.EEXIST:
+				pass
 
 	setup_logging()
 
@@ -61,11 +69,13 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False,
 
 	bench.set_frappe_version(bench_path=path)
 	if bench.FRAPPE_VERSION > 5:
-		setup_socketio(bench_path=path)
+		update_npm_packages(bench_path=path)
 
 	set_all_patches_executed(bench_path=path)
 	build_assets(bench_path=path)
-	redis.generate_config(path)
+
+	if not skip_redis_config_generation:
+		redis.generate_config(path)
 
 	if not no_procfile:
 		setup_procfile(path)
@@ -119,7 +129,7 @@ def exec_cmd(cmd, cwd='.'):
 
 	logger.info(cmd)
 
-	p = subprocess.Popen(cmd, cwd=cwd, shell=True, stdout=stdout, stderr=stderr)
+	p = subprocess.Popen(cmd, cwd=cwd, shell=True, stdout=stdout, stderr=stderr, universal_newlines=True)
 
 	if async:
 		return_code = print_output(p)
@@ -134,10 +144,12 @@ def setup_env(bench_path='.'):
 	exec_cmd('./env/bin/pip -q install --upgrade pip', cwd=bench_path)
 	exec_cmd('./env/bin/pip -q install wheel', cwd=bench_path)
 	# exec_cmd('./env/bin/pip -q install https://github.com/frappe/MySQLdb1/archive/MySQLdb-1.2.5-patched.tar.gz', cwd=bench_path)
+	exec_cmd('./env/bin/pip -q install six', cwd=bench_path)
 	exec_cmd('./env/bin/pip -q install -e git+https://github.com/frappe/python-pdfkit.git#egg=pdfkit', cwd=bench_path)
 
 def setup_socketio(bench_path='.'):
-	exec_cmd("npm install socket.io redis express superagent cookie", cwd=bench_path)
+	exec_cmd("npm install socket.io redis express superagent cookie babel-core less chokidar \
+		babel-cli babel-preset-es2015 babel-preset-es2016 babel-preset-es2017 babel-preset-babili", cwd=bench_path)
 
 def patch_sites(bench_path='.'):
 	bench.set_frappe_version(bench_path=bench_path)
@@ -352,6 +364,7 @@ def set_default_site(site, bench_path='.'):
 			cwd=os.path.join(bench_path, 'sites'))
 
 def update_requirements(bench_path='.'):
+	print('Updating Python libraries...')
 	pip = os.path.join(bench_path, 'env', 'bin', 'pip')
 
 	# upgrade pip to latest
@@ -367,6 +380,38 @@ def update_requirements(bench_path='.'):
 		req_file = os.path.join(apps_dir, app, 'requirements.txt')
 		install_requirements(pip, req_file)
 
+def update_npm_packages(bench_path='.'):
+	print('Updating node libraries...')
+	apps_dir = os.path.join(bench_path, 'apps')
+	package_json = {}
+
+	for app in os.listdir(apps_dir):
+		package_json_path = os.path.join(apps_dir, app, 'package.json')
+
+		if os.path.exists(package_json_path):
+			with open(package_json_path, "r") as f:
+				app_package_json = json.loads(f.read())
+				# package.json is usually a dict in a dict
+				for key, value in iteritems(app_package_json):
+					if not key in package_json:
+						package_json[key] = value
+					else:
+						if isinstance(value, dict):
+							package_json[key].update(value)
+						elif isinstance(value, list):
+							package_json[key].extend(value)
+						else:
+							package_json[key] = value
+
+	if package_json is {}:
+		with open(os.path.join(os.path.dirname(__file__), 'package.json'), 'r') as f:
+			package_json = json.loads(f.read())
+
+	with open(os.path.join(bench_path, 'package.json'), 'w') as f:
+		f.write(json.dumps(package_json, indent=1, sort_keys=True))
+
+	exec_cmd('npm install', cwd=bench_path)
+
 def install_requirements(pip, req_file):
 	if os.path.exists(req_file):
 		exec_cmd("{pip} install -q -r {req_file}".format(pip=pip, req_file=req_file))

docs/branch_details.md

@@ -0,0 +1,13 @@
+### ERPNext/Frappe Branching
+
+#### Branch Description
+ - `develop` Branch: All new feature developments will go in develop branch
+ - `staging` Branch: This branch serves as a release candidate. Before a week, release team will pull the feature from develop branch to staging branch.
+    EG: if the feature is in 25 July's milestone then it should go in staging on 19th July.
+ - `master` Branch: Community release.
+ - `hotfix` Branch: mainly define for support issues. This will include bugs or any high priority task like security patches.
+
+#### Where to send PR?
+ - If you are working on a new feature, then PR should point to develop branch
+ - If you are working on support issue / bug / error report, then PR should point to hotfix brach
+ - While performing testing on Staging branch, if any fix needed then only send that fix PR to staging.

docs/contribution_guidelines.md

@@ -0,0 +1,46 @@
+# Contribution Guidelines
+
+### Introduction (for first timers)
+
+Thank you for your interesting in contributing to an open source project! Our world works on people taking initiative to contribute to the "commons" and contributing to open source means you are contributing to make things better for not only yourself, but everyone else too! So thank you for taking this initiative.
+
+Great projects also work because of great quality. Open source or not, the user really cares that things should work as they are advertised, and consistently. New features should follow the same pattern and so that users don't have to learn things again and again. 
+
+Developers who maintain open source also expect that you follow certain guidelines. These guidelines ensure that developers are able quickly give feedback on your contribution and how to make it better. Most probably you might have to go back and change a few things, but it will be in th interest of making this process better for everyone. So do be prepared for some back and forth.
+
+Happy contributing!
+
+### Feedback Policy
+
+We will strive for a "Zero Pull Request Pending" policy, inspired by "Zero Inbox". This means, that if the pull request is good, it will be merged within a day and if it does not meet the requirements, it will be closed.
+
+### Design Guides
+
+Please read the following design guidelines carefully when contributing:
+
+1. [Form Design Guidelines](https://github.com/frappe/erpnext/wiki/Form-Design-Guidelines)
+1. [How to break large contributions into smaller ones](https://github.com/frappe/erpnext/wiki/Cascading-Pull-Requests)
+
+### Pull Request Requirements
+
+1. **Test Cases:** Important to add test cases, even if its a very simple one that just calls the function. For UI, till we don't have Selenium testing setup, we need to see a screenshot / animated GIF.
+1. **UX:** If your change involves user experience, add a screenshot / narration / animated GIF.
+1. **Documentation:** Test Case must involve updating necessary documentation
+1. **Explanation:** Include explanation if there is a design change, explain the use case and why this suggested change is better. If you are including a new library or replacing one, please give sufficient reference of why the suggested library is better.
+1. **Demo:** Remember to update the demo script so that data related your feature is included in the demo.
+1. **Failing Tests:** This is simple, you must make sure all automated tests are passing.
+1. **Very Large Contribution:** It is very hard to accept and merge very large contributions, because there are too many lines of code to check and its implications can be large and unexpected. They way to contribute big features is to build them part by part. We can understand there are exceptions, but in most cases try and keep your pull-request to **30 lines of code** excluding tests and config files. **Use [Cascading Pull Requests](https://github.com/frappe/erpnext/wiki/Cascading-Pull-Requests)** for large features.
+1. **Incomplete Contributions must be hidden:** If the contribution is WIP or incomplete - which will most likely be the case, you can send small PRs as long as the user is not exposed to unfinished functionality. This will ensure that your code does not have build or other collateral issues. But these features must remain completely hidden to the user.
+1. **Incorrect Patches:** If your design involves schema change and you must include patches that update the data as per your new schema.
+1. **Incorrect Naming:** The naming of variables, models, fields etc must be consistent as per the existing design and semantics used in the system.
+1. **Translated Strings:** All user facing strings / text must be wrapped in the `__("")` function in javascript and `_("")` function in Python, so that it is shown as translated to the user.
+1. **Deprecated API:** The API used in the pull request must be the latest recommended methods and usage of globals like `cur_frm` must be avoided.
+1. **Whitespace and indentation:** The ERPNext and Frappe Project uses tabs (I know and we are sorry, but its too much effort to change it now and we don't want to lose the history). The indentation must be consistent whether you are writing Javascript or Python. Multi-line strings or expressions must also be consistently indented, not hanging like a bee hive at the end of the line. We just think the code looks a lot more stable that way.
+
+#### What if my Pull Request is closed?
+
+Don't worry, fix the problem and re-open it!
+
+#### Why do we follow this policy?
+
+This is because ERPNext is at a stage where it is being used by thousands of companies and introducing breaking changes can be harmful for everyone. Also we do not want to stop the speed of contributions and the best way to encourage contributors is to give fast feedback.

docs/release_policy.md

@@ -0,0 +1,63 @@
+# Release Policy
+
+#### Definitions:
+ - `develop` Branch: All new feature developments will go in develop branch
+ - `staging` Branch: This branch serves as a release candidate. Before a week, release team will pull the feature from develop branch to staging branch.
+    EG: if the feature is in 25 July's milestone then it should go in staging on 19th July.
+ - `master` Branch: `master` branch serves as a stable branch. This will use as production deployment.
+ - `hotfix` Branch: mainly define for support issues. This will include bugs or any high priority task like security patches.
+
+#### Create release from staging
+- On Tuesday, we will release from staging to master.
+
+- Versioning: Given a version number MAJOR.MINOR.PATCH, increment the:
+  - MAJOR version when you make incompatible API changes,
+  - MINOR version when you add functionality in a backwards-compatible manner, and
+  - PATCH version when you make backwards-compatible bug fixes.
+
+- Impact on branches:
+  - merge staging branch to master
+  - push merge commit back to staging branch
+  - push merge commit to develop branch
+  - push merge commit to hotfix branch
+
+- Use release command to create release,
+``` usage: bench release APP patch|minor|major --from-branch staging ```
+
+---
+
+#### Create staging branch
+
+- On Wednesday morning, `develop` will be merge into `staging`. `staging` branch is a release candidate. All new features will first go from `develop` to `staging` and then `staging` to `master`.
+
+- Use the prepare-staging command to create staging branch
+```usage: bench prepare-staging APP```
+
+- Impact on branches?
+  - merge all commits from develop branch to staging
+  - push merge commit back to develop
+
+- QA will use staging for testing.
+
+- Deploy staging branch on frappe.io, erpnext.org, frappe.erpnext.com. 
+
+- Only regression and security fixes can be cherry-picked into staging
+
+- Create a discuss post on what all new features or fixes going in next version.
+
+---
+
+#### Create release from hotfix
+- Depending on priority, hotfix release will take place.
+
+- Versioning:
+  - PATCH version when you make backwards-compatible bug fixes.
+
+- Impact on branches:
+  - merge hotfix branch to master
+  - push merge commit back to staging branch
+  - push merge commit to develop branch
+  - push merge commit to staging branch
+
+- Use release command to create release,
+``` usage: bench release APP patch --from-branch hotfix ```

docs/releasing_frappe_erpext.md

@@ -0,0 +1,39 @@
+# Releasing Frappe ERPNext
+
+* Make a new bench dedicated for releasing
+```
+bench init release-bench --frappe-path git@github.com:frappe/frappe.git
+```
+
+* Get ERPNext in the release bench
+```
+bench get-app erpnext git@github.com:frappe/erpnext.git
+```
+
+* Configure as release bench. Add this to the common_site_config.json
+```
+"release_bench": true,
+```
+
+* Add branches to update in common_site_config.json
+```
+"branches_to_update": {
+    "staging": ["develop", "hotfix"],
+    "hotfix": ["develop", "staging"]
+}
+```
+
+* Use the release commands to release
+```
+Usage: bench release [OPTIONS] APP BUMP_TYPE
+```
+
+* Arguments :
+  * _APP_ App name e.g [frappe|erpnext|yourapp]
+  * _BUMP_TYPE_ [major|minor|patch|stable|prerelease]
+* Options:
+  * --from-branch git develop branch, default is develop
+  * --to-branch git master branch, default is master
+  * --remote git remote, default is upstream
+  * --owner git owner, default is frappe
+  * --repo-name git repo name if different from app name

playbooks/develop/create_user.yml

@@ -10,14 +10,24 @@
       file:
         path: '/home/{{ frappe_user }}'
         mode: 'o+rx'
+        owner: '{{ frappe_user }}'
+        group: '{{ frappe_user }}'
+        recurse: yes
       when: ansible_distribution == 'Ubuntu' or ansible_distribution == 'CentOS' or ansible_distribution == 'Debian'
 
     - name: Set home folder perms
       file:
         path: '/Users/{{ frappe_user }}'
         mode: 'o+rx'
+        owner: '{{ frappe_user }}'
+        group: '{{ frappe_user }}'
+        recurse: yes
       when: ansible_distribution == 'MacOSX'
 
     - name: Set /tmp/.bench folder perms
-      command: 'chown -R {{ frappe_user }}:{{ frappe_user }} {{ repo_path }}'
+      file:
+        path: '{{ repo_path }}'
+        owner: '{{ frappe_user }}'
+        group: '{{ frappe_user }}'
+        recurse: yes
       when: ansible_distribution == 'Ubuntu' or ansible_distribution == 'CentOS' or ansible_distribution == 'Debian'

playbooks/develop/includes/setup_bench.yml

@@ -18,6 +18,12 @@
     become: yes
     become_user: root
 
+  - name: Overwrite bench if required
+    file:
+      state: absent
+      path: "{{ bench_path }}"
+    when: overwrite
+
   - name: Check whether bench exists
     stat: path="{{ bench_path }}"
     register: bench_stat

playbooks/develop/includes/setup_dev_env.yml

@@ -1,10 +1,4 @@
 ---
- # Setup Socketio
- - name: setup procfile
-   command: bench setup socketio
-   args:
-     creates: "{{ bench_path }}/node_modules"
-     chdir: "{{ bench_path }}"
 
  # Setup Procfile
  - name: setup procfile

playbooks/develop/includes/wkhtmltopdf.yml

@@ -1,6 +1,6 @@
 ---
   - name: download wkthmltox linux
-    get_url: url=http://download.gna.org/wkhtmltopdf/0.12/0.12.3/wkhtmltox-0.12.3_linux-generic-{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.tar.xz dest=/tmp/wkhtmltox.tar.xz
+    get_url: url=https://github.com/frappe/wkhtmltopdf/raw/master/wkhtmltox-0.12.3_linux-generic-{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.tar.xz dest=/tmp/wkhtmltox.tar.xz
   
   - name: Creates directory
     file: path=/tmp/wkhtmltox state=directory

playbooks/install.py

@@ -61,7 +61,7 @@ def install_bench(args):
 	# Restricting ansible version due to following bug in ansible 2.1
 	# https://github.com/ansible/ansible-modules-core/issues/3752
 	success = run_os_command({
-		'pip': "sudo pip install 'ansible==2.0.2.0'"
+		'pip': "sudo pip install ansible"
 	})
 
 	if not success:
@@ -97,7 +97,7 @@ def install_bench(args):
 	extra_vars.update(repo_path=repo_path)
 	run_playbook('develop/create_user.yml', extra_vars=extra_vars)
 
-	extra_vars.update(get_passwords(args.run_travis or args.without_bench_setup))
+	extra_vars.update(get_passwords(args))
 	if args.production:
 		extra_vars.update(max_worker_connections=multiprocessing.cpu_count() * 1024)
 
@@ -229,9 +229,31 @@ def could_not_install(package):
 def is_sudo_user():
 	return os.geteuid() == 0
 
-def get_passwords(ignore_prompt=False):
+
+def get_passwords(args):
+	"""
+	Returns a dict of passwords for further use
+	and creates passwords.txt in the bench user's home directory
+	"""
+
+	ignore_prompt = args.run_travis or args.without_bench_setup
+	mysql_root_password, admin_password = '', ''
+	passwords_file_path = os.path.join(os.path.expanduser('~' + args.user), 'passwords.txt')
+
 	if not ignore_prompt:
-		mysql_root_password, admin_password = '', ''
+		# set passwords from existing passwords.txt
+		if os.path.isfile(passwords_file_path):
+			with open(passwords_file_path, 'r') as f:
+				passwords = json.load(f)
+				mysql_root_password, admin_password = passwords['mysql_root_password'], passwords['admin_password']
+
+		# set passwords from cli args
+		if args.mysql_root_password:
+			mysql_root_password = args.mysql_root_password
+		if args.admin_password:
+			admin_password = args.admin_password
+
+		# prompt for passwords
 		pass_set = True
 		while pass_set:
 			# mysql root password
@@ -262,7 +284,6 @@ def get_passwords(ignore_prompt=False):
 	}
 
 	if not ignore_prompt:
-		passwords_file_path = os.path.join(os.path.expanduser('~'), 'passwords.txt')
 		with open(passwords_file_path, 'w') as f:
 			json.dump(passwords, f, indent=1)
 
@@ -270,6 +291,7 @@ def get_passwords(ignore_prompt=False):
 
 	return passwords
 
+
 def get_extra_vars_json(extra_args):
 	# We need to pass production as extra_vars to the playbook to execute conditionals in the
 	# playbook. Extra variables can passed as json or key=value pair. Here, we will use JSON.
@@ -335,6 +357,14 @@ def parse_commandline_args():
 	parser.add_argument('--without-bench-setup', dest='without_bench_setup', action='store_true', default=False,
 		help=argparse.SUPPRESS)
 	
+	# whether to overwrite an existing bench
+	parser.add_argument('--overwrite', dest='overwrite', action='store_true', default=False,
+		help='Whether to overwrite an existing bench')
+
+	# set passwords
+	parser.add_argument('--mysql-root-password', dest='mysql_root_password', help='Set mysql root password')
+	parser.add_argument('--admin-password', dest='admin_password', help='Set admin password')
+
 	args = parser.parse_args()
 
 	return args

vm/ansible/roles/wkhtmltopdf/tasks/main.yml

@@ -18,7 +18,7 @@
   when: ansible_os_family == 'Debian'
 
 - name: download wkthmltox linux
-  get_url: url=http://download.gna.org/wkhtmltopdf/0.12/0.12.3/wkhtmltox-0.12.3_linux-generic-{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.tar.xz dest=/tmp/wkhtmltox.tar.xz
+  get_url: url=https://github.com/frappe/wkhtmltopdf/raw/master/wkhtmltox-0.12.3_linux-generic-{{ "amd64" if ansible_architecture == "x86_64" else "i386"}}.tar.xz dest=/tmp/wkhtmltox.tar.xz
 
 - name: unarchive wkhtmltopdf
   unarchive: src=/tmp/wkhtmltox.tar.xz dest=/tmp/wkhtmltox