Develop (#463)

* Add arguments for mysql_root_password and admin_password
Consider existing passwords.txt

* Remove duplicated --overwrite after rebase

* python 3 compatible changes for `update_npm_packages` (#434)

* Fix travis (#446)

* python 3 compatible changes for `update_npm_packages`

* install urllib3 pyOpenSSL ndg-httpsclient pyasn1 to fix wkhtmltopdf download failure on Travis

* use latest version of ansible

* test should look for mysqlclient-1.3.10 not mysqlclient-1.3.8

* `exec_cmd` causes TypeError on Python 3 (#451) (#452)

* python 3 compatible changes for `update_npm_packages`

* add universal_newlines=True to Popen

* Added docker specific changes to bench (#460)

* Added skip_bench_mkdir flag

* added skip_bench_mkdir and skip_redis_config_generation

* Removed the brackets around not in the utils.py/redis.generate_config()

* Replace lines 56-61 in utils.py since the tests failed.

* Reinstated lines 56-61

* printing the dirname

* Removed the rogue break

* Update utils.py

.travis.yml

@@ -7,6 +7,8 @@ python:
   - "2.7"
 
 install:
+  - sudo rm /etc/apt/sources.list.d/docker.list
+  - sudo pip install urllib3 pyOpenSSL ndg-httpsclient pyasn1
   - sudo apt-get purge -y mysql-common mysql-server mysql-client
   - sudo apt-get install --only-upgrade -y git
   - mkdir -p ~/.bench

bench/commands/make.py

@@ -10,13 +10,15 @@ import click
 @click.option('--no-backups',is_flag=True, help="Run migrations for all sites in the bench")
 @click.option('--no-auto-update',is_flag=True, help="Build JS and CSS artifacts for the bench")
 @click.option('--verbose',is_flag=True, help="Verbose output during install")
+@click.option('--skip-bench-mkdir', is_flag=True, help="Skip mkdir frappe-bench")
+@click.option('--skip-redis-config-generation', is_flag=True, help="Skip redis config generation if already specifying the common-site-config file")
 def init(path, apps_path, frappe_path, frappe_branch, no_procfile, no_backups,
-		no_auto_update, clone_from, verbose):
+		no_auto_update, clone_from, verbose, skip_bench_mkdir, skip_redis_config_generation):
 	"Create a new bench"
 	from bench.utils import init
 	init(path, apps_path=apps_path, no_procfile=no_procfile, no_backups=no_backups,
 			no_auto_update=no_auto_update, frappe_path=frappe_path, frappe_branch=frappe_branch,
-			verbose=verbose, clone_from=clone_from)
+			verbose=verbose, clone_from=clone_from, skip_bench_mkdir=skip_bench_mkdir, skip_redis_config_generation=skip_redis_config_generation)
 	click.echo('Bench {} initialized'.format(path))
 
 
@@ -54,5 +56,3 @@ def new_site(site, mariadb_root_password=None, admin_password=None):
 	"Create a new site in the bench"
 	from bench.utils import new_site
 	new_site(site, mariadb_root_password=mariadb_root_password, admin_password=admin_password)
-
-

bench/utils.py

@@ -2,6 +2,8 @@ import os, sys, shutil, subprocess, logging, itertools, requests, json, platform
 from distutils.spawn import find_executable
 import bench
 from bench import env
+from six import iteritems
+
 
 class PatchError(Exception):
 	pass
@@ -25,21 +27,27 @@ def get_env_cmd(cmd, bench_path='.'):
 
 def init(path, apps_path=None, no_procfile=False, no_backups=False,
 		no_auto_update=False, frappe_path=None, frappe_branch=None, wheel_cache_dir=None,
-		verbose=False, clone_from=None):
+		verbose=False, clone_from=None, skip_bench_mkdir=False, skip_redis_config_generation=False):
 	from .app import get_app, install_apps_from_path
 	from .config.common_site_config import make_config
 	from .config import redis
 	from .config.procfile import setup_procfile
 	from bench.patches import set_all_patches_executed
 
-	if os.path.exists(path):
-		print('Directory {} already exists!'.format(path))
-		raise Exception("Site directory already exists")
-		# sys.exit(1)
+	if(skip_bench_mkdir):
+		pass
+	else:
+		if os.path.exists(path):
+			print('Directory {} already exists!'.format(path))
+			raise Exception("Site directory already exists")
+		os.makedirs(path)
 
-	os.makedirs(path)
 	for dirname in folders_in_bench:
-		os.mkdir(os.path.join(path, dirname))
+		try:
+			os.makedirs(os.path.join(path, dirname))
+		except OSError, e:
+			if e.errno != os.errno.EEXIST:
+				pass
 
 	setup_logging()
 
@@ -65,7 +73,9 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False,
 
 	set_all_patches_executed(bench_path=path)
 	build_assets(bench_path=path)
-	redis.generate_config(path)
+
+	if not skip_redis_config_generation:
+		redis.generate_config(path)
 
 	if not no_procfile:
 		setup_procfile(path)
@@ -119,7 +129,7 @@ def exec_cmd(cmd, cwd='.'):
 
 	logger.info(cmd)
 
-	p = subprocess.Popen(cmd, cwd=cwd, shell=True, stdout=stdout, stderr=stderr)
+	p = subprocess.Popen(cmd, cwd=cwd, shell=True, stdout=stdout, stderr=stderr, universal_newlines=True)
 
 	if async:
 		return_code = print_output(p)
@@ -411,7 +421,7 @@ def update_npm_packages(bench_path='.'):
 			with open(package_json_path, "r") as f:
 				app_package_json = json.loads(f.read())
 				# package.json is usually a dict in a dict
-				for key, value in app_package_json.iteritems():
+				for key, value in iteritems(app_package_json):
 					if not key in package_json:
 						package_json[key] = value
 					else:
@@ -421,7 +431,7 @@ def update_npm_packages(bench_path='.'):
 							package_json[key].extend(value)
 						else:
 							package_json[key] = value
-	
+
 	if package_json is {}:
 		with open(os.path.join(os.path.dirname(__file__), 'package.json'), 'r') as f:
 			package_json = json.loads(f.read())
@@ -651,7 +661,7 @@ def update_translations(app, lang):
 				f.flush()
 
 	print('downloaded for', app, lang)
-	
+
 def download_chart_of_accounts():
 	charts_dir = os.path.join('apps', "erpnext", "erpnext", 'accounts', 'chart_of_accounts', "submitted")
 	csv_file = os.path.join(translations_dir, lang + '.csv')

playbooks/install.py

@@ -61,7 +61,7 @@ def install_bench(args):
 	# Restricting ansible version due to following bug in ansible 2.1
 	# https://github.com/ansible/ansible-modules-core/issues/3752
 	success = run_os_command({
-		'pip': "sudo pip install 'ansible==2.0.2.0'"
+		'pip': "sudo pip install ansible"
 	})
 
 	if not success:
@@ -97,7 +97,7 @@ def install_bench(args):
 	extra_vars.update(repo_path=repo_path)
 	run_playbook('develop/create_user.yml', extra_vars=extra_vars)
 
-	extra_vars.update(get_passwords(args.run_travis or args.without_bench_setup))
+	extra_vars.update(get_passwords(args))
 	if args.production:
 		extra_vars.update(max_worker_connections=multiprocessing.cpu_count() * 1024)
 
@@ -229,9 +229,31 @@ def could_not_install(package):
 def is_sudo_user():
 	return os.geteuid() == 0
 
-def get_passwords(ignore_prompt=False):
+
+def get_passwords(args):
+	"""
+	Returns a dict of passwords for further use
+	and creates passwords.txt in the bench user's home directory
+	"""
+
+	ignore_prompt = args.run_travis or args.without_bench_setup
+	mysql_root_password, admin_password = '', ''
+	passwords_file_path = os.path.join(os.path.expanduser('~' + args.user), 'passwords.txt')
+
 	if not ignore_prompt:
-		mysql_root_password, admin_password = '', ''
+		# set passwords from existing passwords.txt
+		if os.path.isfile(passwords_file_path):
+			with open(passwords_file_path, 'r') as f:
+				passwords = json.load(f)
+				mysql_root_password, admin_password = passwords['mysql_root_password'], passwords['admin_password']
+
+		# set passwords from cli args
+		if args.mysql_root_password:
+			mysql_root_password = args.mysql_root_password
+		if args.admin_password:
+			admin_password = args.admin_password
+
+		# prompt for passwords
 		pass_set = True
 		while pass_set:
 			# mysql root password
@@ -262,7 +284,6 @@ def get_passwords(ignore_prompt=False):
 	}
 
 	if not ignore_prompt:
-		passwords_file_path = os.path.join(os.path.expanduser('~'), 'passwords.txt')
 		with open(passwords_file_path, 'w') as f:
 			json.dump(passwords, f, indent=1)
 
@@ -270,6 +291,7 @@ def get_passwords(ignore_prompt=False):
 
 	return passwords
 
+
 def get_extra_vars_json(extra_args):
 	# We need to pass production as extra_vars to the playbook to execute conditionals in the
 	# playbook. Extra variables can passed as json or key=value pair. Here, we will use JSON.
@@ -334,11 +356,15 @@ def parse_commandline_args():
 
 	parser.add_argument('--without-bench-setup', dest='without_bench_setup', action='store_true', default=False,
 		help=argparse.SUPPRESS)
-
+	
 	# whether to overwrite an existing bench
 	parser.add_argument('--overwrite', dest='overwrite', action='store_true', default=False,
 		help='Whether to overwrite an existing bench')
 
+	# set passwords
+	parser.add_argument('--mysql-root-password', dest='mysql_root_password', help='Set mysql root password')
+	parser.add_argument('--admin-password', dest='admin_password', help='Set admin password')
+
 	args = parser.parse_args()
 
 	return args