[fix] bench setup add-domain/remove-domain/sync-domains + [cleanup] moved functions to site_config.py

bench/commands/setup.py

@@ -1,4 +1,4 @@
-import click, sys
+import click, sys, json
 
 @click.group()
 def setup():
@@ -15,20 +15,24 @@ def setup_sudoers(user):
 
 
 @click.command('nginx')
-@click.option('--force', help='Force regeneration of nginx config file', default=False, is_flag=True)
-def setup_nginx(force=None):
+@click.option('--yes', help='Yes to regeneration of nginx config file', default=False, is_flag=True)
+def setup_nginx(yes=False):
 	"generate config for nginx"
 	from bench.config.nginx import make_nginx_conf
-	make_nginx_conf(bench_path=".", force=force)
+	make_nginx_conf(bench_path=".", yes=yes)
 
+@click.command('reload-nginx')
+def reload_nginx():
+	from bench.config.production_setup import reload_nginx
+	reload_nginx()
 
 @click.command('supervisor')
 @click.option('--user')
-@click.option('--force', help='Force regeneration of supervisor config', is_flag=True, default=False)
-def setup_supervisor(user=None, force=None):
+@click.option('--yes', help='Yes to regeneration of supervisor config', is_flag=True, default=False)
+def setup_supervisor(user=None, yes=False):
 	"generate config for supervisor with an optional user argument"
 	from bench.config.supervisor import generate_supervisor_config
-	generate_supervisor_config(bench_path=".", user=user, force=force)
+	generate_supervisor_config(bench_path=".", user=user, yes=yes)
 
 @click.command('redis')
 def setup_redis():
@@ -40,7 +44,7 @@ def setup_redis():
 @click.command('fonts')
 def setup_fonts():
 	"Add frappe fonts to system"
-	from bench.config.fonts import setup_fonts
+	from bench.utils import setup_fonts
 	setup_fonts()
 
 
@@ -101,27 +105,53 @@ def setup_config():
 	make_config('.')
 
 
-@click.command('domain')
+@click.command('add-domain')
 @click.argument('domain')
-@click.option('--site')
-@click.option('--ssl-certificate-path', help="Path to SSL certificate")
-@click.option('--ssl-certificate-key', help="Path to SSL certificate key")
-def setup_domain(site, domain, ssl_certificate_path=None, ssl_certificate_key=None):
-	"Add custom domain to site"
-	from bench.utils import get_site_domains, update_site_domains
+@click.option('--site', prompt=True)
+@click.option('--ssl-certificate', help="Absolute path to SSL Certificate")
+@click.option('--ssl-certificate-key', help="Absolute path to SSL Certificate Key")
+def add_domain(domain, site=None, ssl_certificate=None, ssl_certificate_key=None):
+	"""Add custom domain to site"""
+	from bench.config.site_config import add_domain
+
 	if not site:
 		print "Please specify site"
 		sys.exit(1)
 
-	domains = get_site_domains(site)
-	if ssl_certificate_key and ssl_certificate_path:
-		domain = {
-			'domain' : domain,
-			'ssl_certificate_path': ssl_certificate_path,
-			'ssl_certificate_key': ssl_certificate_key
-			}
-	update_site_domains(site, domain)
+	add_domain(site, domain, ssl_certificate, ssl_certificate_key, bench_path='.')
 
+@click.command('remove-domain')
+@click.argument('domain')
+@click.option('--site', prompt=True)
+def remove_domain(domain, site=None):
+	"""Remove custom domain from a site"""
+	from bench.config.site_config import remove_domain
+
+	if not site:
+		print "Please specify site"
+		sys.exit(1)
+
+	remove_domain(site, domain, bench_path='.')
+
+@click.command('sync-domains')
+@click.argument('domains')
+@click.option('--site', prompt=True)
+def sync_domains(domains, site=None):
+	from bench.config.site_config import sync_domains
+
+	if not site:
+		print "Please specify site"
+		sys.exit(1)
+
+	domains = json.loads(domains)
+	if not isinstance(domains, list):
+		print "Domains should be a json list of strings or dictionaries"
+		sys.exit(1)
+
+	changed = sync_domains(site, domains, bench_path='.')
+
+	# if changed, success, else failure
+	sys.exit(0 if changed else 1)
 
 setup.add_command(setup_sudoers)
 setup.add_command(setup_nginx)
@@ -136,4 +166,6 @@ setup.add_command(setup_procfile)
 setup.add_command(setup_socketio)
 setup.add_command(setup_config)
 setup.add_command(setup_fonts)
-setup.add_command(setup_domain)
+setup.add_command(add_domain)
+setup.add_command(remove_domain)
+setup.add_command(sync_domains)

bench/commands/utils.py

@@ -23,7 +23,7 @@ def restart():
 @click.argument('port', type=int)
 def set_nginx_port(site, port):
 	"Set nginx port for site"
-	from bench.utils import set_nginx_port
+	from bench.config.site_config import set_nginx_port
 	set_nginx_port(site, port)
 
 
@@ -32,7 +32,7 @@ def set_nginx_port(site, port):
 @click.argument('ssl-certificate-path')
 def set_ssl_certificate(site, ssl_certificate_path):
 	"Set ssl certificate path for site"
-	from bench.utils import set_ssl_certificate
+	from bench.config.site_config import set_ssl_certificate
 	set_ssl_certificate(site, ssl_certificate_path)
 
 
@@ -41,7 +41,7 @@ def set_ssl_certificate(site, ssl_certificate_path):
 @click.argument('ssl-certificate-key-path')
 def set_ssl_certificate_key(site, ssl_certificate_key_path):
 	"Set ssl certificate private key path for site"
-	from bench.utils import set_ssl_certificate_key
+	from bench.config.site_config import set_ssl_certificate_key
 	set_ssl_certificate_key(site, ssl_certificate_key_path)
 
 
@@ -50,7 +50,7 @@ def set_ssl_certificate_key(site, ssl_certificate_key_path):
 @click.argument('url-root')
 def set_url_root(site, url_root):
 	"Set url root for site"
-	from bench.utils import set_url_root
+	from bench.config.site_config import set_url_root
 	set_url_root(site, url_root)
 
 

bench/config/fonts.py

@@ -1,17 +0,0 @@
-import os, shutil
-from bench.utils import exec_cmd
-
-def setup_fonts():
-	fonts_path = os.path.join('/tmp', 'fonts')
-
-	exec_cmd("git clone https://github.com/frappe/fonts.git", cwd='/tmp')
-	os.rename('/usr/share/fonts', '/usr/share/fonts_backup')
-	os.rename('/etc/fonts', '/etc/fonts_backup')
-	os.rename(os.path.join(fonts_path, 'usr_share_fonts'), '/usr/share/fonts')
-	os.rename(os.path.join(fonts_path, 'etc_fonts'), '/etc/fonts')
-	shutil.rmtree(fonts_path)
-	exec_cmd("fc-cache -fv")
-
-
-
-

bench/config/nginx.py

@@ -1,7 +1,7 @@
 import os, json, click, random, string
-from bench.utils import get_sites, get_bench_name
+from bench.utils import get_sites, get_bench_name, exec_cmd
 
-def make_nginx_conf(bench_path, force=False):
+def make_nginx_conf(bench_path, yes=False):
 	from bench import env
 	from bench.config.common_site_config import get_config
 
@@ -25,7 +25,7 @@ def make_nginx_conf(bench_path, force=False):
 	})
 
 	conf_path = os.path.join(bench_path, "config", "nginx.conf")
-	if not force and os.path.exists(conf_path):
+	if not yes and os.path.exists(conf_path):
 		click.confirm('nginx.conf already exists and this will overwrite it. Do you want to continue?',
 			abort=True)
 
@@ -84,6 +84,7 @@ def prepare_sites(config, bench_path):
 
 def get_sites_with_config(bench_path):
 	from bench.config.common_site_config import get_config
+	from bench.config.site_config import get_site_config
 
 	sites = get_sites(bench_path=bench_path)
 	dns_multitenant = get_config(bench_path).get('dns_multitenant')
@@ -146,6 +147,3 @@ def use_wildcard_certificate(bench_path, ret):
 			site['ssl_certificate_key'] = ssl_certificate_key
 			site['wildcard'] = 1
 
-def get_site_config(site, bench_path='.'):
-	with open(os.path.join(bench_path, 'sites', site, 'site_config.json')) as f:
-		return json.load(f)

bench/config/procfile.py

@@ -3,10 +3,10 @@ from bench.utils import find_executable
 from bench.app import use_rq
 from bench.config.common_site_config import get_config
 
-def setup_procfile(bench_path, force=False):
+def setup_procfile(bench_path, yes=False):
 	config = get_config(bench_path=bench_path)
 	procfile_path = os.path.join(bench_path, 'Procfile')
-	if not force and os.path.exists(procfile_path):
+	if not yes and os.path.exists(procfile_path):
 		click.confirm('A Procfile already exists and this will overwrite it. Do you want to continue?',
 			abort=True)
 

bench/config/production_setup.py

@@ -23,11 +23,12 @@ def setup_production(user, bench_path='.'):
 	if not os.path.islink(nginx_conf):
 		os.symlink(os.path.abspath(os.path.join(bench_path, 'config', 'nginx.conf')), nginx_conf)
 
-	exec_cmd('sudo supervisorctl reload')
+	update_supervisor()
+
 	if os.environ.get('NO_SERVICE_RESTART'):
 		return
 
-	service('nginx', 'restart')
+	reload_nginx()
 
 def disable_production(bench_path='.'):
 	bench_name = get_bench_name(bench_path)
@@ -49,7 +50,7 @@ def disable_production(bench_path='.'):
 	if os.path.islink(nginx_conf):
 		os.unlink(nginx_conf)
 
-	service('nginx', 'reload')
+	reload_nginx()
 
 def service(service, option):
 	if os.path.basename(get_program(['systemctl']) or '') == 'systemctl' and is_running_systemd():
@@ -61,7 +62,7 @@ def service(service, option):
 		service_manager = os.environ.get("BENCH_SERVICE_MANAGER")
 		if service_manager:
 			service_manager_command = (os.environ.get("BENCH_SERVICE_MANAGER_COMMAND")
-				or "{service_manager} restart {service}").format(service_manager=service_manager, service=service)
+				or "{service_manager} {option} {service}").format(service_manager=service_manager, service=service, option=option)
 			exec_cmd(service_manager_command)
 
 		else:
@@ -92,3 +93,11 @@ def is_running_systemd():
 	elif comm == "systemd":
 		return True
 	return False
+
+def update_supervisor():
+	exec_cmd('sudo supervisorctl reread')
+	exec_cmd('sudo supervisorctl update')
+
+def reload_nginx():
+	exec_cmd(['sudo', 'nginx', '-t'])
+	service('nginx', 'reload')

bench/config/site_config.py

@@ -0,0 +1,101 @@
+import os, json
+from bench.utils import get_sites
+from bench.config.nginx import make_nginx_conf
+from collections import defaultdict
+
+def get_site_config(site, bench_path='.'):
+	config_path = os.path.join(bench_path, 'sites', site, 'site_config.json')
+	if not os.path.exists(config_path):
+		return {}
+	with open(config_path) as f:
+		return json.load(f)
+
+def put_site_config(site, config, bench_path='.'):
+	config_path = os.path.join(bench_path, 'sites', site, 'site_config.json')
+	with open(config_path, 'w') as f:
+		return json.dump(config, f, indent=1)
+
+def update_site_config(site, new_config, bench_path='.'):
+	config = get_site_config(site, bench_path=bench_path)
+	config.update(new_config)
+	put_site_config(site, config, bench_path=bench_path)
+
+def set_nginx_port(site, port, bench_path='.', gen_config=True):
+	set_site_config_nginx_property(site, {"nginx_port": port}, bench_path=bench_path, gen_config=gen_config)
+
+def set_ssl_certificate(site, ssl_certificate, bench_path='.', gen_config=True):
+	set_site_config_nginx_property(site, {"ssl_certificate": ssl_certificate}, bench_path=bench_path, gen_config=gen_config)
+
+def set_ssl_certificate_key(site, ssl_certificate_key, bench_path='.', gen_config=True):
+	set_site_config_nginx_property(site, {"ssl_certificate_key": ssl_certificate_key}, bench_path=bench_path, gen_config=gen_config)
+
+def set_site_config_nginx_property(site, config, bench_path='.', gen_config=True):
+	if site not in get_sites(bench_path=bench_path):
+		raise Exception("No such site")
+	update_site_config(site, config, bench_path=bench_path)
+	if gen_config:
+		make_nginx_conf(bench_path=bench_path)
+
+def set_url_root(site, url_root, bench_path='.'):
+	update_site_config(site, {"host_name": url_root}, bench_path=bench_path)
+
+def add_domain(site, domain, ssl_certificate, ssl_certificate_key, bench_path='.'):
+	domains = get_domains(site, bench_path)
+	for d in domains:
+		if (isinstance(d, dict) and d['domain']==domain) or d==domain:
+			print "Domain {0} already exists".format(domain)
+			return
+
+	if ssl_certificate_key and ssl_certificate:
+		domain = {
+			'domain' : domain,
+			'ssl_certificate': ssl_certificate,
+			'ssl_certificate_key': ssl_certificate_key
+		}
+
+	domains.append(domain)
+	update_site_config(site, { "domains": domains }, bench_path=bench_path)
+
+def remove_domain(site, domain, bench_path='.'):
+	domains = get_domains(site, bench_path)
+	for i, d in enumerate(domains):
+		if (isinstance(d, dict) and d['domain']==domain) or d==domain:
+			domains.remove(d)
+			break
+
+	update_site_config(site, { 'domains': domains }, bench_path=bench_path)
+
+def sync_domains(site, domains, bench_path='.'):
+	"""Checks if there is a change in domains. If yes, updates the domains list."""
+	changed = False
+	existing_domains = get_domains_dict(get_domains(site, bench_path))
+	new_domains = get_domains_dict(domains)
+
+	if set(existing_domains.keys()) != set(new_domains.keys()):
+		changed = True
+
+	else:
+		for d in existing_domains.values():
+			if d != new_domains.get(d['domain']):
+				changed = True
+				break
+
+	if changed:
+		# replace existing domains with this one
+		update_site_config(site, { 'domains': domains }, bench_path='.')
+
+	return changed
+
+def get_domains(site, bench_path='.'):
+	return get_site_config(site, bench_path=bench_path).get('domains') or []
+
+def get_domains_dict(domains):
+	domains_dict = defaultdict(dict)
+	for d in domains:
+		if isinstance(d, basestring):
+			domains_dict[d] = { 'domain': d }
+
+		elif isinstance(d, dict):
+			domains_dict[d['domain']] = d
+
+	return domains_dict

bench/config/supervisor.py

@@ -1,7 +1,7 @@
 import os, getpass, click
 import bench
 
-def generate_supervisor_config(bench_path, user=None, force=False):
+def generate_supervisor_config(bench_path, user=None, yes=False):
 	from bench.app import get_current_frappe_version, use_rq
 	from bench.utils import get_bench_name, find_executable
 	from bench.config.common_site_config import get_config, update_config, get_gunicorn_workers
@@ -34,7 +34,7 @@ def generate_supervisor_config(bench_path, user=None, force=False):
 	})
 
 	conf_path = os.path.join(bench_path, 'config', 'supervisor.conf')
-	if not force and os.path.exists(conf_path):
+	if not yes and os.path.exists(conf_path):
 		click.confirm('supervisor.conf already exists and this will overwrite it. Do you want to continue?',
 			abort=True)
 

bench/patches/v3/celery_to_rq.py

@@ -18,8 +18,8 @@ def execute(bench_path):
 		'Do you want to continue?',
 		abort=True)
 
-	setup_procfile(bench_path, force=True)
+	setup_procfile(bench_path, yes=True)
 
 	# if production setup
 	if os.path.exists(os.path.join(bench_path, 'config', 'supervisor.conf')):
-		generate_supervisor_config(bench_path, force=True)
+		generate_supervisor_config(bench_path, yes=True)

bench/utils.py

@@ -1,28 +1,16 @@
-import os
-import sys
-import subprocess
-import logging
-import itertools
-import requests
-import json
-import platform
-import select
-import multiprocessing
+import os, sys, shutil, subprocess, logging, itertools, requests, json, platform, select, pwd, grp, multiprocessing
 from distutils.spawn import find_executable
-import pwd, grp
 import bench
 from bench import env
 
 class PatchError(Exception):
 	pass
 
-
 class CommandFailedError(Exception):
 	pass
 
 logger = logging.getLogger(__name__)
 
-
 folders_in_bench = ('apps', 'sites', 'config', 'logs', 'config/pids')
 
 def get_frappe(bench_path='.'):
@@ -322,51 +310,6 @@ def restart_supervisor_processes(bench_path='.'):
 
 		exec_cmd('sudo supervisorctl restart {group}'.format(group=group), cwd=bench_path)
 
-def get_site_config(site, bench_path='.'):
-	config_path = os.path.join(bench_path, 'sites', site, 'site_config.json')
-	if not os.path.exists(config_path):
-		return {}
-	with open(config_path) as f:
-		return json.load(f)
-
-def put_site_config(site, config, bench_path='.'):
-	config_path = os.path.join(bench_path, 'sites', site, 'site_config.json')
-	with open(config_path, 'w') as f:
-		return json.dump(config, f, indent=1)
-
-def update_site_config(site, new_config, bench_path='.'):
-	config = get_site_config(site, bench_path=bench_path)
-	config.update(new_config)
-	put_site_config(site, config, bench_path=bench_path)
-
-def get_site_domains(site, bench_path='.'):
-	return get_site_config(site, bench_path).get("domains") or []
-
-def update_site_domains(site, domain, bench_path='.'):
-	domains = get_site_domains(site, bench_path)
-	domains.append(domain)
-	update_site_config(site, {"domains": domains})
-
-def set_nginx_port(site, port, bench_path='.', gen_config=True):
-	set_site_config_nginx_property(site, {"nginx_port": port}, bench_path=bench_path, gen_config=gen_config)
-
-def set_ssl_certificate(site, ssl_certificate, bench_path='.', gen_config=True):
-	set_site_config_nginx_property(site, {"ssl_certificate": ssl_certificate}, bench_path=bench_path, gen_config=gen_config)
-
-def set_ssl_certificate_key(site, ssl_certificate_key, bench_path='.', gen_config=True):
-	set_site_config_nginx_property(site, {"ssl_certificate_key": ssl_certificate_key}, bench_path=bench_path, gen_config=gen_config)
-
-def set_site_config_nginx_property(site, config, bench_path='.', gen_config=True):
-	from .config.nginx import make_nginx_conf
-	if site not in get_sites(bench_path=bench_path):
-		raise Exception("No such site")
-	update_site_config(site, config, bench_path=bench_path)
-	if gen_config:
-		make_nginx_conf(bench_path=bench_path)
-
-def set_url_root(site, url_root, bench_path='.'):
-	update_site_config(site, {"host_name": url_root}, bench_path=bench_path)
-
 def set_default_site(site, bench_path='.'):
 	if not site in get_sites(bench_path=bench_path):
 		raise Exception("Site not in bench")
@@ -668,3 +611,14 @@ def validate_pillow_dependencies(bench_path, requirements):
 
 def get_bench_name(bench_path):
 	return os.path.basename(os.path.abspath(bench_path))
+
+def setup_fonts():
+	fonts_path = os.path.join('/tmp', 'fonts')
+
+	exec_cmd("git clone https://github.com/frappe/fonts.git", cwd='/tmp')
+	os.rename('/usr/share/fonts', '/usr/share/fonts_backup')
+	os.rename('/etc/fonts', '/etc/fonts_backup')
+	os.rename(os.path.join(fonts_path, 'usr_share_fonts'), '/usr/share/fonts')
+	os.rename(os.path.join(fonts_path, 'etc_fonts'), '/etc/fonts')
+	shutil.rmtree(fonts_path)
+	exec_cmd("fc-cache -fv")