From 961606fe4b6a1aaab6e4aaac6d2f54523c546a3f Mon Sep 17 00:00:00 2001 From: Anand Doshi Date: Thu, 7 Jul 2016 20:09:13 +0530 Subject: [PATCH] [fix] bench setup add-domain/remove-domain/sync-domains + [cleanup] moved functions to site_config.py --- bench/commands/setup.py | 80 ++++++++++++++++-------- bench/commands/utils.py | 8 +-- bench/config/fonts.py | 17 ------ bench/config/nginx.py | 10 ++- bench/config/procfile.py | 8 +-- bench/config/production_setup.py | 17 ++++-- bench/config/site_config.py | 101 +++++++++++++++++++++++++++++++ bench/config/supervisor.py | 4 +- bench/patches/v3/celery_to_rq.py | 4 +- bench/utils.py | 70 ++++----------------- 10 files changed, 198 insertions(+), 121 deletions(-) delete mode 100644 bench/config/fonts.py create mode 100644 bench/config/site_config.py diff --git a/bench/commands/setup.py b/bench/commands/setup.py index 6c59bfcb..2c9f3e04 100644 --- a/bench/commands/setup.py +++ b/bench/commands/setup.py @@ -1,4 +1,4 @@ -import click, sys +import click, sys, json @click.group() def setup(): @@ -15,20 +15,24 @@ def setup_sudoers(user): @click.command('nginx') -@click.option('--force', help='Force regeneration of nginx config file', default=False, is_flag=True) -def setup_nginx(force=None): +@click.option('--yes', help='Yes to regeneration of nginx config file', default=False, is_flag=True) +def setup_nginx(yes=False): "generate config for nginx" from bench.config.nginx import make_nginx_conf - make_nginx_conf(bench_path=".", force=force) + make_nginx_conf(bench_path=".", yes=yes) +@click.command('reload-nginx') +def reload_nginx(): + from bench.config.production_setup import reload_nginx + reload_nginx() @click.command('supervisor') @click.option('--user') -@click.option('--force', help='Force regeneration of supervisor config', is_flag=True, default=False) -def setup_supervisor(user=None, force=None): +@click.option('--yes', help='Yes to regeneration of supervisor config', is_flag=True, default=False) +def setup_supervisor(user=None, yes=False): "generate config for supervisor with an optional user argument" from bench.config.supervisor import generate_supervisor_config - generate_supervisor_config(bench_path=".", user=user, force=force) + generate_supervisor_config(bench_path=".", user=user, yes=yes) @click.command('redis') def setup_redis(): @@ -40,7 +44,7 @@ def setup_redis(): @click.command('fonts') def setup_fonts(): "Add frappe fonts to system" - from bench.config.fonts import setup_fonts + from bench.utils import setup_fonts setup_fonts() @@ -101,27 +105,53 @@ def setup_config(): make_config('.') -@click.command('domain') +@click.command('add-domain') @click.argument('domain') -@click.option('--site') -@click.option('--ssl-certificate-path', help="Path to SSL certificate") -@click.option('--ssl-certificate-key', help="Path to SSL certificate key") -def setup_domain(site, domain, ssl_certificate_path=None, ssl_certificate_key=None): - "Add custom domain to site" - from bench.utils import get_site_domains, update_site_domains +@click.option('--site', prompt=True) +@click.option('--ssl-certificate', help="Absolute path to SSL Certificate") +@click.option('--ssl-certificate-key', help="Absolute path to SSL Certificate Key") +def add_domain(domain, site=None, ssl_certificate=None, ssl_certificate_key=None): + """Add custom domain to site""" + from bench.config.site_config import add_domain + if not site: print "Please specify site" sys.exit(1) - domains = get_site_domains(site) - if ssl_certificate_key and ssl_certificate_path: - domain = { - 'domain' : domain, - 'ssl_certificate_path': ssl_certificate_path, - 'ssl_certificate_key': ssl_certificate_key - } - update_site_domains(site, domain) + add_domain(site, domain, ssl_certificate, ssl_certificate_key, bench_path='.') +@click.command('remove-domain') +@click.argument('domain') +@click.option('--site', prompt=True) +def remove_domain(domain, site=None): + """Remove custom domain from a site""" + from bench.config.site_config import remove_domain + + if not site: + print "Please specify site" + sys.exit(1) + + remove_domain(site, domain, bench_path='.') + +@click.command('sync-domains') +@click.argument('domains') +@click.option('--site', prompt=True) +def sync_domains(domains, site=None): + from bench.config.site_config import sync_domains + + if not site: + print "Please specify site" + sys.exit(1) + + domains = json.loads(domains) + if not isinstance(domains, list): + print "Domains should be a json list of strings or dictionaries" + sys.exit(1) + + changed = sync_domains(site, domains, bench_path='.') + + # if changed, success, else failure + sys.exit(0 if changed else 1) setup.add_command(setup_sudoers) setup.add_command(setup_nginx) @@ -136,4 +166,6 @@ setup.add_command(setup_procfile) setup.add_command(setup_socketio) setup.add_command(setup_config) setup.add_command(setup_fonts) -setup.add_command(setup_domain) +setup.add_command(add_domain) +setup.add_command(remove_domain) +setup.add_command(sync_domains) diff --git a/bench/commands/utils.py b/bench/commands/utils.py index 3ebab0bf..a2b30960 100644 --- a/bench/commands/utils.py +++ b/bench/commands/utils.py @@ -23,7 +23,7 @@ def restart(): @click.argument('port', type=int) def set_nginx_port(site, port): "Set nginx port for site" - from bench.utils import set_nginx_port + from bench.config.site_config import set_nginx_port set_nginx_port(site, port) @@ -32,7 +32,7 @@ def set_nginx_port(site, port): @click.argument('ssl-certificate-path') def set_ssl_certificate(site, ssl_certificate_path): "Set ssl certificate path for site" - from bench.utils import set_ssl_certificate + from bench.config.site_config import set_ssl_certificate set_ssl_certificate(site, ssl_certificate_path) @@ -41,7 +41,7 @@ def set_ssl_certificate(site, ssl_certificate_path): @click.argument('ssl-certificate-key-path') def set_ssl_certificate_key(site, ssl_certificate_key_path): "Set ssl certificate private key path for site" - from bench.utils import set_ssl_certificate_key + from bench.config.site_config import set_ssl_certificate_key set_ssl_certificate_key(site, ssl_certificate_key_path) @@ -50,7 +50,7 @@ def set_ssl_certificate_key(site, ssl_certificate_key_path): @click.argument('url-root') def set_url_root(site, url_root): "Set url root for site" - from bench.utils import set_url_root + from bench.config.site_config import set_url_root set_url_root(site, url_root) diff --git a/bench/config/fonts.py b/bench/config/fonts.py deleted file mode 100644 index f6885ab3..00000000 --- a/bench/config/fonts.py +++ /dev/null @@ -1,17 +0,0 @@ -import os, shutil -from bench.utils import exec_cmd - -def setup_fonts(): - fonts_path = os.path.join('/tmp', 'fonts') - - exec_cmd("git clone https://github.com/frappe/fonts.git", cwd='/tmp') - os.rename('/usr/share/fonts', '/usr/share/fonts_backup') - os.rename('/etc/fonts', '/etc/fonts_backup') - os.rename(os.path.join(fonts_path, 'usr_share_fonts'), '/usr/share/fonts') - os.rename(os.path.join(fonts_path, 'etc_fonts'), '/etc/fonts') - shutil.rmtree(fonts_path) - exec_cmd("fc-cache -fv") - - - - diff --git a/bench/config/nginx.py b/bench/config/nginx.py index 6570597b..be266e37 100644 --- a/bench/config/nginx.py +++ b/bench/config/nginx.py @@ -1,7 +1,7 @@ import os, json, click, random, string -from bench.utils import get_sites, get_bench_name +from bench.utils import get_sites, get_bench_name, exec_cmd -def make_nginx_conf(bench_path, force=False): +def make_nginx_conf(bench_path, yes=False): from bench import env from bench.config.common_site_config import get_config @@ -25,7 +25,7 @@ def make_nginx_conf(bench_path, force=False): }) conf_path = os.path.join(bench_path, "config", "nginx.conf") - if not force and os.path.exists(conf_path): + if not yes and os.path.exists(conf_path): click.confirm('nginx.conf already exists and this will overwrite it. Do you want to continue?', abort=True) @@ -84,6 +84,7 @@ def prepare_sites(config, bench_path): def get_sites_with_config(bench_path): from bench.config.common_site_config import get_config + from bench.config.site_config import get_site_config sites = get_sites(bench_path=bench_path) dns_multitenant = get_config(bench_path).get('dns_multitenant') @@ -146,6 +147,3 @@ def use_wildcard_certificate(bench_path, ret): site['ssl_certificate_key'] = ssl_certificate_key site['wildcard'] = 1 -def get_site_config(site, bench_path='.'): - with open(os.path.join(bench_path, 'sites', site, 'site_config.json')) as f: - return json.load(f) diff --git a/bench/config/procfile.py b/bench/config/procfile.py index fe398ca6..3e4183e9 100755 --- a/bench/config/procfile.py +++ b/bench/config/procfile.py @@ -1,12 +1,12 @@ import bench, os, click -from bench.utils import find_executable -from bench.app import use_rq +from bench.utils import find_executable +from bench.app import use_rq from bench.config.common_site_config import get_config -def setup_procfile(bench_path, force=False): +def setup_procfile(bench_path, yes=False): config = get_config(bench_path=bench_path) procfile_path = os.path.join(bench_path, 'Procfile') - if not force and os.path.exists(procfile_path): + if not yes and os.path.exists(procfile_path): click.confirm('A Procfile already exists and this will overwrite it. Do you want to continue?', abort=True) diff --git a/bench/config/production_setup.py b/bench/config/production_setup.py index 9cbd3c39..302df30e 100755 --- a/bench/config/production_setup.py +++ b/bench/config/production_setup.py @@ -23,11 +23,12 @@ def setup_production(user, bench_path='.'): if not os.path.islink(nginx_conf): os.symlink(os.path.abspath(os.path.join(bench_path, 'config', 'nginx.conf')), nginx_conf) - exec_cmd('sudo supervisorctl reload') + update_supervisor() + if os.environ.get('NO_SERVICE_RESTART'): return - service('nginx', 'restart') + reload_nginx() def disable_production(bench_path='.'): bench_name = get_bench_name(bench_path) @@ -49,7 +50,7 @@ def disable_production(bench_path='.'): if os.path.islink(nginx_conf): os.unlink(nginx_conf) - service('nginx', 'reload') + reload_nginx() def service(service, option): if os.path.basename(get_program(['systemctl']) or '') == 'systemctl' and is_running_systemd(): @@ -61,7 +62,7 @@ def service(service, option): service_manager = os.environ.get("BENCH_SERVICE_MANAGER") if service_manager: service_manager_command = (os.environ.get("BENCH_SERVICE_MANAGER_COMMAND") - or "{service_manager} restart {service}").format(service_manager=service_manager, service=service) + or "{service_manager} {option} {service}").format(service_manager=service_manager, service=service, option=option) exec_cmd(service_manager_command) else: @@ -92,3 +93,11 @@ def is_running_systemd(): elif comm == "systemd": return True return False + +def update_supervisor(): + exec_cmd('sudo supervisorctl reread') + exec_cmd('sudo supervisorctl update') + +def reload_nginx(): + exec_cmd(['sudo', 'nginx', '-t']) + service('nginx', 'reload') diff --git a/bench/config/site_config.py b/bench/config/site_config.py new file mode 100644 index 00000000..a61c1fa3 --- /dev/null +++ b/bench/config/site_config.py @@ -0,0 +1,101 @@ +import os, json +from bench.utils import get_sites +from bench.config.nginx import make_nginx_conf +from collections import defaultdict + +def get_site_config(site, bench_path='.'): + config_path = os.path.join(bench_path, 'sites', site, 'site_config.json') + if not os.path.exists(config_path): + return {} + with open(config_path) as f: + return json.load(f) + +def put_site_config(site, config, bench_path='.'): + config_path = os.path.join(bench_path, 'sites', site, 'site_config.json') + with open(config_path, 'w') as f: + return json.dump(config, f, indent=1) + +def update_site_config(site, new_config, bench_path='.'): + config = get_site_config(site, bench_path=bench_path) + config.update(new_config) + put_site_config(site, config, bench_path=bench_path) + +def set_nginx_port(site, port, bench_path='.', gen_config=True): + set_site_config_nginx_property(site, {"nginx_port": port}, bench_path=bench_path, gen_config=gen_config) + +def set_ssl_certificate(site, ssl_certificate, bench_path='.', gen_config=True): + set_site_config_nginx_property(site, {"ssl_certificate": ssl_certificate}, bench_path=bench_path, gen_config=gen_config) + +def set_ssl_certificate_key(site, ssl_certificate_key, bench_path='.', gen_config=True): + set_site_config_nginx_property(site, {"ssl_certificate_key": ssl_certificate_key}, bench_path=bench_path, gen_config=gen_config) + +def set_site_config_nginx_property(site, config, bench_path='.', gen_config=True): + if site not in get_sites(bench_path=bench_path): + raise Exception("No such site") + update_site_config(site, config, bench_path=bench_path) + if gen_config: + make_nginx_conf(bench_path=bench_path) + +def set_url_root(site, url_root, bench_path='.'): + update_site_config(site, {"host_name": url_root}, bench_path=bench_path) + +def add_domain(site, domain, ssl_certificate, ssl_certificate_key, bench_path='.'): + domains = get_domains(site, bench_path) + for d in domains: + if (isinstance(d, dict) and d['domain']==domain) or d==domain: + print "Domain {0} already exists".format(domain) + return + + if ssl_certificate_key and ssl_certificate: + domain = { + 'domain' : domain, + 'ssl_certificate': ssl_certificate, + 'ssl_certificate_key': ssl_certificate_key + } + + domains.append(domain) + update_site_config(site, { "domains": domains }, bench_path=bench_path) + +def remove_domain(site, domain, bench_path='.'): + domains = get_domains(site, bench_path) + for i, d in enumerate(domains): + if (isinstance(d, dict) and d['domain']==domain) or d==domain: + domains.remove(d) + break + + update_site_config(site, { 'domains': domains }, bench_path=bench_path) + +def sync_domains(site, domains, bench_path='.'): + """Checks if there is a change in domains. If yes, updates the domains list.""" + changed = False + existing_domains = get_domains_dict(get_domains(site, bench_path)) + new_domains = get_domains_dict(domains) + + if set(existing_domains.keys()) != set(new_domains.keys()): + changed = True + + else: + for d in existing_domains.values(): + if d != new_domains.get(d['domain']): + changed = True + break + + if changed: + # replace existing domains with this one + update_site_config(site, { 'domains': domains }, bench_path='.') + + return changed + +def get_domains(site, bench_path='.'): + return get_site_config(site, bench_path=bench_path).get('domains') or [] + +def get_domains_dict(domains): + domains_dict = defaultdict(dict) + for d in domains: + if isinstance(d, basestring): + domains_dict[d] = { 'domain': d } + + elif isinstance(d, dict): + domains_dict[d['domain']] = d + + return domains_dict diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 0924e018..27396b15 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -1,7 +1,7 @@ import os, getpass, click import bench -def generate_supervisor_config(bench_path, user=None, force=False): +def generate_supervisor_config(bench_path, user=None, yes=False): from bench.app import get_current_frappe_version, use_rq from bench.utils import get_bench_name, find_executable from bench.config.common_site_config import get_config, update_config, get_gunicorn_workers @@ -34,7 +34,7 @@ def generate_supervisor_config(bench_path, user=None, force=False): }) conf_path = os.path.join(bench_path, 'config', 'supervisor.conf') - if not force and os.path.exists(conf_path): + if not yes and os.path.exists(conf_path): click.confirm('supervisor.conf already exists and this will overwrite it. Do you want to continue?', abort=True) diff --git a/bench/patches/v3/celery_to_rq.py b/bench/patches/v3/celery_to_rq.py index 301cd956..035730a6 100644 --- a/bench/patches/v3/celery_to_rq.py +++ b/bench/patches/v3/celery_to_rq.py @@ -18,8 +18,8 @@ def execute(bench_path): 'Do you want to continue?', abort=True) - setup_procfile(bench_path, force=True) + setup_procfile(bench_path, yes=True) # if production setup if os.path.exists(os.path.join(bench_path, 'config', 'supervisor.conf')): - generate_supervisor_config(bench_path, force=True) + generate_supervisor_config(bench_path, yes=True) diff --git a/bench/utils.py b/bench/utils.py index ba96ffb3..a94fd93c 100644 --- a/bench/utils.py +++ b/bench/utils.py @@ -1,28 +1,16 @@ -import os -import sys -import subprocess -import logging -import itertools -import requests -import json -import platform -import select -import multiprocessing +import os, sys, shutil, subprocess, logging, itertools, requests, json, platform, select, pwd, grp, multiprocessing from distutils.spawn import find_executable -import pwd, grp import bench from bench import env class PatchError(Exception): pass - class CommandFailedError(Exception): pass logger = logging.getLogger(__name__) - folders_in_bench = ('apps', 'sites', 'config', 'logs', 'config/pids') def get_frappe(bench_path='.'): @@ -322,51 +310,6 @@ def restart_supervisor_processes(bench_path='.'): exec_cmd('sudo supervisorctl restart {group}'.format(group=group), cwd=bench_path) -def get_site_config(site, bench_path='.'): - config_path = os.path.join(bench_path, 'sites', site, 'site_config.json') - if not os.path.exists(config_path): - return {} - with open(config_path) as f: - return json.load(f) - -def put_site_config(site, config, bench_path='.'): - config_path = os.path.join(bench_path, 'sites', site, 'site_config.json') - with open(config_path, 'w') as f: - return json.dump(config, f, indent=1) - -def update_site_config(site, new_config, bench_path='.'): - config = get_site_config(site, bench_path=bench_path) - config.update(new_config) - put_site_config(site, config, bench_path=bench_path) - -def get_site_domains(site, bench_path='.'): - return get_site_config(site, bench_path).get("domains") or [] - -def update_site_domains(site, domain, bench_path='.'): - domains = get_site_domains(site, bench_path) - domains.append(domain) - update_site_config(site, {"domains": domains}) - -def set_nginx_port(site, port, bench_path='.', gen_config=True): - set_site_config_nginx_property(site, {"nginx_port": port}, bench_path=bench_path, gen_config=gen_config) - -def set_ssl_certificate(site, ssl_certificate, bench_path='.', gen_config=True): - set_site_config_nginx_property(site, {"ssl_certificate": ssl_certificate}, bench_path=bench_path, gen_config=gen_config) - -def set_ssl_certificate_key(site, ssl_certificate_key, bench_path='.', gen_config=True): - set_site_config_nginx_property(site, {"ssl_certificate_key": ssl_certificate_key}, bench_path=bench_path, gen_config=gen_config) - -def set_site_config_nginx_property(site, config, bench_path='.', gen_config=True): - from .config.nginx import make_nginx_conf - if site not in get_sites(bench_path=bench_path): - raise Exception("No such site") - update_site_config(site, config, bench_path=bench_path) - if gen_config: - make_nginx_conf(bench_path=bench_path) - -def set_url_root(site, url_root, bench_path='.'): - update_site_config(site, {"host_name": url_root}, bench_path=bench_path) - def set_default_site(site, bench_path='.'): if not site in get_sites(bench_path=bench_path): raise Exception("Site not in bench") @@ -668,3 +611,14 @@ def validate_pillow_dependencies(bench_path, requirements): def get_bench_name(bench_path): return os.path.basename(os.path.abspath(bench_path)) + +def setup_fonts(): + fonts_path = os.path.join('/tmp', 'fonts') + + exec_cmd("git clone https://github.com/frappe/fonts.git", cwd='/tmp') + os.rename('/usr/share/fonts', '/usr/share/fonts_backup') + os.rename('/etc/fonts', '/etc/fonts_backup') + os.rename(os.path.join(fonts_path, 'usr_share_fonts'), '/usr/share/fonts') + os.rename(os.path.join(fonts_path, 'etc_fonts'), '/etc/fonts') + shutil.rmtree(fonts_path) + exec_cmd("fc-cache -fv")