[cleanup] nginx config for multi-bench

bench/cli.py

@@ -12,14 +12,15 @@ from .utils import set_url_root as _set_url_root
 from .utils import set_default_site as _set_default_site
 from .utils import (build_assets, patch_sites, exec_cmd, update_bench, get_env_cmd, get_frappe, setup_logging,
 					get_config, update_config, restart_supervisor_processes, put_config, default_config, update_requirements,
-					backup_all_sites, backup_site, get_sites, prime_wheel_cache, is_root, set_mariadb_host, drop_privileges,
+					backup_all_sites, backup_site, get_sites, is_root, set_mariadb_host, drop_privileges,
 					fix_file_perms, fix_prod_setup_perms, set_ssl_certificate, set_ssl_certificate_key,
 					get_cmd_output, post_upgrade, get_bench_name,
 					pre_upgrade, validate_upgrade, PatchError, download_translations_p, setup_socketio, before_update)
 from .app import get_app as _get_app
 from .app import new_app as _new_app
 from .app import pull_all_apps, get_apps, get_current_frappe_version, is_version_upgrade, switch_to_v4, switch_to_v5, switch_to_master, switch_to_develop
-from .config import generate_nginx_config, generate_supervisor_config, generate_redis_cache_config, generate_redis_async_broker_config, generate_redis_celery_broker_config
+from .config import generate_supervisor_config, generate_redis_cache_config, generate_redis_async_broker_config, generate_redis_celery_broker_config
+from .config.nginx import make_nginx_conf
 from .production_setup import setup_production as _setup_production
 from .migrate_to_v5 import migrate_to_v5
 import os
@@ -402,11 +403,6 @@ def _backup_all_sites():
 	"backup all sites"
 	backup_all_sites(bench='.')
 
-@click.command('prime-wheel-cache')
-def _prime_wheel_cache():
-	"Update wheel cache"
-	prime_wheel_cache(bench='.')
-
 @click.command('release')
 @click.argument('app', type=click.Choice(['frappe', 'erpnext', 'erpnext_shopify', 'paypal_integration']))
 @click.argument('bump-type', type=click.Choice(['major', 'minor', 'patch']))
@@ -433,7 +429,7 @@ def setup_sudoers(user):
 @click.command('nginx')
 def setup_nginx():
 	"generate config for nginx"
-	generate_nginx_config()
+	make_nginx_conf(bench=".")
 
 @click.command('supervisor')
 def setup_supervisor():
@@ -631,7 +627,6 @@ bench.add_command(_switch_to_v5)
 bench.add_command(shell)
 bench.add_command(_backup_all_sites)
 bench.add_command(_backup_site)
-bench.add_command(_prime_wheel_cache)
 bench.add_command(_release)
 bench.add_command(patch)
 bench.add_command(set_url_root)

bench/config/__init__.py

@@ -6,9 +6,9 @@ import shutil
 import socket
 from distutils.spawn import find_executable
 from jinja2 import Environment, PackageLoader
-from .utils import get_sites, get_config, update_config, get_redis_version, update_common_site_config, get_bench_name
+from bench.utils import get_sites, get_config, update_config, get_redis_version, update_common_site_config, get_bench_name
 
-env = Environment(loader=PackageLoader('bench', 'templates'), trim_blocks=True)
+env = Environment(loader=PackageLoader('bench.config'), trim_blocks=True)
 
 def write_config_file(bench, file_name, config):
 	config_path = os.path.join(bench, 'config')
@@ -23,7 +23,7 @@ def write_config_file(bench, file_name, config):
 		f.write(config)
 
 def generate_supervisor_config(bench='.', user=None):
-	from .app import get_current_frappe_version
+	from bench.app import get_current_frappe_version
 	template = env.get_template('supervisor.conf')
 	bench_dir = os.path.abspath(bench)
 	sites_dir = os.path.join(bench_dir, "sites")
@@ -51,10 +51,6 @@ def generate_supervisor_config(bench='.', user=None):
 	write_config_file(bench, 'supervisor.conf', config)
 	update_config({'restart_supervisor_on_update': True})
 
-def get_site_config(site, bench='.'):
-	with open(os.path.join(bench, 'sites', site, 'site_config.json')) as f:
-		return json.load(f)
-
 def generate_common_site_config(bench='.'):
 	'''Generates the default common_site_config.json while a new bench is created'''
 	config = get_config(bench=bench)
@@ -76,47 +72,34 @@ def generate_common_site_config(bench='.'):
 	if common_site_config:
 		update_common_site_config(common_site_config, bench=bench)
 
-def get_sites_with_config(bench='.'):
+# def generate_nginx_config(bench='.'):
-	sites = get_sites(bench=bench)
+# 	template = env.get_template('nginx.conf')
-	ret = []
+# 	bench_dir = os.path.abspath(bench)
-	for site in sites:
+# 	sites_dir = os.path.join(bench_dir, "sites")
-		site_config = get_site_config(site, bench=bench)
+# 	sites = get_sites_with_config(bench=bench)
-		ret.append({
+# 	user = getpass.getuser()
-			"name": site,
+# 	config = get_config(bench)
-			"port": site_config.get('nginx_port'),
+#
-			"ssl_certificate": site_config.get('ssl_certificate'),
+# 	if config.get('serve_default_site'):
-			"ssl_certificate_key": site_config.get('ssl_certificate_key')
+# 		try:
-		})
+# 			with open("sites/currentsite.txt") as f:
-	return ret
+# 				default_site = {'name': f.read().strip()}
-
+# 		except IOError:
-def generate_nginx_config(bench='.'):
+# 			default_site = None
-	template = env.get_template('nginx.conf')
+# 	else:
-	bench_dir = os.path.abspath(bench)
+# 		default_site = None
-	sites_dir = os.path.join(bench_dir, "sites")
+#
-	sites = get_sites_with_config(bench=bench)
+# 	config = template.render(**{
-	user = getpass.getuser()
+# 		"sites_dir": sites_dir,
-	config = get_config(bench)
+# 		"http_timeout": config.get("http_timeout", 120),
-
+# 		"default_site": default_site,
-	if config.get('serve_default_site'):
+# 		"dns_multitenant": config.get('dns_multitenant'),
-		try:
+# 		"sites": sites,
-			with open("sites/currentsite.txt") as f:
+# 		"webserver_port": config.get('webserver_port', 8000),
-				default_site = {'name': f.read().strip()}
+# 		"socketio_port": config.get('socketio_port', 3000),
-		except IOError:
+# 		"bench_name": get_bench_name(bench)
-			default_site = None
+# 	})
-	else:
+# 	write_config_file(bench, 'nginx.conf', config)
-		default_site = None
-
-	config = template.render(**{
-		"sites_dir": sites_dir,
-		"http_timeout": config.get("http_timeout", 120),
-		"default_site": default_site,
-		"dns_multitenant": config.get('dns_multitenant'),
-		"sites": sites,
-		"webserver_port": config.get('webserver_port', 8000),
-		"socketio_port": config.get('socketio_port', 3000),
-		"bench_name": get_bench_name(bench)
-	})
-	write_config_file(bench, 'nginx.conf', config)
 
 def generate_redis_celery_broker_config(bench='.'):
 	"""Redis that is used for queueing celery tasks"""

bench/config/nginx.py

@@ -0,0 +1,72 @@
+import os
+import json
+from bench.utils import get_sites, get_config, get_bench_name
+
+def make_nginx_conf(bench):
+	from bench.config import env, write_config_file
+
+	template = env.get_template('nginx.conf')
+	bench_path = os.path.abspath(bench)
+	sites_path = os.path.join(bench_path, "sites")
+
+	config = get_config(bench)
+	sites = prepare_sites(config, bench)
+
+	nginx_conf = template.render(**{
+		"sites_path": sites_path,
+		"http_timeout": config.get("http_timeout"),
+		"sites": sites,
+		"webserver_port": config.get('webserver_port'),
+		"socketio_port": config.get('socketio_port'),
+		"bench_name": get_bench_name(bench)
+	})
+
+	write_config_file(bench, 'nginx.conf', nginx_conf)
+
+def prepare_sites(config, bench):
+	sites = {
+		"that_use_dns": [],
+		"that_use_ssl": [],
+		"that_use_port": []
+	}
+	ports_in_use = {}
+	dns_multitenant = config.get('dns_multitenant')
+
+	for site in get_sites_with_config(bench=bench):
+		if dns_multitenant:
+			# assumes site's folder name is same as the domain name
+
+			if site.get("ssl_certificate") and site.get("ssl_certificate_key"):
+				sites["that_use_ssl"].append(site)
+
+			else:
+				sites["that_use_dns"].append(site["name"])
+
+		else:
+			if not site.get("port"):
+				site["port"] = 80
+
+			if site["port"] in ports_in_use:
+				raise Exception("Port {0} is being used by another site {1}".format(site["port"], ports_in_use[site["port"]]))
+
+			ports_in_use[site["port"]] = site["name"]
+			sites["that_use_port"].append(site)
+
+	return sites
+
+def get_sites_with_config(bench):
+	sites = get_sites(bench=bench)
+	ret = []
+	for site in sites:
+		site_config = get_site_config(site, bench=bench)
+		ret.append({
+			"name": site,
+			"port": site_config.get('nginx_port'),
+			"ssl_certificate": site_config.get('ssl_certificate'),
+			"ssl_certificate_key": site_config.get('ssl_certificate_key')
+		})
+	return ret
+
+def get_site_config(site, bench='.'):
+	with open(os.path.join(bench, 'sites', site, 'site_config.json')) as f:
+		return json.load(f)

bench/config/templates/nginx.conf

@@ -0,0 +1,106 @@
+{% macro server_block(bench_name, port, server_names, sites_path, ssl_certificate, ssl_certificate_key) %}
+
+{%- set site_name = server_names[0] if (server_names|length)==1 else "$host" -%}
+
+server {
+	listen {{ port }};
+	server_name
+		{% for name in server_names -%}
+		{{ name }}
+		{% endfor -%}
+		;
+
+	client_max_body_size 4G;
+	keepalive_timeout 5;
+	sendfile on;
+	root {{ sites_path }};
+
+	{% if ssl_certificate and ssl_certificate_key %}
+	ssl on;
+	ssl_certificate      {{ ssl_certificate }};
+	ssl_certificate_key  {{ ssl_certificate_key }};
+	ssl_session_timeout  5m;
+	ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
+	ssl_ciphers "EECDH+ECDSA+AESGCM EECDH+aRSA+AESGCM EECDH+ECDSA+SHA384 EECDH+ECDSA+SHA256 EECDH+aRSA+SHA384 EECDH+aRSA+SHA256 EECDH+aRSA+RC4 EECDH EDH+aRSA RC4 !aNULL !eNULL !LOW !3DES !MD5 !EXP !PSK !SRP !DSS";
+	ssl_prefer_server_ciphers   on;
+	{% endif %}
+
+	location /assets {
+		try_files $uri =404;
+	}
+
+	location ~ ^/protected/(.*) {
+		internal;
+		try_files /{{ site_name }}/$1 =404;
+	}
+
+	location /socket.io {
+		proxy_http_version 1.1;
+		proxy_set_header Upgrade $http_upgrade;
+		proxy_set_header Connection "upgrade";
+		proxy_set_header X-Frappe-Site-Name {{ site_name }};
+		proxy_set_header Origin $scheme://$http_host;
+		proxy_set_header Host $host;
+
+		proxy_pass http://{{ bench_name }}-socketio-server;
+	}
+
+	location / {
+		try_files /{{ site_name }}/public/$uri @webserver;
+	}
+
+	location @webserver {
+		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+		proxy_set_header X-Forwarded-Proto $scheme;
+		proxy_set_header X-Frappe-Site-Name {{ site_name }};
+		proxy_set_header Host $host;
+		proxy_set_header X-Use-X-Accel-Redirect True;
+		proxy_read_timeout {{ http_timeout or 120 }};
+		proxy_redirect off;
+
+		proxy_pass  http://{{ bench_name }}-frappe;
+	}
+}
+
+{% if ssl_certificate and ssl_certificate_key %}
+	# http to https redirect for {{ server_names[0] }}
+	server {
+	    listen 80;
+		server_name {{ server_names[0] }};
+	    return 301 https://$host$request_uri?$query_string;
+	}
+
+{% endif %}
+
+{# keep the empty line above for a pleasant rendering #}
+{% endmacro %}
+
+upstream {{ bench_name }}-frappe {
+    server 127.0.0.1:{{ webserver_port or 8000 }} fail_timeout=0;
+}
+
+upstream {{ bench_name}}-socketio-server {
+    server 127.0.0.1:{{ socketio_port or 3000 }} fail_timeout=0;
+}
+
+{% if sites.that_use_dns -%}
+
+	{{ server_block(bench_name, 80, sites.that_use_dns, sites_path) }}
+
+{%- endif %}
+
+{%- if sites.that_use_ssl -%}
+	{% for site in sites.that_use_ssl -%}
+
+		{{ server_block(bench_name, 443, [site.name], sites_path, site.ssl_certificate, site.ssl_certificate_key) }}
+
+	{% endfor %}
+{%- endif %}
+
+{% if sites.that_use_port -%}
+	{%- for site in sites.that_use_port -%}
+
+		{{ server_block(bench_name, site.port, [site.name], sites_path) }}
+
+	{%- endfor %}
+{% endif %}

bench/production_setup.py

@@ -1,5 +1,6 @@
 from .utils import get_program, exec_cmd, get_cmd_output, fix_prod_setup_perms, get_config, get_bench_name
-from .config import generate_nginx_config, generate_supervisor_config
+from .config import generate_supervisor_config
+from .config.nginx import make_nginx_conf
 from jinja2 import Environment, PackageLoader
 import os
 import shutil
@@ -46,24 +47,16 @@ def is_running_systemd():
 		return True
 	return False
 
-def copy_default_nginx_config():
-	shutil.copy(os.path.join(os.path.dirname(__file__), 'templates', 'nginx_default.conf'), '/etc/nginx/nginx.conf')
-
 def setup_production(user, bench='.'):
 	generate_supervisor_config(bench=bench, user=user)
-	generate_nginx_config(bench=bench)
+	make_nginx_conf(bench=bench)
 	fix_prod_setup_perms(bench, frappe_user=user)
 	remove_default_nginx_configs()
 
 	bench_name = get_bench_name(bench)
 	nginx_conf = '/etc/nginx/conf.d/{bench_name}.conf'.format(bench_name=bench_name)
 
-	if is_centos7():
+	supervisor_conf_extn = "ini" if is_centos7() else "conf"
-		supervisor_conf_extn = "ini"
-		copy_default_nginx_config()
-	else:
-		supervisor_conf_extn = "conf"
-
 	supervisor_conf = os.path.join(get_supervisor_confdir(), '{bench_name}.{extn}'.format(
 		bench_name=bench_name, extn=supervisor_conf_extn))
 

bench/templates/cached_requirements.txt

@@ -1,26 +0,0 @@
-chardet
-cssmin
-dropbox
-gunicorn
-httplib2
-jinja2
-markdown2
-markupsafe
-mysql-python
-pygeoip
-python-dateutil
-python-memcached
-pytz==2013d
-six
-slugify
-termcolor
-werkzeug
-semantic_version
-rauth>=0.6.2
-requests==1.2.3
-celery
-redis
-selenium
-unidecode
-babel
-pdfkit

bench/templates/nginx.conf

@@ -1,105 +0,0 @@
-upstream {{ bench_name }}-frappe {
-    server 127.0.0.1:{{ webserver_port }} fail_timeout=0;
-}
-
-upstream {{ bench_name}}-socketio-server {
-    server 127.0.0.1:{{ socketio_port }} fail_timeout=0;
-}
-
-{% macro location_block(site, port=80, default=False, server_name=None, sites=None, dns_multitenant=False) -%}
-		keepalive_timeout 5;
-		sendfile on;
-		root {{ sites_dir }};
-
-		location /assets {
-			try_files $uri =404;
-		}
-
-		location ~ ^/protected/(.*) {
-			internal;
-			try_files /{{ "$host" if dns_multitenant else site.name }}/$1 =404;
-		}
-
-		location /socket.io {
-			proxy_pass http://{{ bench_name }}-socketio-server;
-			proxy_http_version 1.1;
-			proxy_set_header Upgrade $http_upgrade;
-			proxy_set_header Connection "upgrade";
-			{% if not dns_multitenant %}
-			proxy_set_header X-Frappe-Site-Name {{ site.name }};
-			{% endif %}
-			proxy_set_header Origin $scheme://$http_host;
-			proxy_set_header Host $host;
-		}
-
-		location / {
-			try_files /{{ "$host" if dns_multitenant else site.name }}/public/$uri @webserver;
-		}
-
-		location @webserver {
-			proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-			proxy_set_header X-Forwarded-Proto $scheme;
-			{% if not dns_multitenant %}
-			proxy_set_header X-Frappe-Site-Name {{ site.name }};
-			{% endif %}
-			proxy_set_header Host $host;
-			proxy_set_header X-Use-X-Accel-Redirect True;
-			proxy_read_timeout {{ http_timeout }};
-			proxy_redirect off;
-			proxy_pass  http://{{ bench_name }}-frappe;
-		}
-{%- endmacro %}
-
-{% macro server_name_block(site, default=False, server_name=None, sites=None, dns_multitenant=False) -%}
-		client_max_body_size 4G;
-		{% if dns_multitenant and sites %}
-			server_name {% for site in sites %} {{ site.name }} {% endfor %};
-		{% else %}
-			server_name {{ site.name if not server_name else server_name }};
-		{% endif %}
-{%- endmacro %}
-
-{% macro server_block_http(site, port=80, default=False, server_name=None, sites=None, dns_multitenant=False) -%}
-	server {
-		listen {{ site.port if not default and site.port else port }} {% if default %} default {% endif %};
-		{{ server_name_block(site, default=default, server_name=server_name, sites=sites, dns_multitenant=dns_multitenant) }}
-		{{ location_block(site, port=port, default=default, server_name=server_name, sites=sites, dns_multitenant=dns_multitenant) }}
-	}
-{%- endmacro %}
-
-{% macro server_block_https(site, port=443, default=False, server_name=None, sites=None, dns_multitenant=False) -%}
-	server {
-		listen {{ site.ssl_port if not default and site.ssl_port else port }} {% if default %} default {% endif %};
-		{{ server_name_block(site, default=default, server_name=server_name, sites=sites, dns_multitenant=dns_multitenant) }}
-
-		ssl on;
-		ssl_certificate      {{ site.ssl_certificate }};
-		ssl_certificate_key  {{ site.ssl_certificate_key }};
-		ssl_session_timeout  5m;
-		ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
-		ssl_ciphers "EECDH+ECDSA+AESGCM EECDH+aRSA+AESGCM EECDH+ECDSA+SHA384 EECDH+ECDSA+SHA256 EECDH+aRSA+SHA384 EECDH+aRSA+SHA256 EECDH+aRSA+RC4 EECDH EDH+aRSA RC4 !aNULL !eNULL !LOW !3DES !MD5 !EXP !PSK !SRP !DSS";
-		ssl_prefer_server_ciphers   on;
-
-		{{ location_block(site, port=port, default=default, server_name=server_name, sites=sites, dns_multitenant=dns_multitenant) }}
-	}
-{%- endmacro %}
-
-{% for site in sites %}
-
-{% if site.port %}
-{{ server_block_http(site) }}
-{% endif %}
-
-{% if site.ssl_certificate_key and site.ssl_certificate %}
-{{ server_block_https(site) }}
-{% endif %}
-
-{% endfor %}
-
-{% if default_site %}
-{{ server_block_http(default_site, default=True, server_name="frappe_default_site") }}
-{% endif %}
-
-{% if dns_multitenant and sites %}
-{{ server_block_http(None, default=False, sites=sites, dns_multitenant=True) }}
-{% endif %}

bench/utils.py

@@ -388,12 +388,12 @@ def set_ssl_certificate_key(site, ssl_certificate_key, bench='.', gen_config=Tru
 	set_site_config_nginx_property(site, {"ssl_certificate_key": ssl_certificate_key}, bench=bench, gen_config=gen_config)
 
 def set_site_config_nginx_property(site, config, bench='.', gen_config=True):
-	from .config import generate_nginx_config
+	from .config.nginx import make_nginx_conf
 	if site not in get_sites(bench=bench):
 		raise Exception("No such site")
 	update_site_config(site, config, bench=bench)
 	if gen_config:
-		generate_nginx_config(bench=bench)
+		make_nginx_conf(bench=bench)
 
 def set_url_root(site, url_root, bench='.'):
 	update_site_config(site, {"host_name": url_root}, bench=bench)
@@ -427,18 +427,6 @@ def backup_all_sites(bench='.'):
 	for site in get_sites(bench=bench):
 		backup_site(site, bench=bench)
 
-def prime_wheel_cache(bench='.'):
-	conf = get_config(bench=bench)
-	wheel_cache_dir = conf.get('wheel_cache_dir')
-	if not wheel_cache_dir:
-		raise Exception("Wheel cache dir not configured")
-	requirements = os.path.join(os.path.dirname(__file__), 'templates', 'cached_requirements.txt')
-	cmd =  "{pip} wheel --find-links {wheelhouse} --wheel-dir {wheelhouse} -r {requirements}".format(
-				pip=os.path.join(bench, 'env', 'bin', 'pip'),
-				wheelhouse=wheel_cache_dir,
-				requirements=requirements)
-	exec_cmd(cmd)
-
 def is_root():
 	if os.getuid() == 0:
 		return True
@@ -581,7 +569,8 @@ def pre_upgrade(from_ver, to_ver, bench='.'):
 				exec_cmd("{pip} install --upgrade -e {app}".format(pip=pip, app=cwd))
 
 def post_upgrade(from_ver, to_ver, bench='.'):
-	from .config import generate_nginx_config, generate_supervisor_config, generate_redis_cache_config, generate_redis_async_broker_config
+	from .config import generate_supervisor_config, generate_redis_cache_config, generate_redis_async_broker_config
+	from .config.nginx import make_nginx_conf
 	conf = get_config(bench=bench)
 	print "-"*80
 	print "Your bench was upgraded to version {0}".format(to_ver)
@@ -589,7 +578,7 @@ def post_upgrade(from_ver, to_ver, bench='.'):
 	if conf.get('restart_supervisor_on_update'):
 		generate_redis_cache_config(bench=bench)
 		generate_supervisor_config(bench=bench)
-		generate_nginx_config(bench=bench)
+		make_nginx_conf(bench=bench)
 
 		if from_ver == 4 and to_ver == 5:
 			setup_backups(bench=bench)