accounting/bench
2
0
Fork 0
mirror of https://github.com/frappe/bench.git synced 2025-01-24 07:28:25 +00:00
Code Issues Releases Activity

[cleanup] nginx config for multi-bench

Browse Source
This commit is contained in:
Anand Doshi 2016-03-14 20:00:50 +05:30
parent 8efde1f491
commit 4286c99a0f
13 changed files with 222 additions and 215 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
bench/cli.py
View File

@ -12,14 +12,15 @@ from .utils import set_url_root as _set_url_root
from .utils import set_default_site as _set_default_site
from .utils import (build_assets, patch_sites, exec_cmd, update_bench, get_env_cmd, get_frappe, setup_logging,
get_config, update_config, restart_supervisor_processes, put_config, default_config, update_requirements,
backup_all_sites, backup_site, get_sites, prime_wheel_cache, is_root, set_mariadb_host, drop_privileges,
backup_all_sites, backup_site, get_sites, is_root, set_mariadb_host, drop_privileges,
fix_file_perms, fix_prod_setup_perms, set_ssl_certificate, set_ssl_certificate_key,
get_cmd_output, post_upgrade, get_bench_name,
pre_upgrade, validate_upgrade, PatchError, download_translations_p, setup_socketio, before_update)
from .app import get_app as _get_app
from .app import new_app as _new_app
from .app import pull_all_apps, get_apps, get_current_frappe_version, is_version_upgrade, switch_to_v4, switch_to_v5, switch_to_master, switch_to_develop
from .config import generate_nginx_config, generate_supervisor_config, generate_redis_cache_config, generate_redis_async_broker_config, generate_redis_celery_broker_config
from .config import generate_supervisor_config, generate_redis_cache_config, generate_redis_async_broker_config, generate_redis_celery_broker_config
from .config.nginx import make_nginx_conf
from .production_setup import setup_production as _setup_production
from .migrate_to_v5 import migrate_to_v5
import os
@ -402,11 +403,6 @@ def _backup_all_sites():
"backup all sites"
backup_all_sites(bench='.')
@click.command('prime-wheel-cache')
def _prime_wheel_cache():
"Update wheel cache"
prime_wheel_cache(bench='.')
@click.command('release')
@click.argument('app', type=click.Choice(['frappe', 'erpnext', 'erpnext_shopify', 'paypal_integration']))
@click.argument('bump-type', type=click.Choice(['major', 'minor', 'patch']))
@ -433,7 +429,7 @@ def setup_sudoers(user):
@click.command('nginx')
def setup_nginx():
"generate config for nginx"
generate_nginx_config()
make_nginx_conf(bench=".")
@click.command('supervisor')
def setup_supervisor():
@ -631,7 +627,6 @@ bench.add_command(_switch_to_v5)
bench.add_command(shell)
bench.add_command(_backup_all_sites)
bench.add_command(_backup_site)
bench.add_command(_prime_wheel_cache)
bench.add_command(_release)
bench.add_command(patch)
bench.add_command(set_url_root)

79
bench/config.py → bench/config/__init__.py
View File

@ -6,9 +6,9 @@ import shutil
import socket
from distutils.spawn import find_executable
from jinja2 import Environment, PackageLoader
from .utils import get_sites, get_config, update_config, get_redis_version, update_common_site_config, get_bench_name
from bench.utils import get_sites, get_config, update_config, get_redis_version, update_common_site_config, get_bench_name
env = Environment(loader=PackageLoader('bench', 'templates'), trim_blocks=True)
env = Environment(loader=PackageLoader('bench.config'), trim_blocks=True)
def write_config_file(bench, file_name, config):
config_path = os.path.join(bench, 'config')
@ -23,7 +23,7 @@ def write_config_file(bench, file_name, config):
f.write(config)
def generate_supervisor_config(bench='.', user=None):
from .app import get_current_frappe_version
from bench.app import get_current_frappe_version
template = env.get_template('supervisor.conf')
bench_dir = os.path.abspath(bench)
sites_dir = os.path.join(bench_dir, "sites")
@ -51,10 +51,6 @@ def generate_supervisor_config(bench='.', user=None):
write_config_file(bench, 'supervisor.conf', config)
update_config({'restart_supervisor_on_update': True})
def get_site_config(site, bench='.'):
with open(os.path.join(bench, 'sites', site, 'site_config.json')) as f:
return json.load(f)
def generate_common_site_config(bench='.'):
'''Generates the default common_site_config.json while a new bench is created'''
config = get_config(bench=bench)
@ -76,47 +72,34 @@ def generate_common_site_config(bench='.'):
if common_site_config:
update_common_site_config(common_site_config, bench=bench)
def get_sites_with_config(bench='.'):
sites = get_sites(bench=bench)
ret = []
for site in sites:
site_config = get_site_config(site, bench=bench)
ret.append({
"name": site,
"port": site_config.get('nginx_port'),
"ssl_certificate": site_config.get('ssl_certificate'),
"ssl_certificate_key": site_config.get('ssl_certificate_key')
})
return ret
def generate_nginx_config(bench='.'):
template = env.get_template('nginx.conf')
bench_dir = os.path.abspath(bench)
sites_dir = os.path.join(bench_dir, "sites")
sites = get_sites_with_config(bench=bench)
user = getpass.getuser()
config = get_config(bench)
if config.get('serve_default_site'):
try:
with open("sites/currentsite.txt") as f:
default_site = {'name': f.read().strip()}
except IOError:
default_site = None
else:
default_site = None
config = template.render(**{
"sites_dir": sites_dir,
"http_timeout": config.get("http_timeout", 120),
"default_site": default_site,
"dns_multitenant": config.get('dns_multitenant'),
"sites": sites,
"webserver_port": config.get('webserver_port', 8000),
"socketio_port": config.get('socketio_port', 3000),
"bench_name": get_bench_name(bench)
})
write_config_file(bench, 'nginx.conf', config)
# def generate_nginx_config(bench='.'):
# template = env.get_template('nginx.conf')
# bench_dir = os.path.abspath(bench)
# sites_dir = os.path.join(bench_dir, "sites")
# sites = get_sites_with_config(bench=bench)
# user = getpass.getuser()
# config = get_config(bench)
#
# if config.get('serve_default_site'):
# try:
# with open("sites/currentsite.txt") as f:
# default_site = {'name': f.read().strip()}
# except IOError:
# default_site = None
# else:
# default_site = None
#
# config = template.render(**{
# "sites_dir": sites_dir,
# "http_timeout": config.get("http_timeout", 120),
# "default_site": default_site,
# "dns_multitenant": config.get('dns_multitenant'),
# "sites": sites,
# "webserver_port": config.get('webserver_port', 8000),
# "socketio_port": config.get('socketio_port', 3000),
# "bench_name": get_bench_name(bench)
# })
# write_config_file(bench, 'nginx.conf', config)
def generate_redis_celery_broker_config(bench='.'):
"""Redis that is used for queueing celery tasks"""

72
bench/config/nginx.py Normal file
View File

@ -0,0 +1,72 @@
import os
import json
from bench.utils import get_sites, get_config, get_bench_name
def make_nginx_conf(bench):
from bench.config import env, write_config_file
template = env.get_template('nginx.conf')
bench_path = os.path.abspath(bench)
sites_path = os.path.join(bench_path, "sites")
config = get_config(bench)
sites = prepare_sites(config, bench)
nginx_conf = template.render(**{
"sites_path": sites_path,
"http_timeout": config.get("http_timeout"),
"sites": sites,
"webserver_port": config.get('webserver_port'),
"socketio_port": config.get('socketio_port'),
"bench_name": get_bench_name(bench)
})
write_config_file(bench, 'nginx.conf', nginx_conf)
def prepare_sites(config, bench):
sites = {
"that_use_dns": [],
"that_use_ssl": [],
"that_use_port": []
}
ports_in_use = {}
dns_multitenant = config.get('dns_multitenant')
for site in get_sites_with_config(bench=bench):
if dns_multitenant:
# assumes site's folder name is same as the domain name
if site.get("ssl_certificate") and site.get("ssl_certificate_key"):
sites["that_use_ssl"].append(site)
else:
sites["that_use_dns"].append(site["name"])
else:
if not site.get("port"):
site["port"] = 80
if site["port"] in ports_in_use:
raise Exception("Port {0} is being used by another site {1}".format(site["port"], ports_in_use[site["port"]]))
ports_in_use[site["port"]] = site["name"]
sites["that_use_port"].append(site)
return sites
def get_sites_with_config(bench):
sites = get_sites(bench=bench)
ret = []
for site in sites:
site_config = get_site_config(site, bench=bench)
ret.append({
"name": site,
"port": site_config.get('nginx_port'),
"ssl_certificate": site_config.get('ssl_certificate'),
"ssl_certificate_key": site_config.get('ssl_certificate_key')
})
return ret
def get_site_config(site, bench='.'):
with open(os.path.join(bench, 'sites', site, 'site_config.json')) as f:
return json.load(f)

106
bench/config/templates/nginx.conf Normal file
View File

@ -0,0 +1,106 @@
{% macro server_block(bench_name, port, server_names, sites_path, ssl_certificate, ssl_certificate_key) %}
{%- set site_name = server_names[0] if (server_names|length)==1 else "$host" -%}
server {
listen {{ port }};
server_name
{% for name in server_names -%}
{{ name }}
{% endfor -%}
;
client_max_body_size 4G;
keepalive_timeout 5;
sendfile on;
root {{ sites_path }};
{% if ssl_certificate and ssl_certificate_key %}
ssl on;
ssl_certificate {{ ssl_certificate }};
ssl_certificate_key {{ ssl_certificate_key }};
ssl_session_timeout 5m;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers "EECDH+ECDSA+AESGCM EECDH+aRSA+AESGCM EECDH+ECDSA+SHA384 EECDH+ECDSA+SHA256 EECDH+aRSA+SHA384 EECDH+aRSA+SHA256 EECDH+aRSA+RC4 EECDH EDH+aRSA RC4 !aNULL !eNULL !LOW !3DES !MD5 !EXP !PSK !SRP !DSS";
ssl_prefer_server_ciphers on;
{% endif %}
location /assets {
try_files $uri =404;
}
location ~ ^/protected/(.*) {
internal;
try_files /{{ site_name }}/$1 =404;
}
location /socket.io {
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header X-Frappe-Site-Name {{ site_name }};
proxy_set_header Origin $scheme://$http_host;
proxy_set_header Host $host;
proxy_pass http://{{ bench_name }}-socketio-server;
}
location / {
try_files /{{ site_name }}/public/$uri @webserver;
}
location @webserver {
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Frappe-Site-Name {{ site_name }};
proxy_set_header Host $host;
proxy_set_header X-Use-X-Accel-Redirect True;
proxy_read_timeout {{ http_timeout or 120 }};
proxy_redirect off;
proxy_pass http://{{ bench_name }}-frappe;
}
}
{% if ssl_certificate and ssl_certificate_key %}
# http to https redirect for {{ server_names[0] }}
server {
listen 80;
server_name {{ server_names[0] }};
return 301 https://$host$request_uri?$query_string;
}
{% endif %}
{# keep the empty line above for a pleasant rendering #}
{% endmacro %}
upstream {{ bench_name }}-frappe {
server 127.0.0.1:{{ webserver_port or 8000 }} fail_timeout=0;
}
upstream {{ bench_name}}-socketio-server {
server 127.0.0.1:{{ socketio_port or 3000 }} fail_timeout=0;
}
{% if sites.that_use_dns -%}
{{ server_block(bench_name, 80, sites.that_use_dns, sites_path) }}
{%- endif %}
{%- if sites.that_use_ssl -%}
{% for site in sites.that_use_ssl -%}
{{ server_block(bench_name, 443, [site.name], sites_path, site.ssl_certificate, site.ssl_certificate_key) }}
{% endfor %}
{%- endif %}
{% if sites.that_use_port -%}
{%- for site in sites.that_use_port -%}
{{ server_block(bench_name, site.port, [site.name], sites_path) }}
{%- endfor %}
{% endif %}

0
bench/templates/nginx_default.conf → bench/config/templates/nginx_default.conf
View File

0
bench/templates/redis_async_broker.conf → bench/config/templates/redis_async_broker.conf
View File

0
bench/templates/redis_cache.conf → bench/config/templates/redis_cache.conf
View File

0
bench/templates/redis_celery_broker.conf → bench/config/templates/redis_celery_broker.conf
View File

0
bench/templates/supervisor.conf → bench/config/templates/supervisor.conf
View File

15
bench/production_setup.py
View File

@ -1,5 +1,6 @@
from .utils import get_program, exec_cmd, get_cmd_output, fix_prod_setup_perms, get_config, get_bench_name
from .config import generate_nginx_config, generate_supervisor_config
from .config import generate_supervisor_config
from .config.nginx import make_nginx_conf
from jinja2 import Environment, PackageLoader
import os
import shutil
@ -46,24 +47,16 @@ def is_running_systemd():
return True
return False
def copy_default_nginx_config():
shutil.copy(os.path.join(os.path.dirname(__file__), 'templates', 'nginx_default.conf'), '/etc/nginx/nginx.conf')
def setup_production(user, bench='.'):
generate_supervisor_config(bench=bench, user=user)
generate_nginx_config(bench=bench)
make_nginx_conf(bench=bench)
fix_prod_setup_perms(bench, frappe_user=user)
remove_default_nginx_configs()
bench_name = get_bench_name(bench)
nginx_conf = '/etc/nginx/conf.d/{bench_name}.conf'.format(bench_name=bench_name)
if is_centos7():
supervisor_conf_extn = "ini"
copy_default_nginx_config()
else:
supervisor_conf_extn = "conf"
supervisor_conf_extn = "ini" if is_centos7() else "conf"
supervisor_conf = os.path.join(get_supervisor_confdir(), '{bench_name}.{extn}'.format(
bench_name=bench_name, extn=supervisor_conf_extn))

26
bench/templates/cached_requirements.txt
View File

@ -1,26 +0,0 @@
chardet
cssmin
dropbox
gunicorn
httplib2
jinja2
markdown2
markupsafe
mysql-python
pygeoip
python-dateutil
python-memcached
pytz==2013d
six
slugify
termcolor
werkzeug
semantic_version
rauth>=0.6.2
requests==1.2.3
celery
redis
selenium
unidecode
babel
pdfkit

105
bench/templates/nginx.conf
View File

@ -1,105 +0,0 @@
upstream {{ bench_name }}-frappe {
server 127.0.0.1:{{ webserver_port }} fail_timeout=0;
}
upstream {{ bench_name}}-socketio-server {
server 127.0.0.1:{{ socketio_port }} fail_timeout=0;
}
{% macro location_block(site, port=80, default=False, server_name=None, sites=None, dns_multitenant=False) -%}
keepalive_timeout 5;
sendfile on;
root {{ sites_dir }};
location /assets {
try_files $uri =404;
}
location ~ ^/protected/(.*) {
internal;
try_files /{{ "$host" if dns_multitenant else site.name }}/$1 =404;
}
location /socket.io {
proxy_pass http://{{ bench_name }}-socketio-server;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
{% if not dns_multitenant %}
proxy_set_header X-Frappe-Site-Name {{ site.name }};
{% endif %}
proxy_set_header Origin $scheme://$http_host;
proxy_set_header Host $host;
}
location / {
try_files /{{ "$host" if dns_multitenant else site.name }}/public/$uri @webserver;
}
location @webserver {
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
{% if not dns_multitenant %}
proxy_set_header X-Frappe-Site-Name {{ site.name }};
{% endif %}
proxy_set_header Host $host;
proxy_set_header X-Use-X-Accel-Redirect True;
proxy_read_timeout {{ http_timeout }};
proxy_redirect off;
proxy_pass http://{{ bench_name }}-frappe;
}
{%- endmacro %}
{% macro server_name_block(site, default=False, server_name=None, sites=None, dns_multitenant=False) -%}
client_max_body_size 4G;
{% if dns_multitenant and sites %}
server_name {% for site in sites %} {{ site.name }} {% endfor %};
{% else %}
server_name {{ site.name if not server_name else server_name }};
{% endif %}
{%- endmacro %}
{% macro server_block_http(site, port=80, default=False, server_name=None, sites=None, dns_multitenant=False) -%}
server {
listen {{ site.port if not default and site.port else port }} {% if default %} default {% endif %};
{{ server_name_block(site, default=default, server_name=server_name, sites=sites, dns_multitenant=dns_multitenant) }}
{{ location_block(site, port=port, default=default, server_name=server_name, sites=sites, dns_multitenant=dns_multitenant) }}
}
{%- endmacro %}
{% macro server_block_https(site, port=443, default=False, server_name=None, sites=None, dns_multitenant=False) -%}
server {
listen {{ site.ssl_port if not default and site.ssl_port else port }} {% if default %} default {% endif %};
{{ server_name_block(site, default=default, server_name=server_name, sites=sites, dns_multitenant=dns_multitenant) }}
ssl on;
ssl_certificate {{ site.ssl_certificate }};
ssl_certificate_key {{ site.ssl_certificate_key }};
ssl_session_timeout 5m;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers "EECDH+ECDSA+AESGCM EECDH+aRSA+AESGCM EECDH+ECDSA+SHA384 EECDH+ECDSA+SHA256 EECDH+aRSA+SHA384 EECDH+aRSA+SHA256 EECDH+aRSA+RC4 EECDH EDH+aRSA RC4 !aNULL !eNULL !LOW !3DES !MD5 !EXP !PSK !SRP !DSS";
ssl_prefer_server_ciphers on;
{{ location_block(site, port=port, default=default, server_name=server_name, sites=sites, dns_multitenant=dns_multitenant) }}
}
{%- endmacro %}
{% for site in sites %}
{% if site.port %}
{{ server_block_http(site) }}
{% endif %}
{% if site.ssl_certificate_key and site.ssl_certificate %}
{{ server_block_https(site) }}
{% endif %}
{% endfor %}
{% if default_site %}
{{ server_block_http(default_site, default=True, server_name="frappe_default_site") }}
{% endif %}
{% if dns_multitenant and sites %}
{{ server_block_http(None, default=False, sites=sites, dns_multitenant=True) }}
{% endif %}

21
bench/utils.py
View File

@ -388,12 +388,12 @@ def set_ssl_certificate_key(site, ssl_certificate_key, bench='.', gen_config=Tru
set_site_config_nginx_property(site, {"ssl_certificate_key": ssl_certificate_key}, bench=bench, gen_config=gen_config)
def set_site_config_nginx_property(site, config, bench='.', gen_config=True):
from .config import generate_nginx_config
from .config.nginx import make_nginx_conf
if site not in get_sites(bench=bench):
raise Exception("No such site")
update_site_config(site, config, bench=bench)
if gen_config:
generate_nginx_config(bench=bench)
make_nginx_conf(bench=bench)
def set_url_root(site, url_root, bench='.'):
update_site_config(site, {"host_name": url_root}, bench=bench)
@ -427,18 +427,6 @@ def backup_all_sites(bench='.'):
for site in get_sites(bench=bench):
backup_site(site, bench=bench)
def prime_wheel_cache(bench='.'):
conf = get_config(bench=bench)
wheel_cache_dir = conf.get('wheel_cache_dir')
if not wheel_cache_dir:
raise Exception("Wheel cache dir not configured")
requirements = os.path.join(os.path.dirname(__file__), 'templates', 'cached_requirements.txt')
cmd = "{pip} wheel --find-links {wheelhouse} --wheel-dir {wheelhouse} -r {requirements}".format(
pip=os.path.join(bench, 'env', 'bin', 'pip'),
wheelhouse=wheel_cache_dir,
requirements=requirements)
exec_cmd(cmd)
def is_root():
if os.getuid() == 0:
return True
@ -581,7 +569,8 @@ def pre_upgrade(from_ver, to_ver, bench='.'):
exec_cmd("{pip} install --upgrade -e {app}".format(pip=pip, app=cwd))
def post_upgrade(from_ver, to_ver, bench='.'):
from .config import generate_nginx_config, generate_supervisor_config, generate_redis_cache_config, generate_redis_async_broker_config
from .config import generate_supervisor_config, generate_redis_cache_config, generate_redis_async_broker_config
from .config.nginx import make_nginx_conf
conf = get_config(bench=bench)
print "-"*80
print "Your bench was upgraded to version {0}".format(to_ver)
@ -589,7 +578,7 @@ def post_upgrade(from_ver, to_ver, bench='.'):
if conf.get('restart_supervisor_on_update'):
generate_redis_cache_config(bench=bench)
generate_supervisor_config(bench=bench)
generate_nginx_config(bench=bench)
make_nginx_conf(bench=bench)
if from_ver == 4 and to_ver == 5:
setup_backups(bench=bench)