From d82a67be9e6045f19a347e89b6ee4e57cd51e759 Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Tue, 15 Mar 2016 16:57:48 +0530 Subject: [PATCH] [cleanup] redis, supervisor, config --- bench/__init__.py | 3 + bench/app.py | 4 +- bench/cli.py | 37 ++--- bench/config/__init__.py | 146 ------------------ bench/config/common_site_config.py | 103 ++++++++++++ bench/config/nginx.py | 3 +- bench/config/procfile.py | 9 ++ bench/config/redis.py | 55 +++++++ bench/config/supervisor.py | 36 +++++ bench/config/templates/Procfile | 10 ++ ...is_celery_broker.conf => redis_queue.conf} | 0 ..._async_broker.conf => redis_socketio.conf} | 0 bench/production_setup.py | 6 +- bench/release.py | 3 +- bench/tests/test_init.py | 8 +- bench/utils.py | 119 ++------------ 16 files changed, 251 insertions(+), 291 deletions(-) create mode 100644 bench/config/common_site_config.py create mode 100644 bench/config/procfile.py create mode 100644 bench/config/redis.py create mode 100644 bench/config/supervisor.py create mode 100644 bench/config/templates/Procfile rename bench/config/templates/{redis_celery_broker.conf => redis_queue.conf} (100%) rename bench/config/templates/{redis_async_broker.conf => redis_socketio.conf} (100%) diff --git a/bench/__init__.py b/bench/__init__.py index e69de29b..e8566755 100644 --- a/bench/__init__.py +++ b/bench/__init__.py @@ -0,0 +1,3 @@ +from jinja2 import Environment, PackageLoader + +env = Environment(loader=PackageLoader('bench.config'), trim_blocks=True) diff --git a/bench/app.py b/bench/app.py index bf8717c9..27fbdef5 100644 --- a/bench/app.py +++ b/bench/app.py @@ -1,5 +1,7 @@ import os -from .utils import exec_cmd, get_frappe, check_git_for_shallow_clone, get_config, build_assets, restart_supervisor_processes, get_cmd_output, run_frappe_cmd +from .utils import (exec_cmd, get_frappe, check_git_for_shallow_clone, build_assets, + restart_supervisor_processes, get_cmd_output, run_frappe_cmd) +from .config.common_site_config import get_config import logging import requests diff --git a/bench/cli.py b/bench/cli.py index 88a152d8..83ce54fc 100644 --- a/bench/cli.py +++ b/bench/cli.py @@ -6,12 +6,11 @@ from .utils import setup_backups as _setup_backups from .utils import setup_auto_update as _setup_auto_update from .utils import setup_sudoers as _setup_sudoers from .utils import start as _start -from .utils import setup_procfile as _setup_procfile from .utils import set_nginx_port as _set_nginx_port from .utils import set_url_root as _set_url_root from .utils import set_default_site as _set_default_site from .utils import (build_assets, patch_sites, exec_cmd, update_bench, get_env_cmd, get_frappe, setup_logging, - get_config, update_config, restart_supervisor_processes, put_config, default_config, update_requirements, + restart_supervisor_processes, update_requirements, backup_all_sites, backup_site, get_sites, is_root, set_mariadb_host, drop_privileges, fix_file_perms, fix_prod_setup_perms, set_ssl_certificate, set_ssl_certificate_key, get_cmd_output, post_upgrade, get_bench_name, @@ -19,10 +18,11 @@ from .utils import (build_assets, patch_sites, exec_cmd, update_bench, get_env_c from .app import get_app as _get_app from .app import new_app as _new_app from .app import pull_all_apps, get_apps, get_current_frappe_version, is_version_upgrade, switch_to_v4, switch_to_v5, switch_to_master, switch_to_develop -from .config import generate_supervisor_config, generate_redis_cache_config, generate_redis_async_broker_config, generate_redis_celery_broker_config from .config.nginx import make_nginx_conf from .production_setup import setup_production as _setup_production from .migrate_to_v5 import migrate_to_v5 +from .config.common_site_config import get_config, make_config, update_config + import os import sys import logging @@ -434,22 +434,14 @@ def setup_nginx(): @click.command('supervisor') def setup_supervisor(): "generate config for supervisor" + from bench.config.supervisor import generate_supervisor_config generate_supervisor_config() -@click.command('redis-cache') -def setup_redis_cache(): +@click.command('redis') +def setup_redis(): "generate config for redis cache" - generate_redis_cache_config() - -@click.command('redis-async-broker') -def setup_redis_async_broker(): - "generate config for redis async broker" - generate_redis_async_broker_config() - -@click.command('redis-celery-broker') -def setup_redis_celery_broker(): - "generate config for redis celery broker" - generate_redis_celery_broker_config() + from bench.config.redis import generate_config + generate_config('.') @click.command('production') @click.argument('user') @@ -477,11 +469,10 @@ def setup_env(): _setup_env() @click.command('procfile') -@click.option('--with-watch', is_flag=True) -@click.option('--with-celery-broker', is_flag=True) -def setup_procfile(with_celery_broker, with_watch): +def setup_procfile(): "Setup Procfile for bench start" - _setup_procfile(with_celery_broker, with_watch) + from bench.config.procfile import setup_procfile + setup_procfile('.') @click.command('socketio') def _setup_socketio(): @@ -491,14 +482,12 @@ def _setup_socketio(): @click.command('config') def setup_config(): "overwrite or make config.json" - put_config(default_config) + make_config('.') setup.add_command(setup_nginx) setup.add_command(setup_sudoers) setup.add_command(setup_supervisor) -setup.add_command(setup_redis_cache) -setup.add_command(setup_redis_async_broker) -setup.add_command(setup_redis_celery_broker) +setup.add_command(setup_redis) setup.add_command(setup_auto_update) setup.add_command(setup_dnsmasq) setup.add_command(setup_backups) diff --git a/bench/config/__init__.py b/bench/config/__init__.py index b1e86841..e69de29b 100644 --- a/bench/config/__init__.py +++ b/bench/config/__init__.py @@ -1,146 +0,0 @@ -import os -import getpass -import json -import subprocess -import shutil -import socket -from distutils.spawn import find_executable -from jinja2 import Environment, PackageLoader -from bench.utils import get_sites, get_config, update_config, get_redis_version, update_common_site_config, get_bench_name - -env = Environment(loader=PackageLoader('bench.config'), trim_blocks=True) - -def write_config_file(bench, file_name, config): - config_path = os.path.join(bench, 'config') - file_path = os.path.join(config_path, file_name) - number = (len([path for path in os.listdir(config_path) if path.startswith(file_name)]) -1 ) or '' - if number: - number = '.' + str(number) - if os.path.exists(file_path): - shutil.move(file_path, file_path + '.save' + number) - - with open(file_path, 'wb') as f: - f.write(config) - -def generate_supervisor_config(bench='.', user=None): - from bench.app import get_current_frappe_version - template = env.get_template('supervisor.conf') - bench_dir = os.path.abspath(bench) - sites_dir = os.path.join(bench_dir, "sites") - sites = get_sites(bench=bench) - if not user: - user = getpass.getuser() - - config = get_config(bench=bench) - - config = template.render(**{ - "bench_dir": bench_dir, - "sites_dir": sites_dir, - "user": user, - "http_timeout": config.get("http_timeout", 120), - "redis_server": find_executable('redis-server'), - "node": find_executable('node') or find_executable('nodejs'), - "redis_cache_config": os.path.join(bench_dir, 'config', 'redis_cache.conf'), - "redis_async_broker_config": os.path.join(bench_dir, 'config', 'redis_async_broker.conf'), - "redis_celery_broker_config": os.path.join(bench_dir, 'config', 'redis_celery_broker.conf'), - "frappe_version": get_current_frappe_version(), - "webserver_port": config.get('webserver_port', 8000), - "gunicorn_workers": config.get('gunicorn_workers', 2), - "bench_name": get_bench_name(bench) - }) - write_config_file(bench, 'supervisor.conf', config) - update_config({'restart_supervisor_on_update': True}) - -def generate_common_site_config(bench='.'): - '''Generates the default common_site_config.json while a new bench is created''' - config = get_config(bench=bench) - common_site_config = {} - - for bench_config_field, site_config_field in ( - ("redis_celery_broker_port", "celery_broker"), - ("redis_async_broker_port", "async_redis_server"), - ("redis_cache_port", "cache_redis_server") - ): - - port = config.get(bench_config_field) - if config.get(bench_config_field): - redis_url = "redis://localhost:{0}".format(port) - common_site_config[site_config_field] = redis_url - - # TODO Optionally we need to add the host or domain name in case dns_multitenant is false - - if common_site_config: - update_common_site_config(common_site_config, bench=bench) - -# def generate_nginx_config(bench='.'): -# template = env.get_template('nginx.conf') -# bench_dir = os.path.abspath(bench) -# sites_dir = os.path.join(bench_dir, "sites") -# sites = get_sites_with_config(bench=bench) -# user = getpass.getuser() -# config = get_config(bench) -# -# if config.get('serve_default_site'): -# try: -# with open("sites/currentsite.txt") as f: -# default_site = {'name': f.read().strip()} -# except IOError: -# default_site = None -# else: -# default_site = None -# -# config = template.render(**{ -# "sites_dir": sites_dir, -# "http_timeout": config.get("http_timeout", 120), -# "default_site": default_site, -# "dns_multitenant": config.get('dns_multitenant'), -# "sites": sites, -# "webserver_port": config.get('webserver_port', 8000), -# "socketio_port": config.get('socketio_port', 3000), -# "bench_name": get_bench_name(bench) -# }) -# write_config_file(bench, 'nginx.conf', config) - -def generate_redis_celery_broker_config(bench='.'): - """Redis that is used for queueing celery tasks""" - _generate_redis_config( - template_name='redis_celery_broker.conf', - context={ - "port": get_config(bench).get('redis_celery_broker_port', '11311'), - "bench_path": os.path.abspath(bench), - }, - bench=bench - ) - -def generate_redis_async_broker_config(bench='.'): - """Redis that is used to do pub/sub""" - _generate_redis_config( - template_name='redis_async_broker.conf', - context={ - "port": get_config(bench).get('redis_async_broker_port', '12311'), - }, - bench=bench - ) - -def generate_redis_cache_config(bench='.'): - """Redis that is used and optimized for caching""" - config = get_config(bench=bench) - - _generate_redis_config( - template_name='redis_cache.conf', - context={ - "maxmemory": config.get('cache_maxmemory', '50'), - "port": config.get('redis_cache_port', '13311'), - "redis_version": get_redis_version(), - }, - bench=bench - ) - -def _generate_redis_config(template_name, context, bench): - template = env.get_template(template_name) - - if "pid_path" not in context: - context["pid_path"] = os.path.abspath(os.path.join(bench, "config", "pids")) - - redis_config = template.render(**context) - write_config_file(bench, template_name, redis_config) diff --git a/bench/config/common_site_config.py b/bench/config/common_site_config.py new file mode 100644 index 00000000..b61cd8b8 --- /dev/null +++ b/bench/config/common_site_config.py @@ -0,0 +1,103 @@ +import os +import multiprocessing +import getpass +import json +import urlparse + +default_config = { + 'restart_supervisor_on_update': False, + 'auto_update': False, + 'serve_default_site': True, + 'rebase_on_pull': False, + 'update_bench_on_update': True, + 'frappe_user': getpass.getuser(), + 'shallow_clone': True, +} + +def make_config(bench_path): + bench_config = get_config(bench_path) + bench_config.update(default_config) + bench_config.update(get_gunicorn_workers()) + update_config_for_frappe(bench_config, bench_path) + + put_config(bench_config, bench_path) + +def get_config(bench): + config_path = get_config_path(bench) + if not os.path.exists(config_path): + return {} + with open(config_path) as f: + return json.load(f) + +def put_config(config, bench='.'): + config_path = get_config_path(bench) + with open(config_path, 'w') as f: + return json.dump(config, f, indent=1) + +def get_config_path(bench): + return os.path.join(bench, 'sites', 'common_site_config.json') + +def update_config(new_config, bench='.'): + config = get_config(bench=bench) + config.update(new_config) + put_config(config, bench=bench) + +def get_gunicorn_workers(): + '''This function will return the maximum workers that can be started depending upon + number of cpu's present on the machine''' + return { + "gunicorn_workers": multiprocessing.cpu_count() + } + +def update_config_for_frappe(config, bench_path): + ports = make_ports(bench_path) + + for key in ('redis_cache', 'redis_queue', 'redis_socketio'): + if key not in config: + config[key] = "redis://localhost:{0}".format(ports[key]) + + for key in ('webserver_port', 'socketio_port'): + if key not in config: + config[key] = ports[key] + + + # TODO Optionally we need to add the host or domain name in case dns_multitenant is false + +def make_ports(bench_path): + benches_path = os.path.dirname(os.path.abspath(bench_path)) + + default_ports = { + "webserver_port": 8000, + "socketio_port": 9000, + "redis_queue": 11000, + "redis_socketio": 12000, + "redis_cache": 13000 + } + + # collect all existing ports + existing_ports = {} + for folder in os.listdir(benches_path): + bench = os.path.join(benches_path, folder) + if os.path.isdir(bench): + bench_config = get_config(bench) + for key in default_ports.keys(): + value = bench_config.get(key) + + # extract port from redis url + if value and (key in ('redis_cache', 'redis_queue', 'redis_socketio')): + value = urlparse.urlparse(value).port + + if value: + existing_ports.setdefault(key, []).append(value) + + # new port value = max of existing port value + 1 + ports = {} + for key, value in default_ports.items(): + existing_value = existing_ports.get(key, []) + if existing_value: + value = max(existing_value) + 1 + + ports[key] = value + + return ports + \ No newline at end of file diff --git a/bench/config/nginx.py b/bench/config/nginx.py index b5df904a..4fc92820 100644 --- a/bench/config/nginx.py +++ b/bench/config/nginx.py @@ -1,9 +1,10 @@ import os import json -from bench.utils import get_sites, get_config, get_bench_name +from bench.utils import get_sites, get_bench_name def make_nginx_conf(bench): from bench.config import env, write_config_file + from bench.config.common_site_config import get_config template = env.get_template('nginx.conf') bench_path = os.path.abspath(bench) diff --git a/bench/config/procfile.py b/bench/config/procfile.py new file mode 100644 index 00000000..ec5c0141 --- /dev/null +++ b/bench/config/procfile.py @@ -0,0 +1,9 @@ +import bench, os +from bench.utils import find_executable + +def setup_procfile(bench_path): + procfile = bench.env.get_template('Procfile').render(node=find_executable("node") \ + or find_executable("nodejs")) + + with open(os.path.join(bench_path, 'Procfile'), 'w') as f: + f.write(procfile) diff --git a/bench/config/redis.py b/bench/config/redis.py new file mode 100644 index 00000000..06288cf0 --- /dev/null +++ b/bench/config/redis.py @@ -0,0 +1,55 @@ +from .common_site_config import get_config +import re, os, subprocess, urlparse +import bench + +def generate_config(bench_path): + config = get_config(bench_path) + + ports = {} + for key in ('redis_cache', 'redis_queue', 'redis_socketio'): + ports[key] = urlparse.urlparse(config[key]).port + + write_redis_config( + template_name='redis_queue.conf', + context={ + "port": ports['redis_queue'], + "bench_path": os.path.abspath(bench_path), + }, + bench_path=bench_path + ) + + write_redis_config( + template_name='redis_socketio.conf', + context={ + "port": ports['redis_socketio'], + }, + bench_path=bench_path + ) + + write_redis_config( + template_name='redis_cache.conf', + context={ + "maxmemory": config.get('cache_maxmemory', '50'), + "port": ports['redis_cache'], + "redis_version": get_redis_version(), + }, + bench_path=bench_path + ) + +def write_redis_config(template_name, context, bench_path): + template = bench.env.get_template(template_name) + + if "pid_path" not in context: + context["pid_path"] = os.path.abspath(os.path.join(bench_path, "config", "pids")) + + with open(os.path.join(bench_path, 'config', template_name), 'w') as f: + f.write(template.render(**context)) + +def get_redis_version(): + version_string = subprocess.check_output('redis-server --version', shell=True).strip() + if re.search("Redis server version 2.4", version_string): + return "2.4" + if re.search("Redis server v=2.6", version_string): + return "2.6" + if re.search("Redis server v=2.8", version_string): + return "2.8" diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py new file mode 100644 index 00000000..9cfa8352 --- /dev/null +++ b/bench/config/supervisor.py @@ -0,0 +1,36 @@ +import os, getpass, bench + +def generate_supervisor_config(bench_path, user=None): + from bench.app import get_current_frappe_version + from bench.utils import get_bench_name, find_executable + from bench.config.common_site_config import get_config, update_config + + template = bench.env.get_template('supervisor.conf') + if not user: + user = getpass.getuser() + + config = get_config(bench_path=bench_path) + + bench_dir = os.path.abspath(bench_path) + + config = template.render(**{ + "bench_dir": bench_dir, + "sites_dir": os.path.join(bench_dir, 'sites'), + "user": user, + "http_timeout": config.get("http_timeout", 120), + "redis_server": find_executable('redis-server'), + "node": find_executable('node') or find_executable('nodejs'), + "redis_cache_config": os.path.join(bench_dir, 'config', 'redis_cache.conf'), + "redis_async_broker_config": os.path.join(bench_dir, 'config', 'redis_async_broker.conf'), + "redis_celery_broker_config": os.path.join(bench_dir, 'config', 'redis_celery_broker.conf'), + "frappe_version": get_current_frappe_version(), + "webserver_port": config.get('webserver_port', 8000), + "gunicorn_workers": config.get('gunicorn_workers', 2), + "bench_name": get_bench_name(bench_path) + }) + + with open(os.path.join(bench_path, 'config', 'supervisor.conf'), 'w') as f: + f.write(config) + + update_config({'restart_supervisor_on_update': True}) + diff --git a/bench/config/templates/Procfile b/bench/config/templates/Procfile new file mode 100644 index 00000000..edabd79e --- /dev/null +++ b/bench/config/templates/Procfile @@ -0,0 +1,10 @@ +redis_cache: redis-server config/redis_cache.conf +redis_socketio: redis-server config/redis_socketio.conf +redis_queue: redis-server config/redis_queue.conf +web: bench serve +socketio: {{ node }} apps/frappe/socketio.js +worker: sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app worker' +longjob_worker: sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app -n longjobs@%h worker' +async_worker: sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app -n async@%h worker' +workerbeat: sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app beat -s scheduler.schedule' +watch: bench watch diff --git a/bench/config/templates/redis_celery_broker.conf b/bench/config/templates/redis_queue.conf similarity index 100% rename from bench/config/templates/redis_celery_broker.conf rename to bench/config/templates/redis_queue.conf diff --git a/bench/config/templates/redis_async_broker.conf b/bench/config/templates/redis_socketio.conf similarity index 100% rename from bench/config/templates/redis_async_broker.conf rename to bench/config/templates/redis_socketio.conf diff --git a/bench/production_setup.py b/bench/production_setup.py index b3d7e1d5..17c273c7 100644 --- a/bench/production_setup.py +++ b/bench/production_setup.py @@ -1,9 +1,7 @@ -from .utils import get_program, exec_cmd, get_cmd_output, fix_prod_setup_perms, get_config, get_bench_name -from .config import generate_supervisor_config +from .utils import get_program, exec_cmd, get_cmd_output, fix_prod_setup_perms, get_bench_name +from .config.supervisor import generate_supervisor_config from .config.nginx import make_nginx_conf -from jinja2 import Environment, PackageLoader import os -import shutil def restart_service(service): if os.path.basename(get_program(['systemctl']) or '') == 'systemctl' and is_running_systemd(): diff --git a/bench/release.py b/bench/release.py index 5bbbe637..384c0e1e 100755 --- a/bench/release.py +++ b/bench/release.py @@ -12,7 +12,8 @@ import re from requests.auth import HTTPBasicAuth import requests.exceptions from time import sleep -from .utils import get_config +from .config.common_site_config import get_config + github_username = None github_password = None diff --git a/bench/tests/test_init.py b/bench/tests/test_init.py index 923ec0c9..20903009 100644 --- a/bench/tests/test_init.py +++ b/bench/tests/test_init.py @@ -37,8 +37,8 @@ class TestBenchInit(unittest.TestCase): self.assert_ports("test-bench-1", { "webserver_port": 8000, "socketio_port": 9000, - "redis_celery_broker_port": 11000, - "redis_async_broker_port": 12000, + "redis_queue_port": 11000, + "redis_socketio_port": 12000, "redis_cache_port": 13000 }) @@ -54,8 +54,8 @@ class TestBenchInit(unittest.TestCase): self.assert_ports("test-bench-2", { "webserver_port": 8001, "socketio_port": 9001, - "redis_celery_broker_port": 11001, - "redis_async_broker_port": 12001, + "redis_queue_port": 11001, + "redis_socketio_port": 12001, "redis_cache_port": 13001 }) diff --git a/bench/utils.py b/bench/utils.py index e686110d..5cd8bb64 100644 --- a/bench/utils.py +++ b/bench/utils.py @@ -1,5 +1,4 @@ import os -import re import sys import subprocess import getpass @@ -49,7 +48,9 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, no_auto_update=False, frappe_path=None, frappe_branch=None, wheel_cache_dir=None, verbose=False): from .app import get_app, install_apps_from_path - from .config import generate_redis_cache_config, generate_redis_async_broker_config, generate_redis_celery_broker_config, generate_common_site_config + from .config.common_site_config import make_config + from .config import redis + from .config.procfile import setup_procfile global FRAPPE_VERSION if os.path.exists(path): @@ -65,10 +66,7 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, setup_env(bench=path) - bench_config = make_bench_config() - put_config(bench_config, bench=path) - - generate_common_site_config(bench=path) + make_config(path) if not frappe_path: frappe_path = 'https://github.com/frappe/frappe.git' @@ -82,62 +80,15 @@ def init(path, apps_path=None, no_procfile=False, no_backups=False, setup_socketio(bench=path) build_assets(bench=path) - generate_redis_celery_broker_config(bench=path) - generate_redis_cache_config(bench=path) - generate_redis_async_broker_config(bench=path) + redis.generate_config(path) if not no_procfile: - setup_procfile(bench=path) + setup_procfile(path) if not no_backups: setup_backups(bench=path) if not no_auto_update: setup_auto_update(bench=path) -def make_bench_config(): - bench_config = {} - bench_config.update(default_config) - bench_config.update(make_ports()) - bench_config.update(get_gunicorn_workers()) - - return bench_config - -def get_gunicorn_workers(): - '''This function will return the maximum workers that can be started depending upon - number of cpu's present on the machine''' - return { - "gunicorn_workers": multiprocessing.cpu_count() - } - -def make_ports(benches_path="."): - default_ports = { - "webserver_port": 8000, - "socketio_port": 9000, - "redis_celery_broker_port": 11000, - "redis_async_broker_port": 12000, - "redis_cache_port": 13000 - } - - # collect all existing ports - existing_ports = {} - for folder in os.listdir(benches_path): - bench = os.path.join(benches_path, folder) - if os.path.isdir(bench): - bench_config = get_config(bench) - for key in default_ports.keys(): - value = bench_config.get(key) - if value: - existing_ports.setdefault(key, []).append(value) - - # new port value = max of existing port value + 1 - ports = {} - for key, value in default_ports.items(): - existing_value = existing_ports.get(key, []) - if existing_value: - value = max(existing_value) + 1 - - ports[key] = value - - return ports def exec_cmd(cmd, cwd='.'): from .cli import from_command_line @@ -168,32 +119,6 @@ def setup_env(bench='.'): def setup_socketio(bench='.'): exec_cmd("npm install socket.io redis express superagent cookie", cwd=bench) -def setup_procfile(with_celery_broker=False, with_watch=False, bench='.'): - from .app import get_current_frappe_version - frappe_version = get_current_frappe_version() - procfile_contents = { - 'web': "./env/bin/frappe --serve --sites_path sites", - 'worker': "sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app worker'", - 'longjob_worker': "sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app -n longjobs@%h worker'", - 'async_worker': "sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app -n async@%h worker'", - 'workerbeat': "sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app beat -s scheduler.schedule'" - } - if frappe_version > 4: - procfile_contents['redis_cache'] = "redis-server config/redis_cache.conf" - procfile_contents['redis_async_broker'] = "redis-server config/redis_async_broker.conf" - procfile_contents['web'] = "bench serve" - if with_celery_broker: - procfile_contents['redis_celery'] = "redis-server" - if with_watch: - procfile_contents['watch'] = "bench watch" - if frappe_version > 5: - procfile_contents['socketio'] = "{0} apps/frappe/socketio.js".format(find_executable("node") or find_executable("nodejs")) - - procfile = '\n'.join(["{0}: {1}".format(k, v) for k, v in procfile_contents.items()]) - - with open(os.path.join(bench, 'Procfile'), 'w') as f: - f.write(procfile) - def new_site(site, mariadb_root_password=None, admin_password=None, bench='.'): import hashlib logger.info('creating new site {}'.format(site)) @@ -293,22 +218,6 @@ def setup_logging(bench='.'): logger.addHandler(hdlr) logger.setLevel(logging.DEBUG) -def get_config(bench): - config_path = os.path.join(bench, 'config.json') - if not os.path.exists(config_path): - return {} - with open(config_path) as f: - return json.load(f) - -def put_config(config, bench='.'): - with open(os.path.join(bench, 'config.json'), 'w') as f: - return json.dump(config, f, indent=1) - -def update_config(new_config, bench='.'): - config = get_config(bench=bench) - config.update(new_config) - put_config(config, bench=bench) - def get_program(programs): program = None for p in programs: @@ -355,6 +264,7 @@ def get_cmd_output(cmd, cwd='.'): raise def restart_supervisor_processes(bench='.'): + from .config.common_site_config import get_config conf = get_config(bench=bench) bench_name = get_bench_name(bench) cmd = conf.get('supervisor_restart_cmd', @@ -471,6 +381,7 @@ def drop_privileges(uid_name='nobody', gid_name='nogroup'): os.umask(022) def fix_prod_setup_perms(bench='.', frappe_user=None): + from .config.common_site_config import get_config files = [ "logs/web.error.log", "logs/web.log", @@ -507,15 +418,6 @@ def fix_file_perms(): if not _file.startswith('activate'): os.chmod(os.path.join(bin_dir, _file), 0755) -def get_redis_version(): - version_string = subprocess.check_output('redis-server --version', shell=True).strip() - if re.search("Redis server version 2.4", version_string): - return "2.4" - if re.search("Redis server v=2.6", version_string): - return "2.6" - if re.search("Redis server v=2.8", version_string): - return "2.8" - def get_current_frappe_version(bench='.'): from .app import get_current_frappe_version as fv return fv(bench=bench) @@ -569,6 +471,7 @@ def pre_upgrade(from_ver, to_ver, bench='.'): exec_cmd("{pip} install --upgrade -e {app}".format(pip=pip, app=cwd)) def post_upgrade(from_ver, to_ver, bench='.'): + from .config.common_site_config import get_config from .config import generate_supervisor_config, generate_redis_cache_config, generate_redis_async_broker_config from .config.nginx import make_nginx_conf conf = get_config(bench=bench) @@ -593,10 +496,6 @@ def post_upgrade(from_ver, to_ver, bench='.'): print "sudo service nginx restart" print "sudo supervisorctl reload" - if to_ver >= 5: - # For dev server. Always set this up incase someone wants to start a dev server. - setup_procfile(bench=bench) - def update_translations_p(args): try: update_translations(*args)