diff --git a/README.md b/README.md index 837724e8..f1baf13f 100755 --- a/README.md +++ b/README.md @@ -15,8 +15,8 @@ Production vs Development | Production | Development | |--------------------------------------------------------------------------|-------------------------------------------------------------------| -| The Production setup uses Nginx and Supervisor | The development setup uses Socketio. | -| This setup isn't meant for instant updates in code. | Any code changes will be reflected instantly. | +| The Production setup uses Nginx and Supervisor | The development setup uses Socketio. | +| This setup isn't meant for instant updates in code. | Any code changes will be reflected instantly. | | Background services handle all the work, and they start with the system. | You need to explicitly start your server by running `bench start` | | Uses Celery for job queuing | Uses RQ for queuing | | Installs with master branch | Installs with develop branch | diff --git a/bench/config/procfile.py b/bench/config/procfile.py index 8fea1849..726cc152 100644 --- a/bench/config/procfile.py +++ b/bench/config/procfile.py @@ -1,14 +1,20 @@ import bench, os, click from bench.utils import find_executable +from bench.app import get_current_frappe_version, get_current_branch +from bench.config.common_site_config import get_config def setup_procfile(bench_path, force=False): + config = get_config(bench=bench_path) procfile_path = os.path.join(bench_path, 'Procfile') if not force and os.path.exists(procfile_path): click.confirm('A Procfile already exists and this will overwrite it. Do you want to continue?', abort=True) - procfile = bench.env.get_template('Procfile').render(node=find_executable("node") \ - or find_executable("nodejs")) + procfile = bench.env.get_template('Procfile').render( + node=find_executable("node") or find_executable("nodejs"), + frappe_version=get_current_frappe_version(), + frappe_branch=get_current_branch('frappe', bench_path), + webserver_port=config.get('webserver_port')) with open(procfile_path, 'w') as f: f.write(procfile) diff --git a/bench/config/supervisor.py b/bench/config/supervisor.py index 28339341..30eaadff 100644 --- a/bench/config/supervisor.py +++ b/bench/config/supervisor.py @@ -2,7 +2,7 @@ import os, getpass, click import bench def generate_supervisor_config(bench_path, user=None, force=False): - from bench.app import get_current_frappe_version + from bench.app import get_current_frappe_version, get_current_branch from bench.utils import get_bench_name, find_executable from bench.config.common_site_config import get_config, update_config, get_gunicorn_workers @@ -18,13 +18,14 @@ def generate_supervisor_config(bench_path, user=None, force=False): "bench_dir": bench_dir, "sites_dir": os.path.join(bench_dir, 'sites'), "user": user, + "frappe_version": get_current_frappe_version(), + "frappe_branch": get_current_branch('frappe', bench_path), "http_timeout": config.get("http_timeout", 120), "redis_server": find_executable('redis-server'), "node": find_executable('node') or find_executable('nodejs'), "redis_cache_config": os.path.join(bench_dir, 'config', 'redis_cache.conf'), "redis_socketio_config": os.path.join(bench_dir, 'config', 'redis_socketio.conf'), "redis_queue_config": os.path.join(bench_dir, 'config', 'redis_queue.conf'), - "frappe_version": get_current_frappe_version(), "webserver_port": config.get('webserver_port', 8000), "gunicorn_workers": config.get('gunicorn_workers', get_gunicorn_workers()["gunicorn_workers"]), "bench_name": get_bench_name(bench_path), diff --git a/bench/config/templates/Procfile b/bench/config/templates/Procfile index 4d73ee41..a01f2016 100644 --- a/bench/config/templates/Procfile +++ b/bench/config/templates/Procfile @@ -1,10 +1,19 @@ +{%- set use_rq = (frappe_branch=='develop' or frappe_version >= 7) -%} redis_cache: redis-server config/redis_cache.conf redis_socketio: redis-server config/redis_socketio.conf redis_queue: redis-server config/redis_queue.conf -web: bench serve +web: bench serve {% if webserver_port -%} --port {{ webserver_port }} {%- endif %} + socketio: {{ node }} apps/frappe/socketio.js watch: bench watch +{% if use_rq -%} schedule: bench schedule worker_short: bench worker --queue short worker_long: bench worker --queue long worker_default: bench worker --queue default +{% else %} +workerbeat: sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app beat -s scheduler.schedule' +worker: sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app worker -n jobs@%h -Ofair --soft-time-limit 360 --time-limit 390' +longjob_worker: sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app worker -n longjobs@%h -Ofair --soft-time-limit 1500 --time-limit 1530' +async_worker: sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app worker -n async@%h -Ofair --soft-time-limit 1500 --time-limit 1530' +{%- endif %} diff --git a/bench/config/templates/supervisor.conf b/bench/config/templates/supervisor.conf index 3ca4a6de..74c0a9d0 100644 --- a/bench/config/templates/supervisor.conf +++ b/bench/config/templates/supervisor.conf @@ -1,3 +1,4 @@ +{%- set use_rq = (frappe_branch=='develop' or frappe_version >= 7) -%} ; Notes: ; priority=1 --> Lower priorities indicate programs that start first and shut down last ; killasgroup=true --> send kill signal to child processes too @@ -12,6 +13,7 @@ stderr_logfile={{ bench_dir }}/logs/web.error.log user={{ user }} directory={{ sites_dir }} +{% if use_rq %} [program:{{ bench_name }}-frappe-schedule] command=bench schedule priority=3 @@ -64,6 +66,55 @@ killasgroup=true numprocs={{ background_workers }} process_name=%(program_name)s-%(process_num)d +{% else %} +[program:{{ bench_name }}-frappe-workerbeat] +command={{ bench_dir }}/env/bin/python -m frappe.celery_app beat -s beat.schedule +priority=3 +autostart=true +autorestart=true +stdout_logfile={{ bench_dir }}/logs/workerbeat.log +stderr_logfile={{ bench_dir }}/logs/workerbeat.error.log +user={{ user }} +directory={{ sites_dir }} + +[program:{{ bench_name }}-frappe-worker] +command={{ bench_dir }}/env/bin/python -m frappe.celery_app worker -n jobs@%%h -Ofair --soft-time-limit 360 --time-limit 390 --loglevel INFO +priority=4 +autostart=true +autorestart=true +stdout_logfile={{ bench_dir }}/logs/worker.log +stderr_logfile={{ bench_dir }}/logs/worker.error.log +user={{ user }} +stopwaitsecs=400 +directory={{ sites_dir }} +killasgroup=true + +[program:{{ bench_name }}-frappe-longjob-worker] +command={{ bench_dir }}/env/bin/python -m frappe.celery_app worker -n longjobs@%%h -Ofair --soft-time-limit 1500 --time-limit 1530 --loglevel INFO +priority=2 +autostart=true +autorestart=true +stdout_logfile={{ bench_dir }}/logs/worker.log +stderr_logfile={{ bench_dir }}/logs/worker.error.log +user={{ user }} +stopwaitsecs=1540 +directory={{ sites_dir }} +killasgroup=true + +[program:{{ bench_name }}-frappe-async-worker] +command={{ bench_dir }}/env/bin/python -m frappe.celery_app worker -n async@%%h -Ofair --soft-time-limit 1500 --time-limit 1530 --loglevel INFO +priority=2 +autostart=true +autorestart=true +stdout_logfile={{ bench_dir }}/logs/worker.log +stderr_logfile={{ bench_dir }}/logs/worker.error.log +user={{ user }} +stopwaitsecs=1540 +directory={{ sites_dir }} +killasgroup=true + +{% endif %} + [program:{{ bench_name }}-redis-cache] command={{ redis_server }} {{ redis_cache_config }} priority=1 @@ -112,8 +163,17 @@ directory={{ bench_dir }} [group:{{ bench_name }}-web] programs={{ bench_name }}-frappe-web {%- if node -%} ,{{ bench_name }}-node-socketio {%- endif%} +{% if use_rq %} + [group:{{ bench_name }}-workers] programs={{ bench_name }}-frappe-schedule,{{ bench_name }}-frappe-default-worker,{{ bench_name }}-frappe-short-worker,{{ bench_name }}-frappe-long-worker +{% else %} + +[group:{{ bench_name }}-workers] +programs={{ bench_name }}-frappe-workerbeat,{{ bench_name }}-frappe-worker,{{ bench_name }}-frappe-longjob-worker,{{ bench_name }}-frappe-async-worker + +{% endif %} + [group:{{ bench_name }}-redis] programs={{ bench_name }}-redis-cache,{{ bench_name }}-redis-queue {%- if frappe_version > 5 -%} ,{{ bench_name }}-redis-socketio {%- endif %} diff --git a/bench/patches/__init__.py b/bench/patches/__init__.py index 61013f5b..5a504737 100644 --- a/bench/patches/__init__.py +++ b/bench/patches/__init__.py @@ -18,8 +18,10 @@ def run(bench_path): if patch not in executed_patches: module = importlib.import_module(patch.split()[0]) execute = getattr(module, 'execute') - execute(bench_path) - executed_patches.append(patch) + result = execute(bench_path) + + if result != False: + executed_patches.append(patch) finally: with open(target_patch_file, 'w') as f: diff --git a/bench/patches/v3/celery_to_rq.py b/bench/patches/v3/celery_to_rq.py index 94472995..301cd956 100644 --- a/bench/patches/v3/celery_to_rq.py +++ b/bench/patches/v3/celery_to_rq.py @@ -1,8 +1,17 @@ import click, os from bench.config.procfile import setup_procfile from bench.config.supervisor import generate_supervisor_config +from bench.app import get_current_frappe_version, get_current_branch def execute(bench_path): + frappe_branch = get_current_branch('frappe', bench_path) + frappe_version = get_current_frappe_version(bench_path) + + if not (frappe_branch=='develop' or frappe_version >= 7): + # not version 7+ + # prevent running this patch + return False + click.confirm('\nThis update will remove Celery config and prepare the bench to use Python RQ.\n' 'And it will overwrite Procfile and supervisor.conf.\n' 'If you don\'t know what this means, type Y ;)\n\n' diff --git a/bench/utils.py b/bench/utils.py index 9445fa8a..8a0b981e 100644 --- a/bench/utils.py +++ b/bench/utils.py @@ -271,18 +271,26 @@ def restart_supervisor_processes(bench='.'): from .config.common_site_config import get_config conf = get_config(bench=bench) bench_name = get_bench_name(bench) - cmd = conf.get('supervisor_restart_cmd', - 'sudo supervisorctl restart {bench_name}-web: {bench_name}-workers:'.format(bench_name=bench_name)) - try: + cmd = conf.get('supervisor_restart_cmd') + if cmd: exec_cmd(cmd, cwd=bench) - except CommandFailedError: - if '{bench_name}-workers:'.format(bench_name=bench_name) in cmd: - # backward compatibility - exec_cmd('sudo supervisorctl restart frappe:', cwd=bench) + else: + supervisor_status = subprocess.check_output(['sudo', 'supervisorctl', 'status'], cwd=bench) + + if '{bench_name}-workers:'.format(bench_name=bench_name) in supervisor_status: + group = '{bench_name}-web: {bench_name}-workers:'.format(bench_name=bench_name) + + # backward compatibility + elif '{bench_name}-processes:'.format(bench_name=bench_name) in supervisor_status: + group = '{bench_name}-processes:'.format(bench_name=bench_name) + + # backward compatibility else: - raise + group = 'frappe:' + + exec_cmd('sudo supervisorctl restart {group}'.format(group=group), cwd=bench) def get_site_config(site, bench='.'): config_path = os.path.join(bench, 'sites', site, 'site_config.json')