accounting/bench
2
0
Fork 0
mirror of https://github.com/frappe/bench.git synced 2025-01-10 00:37:51 +00:00
Code Issues Releases Activity

feat: Multi queue consumption (#1475)

Browse Source 
Ref: https://frappe.io/blog/engineering/reducing-memory-footprint-of-frappe-framework
This commit is contained in:
Ankush Menat 2023-08-07 14:28:02 +05:30 committed by GitHub
parent 87efb75a08
commit 5a2c052b9b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 35 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
bench/config/supervisor.py
View File

@ -3,21 +3,20 @@ import getpass
import logging
import os
# imports - module imports
import bench
from bench.app import use_rq
from bench.utils import get_bench_name, which
from bench.bench import Bench
from bench.config.common_site_config import (
update_config,
get_gunicorn_workers,
get_default_max_requests,
compute_max_requests_jitter,
)
# imports - third party imports
import click
# imports - module imports
import bench
from bench.app import use_rq
from bench.bench import Bench
from bench.config.common_site_config import (
compute_max_requests_jitter,
get_default_max_requests,
get_gunicorn_workers,
update_config,
)
from bench.utils import get_bench_name, which
logger = logging.getLogger(bench.PROJECT_NAME)
@ -58,6 +57,7 @@ def generate_supervisor_config(bench_path, user=None, yes=False, skip_redis=Fals
"bench_cmd": which("bench"),
"skip_redis": skip_redis,
"workers": config.get("workers", {}),
"multi_queue_consumption": can_enable_multi_queue_consumption(bench_path),
}
)
@ -90,6 +90,21 @@ def get_supervisord_conf():
return possibility
def can_enable_multi_queue_consumption(bench_path: str) -> bool:
try:
from semantic_version import Version
from bench.utils.app import get_current_version
supported_version = Version(major=14, minor=18, patch=0)
frappe_version = Version(get_current_version("frappe", bench_path=bench_path))
return frappe_version > supported_version
except Exception:
return False
def check_supervisord_config(user=None):
"""From bench v5.x, we're moving to supervisor running as user"""
# i don't think bench should be responsible for this but we're way past this now...

62
bench/config/templates/supervisor.conf
View File

@ -12,7 +12,6 @@ stderr_logfile={{ bench_dir }}/logs/web.error.log
user={{ user }}
directory={{ sites_dir }}
{% if use_rq %}
[program:{{ bench_name }}-frappe-schedule]
command={{ bench_cmd }} schedule
priority=3
@ -23,6 +22,7 @@ stderr_logfile={{ bench_dir }}/logs/schedule.error.log
user={{ user }}
directory={{ bench_dir }}
{% if not multi_queue_consumption %}
[program:{{ bench_name }}-frappe-default-worker]
command={{ bench_cmd }} worker --queue default
priority=4
@ -36,9 +36,10 @@ directory={{ bench_dir }}
killasgroup=true
numprocs={{ background_workers }}
process_name=%(program_name)s-%(process_num)d
{% endif %}
[program:{{ bench_name }}-frappe-short-worker]
command={{ bench_cmd }} worker --queue short
command={{ bench_cmd }} worker --queue short{{',default' if multi_queue_consumption else ''}}
priority=4
autostart=true
autorestart=true
@ -52,7 +53,7 @@ numprocs={{ background_workers }}
process_name=%(program_name)s-%(process_num)d
[program:{{ bench_name }}-frappe-long-worker]
command={{ bench_cmd }} worker --queue long
command={{ bench_cmd }} worker --queue long{{',default,short' if multi_queue_consumption else ''}}
priority=4
autostart=true
autorestart=true
@ -81,54 +82,6 @@ numprocs={{ worker_details["background_workers"] or background_workers }}
process_name=%(program_name)s-%(process_num)d
{% endfor %}
{% else %}
[program:{{ bench_name }}-frappe-workerbeat]
command={{ bench_dir }}/env/bin/python -m frappe.celery_app beat -s beat.schedule
priority=3
autostart=true
autorestart=true
stdout_logfile={{ bench_dir }}/logs/workerbeat.log
stderr_logfile={{ bench_dir }}/logs/workerbeat.error.log
user={{ user }}
directory={{ sites_dir }}
[program:{{ bench_name }}-frappe-worker]
command={{ bench_dir }}/env/bin/python -m frappe.celery_app worker -n jobs@%%h -Ofair --soft-time-limit 360 --time-limit 390 --loglevel INFO
priority=4
autostart=true
autorestart=true
stdout_logfile={{ bench_dir }}/logs/worker.log
stderr_logfile={{ bench_dir }}/logs/worker.error.log
user={{ user }}
stopwaitsecs=400
directory={{ sites_dir }}
killasgroup=true
[program:{{ bench_name }}-frappe-longjob-worker]
command={{ bench_dir }}/env/bin/python -m frappe.celery_app worker -n longjobs@%%h -Ofair --soft-time-limit 1500 --time-limit 1530 --loglevel INFO
priority=2
autostart=true
autorestart=true
stdout_logfile={{ bench_dir }}/logs/worker.log
stderr_logfile={{ bench_dir }}/logs/worker.error.log
user={{ user }}
stopwaitsecs=1540
directory={{ sites_dir }}
killasgroup=true
[program:{{ bench_name }}-frappe-async-worker]
command={{ bench_dir }}/env/bin/python -m frappe.celery_app worker -n async@%%h -Ofair --soft-time-limit 1500 --time-limit 1530 --loglevel INFO
priority=2
autostart=true
autorestart=true
stdout_logfile={{ bench_dir }}/logs/worker.log
stderr_logfile={{ bench_dir }}/logs/worker.error.log
user={{ user }}
stopwaitsecs=1540
directory={{ sites_dir }}
killasgroup=true
{% endif %}
{% if not skip_redis %}
[program:{{ bench_name }}-redis-cache]
@ -167,15 +120,16 @@ directory={{ bench_dir }}
[group:{{ bench_name }}-web]
programs={{ bench_name }}-frappe-web {%- if node -%} ,{{ bench_name }}-node-socketio {%- endif%}
{% if use_rq %}
{% if multi_queue_consumption %}
[group:{{ bench_name }}-workers]
programs={{ bench_name }}-frappe-schedule,{{ bench_name }}-frappe-default-worker,{{ bench_name }}-frappe-short-worker,{{ bench_name }}-frappe-long-worker{%- for worker_name in workers -%},{{ bench_name }}-frappe-{{ worker_name }}-worker{%- endfor %}
programs={{ bench_name }}-frappe-schedule,{{ bench_name }}-frappe-short-worker,{{ bench_name }}-frappe-long-worker{%- for worker_name in workers -%},{{ bench_name }}-frappe-{{ worker_name }}-worker{%- endfor %}
{% else %}
[group:{{ bench_name }}-workers]
programs={{ bench_name }}-frappe-workerbeat,{{ bench_name }}-frappe-worker,{{ bench_name }}-frappe-longjob-worker,{{ bench_name }}-frappe-async-worker{%- for worker_name in workers -%},{{ bench_name }}-frappe-{{ worker_name }}-worker{%- endfor %}
programs={{ bench_name }}-frappe-schedule,{{ bench_name }}-frappe-default-worker,{{ bench_name }}-frappe-short-worker,{{ bench_name }}-frappe-long-worker{%- for worker_name in workers -%},{{ bench_name }}-frappe-{{ worker_name }}-worker{%- endfor %}
{% endif %}