accounting/bench
2
0
Fork 0
mirror of https://github.com/frappe/bench.git synced 2025-01-10 00:37:51 +00:00
Code Issues Releases Activity

Merge branch 'develop' into staging

Browse Source
This commit is contained in:
Ankush Menat 2023-08-08 22:08:01 +05:30
commit ffe969bdc3
13 changed files with 49 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.github/workflows/ci.yml vendored
View File

@ -114,9 +114,17 @@ jobs:
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- uses: actions/setup-node@v3
if: ${{ matrix.python-version == '3.10' }}
with:
node-version: 18
- uses: actions/setup-node@v3
if: ${{ matrix.python-version == '3.7' }}
with:
node-version: 14
- run: |
wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb;
sudo apt install ./wkhtmltox_0.12.6-1.focal_amd64.deb;

2
bench/config/common_site_config.py
View File

@ -100,7 +100,7 @@ def make_ports(bench_path):
"socketio_port": 9000,
"file_watcher_port": 6787,
"redis_queue": 11000,
"redis_socketio": 12000,
"redis_socketio": 13000,
"redis_cache": 13000,
}

10
bench/config/redis.py
View File

@ -15,7 +15,7 @@ def generate_config(bench_path):
redis_version = get_redis_version()
ports = {}
for key in ("redis_cache", "redis_queue", "redis_socketio"):
for key in ("redis_cache", "redis_queue"):
ports[key] = urlparse(config[key]).port
write_redis_config(
@ -28,12 +28,6 @@ def generate_config(bench_path):
bench_path=bench_path,
)
write_redis_config(
template_name="redis_socketio.conf",
context={"port": ports["redis_socketio"], "redis_version": redis_version},
bench_path=bench_path,
)
write_redis_config(
template_name="redis_cache.conf",
context={
@ -56,10 +50,8 @@ def generate_config(bench_path):
# make ACL files
acl_rq_path = os.path.join(bench_path, "config", "redis_queue.acl")
acl_redis_cache_path = os.path.join(bench_path, "config", "redis_cache.acl")
acl_redis_socketio_path = os.path.join(bench_path, "config", "redis_socketio.acl")
open(acl_rq_path, "a").close()
open(acl_redis_cache_path, "a").close()
open(acl_redis_socketio_path, "a").close()
def write_redis_config(template_name, context, bench_path):

40
bench/config/supervisor.py
View File

@ -3,21 +3,20 @@ import getpass
import logging
import os
# imports - module imports
import bench
from bench.app import use_rq
from bench.utils import get_bench_name, which
from bench.bench import Bench
from bench.config.common_site_config import (
update_config,
get_gunicorn_workers,
get_default_max_requests,
compute_max_requests_jitter,
)
# imports - third party imports
import click
# imports - module imports
import bench
from bench.app import use_rq
from bench.bench import Bench
from bench.config.common_site_config import (
compute_max_requests_jitter,
get_default_max_requests,
get_gunicorn_workers,
update_config,
)
from bench.utils import get_bench_name, which
logger = logging.getLogger(bench.PROJECT_NAME)
@ -48,7 +47,6 @@ def generate_supervisor_config(bench_path, user=None, yes=False, skip_redis=Fals
"redis_server": which("redis-server"),
"node": which("node") or which("nodejs"),
"redis_cache_config": os.path.join(bench_dir, "config", "redis_cache.conf"),
"redis_socketio_config": os.path.join(bench_dir, "config", "redis_socketio.conf"),
"redis_queue_config": os.path.join(bench_dir, "config", "redis_queue.conf"),
"webserver_port": config.get("webserver_port", 8000),
"gunicorn_workers": web_worker_count,
@ -59,6 +57,7 @@ def generate_supervisor_config(bench_path, user=None, yes=False, skip_redis=Fals
"bench_cmd": which("bench"),
"skip_redis": skip_redis,
"workers": config.get("workers", {}),
"multi_queue_consumption": can_enable_multi_queue_consumption(bench_path),
}
)
@ -91,6 +90,21 @@ def get_supervisord_conf():
return possibility
def can_enable_multi_queue_consumption(bench_path: str) -> bool:
try:
from semantic_version import Version
from bench.utils.app import get_current_version
supported_version = Version(major=14, minor=18, patch=0)
frappe_version = Version(get_current_version("frappe", bench_path=bench_path))
return frappe_version > supported_version
except Exception:
return False
def check_supervisord_config(user=None):
"""From bench v5.x, we're moving to supervisor running as user"""
# i don't think bench should be responsible for this but we're way past this now...

15
bench/config/systemd.py
View File

@ -82,7 +82,6 @@ def generate_systemd_config(
"redis_server": which("redis-server"),
"node": which("node") or which("nodejs"),
"redis_cache_config": os.path.join(bench_dir, "config", "redis_cache.conf"),
"redis_socketio_config": os.path.join(bench_dir, "config", "redis_socketio.conf"),
"redis_queue_config": os.path.join(bench_dir, "config", "redis_queue.conf"),
"webserver_port": config.get("webserver_port", 8000),
"gunicorn_workers": web_worker_count,
@ -244,14 +243,10 @@ def setup_redis_config(bench_info, bench_path):
bench_redis_queue_template = bench.config.env().get_template(
"systemd/frappe-bench-redis-queue.service"
)
bench_redis_socketio_template = bench.config.env().get_template(
"systemd/frappe-bench-redis-socketio.service"
)
bench_redis_target_config = bench_redis_target_template.render(**bench_info)
bench_redis_cache_config = bench_redis_cache_template.render(**bench_info)
bench_redis_queue_config = bench_redis_queue_template.render(**bench_info)
bench_redis_socketio_config = bench_redis_socketio_template.render(**bench_info)
bench_redis_target_config_path = os.path.join(
bench_path, "config", "systemd", bench_info.get("bench_name") + "-redis.target"
@ -262,12 +257,6 @@ def setup_redis_config(bench_info, bench_path):
bench_redis_queue_config_path = os.path.join(
bench_path, "config", "systemd", bench_info.get("bench_name") + "-redis-queue.service"
)
bench_redis_socketio_config_path = os.path.join(
bench_path,
"config",
"systemd",
bench_info.get("bench_name") + "-redis-socketio.service",
)
with open(bench_redis_target_config_path, "w") as f:
f.write(bench_redis_target_config)
@ -278,9 +267,6 @@ def setup_redis_config(bench_info, bench_path):
with open(bench_redis_queue_config_path, "w") as f:
f.write(bench_redis_queue_config)
with open(bench_redis_socketio_config_path, "w") as f:
f.write(bench_redis_socketio_config)
def _create_symlinks(bench_path):
bench_dir = os.path.abspath(bench_path)
@ -319,6 +305,5 @@ def get_unit_files(bench_path):
[bench_name + "-node-socketio", ".service"],
[bench_name + "-redis-cache", ".service"],
[bench_name + "-redis-queue", ".service"],
[bench_name + "-redis-socketio", ".service"],
]
return unit_files

1
bench/config/templates/Procfile
View File

@ -1,6 +1,5 @@
{% if not skip_redis %}
redis_cache: redis-server config/redis_cache.conf
redis_socketio: redis-server config/redis_socketio.conf
redis_queue: redis-server config/redis_queue.conf
{% endif %}
web: bench serve {% if webserver_port -%} --port {{ webserver_port }} {%- endif %}

8
bench/config/templates/redis_socketio.conf
View File

@ -1,8 +0,0 @@
dbfilename redis_socketio.rdb
dir {{ pid_path }}
pidfile {{ pid_path }}/redis_socketio.pid
bind 127.0.0.1
port {{ port }}
{% if redis_version and redis_version >= 6.0 %}
aclfile {{ config_path }}/redis_socketio.acl
{% endif %}

76
bench/config/templates/supervisor.conf
View File

@ -12,7 +12,6 @@ stderr_logfile={{ bench_dir }}/logs/web.error.log
user={{ user }}
directory={{ sites_dir }}
{% if use_rq %}
[program:{{ bench_name }}-frappe-schedule]
command={{ bench_cmd }} schedule
priority=3
@ -23,6 +22,7 @@ stderr_logfile={{ bench_dir }}/logs/schedule.error.log
user={{ user }}
directory={{ bench_dir }}
{% if not multi_queue_consumption %}
[program:{{ bench_name }}-frappe-default-worker]
command={{ bench_cmd }} worker --queue default
priority=4
@ -36,9 +36,10 @@ directory={{ bench_dir }}
killasgroup=true
numprocs={{ background_workers }}
process_name=%(program_name)s-%(process_num)d
{% endif %}
[program:{{ bench_name }}-frappe-short-worker]
command={{ bench_cmd }} worker --queue short
command={{ bench_cmd }} worker --queue short{{',default' if multi_queue_consumption else ''}}
priority=4
autostart=true
autorestart=true
@ -52,7 +53,7 @@ numprocs={{ background_workers }}
process_name=%(program_name)s-%(process_num)d
[program:{{ bench_name }}-frappe-long-worker]
command={{ bench_cmd }} worker --queue long
command={{ bench_cmd }} worker --queue long{{',default,short' if multi_queue_consumption else ''}}
priority=4
autostart=true
autorestart=true
@ -81,54 +82,6 @@ numprocs={{ worker_details["background_workers"] or background_workers }}
process_name=%(program_name)s-%(process_num)d
{% endfor %}
{% else %}
[program:{{ bench_name }}-frappe-workerbeat]
command={{ bench_dir }}/env/bin/python -m frappe.celery_app beat -s beat.schedule
priority=3
autostart=true
autorestart=true
stdout_logfile={{ bench_dir }}/logs/workerbeat.log
stderr_logfile={{ bench_dir }}/logs/workerbeat.error.log
user={{ user }}
directory={{ sites_dir }}
[program:{{ bench_name }}-frappe-worker]
command={{ bench_dir }}/env/bin/python -m frappe.celery_app worker -n jobs@%%h -Ofair --soft-time-limit 360 --time-limit 390 --loglevel INFO
priority=4
autostart=true
autorestart=true
stdout_logfile={{ bench_dir }}/logs/worker.log
stderr_logfile={{ bench_dir }}/logs/worker.error.log
user={{ user }}
stopwaitsecs=400
directory={{ sites_dir }}
killasgroup=true
[program:{{ bench_name }}-frappe-longjob-worker]
command={{ bench_dir }}/env/bin/python -m frappe.celery_app worker -n longjobs@%%h -Ofair --soft-time-limit 1500 --time-limit 1530 --loglevel INFO
priority=2
autostart=true
autorestart=true
stdout_logfile={{ bench_dir }}/logs/worker.log
stderr_logfile={{ bench_dir }}/logs/worker.error.log
user={{ user }}
stopwaitsecs=1540
directory={{ sites_dir }}
killasgroup=true
[program:{{ bench_name }}-frappe-async-worker]
command={{ bench_dir }}/env/bin/python -m frappe.celery_app worker -n async@%%h -Ofair --soft-time-limit 1500 --time-limit 1530 --loglevel INFO
priority=2
autostart=true
autorestart=true
stdout_logfile={{ bench_dir }}/logs/worker.log
stderr_logfile={{ bench_dir }}/logs/worker.error.log
user={{ user }}
stopwaitsecs=1540
directory={{ sites_dir }}
killasgroup=true
{% endif %}
{% if not skip_redis %}
[program:{{ bench_name }}-redis-cache]
@ -152,18 +105,6 @@ user={{ user }}
directory={{ sites_dir }}
{% endif %}
{% if not skip_redis %}
[program:{{ bench_name }}-redis-socketio]
command={{ redis_server }} {{ redis_socketio_config }}
priority=1
autostart=true
autorestart=true
stdout_logfile={{ bench_dir }}/logs/redis-socketio.log
stderr_logfile={{ bench_dir }}/logs/redis-socketio.error.log
user={{ user }}
directory={{ sites_dir }}
{% endif %}
{% if node %}
[program:{{ bench_name }}-node-socketio]
command={{ node }} {{ bench_dir }}/apps/frappe/socketio.js
@ -179,19 +120,20 @@ directory={{ bench_dir }}
[group:{{ bench_name }}-web]
programs={{ bench_name }}-frappe-web {%- if node -%} ,{{ bench_name }}-node-socketio {%- endif%}
{% if use_rq %}
{% if multi_queue_consumption %}
[group:{{ bench_name }}-workers]
programs={{ bench_name }}-frappe-schedule,{{ bench_name }}-frappe-default-worker,{{ bench_name }}-frappe-short-worker,{{ bench_name }}-frappe-long-worker{%- for worker_name in workers -%},{{ bench_name }}-frappe-{{ worker_name }}-worker{%- endfor %}
programs={{ bench_name }}-frappe-schedule,{{ bench_name }}-frappe-short-worker,{{ bench_name }}-frappe-long-worker{%- for worker_name in workers -%},{{ bench_name }}-frappe-{{ worker_name }}-worker{%- endfor %}
{% else %}
[group:{{ bench_name }}-workers]
programs={{ bench_name }}-frappe-workerbeat,{{ bench_name }}-frappe-worker,{{ bench_name }}-frappe-longjob-worker,{{ bench_name }}-frappe-async-worker{%- for worker_name in workers -%},{{ bench_name }}-frappe-{{ worker_name }}-worker{%- endfor %}
programs={{ bench_name }}-frappe-schedule,{{ bench_name }}-frappe-default-worker,{{ bench_name }}-frappe-short-worker,{{ bench_name }}-frappe-long-worker{%- for worker_name in workers -%},{{ bench_name }}-frappe-{{ worker_name }}-worker{%- endfor %}
{% endif %}
{% if not skip_redis %}
[group:{{ bench_name }}-redis]
programs={{ bench_name }}-redis-cache,{{ bench_name }}-redis-queue,{{ bench_name }}-redis-socketio
programs={{ bench_name }}-redis-cache,{{ bench_name }}-redis-queue
{% endif %}

12
bench/config/templates/systemd/frappe-bench-redis-socketio.service
View File

@ -1,12 +0,0 @@
[Unit]
Description="{{ bench_name }}-redis-socketio"
PartOf={{ bench_name }}-redis.target
[Service]
User={{ user }}
Group={{ user }}
Restart=always
ExecStart={{ redis_server }} {{ redis_socketio_config }}
StandardOutput=file:{{ bench_dir }}/logs/redis-socketio.log
StandardError=file:{{ bench_dir }}/logs/redis-socketio.error.log
WorkingDirectory={{ sites_dir }}

2
bench/config/templates/systemd/frappe-bench-redis.target
View File

@ -1,6 +1,6 @@
[Unit]
After=network.target
Wants={{ bench_name }}-redis-cache.service {{ bench_name }}-redis-queue.service {{ bench_name }}-redis-socketio.service
Wants={{ bench_name }}-redis-cache.service {{ bench_name }}-redis-queue.service
[Install]
WantedBy=multi-user.target

1
bench/tests/test_base.py
View File

@ -67,7 +67,6 @@ class TestBenchBase(unittest.TestCase):
def assert_config(self, bench_name):
for config, search_key in (
("redis_queue.conf", "redis_queue.rdb"),
("redis_socketio.conf", "redis_socketio.rdb"),
("redis_cache.conf", "redis_cache.rdb"),
):

4
bench/tests/test_init.py
View File

@ -55,7 +55,7 @@ class TestBenchInit(TestBenchBase):
"socketio_port": 9000,
"file_watcher_port": 6787,
"redis_queue": "redis://localhost:11000",
"redis_socketio": "redis://localhost:12000",
"redis_socketio": "redis://localhost:13000",
"redis_cache": "redis://localhost:13000",
},
)
@ -67,7 +67,7 @@ class TestBenchInit(TestBenchBase):
"socketio_port": 9001,
"file_watcher_port": 6788,
"redis_queue": "redis://localhost:11001",
"redis_socketio": "redis://localhost:12001",
"redis_socketio": "redis://localhost:13001",
"redis_cache": "redis://localhost:13001",
},
)

2
bench/tests/test_setup_production.py
View File

@ -103,7 +103,6 @@ class TestSetupProduction(TestBenchBase):
f"program:{bench_name}-frappe-web",
f"program:{bench_name}-redis-cache",
f"program:{bench_name}-redis-queue",
f"program:{bench_name}-redis-socketio",
f"group:{bench_name}-web",
f"group:{bench_name}-workers",
f"group:{bench_name}-redis",
@ -150,7 +149,6 @@ class TestSetupProduction(TestBenchBase):
# "{bench_name}-web:{bench_name}-node-socketio[\s]+RUNNING",
r"{bench_name}-redis:{bench_name}-redis-cache[\s]+RUNNING",
r"{bench_name}-redis:{bench_name}-redis-queue[\s]+RUNNING",
r"{bench_name}-redis:{bench_name}-redis-socketio[\s]+RUNNING",
]
if use_rq: