Merge branch 'develop' into staging

.github/workflows/ci.yml

@@ -114,9 +114,17 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
+
       - uses: actions/setup-node@v3
+        if: ${{ matrix.python-version == '3.10' }}
+        with:
+          node-version: 18
+
+      - uses: actions/setup-node@v3
+        if: ${{ matrix.python-version == '3.7' }}
         with:
           node-version: 14
+
       - run: |
           wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb;
           sudo apt install ./wkhtmltox_0.12.6-1.focal_amd64.deb;

bench/config/common_site_config.py

@@ -100,7 +100,7 @@ def make_ports(bench_path):
 		"socketio_port": 9000,
 		"file_watcher_port": 6787,
 		"redis_queue": 11000,
-		"redis_socketio": 12000,
+		"redis_socketio": 13000,
 		"redis_cache": 13000,
 	}
 

bench/config/redis.py

@@ -15,7 +15,7 @@ def generate_config(bench_path):
 	redis_version = get_redis_version()
 
 	ports = {}
-	for key in ("redis_cache", "redis_queue", "redis_socketio"):
+	for key in ("redis_cache", "redis_queue"):
 		ports[key] = urlparse(config[key]).port
 
 	write_redis_config(
@@ -28,12 +28,6 @@ def generate_config(bench_path):
 		bench_path=bench_path,
 	)
 
-	write_redis_config(
-		template_name="redis_socketio.conf",
-		context={"port": ports["redis_socketio"], "redis_version": redis_version},
-		bench_path=bench_path,
-	)
-
 	write_redis_config(
 		template_name="redis_cache.conf",
 		context={
@@ -56,10 +50,8 @@ def generate_config(bench_path):
 	# make ACL files
 	acl_rq_path = os.path.join(bench_path, "config", "redis_queue.acl")
 	acl_redis_cache_path = os.path.join(bench_path, "config", "redis_cache.acl")
-	acl_redis_socketio_path = os.path.join(bench_path, "config", "redis_socketio.acl")
 	open(acl_rq_path, "a").close()
 	open(acl_redis_cache_path, "a").close()
-	open(acl_redis_socketio_path, "a").close()
 
 
 def write_redis_config(template_name, context, bench_path):

bench/config/supervisor.py

@@ -3,21 +3,20 @@ import getpass
 import logging
 import os
 
-# imports - module imports
-import bench
-from bench.app import use_rq
-from bench.utils import get_bench_name, which
-from bench.bench import Bench
-from bench.config.common_site_config import (
-	update_config,
-	get_gunicorn_workers,
-	get_default_max_requests,
-	compute_max_requests_jitter,
-)
-
 # imports - third party imports
 import click
 
+# imports - module imports
+import bench
+from bench.app import use_rq
+from bench.bench import Bench
+from bench.config.common_site_config import (
+	compute_max_requests_jitter,
+	get_default_max_requests,
+	get_gunicorn_workers,
+	update_config,
+)
+from bench.utils import get_bench_name, which
 
 logger = logging.getLogger(bench.PROJECT_NAME)
 
@@ -48,7 +47,6 @@ def generate_supervisor_config(bench_path, user=None, yes=False, skip_redis=Fals
 			"redis_server": which("redis-server"),
 			"node": which("node") or which("nodejs"),
 			"redis_cache_config": os.path.join(bench_dir, "config", "redis_cache.conf"),
-			"redis_socketio_config": os.path.join(bench_dir, "config", "redis_socketio.conf"),
 			"redis_queue_config": os.path.join(bench_dir, "config", "redis_queue.conf"),
 			"webserver_port": config.get("webserver_port", 8000),
 			"gunicorn_workers": web_worker_count,
@@ -59,6 +57,7 @@ def generate_supervisor_config(bench_path, user=None, yes=False, skip_redis=Fals
 			"bench_cmd": which("bench"),
 			"skip_redis": skip_redis,
 			"workers": config.get("workers", {}),
+			"multi_queue_consumption": can_enable_multi_queue_consumption(bench_path),
 		}
 	)
 
@@ -91,6 +90,21 @@ def get_supervisord_conf():
 			return possibility
 
 
+def can_enable_multi_queue_consumption(bench_path: str) -> bool:
+	try:
+		from semantic_version import Version
+
+		from bench.utils.app import get_current_version
+
+		supported_version = Version(major=14, minor=18, patch=0)
+
+		frappe_version = Version(get_current_version("frappe", bench_path=bench_path))
+
+		return frappe_version > supported_version
+	except Exception:
+		return False
+
+
 def check_supervisord_config(user=None):
 	"""From bench v5.x, we're moving to supervisor running as user"""
 	# i don't think bench should be responsible for this but we're way past this now...

bench/config/systemd.py

@@ -82,7 +82,6 @@ def generate_systemd_config(
 		"redis_server": which("redis-server"),
 		"node": which("node") or which("nodejs"),
 		"redis_cache_config": os.path.join(bench_dir, "config", "redis_cache.conf"),
-		"redis_socketio_config": os.path.join(bench_dir, "config", "redis_socketio.conf"),
 		"redis_queue_config": os.path.join(bench_dir, "config", "redis_queue.conf"),
 		"webserver_port": config.get("webserver_port", 8000),
 		"gunicorn_workers": web_worker_count,
@@ -244,14 +243,10 @@ def setup_redis_config(bench_info, bench_path):
 	bench_redis_queue_template = bench.config.env().get_template(
 		"systemd/frappe-bench-redis-queue.service"
 	)
-	bench_redis_socketio_template = bench.config.env().get_template(
-		"systemd/frappe-bench-redis-socketio.service"
-	)
 
 	bench_redis_target_config = bench_redis_target_template.render(**bench_info)
 	bench_redis_cache_config = bench_redis_cache_template.render(**bench_info)
 	bench_redis_queue_config = bench_redis_queue_template.render(**bench_info)
-	bench_redis_socketio_config = bench_redis_socketio_template.render(**bench_info)
 
 	bench_redis_target_config_path = os.path.join(
 		bench_path, "config", "systemd", bench_info.get("bench_name") + "-redis.target"
@@ -262,12 +257,6 @@ def setup_redis_config(bench_info, bench_path):
 	bench_redis_queue_config_path = os.path.join(
 		bench_path, "config", "systemd", bench_info.get("bench_name") + "-redis-queue.service"
 	)
-	bench_redis_socketio_config_path = os.path.join(
-		bench_path,
-		"config",
-		"systemd",
-		bench_info.get("bench_name") + "-redis-socketio.service",
-	)
 
 	with open(bench_redis_target_config_path, "w") as f:
 		f.write(bench_redis_target_config)
@@ -278,9 +267,6 @@ def setup_redis_config(bench_info, bench_path):
 	with open(bench_redis_queue_config_path, "w") as f:
 		f.write(bench_redis_queue_config)
 
-	with open(bench_redis_socketio_config_path, "w") as f:
-		f.write(bench_redis_socketio_config)
-
 
 def _create_symlinks(bench_path):
 	bench_dir = os.path.abspath(bench_path)
@@ -319,6 +305,5 @@ def get_unit_files(bench_path):
 		[bench_name + "-node-socketio", ".service"],
 		[bench_name + "-redis-cache", ".service"],
 		[bench_name + "-redis-queue", ".service"],
-		[bench_name + "-redis-socketio", ".service"],
 	]
 	return unit_files

bench/config/templates/Procfile

@@ -1,6 +1,5 @@
 {% if not skip_redis %}
 redis_cache: redis-server config/redis_cache.conf
-redis_socketio: redis-server config/redis_socketio.conf
 redis_queue: redis-server config/redis_queue.conf
 {% endif %}
 web: bench serve {% if webserver_port -%} --port {{ webserver_port }} {%- endif %}

bench/config/templates/redis_socketio.conf

@@ -1,8 +0,0 @@
-dbfilename redis_socketio.rdb
-dir {{ pid_path }}
-pidfile {{ pid_path }}/redis_socketio.pid
-bind 127.0.0.1
-port {{ port }}
-{% if redis_version and redis_version >= 6.0 %}
-aclfile {{ config_path }}/redis_socketio.acl
-{% endif %}

bench/config/templates/supervisor.conf

@@ -12,7 +12,6 @@ stderr_logfile={{ bench_dir }}/logs/web.error.log
 user={{ user }}
 directory={{ sites_dir }}
 
-{% if use_rq %}
 [program:{{ bench_name }}-frappe-schedule]
 command={{ bench_cmd }} schedule
 priority=3
@@ -23,6 +22,7 @@ stderr_logfile={{ bench_dir }}/logs/schedule.error.log
 user={{ user }}
 directory={{ bench_dir }}
 
+{% if not multi_queue_consumption %}
 [program:{{ bench_name }}-frappe-default-worker]
 command={{ bench_cmd }} worker --queue default
 priority=4
@@ -36,9 +36,10 @@ directory={{ bench_dir }}
 killasgroup=true
 numprocs={{ background_workers }}
 process_name=%(program_name)s-%(process_num)d
+{% endif %}
 
 [program:{{ bench_name }}-frappe-short-worker]
-command={{ bench_cmd }} worker --queue short
+command={{ bench_cmd }} worker --queue short{{',default' if multi_queue_consumption else ''}}
 priority=4
 autostart=true
 autorestart=true
@@ -52,7 +53,7 @@ numprocs={{ background_workers }}
 process_name=%(program_name)s-%(process_num)d
 
 [program:{{ bench_name }}-frappe-long-worker]
-command={{ bench_cmd }} worker --queue long
+command={{ bench_cmd }} worker --queue long{{',default,short' if multi_queue_consumption else ''}}
 priority=4
 autostart=true
 autorestart=true
@@ -81,54 +82,6 @@ numprocs={{ worker_details["background_workers"] or background_workers }}
 process_name=%(program_name)s-%(process_num)d
 {% endfor %}
 
-{% else %}
-[program:{{ bench_name }}-frappe-workerbeat]
-command={{ bench_dir }}/env/bin/python -m frappe.celery_app beat -s beat.schedule
-priority=3
-autostart=true
-autorestart=true
-stdout_logfile={{ bench_dir }}/logs/workerbeat.log
-stderr_logfile={{ bench_dir }}/logs/workerbeat.error.log
-user={{ user }}
-directory={{ sites_dir }}
-
-[program:{{ bench_name }}-frappe-worker]
-command={{ bench_dir }}/env/bin/python -m frappe.celery_app worker -n jobs@%%h -Ofair --soft-time-limit 360 --time-limit 390 --loglevel INFO
-priority=4
-autostart=true
-autorestart=true
-stdout_logfile={{ bench_dir }}/logs/worker.log
-stderr_logfile={{ bench_dir }}/logs/worker.error.log
-user={{ user }}
-stopwaitsecs=400
-directory={{ sites_dir }}
-killasgroup=true
-
-[program:{{ bench_name }}-frappe-longjob-worker]
-command={{ bench_dir }}/env/bin/python -m frappe.celery_app worker -n longjobs@%%h -Ofair --soft-time-limit 1500 --time-limit 1530 --loglevel INFO
-priority=2
-autostart=true
-autorestart=true
-stdout_logfile={{ bench_dir }}/logs/worker.log
-stderr_logfile={{ bench_dir }}/logs/worker.error.log
-user={{ user }}
-stopwaitsecs=1540
-directory={{ sites_dir }}
-killasgroup=true
-
-[program:{{ bench_name }}-frappe-async-worker]
-command={{ bench_dir }}/env/bin/python -m frappe.celery_app worker -n async@%%h -Ofair --soft-time-limit 1500 --time-limit 1530 --loglevel INFO
-priority=2
-autostart=true
-autorestart=true
-stdout_logfile={{ bench_dir }}/logs/worker.log
-stderr_logfile={{ bench_dir }}/logs/worker.error.log
-user={{ user }}
-stopwaitsecs=1540
-directory={{ sites_dir }}
-killasgroup=true
-
-{% endif %}
 
 {% if not skip_redis %}
 [program:{{ bench_name }}-redis-cache]
@@ -152,18 +105,6 @@ user={{ user }}
 directory={{ sites_dir }}
 {% endif %}
 
-{% if not skip_redis %}
-[program:{{ bench_name }}-redis-socketio]
-command={{ redis_server }} {{ redis_socketio_config }}
-priority=1
-autostart=true
-autorestart=true
-stdout_logfile={{ bench_dir }}/logs/redis-socketio.log
-stderr_logfile={{ bench_dir }}/logs/redis-socketio.error.log
-user={{ user }}
-directory={{ sites_dir }}
-{% endif %}
-
 {% if node %}
 [program:{{ bench_name }}-node-socketio]
 command={{ node }} {{ bench_dir }}/apps/frappe/socketio.js
@@ -179,19 +120,20 @@ directory={{ bench_dir }}
 [group:{{ bench_name }}-web]
 programs={{ bench_name }}-frappe-web {%- if node -%} ,{{ bench_name }}-node-socketio {%- endif%}
 
-{% if use_rq %}
+
+{% if multi_queue_consumption %}
 
 [group:{{ bench_name }}-workers]
-programs={{ bench_name }}-frappe-schedule,{{ bench_name }}-frappe-default-worker,{{ bench_name }}-frappe-short-worker,{{ bench_name }}-frappe-long-worker{%- for worker_name in workers -%},{{ bench_name }}-frappe-{{ worker_name }}-worker{%- endfor %}
+programs={{ bench_name }}-frappe-schedule,{{ bench_name }}-frappe-short-worker,{{ bench_name }}-frappe-long-worker{%- for worker_name in workers -%},{{ bench_name }}-frappe-{{ worker_name }}-worker{%- endfor %}
 
 {% else %}
 
 [group:{{ bench_name }}-workers]
-programs={{ bench_name }}-frappe-workerbeat,{{ bench_name }}-frappe-worker,{{ bench_name }}-frappe-longjob-worker,{{ bench_name }}-frappe-async-worker{%- for worker_name in workers -%},{{ bench_name }}-frappe-{{ worker_name }}-worker{%- endfor %}
+programs={{ bench_name }}-frappe-schedule,{{ bench_name }}-frappe-default-worker,{{ bench_name }}-frappe-short-worker,{{ bench_name }}-frappe-long-worker{%- for worker_name in workers -%},{{ bench_name }}-frappe-{{ worker_name }}-worker{%- endfor %}
 
 {% endif %}
 
 {% if not skip_redis %}
 [group:{{ bench_name }}-redis]
-programs={{ bench_name }}-redis-cache,{{ bench_name }}-redis-queue,{{ bench_name }}-redis-socketio
+programs={{ bench_name }}-redis-cache,{{ bench_name }}-redis-queue
 {% endif %}

bench/config/templates/systemd/frappe-bench-redis-socketio.service

@@ -1,12 +0,0 @@
-[Unit]
-Description="{{ bench_name }}-redis-socketio"
-PartOf={{ bench_name }}-redis.target
-
-[Service]
-User={{ user }}
-Group={{ user }}
-Restart=always
-ExecStart={{ redis_server }} {{ redis_socketio_config }}
-StandardOutput=file:{{ bench_dir }}/logs/redis-socketio.log
-StandardError=file:{{ bench_dir }}/logs/redis-socketio.error.log
-WorkingDirectory={{ sites_dir }}

bench/config/templates/systemd/frappe-bench-redis.target

@@ -1,6 +1,6 @@
 [Unit]
 After=network.target
-Wants={{ bench_name }}-redis-cache.service {{ bench_name }}-redis-queue.service {{ bench_name }}-redis-socketio.service
+Wants={{ bench_name }}-redis-cache.service {{ bench_name }}-redis-queue.service
 
 [Install]
 WantedBy=multi-user.target

bench/tests/test_base.py

@@ -67,7 +67,6 @@ class TestBenchBase(unittest.TestCase):
 	def assert_config(self, bench_name):
 		for config, search_key in (
 			("redis_queue.conf", "redis_queue.rdb"),
-			("redis_socketio.conf", "redis_socketio.rdb"),
 			("redis_cache.conf", "redis_cache.rdb"),
 		):
 

bench/tests/test_init.py

@@ -55,7 +55,7 @@ class TestBenchInit(TestBenchBase):
 				"socketio_port": 9000,
 				"file_watcher_port": 6787,
 				"redis_queue": "redis://localhost:11000",
-				"redis_socketio": "redis://localhost:12000",
+				"redis_socketio": "redis://localhost:13000",
 				"redis_cache": "redis://localhost:13000",
 			},
 		)
@@ -67,7 +67,7 @@ class TestBenchInit(TestBenchBase):
 				"socketio_port": 9001,
 				"file_watcher_port": 6788,
 				"redis_queue": "redis://localhost:11001",
-				"redis_socketio": "redis://localhost:12001",
+				"redis_socketio": "redis://localhost:13001",
 				"redis_cache": "redis://localhost:13001",
 			},
 		)

bench/tests/test_setup_production.py

@@ -103,7 +103,6 @@ class TestSetupProduction(TestBenchBase):
 				f"program:{bench_name}-frappe-web",
 				f"program:{bench_name}-redis-cache",
 				f"program:{bench_name}-redis-queue",
-				f"program:{bench_name}-redis-socketio",
 				f"group:{bench_name}-web",
 				f"group:{bench_name}-workers",
 				f"group:{bench_name}-redis",
@@ -150,7 +149,6 @@ class TestSetupProduction(TestBenchBase):
 			# "{bench_name}-web:{bench_name}-node-socketio[\s]+RUNNING",
 			r"{bench_name}-redis:{bench_name}-redis-cache[\s]+RUNNING",
 			r"{bench_name}-redis:{bench_name}-redis-queue[\s]+RUNNING",
-			r"{bench_name}-redis:{bench_name}-redis-socketio[\s]+RUNNING",
 		]
 
 		if use_rq: