refactor!: Reuse redis_cache instace for socketio (#1382)

WARNING: Just a POC. You're free to modify your setup to test this out.

Summary:
- After this change bench wont generate config for redis_socketio
  instance.
- Instead we set the same port as redis_cache service for use in
  socketio. i.e. both will share same instance.

Code changes:

- Config will have this difference:

```diff
- "redis_socketio": "redis://localhost:12000",
- "redis_cache": "redis://localhost:13000",
+ "redis_socketio": "redis://localhost:13000",
+ "redis_cache": "redis://localhost:13000",
```

- supervisord/systemd wont start redis_socketio service
- for development setups: redis_socketio service is removed from procfile.

Why?

- Currently this redis instance is used only to ship messages from python
to nodejs server... which is largely stateless (only pub/sub channels)
- This change reduces 1 running process. Less resources for this ~= more
  resources for other running things.

TODO:
- [ ] decicisions - should we even do this? this should be optinal?
- [ ] refactor progressbar in frappe
- [ ] Tests - manual, FC, docker, automated
- [ ] "migration" plan

All dependent deployment projects will likely have to refactor their code to drop this service, shouldn't be too much work.
Example: Frappe cloud, Frappe Docker.

bench/config/common_site_config.py

@@ -100,7 +100,7 @@ def make_ports(bench_path):
 		"socketio_port": 9000,
 		"file_watcher_port": 6787,
 		"redis_queue": 11000,
-		"redis_socketio": 12000,
+		"redis_socketio": 13000,
 		"redis_cache": 13000,
 	}
 

bench/config/redis.py

@@ -15,7 +15,7 @@ def generate_config(bench_path):
 	redis_version = get_redis_version()
 
 	ports = {}
-	for key in ("redis_cache", "redis_queue", "redis_socketio"):
+	for key in ("redis_cache", "redis_queue"):
 		ports[key] = urlparse(config[key]).port
 
 	write_redis_config(
@@ -28,12 +28,6 @@ def generate_config(bench_path):
 		bench_path=bench_path,
 	)
 
-	write_redis_config(
-		template_name="redis_socketio.conf",
-		context={"port": ports["redis_socketio"], "redis_version": redis_version},
-		bench_path=bench_path,
-	)
-
 	write_redis_config(
 		template_name="redis_cache.conf",
 		context={
@@ -56,10 +50,8 @@ def generate_config(bench_path):
 	# make ACL files
 	acl_rq_path = os.path.join(bench_path, "config", "redis_queue.acl")
 	acl_redis_cache_path = os.path.join(bench_path, "config", "redis_cache.acl")
-	acl_redis_socketio_path = os.path.join(bench_path, "config", "redis_socketio.acl")
 	open(acl_rq_path, "a").close()
 	open(acl_redis_cache_path, "a").close()
-	open(acl_redis_socketio_path, "a").close()
 
 
 def write_redis_config(template_name, context, bench_path):

bench/config/supervisor.py

@@ -48,7 +48,6 @@ def generate_supervisor_config(bench_path, user=None, yes=False, skip_redis=Fals
 			"redis_server": which("redis-server"),
 			"node": which("node") or which("nodejs"),
 			"redis_cache_config": os.path.join(bench_dir, "config", "redis_cache.conf"),
-			"redis_socketio_config": os.path.join(bench_dir, "config", "redis_socketio.conf"),
 			"redis_queue_config": os.path.join(bench_dir, "config", "redis_queue.conf"),
 			"webserver_port": config.get("webserver_port", 8000),
 			"gunicorn_workers": web_worker_count,

bench/config/systemd.py

@@ -82,7 +82,6 @@ def generate_systemd_config(
 		"redis_server": which("redis-server"),
 		"node": which("node") or which("nodejs"),
 		"redis_cache_config": os.path.join(bench_dir, "config", "redis_cache.conf"),
-		"redis_socketio_config": os.path.join(bench_dir, "config", "redis_socketio.conf"),
 		"redis_queue_config": os.path.join(bench_dir, "config", "redis_queue.conf"),
 		"webserver_port": config.get("webserver_port", 8000),
 		"gunicorn_workers": web_worker_count,
@@ -244,14 +243,10 @@ def setup_redis_config(bench_info, bench_path):
 	bench_redis_queue_template = bench.config.env().get_template(
 		"systemd/frappe-bench-redis-queue.service"
 	)
-	bench_redis_socketio_template = bench.config.env().get_template(
-		"systemd/frappe-bench-redis-socketio.service"
-	)
 
 	bench_redis_target_config = bench_redis_target_template.render(**bench_info)
 	bench_redis_cache_config = bench_redis_cache_template.render(**bench_info)
 	bench_redis_queue_config = bench_redis_queue_template.render(**bench_info)
-	bench_redis_socketio_config = bench_redis_socketio_template.render(**bench_info)
 
 	bench_redis_target_config_path = os.path.join(
 		bench_path, "config", "systemd", bench_info.get("bench_name") + "-redis.target"
@@ -262,12 +257,6 @@ def setup_redis_config(bench_info, bench_path):
 	bench_redis_queue_config_path = os.path.join(
 		bench_path, "config", "systemd", bench_info.get("bench_name") + "-redis-queue.service"
 	)
-	bench_redis_socketio_config_path = os.path.join(
-		bench_path,
-		"config",
-		"systemd",
-		bench_info.get("bench_name") + "-redis-socketio.service",
-	)
 
 	with open(bench_redis_target_config_path, "w") as f:
 		f.write(bench_redis_target_config)
@@ -278,9 +267,6 @@ def setup_redis_config(bench_info, bench_path):
 	with open(bench_redis_queue_config_path, "w") as f:
 		f.write(bench_redis_queue_config)
 
-	with open(bench_redis_socketio_config_path, "w") as f:
-		f.write(bench_redis_socketio_config)
-
 
 def _create_symlinks(bench_path):
 	bench_dir = os.path.abspath(bench_path)
@@ -319,6 +305,5 @@ def get_unit_files(bench_path):
 		[bench_name + "-node-socketio", ".service"],
 		[bench_name + "-redis-cache", ".service"],
 		[bench_name + "-redis-queue", ".service"],
-		[bench_name + "-redis-socketio", ".service"],
 	]
 	return unit_files

bench/config/templates/Procfile

@@ -1,6 +1,5 @@
 {% if not skip_redis %}
 redis_cache: redis-server config/redis_cache.conf
-redis_socketio: redis-server config/redis_socketio.conf
 redis_queue: redis-server config/redis_queue.conf
 {% endif %}
 web: bench serve {% if webserver_port -%} --port {{ webserver_port }} {%- endif %}

bench/config/templates/redis_socketio.conf

@@ -1,8 +0,0 @@
-dbfilename redis_socketio.rdb
-dir {{ pid_path }}
-pidfile {{ pid_path }}/redis_socketio.pid
-bind 127.0.0.1
-port {{ port }}
-{% if redis_version and redis_version >= 6.0 %}
-aclfile {{ config_path }}/redis_socketio.acl
-{% endif %}

bench/config/templates/supervisor.conf

@@ -152,18 +152,6 @@ user={{ user }}
 directory={{ sites_dir }}
 {% endif %}
 
-{% if not skip_redis %}
-[program:{{ bench_name }}-redis-socketio]
-command={{ redis_server }} {{ redis_socketio_config }}
-priority=1
-autostart=true
-autorestart=true
-stdout_logfile={{ bench_dir }}/logs/redis-socketio.log
-stderr_logfile={{ bench_dir }}/logs/redis-socketio.error.log
-user={{ user }}
-directory={{ sites_dir }}
-{% endif %}
-
 {% if node %}
 [program:{{ bench_name }}-node-socketio]
 command={{ node }} {{ bench_dir }}/apps/frappe/socketio.js
@@ -193,5 +181,5 @@ programs={{ bench_name }}-frappe-workerbeat,{{ bench_name }}-frappe-worker,{{ be
 
 {% if not skip_redis %}
 [group:{{ bench_name }}-redis]
-programs={{ bench_name }}-redis-cache,{{ bench_name }}-redis-queue,{{ bench_name }}-redis-socketio
+programs={{ bench_name }}-redis-cache,{{ bench_name }}-redis-queue
 {% endif %}

bench/config/templates/systemd/frappe-bench-redis-socketio.service

@@ -1,12 +0,0 @@
-[Unit]
-Description="{{ bench_name }}-redis-socketio"
-PartOf={{ bench_name }}-redis.target
-
-[Service]
-User={{ user }}
-Group={{ user }}
-Restart=always
-ExecStart={{ redis_server }} {{ redis_socketio_config }}
-StandardOutput=file:{{ bench_dir }}/logs/redis-socketio.log
-StandardError=file:{{ bench_dir }}/logs/redis-socketio.error.log
-WorkingDirectory={{ sites_dir }}

bench/config/templates/systemd/frappe-bench-redis.target

@@ -1,6 +1,6 @@
 [Unit]
 After=network.target
-Wants={{ bench_name }}-redis-cache.service {{ bench_name }}-redis-queue.service {{ bench_name }}-redis-socketio.service
+Wants={{ bench_name }}-redis-cache.service {{ bench_name }}-redis-queue.service
 
 [Install]
 WantedBy=multi-user.target

bench/tests/test_base.py

@@ -67,7 +67,6 @@ class TestBenchBase(unittest.TestCase):
 	def assert_config(self, bench_name):
 		for config, search_key in (
 			("redis_queue.conf", "redis_queue.rdb"),
-			("redis_socketio.conf", "redis_socketio.rdb"),
 			("redis_cache.conf", "redis_cache.rdb"),
 		):
 

bench/tests/test_init.py

@@ -55,7 +55,7 @@ class TestBenchInit(TestBenchBase):
 				"socketio_port": 9000,
 				"file_watcher_port": 6787,
 				"redis_queue": "redis://localhost:11000",
-				"redis_socketio": "redis://localhost:12000",
+				"redis_socketio": "redis://localhost:13000",
 				"redis_cache": "redis://localhost:13000",
 			},
 		)
@@ -67,7 +67,7 @@ class TestBenchInit(TestBenchBase):
 				"socketio_port": 9001,
 				"file_watcher_port": 6788,
 				"redis_queue": "redis://localhost:11001",
-				"redis_socketio": "redis://localhost:12001",
+				"redis_socketio": "redis://localhost:13001",
 				"redis_cache": "redis://localhost:13001",
 			},
 		)

bench/tests/test_setup_production.py

@@ -103,7 +103,6 @@ class TestSetupProduction(TestBenchBase):
 				f"program:{bench_name}-frappe-web",
 				f"program:{bench_name}-redis-cache",
 				f"program:{bench_name}-redis-queue",
-				f"program:{bench_name}-redis-socketio",
 				f"group:{bench_name}-web",
 				f"group:{bench_name}-workers",
 				f"group:{bench_name}-redis",
@@ -150,7 +149,6 @@ class TestSetupProduction(TestBenchBase):
 			# "{bench_name}-web:{bench_name}-node-socketio[\s]+RUNNING",
 			r"{bench_name}-redis:{bench_name}-redis-cache[\s]+RUNNING",
 			r"{bench_name}-redis:{bench_name}-redis-queue[\s]+RUNNING",
-			r"{bench_name}-redis:{bench_name}-redis-socketio[\s]+RUNNING",
 		]
 
 		if use_rq: