fix: ignore supervisor restart failures where possible (#1400)

* fix: ignore supervisor restart failures where possible

Ignores proc manager restart failure during:
- App install
- App uninstall

Reason:
- You might not have setup prod yet
- This is useful during docker image building where proc manager wont be
  running **yet**.

* ci: dont fail-fast

matrix has no dependency.

.github/workflows/ci.yml

@@ -15,6 +15,7 @@ jobs:
     timeout-minutes: 60
 
     strategy:
+      fail-fast: false
       matrix:
         python-version: [ '3.7', '3.8', '3.9', '3.10' ]
 
@@ -89,6 +90,7 @@ jobs:
     timeout-minutes: 60
 
     strategy:
+      fail-fast: false
       matrix:
         python-version: [ '3.7', '3.10' ]
 

bench/app.py

@@ -586,7 +586,9 @@ def install_app(
 		build_assets(bench_path=bench_path, app=app)
 
 	if restart_bench:
-		bench.reload()
+		# Avoiding exceptions here as production might not be set-up
+		# OR we might just be generating docker images.
+		bench.reload(_raise=False)
 
 
 def pull_apps(apps=None, bench_path=".", reset=False):

bench/bench.py

@@ -44,8 +44,8 @@ logger = logging.getLogger(bench.PROJECT_NAME)
 
 
 class Base:
-	def run(self, cmd, cwd=None):
-		return exec_cmd(cmd, cwd=cwd or self.cwd)
+	def run(self, cmd, cwd=None, _raise=True):
+		return exec_cmd(cmd, cwd=cwd or self.cwd, _raise=_raise)
 
 
 class Validator:
@@ -133,7 +133,7 @@ class Bench(Base, Validator):
 				raise
 		self.apps.sync()
 		# self.build() - removed because it seems unnecessary
-		self.reload()
+		self.reload(_raise=False)
 
 	@step(title="Building Bench Assets", success="Bench Assets Built")
 	def build(self):
@@ -141,16 +141,16 @@ class Bench(Base, Validator):
 		run_frappe_cmd("build", bench_path=self.name)
 
 	@step(title="Reloading Bench Processes", success="Bench Processes Reloaded")
-	def reload(self, web=False, supervisor=True, systemd=True):
+	def reload(self, web=False, supervisor=True, systemd=True, _raise=True):
 		"""If web is True, only web workers are restarted"""
 		conf = self.conf
 
 		if conf.get("developer_mode"):
 			restart_process_manager(bench_path=self.name, web_workers=web)
 		if supervisor or conf.get("restart_supervisor_on_update"):
-			restart_supervisor_processes(bench_path=self.name, web_workers=web)
+			restart_supervisor_processes(bench_path=self.name, web_workers=web, _raise=_raise)
 		if systemd and conf.get("restart_systemd_on_update"):
-			restart_systemd_processes(bench_path=self.name, web_workers=web)
+			restart_systemd_processes(bench_path=self.name, web_workers=web, _raise=_raise)
 
 	def get_installed_apps(self) -> List:
 		"""Returns list of installed apps on bench, not in excluded_apps.txt"""

bench/utils/bench.py

@@ -276,7 +276,7 @@ def patch_sites(bench_path="."):
 			raise PatchError
 
 
-def restart_supervisor_processes(bench_path=".", web_workers=False):
+def restart_supervisor_processes(bench_path=".", web_workers=False, _raise=False):
 	from bench.bench import Bench
 
 	bench = Bench(bench_path)
@@ -285,7 +285,7 @@ def restart_supervisor_processes(bench_path=".", web_workers=False):
 	bench_name = get_bench_name(bench_path)
 
 	if cmd:
-		bench.run(cmd)
+		bench.run(cmd, _raise=_raise)
 
 	else:
 		sudo = ""
@@ -312,18 +312,22 @@ def restart_supervisor_processes(bench_path=".", web_workers=False):
 		else:
 			group = "frappe:"
 
-		bench.run(f"{sudo}supervisorctl restart {group}")
+		failure = bench.run(f"{sudo}supervisorctl restart {group}", _raise=_raise)
+		if failure:
+			log("restarting supervisor failed. Use `bench restart` to retry.", level=3)
 
 
-def restart_systemd_processes(bench_path=".", web_workers=False):
+def restart_systemd_processes(bench_path=".", web_workers=False, _raise=True):
 	bench_name = get_bench_name(bench_path)
 	exec_cmd(
 		f"sudo systemctl stop -- $(systemctl show -p Requires {bench_name}.target | cut"
-		" -d= -f2)"
+		" -d= -f2)",
+		_raise=_raise,
 	)
 	exec_cmd(
 		f"sudo systemctl start -- $(systemctl show -p Requires {bench_name}.target |"
-		" cut -d= -f2)"
+		" cut -d= -f2)",
+		_raise=_raise,
 	)