fix: supervise descendant process groups

stelloauth/rootfs/usr/local/bin/addon-supervisor

@@ -172,6 +172,21 @@ def wait_until_ready(
     raise ReadinessError(f"{name} did not become ready")
 
 
+def process_group_alive(
+    pgid: int,
+    killpg: Callable[[int, int], None] = os.killpg,
+) -> bool:
+    try:
+        killpg(pgid, 0)
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        return True
+    except OSError:
+        return True
+    return True
+
+
 class ProcessManager:
     def __init__(
         self,
@@ -179,6 +194,7 @@ class ProcessManager:
         *,
         popen: Callable[..., subprocess.Popen] = subprocess.Popen,
         killpg: Callable[[int, int], None] = os.killpg,
+        group_alive: Callable[[int], bool] = process_group_alive,
         cloak_probe: Callable[[], None] = probe_cloak,
         stelloauth_probe: Callable[[], None] = probe_stelloauth,
         monotonic: Callable[[], float] = time.monotonic,
@@ -188,12 +204,15 @@ class ProcessManager:
         self.environment = environment
         self._popen = popen
         self._killpg = killpg
+        self._group_alive = group_alive
         self._cloak_probe = cloak_probe
         self._stelloauth_probe = stelloauth_probe
         self._monotonic = monotonic
         self._sleep = sleep
         self._profile_path = profile_path
         self._children: list[tuple[str, subprocess.Popen]] = []
+        self._groups: list[int] = []
+        self._dead_groups: set[int] = set()
         self._stopping = False
         self._term_sent = False
         self._shutdown_deadline: float | None = None
@@ -221,11 +240,9 @@ class ProcessManager:
             start_new_session=True,
         )
         self._children.append((name, process))
+        self._groups.append(process.pid)
         if self._stopping:
-            try:
-                self._killpg(process.pid, signal.SIGTERM)
-            except OSError:
-                pass
+            self._signal_groups(signal.SIGTERM, [process.pid])
         return process
 
     def _first_exited_child(self) -> tuple[str, int] | None:
@@ -238,12 +255,30 @@ class ProcessManager:
     def _startup_stopping(self) -> bool:
         return self._stopping or self._first_exited_child() is not None
 
-    def _signal_running(self, sent_signal: int) -> None:
-        for _name, process in self._children:
-            if process.poll() is not None:
+    def _live_groups(self) -> list[int]:
+        live_groups = []
+        for pgid in self._groups:
+            if pgid in self._dead_groups:
+                continue
+            if self._group_alive(pgid):
+                live_groups.append(pgid)
+            else:
+                self._dead_groups.add(pgid)
+        return live_groups
+
+    def _signal_groups(
+        self, sent_signal: int, groups: list[int] | None = None
+    ) -> None:
+        for pgid in self._live_groups() if groups is None else groups:
+            if pgid in self._dead_groups:
+                continue
+            if groups is not None and not self._group_alive(pgid):
+                self._dead_groups.add(pgid)
                 continue
             try:
-                self._killpg(process.pid, sent_signal)
+                self._killpg(pgid, sent_signal)
+            except ProcessLookupError:
+                self._dead_groups.add(pgid)
             except OSError:
                 pass
 
@@ -251,7 +286,7 @@ class ProcessManager:
         if self._shutdown_deadline is None:
             self._shutdown_deadline = self._monotonic() + 10.0
         if not self._term_sent:
-            self._signal_running(signal.SIGTERM)
+            self._signal_groups(signal.SIGTERM)
             self._term_sent = True
 
     def _reap_all(self) -> None:
@@ -267,16 +302,15 @@ class ProcessManager:
         logging.info("Stopping child processes")
         self._begin_shutdown()
         assert self._shutdown_deadline is not None
-        while (
-            any(process.poll() is None for _name, process in self._children)
-            and self._monotonic() < self._shutdown_deadline
-        ):
+        live_groups = self._live_groups()
+        while live_groups and self._monotonic() < self._shutdown_deadline:
             remaining = self._shutdown_deadline - self._monotonic()
             self._sleep(min(0.25, max(0.0, remaining)))
+            live_groups = self._live_groups()
 
-        if any(process.poll() is None for _name, process in self._children):
+        if live_groups:
             logging.info("Forcing child processes to stop")
-            self._signal_running(signal.SIGKILL)
+            self._signal_groups(signal.SIGKILL, live_groups)
         self._reap_all()
         logging.info("Child processes stopped")