Check processes and restart if dead.

server.py

@@ -15,7 +15,7 @@
 import multiprocessing
 from multiprocessing.queues import Queue
 import time
-from typing import Any
+from typing import Any, Optional
 import json
 from setproctitle import setproctitle
 
@@ -68,21 +68,70 @@ class BAPSicleServer:
     api_to_q: Queue
 
     player: List[multiprocessing.Process] = []
-    websockets_server: multiprocessing.Process
+    websockets_server: Optional[multiprocessing.Process] = None
-    controller_handler: multiprocessing.Process
+    controller_handler: Optional[multiprocessing.Process] = None
-    player_handler: multiprocessing.Process
+    player_handler: Optional[multiprocessing.Process] = None
-    webserver: multiprocessing.Process
+    webserver: Optional[multiprocessing.Process] = None
 
     def __init__(self):
 
         self.startServer()
 
-        terminator = Terminator()
+        self.check_processes()
-        while not terminator.terminate:
-            time.sleep(1)
 
         self.stopServer()
 
+    def check_processes(self):
+
+        terminator = Terminator()
+        log_function = self.logger.log.info
+        while not terminator.terminate:
+
+            # Note, state here will become a copy in the process.
+            # callbacks will not passthrough :/
+
+            for channel in range(self.state.state["num_channels"]):
+                if not self.player[channel] or not self.player[channel].is_alive():
+                    log_function("Player {} not running, (re)starting.".format(channel))
+                    self.player[channel] = multiprocessing.Process(
+                        target=player.Player,
+                        args=(channel, self.player_to_q[channel], self.player_from_q[channel], self.state)
+                    )
+                    self.player[channel].start()
+
+            if not self.player_handler or not self.player_handler.is_alive():
+                log_function("Player Handler not running, (re)starting.")
+                self.player_handler = multiprocessing.Process(
+                    target=PlayerHandler,
+                    args=(self.player_from_q, self.websocket_to_q, self.ui_to_q, self.controller_to_q),
+                )
+                self.player_handler.start()
+
+            if not self.websockets_server or not self.websockets_server.is_alive():
+                log_function("Websocket Server not running, (re)starting.")
+                self.websockets_server = multiprocessing.Process(
+                    target=WebsocketServer, args=(self.player_to_q, self.websocket_to_q, self.state)
+                )
+                self.websockets_server.start()
+
+            if not self.webserver or not self.webserver.is_alive():
+                log_function("Webserver not running, (re)starting.")
+                self.webserver = multiprocessing.Process(
+                    target=WebServer, args=(self.player_to_q, self.ui_to_q, self.state)
+                )
+                self.webserver.start()
+
+            if not self.controller_handler or not self.controller_handler.is_alive():
+                log_function("Controller Handler not running, (re)starting.")
+                self.controller_handler = multiprocessing.Process(
+                    target=MattchBox, args=(self.player_to_q, self.controller_to_q, self.state)
+                )
+                self.controller_handler.start()
+
+            # After first starting processes, switch logger to error, since any future starts will have been failures.
+            log_function = self.logger.log.error
+            time.sleep(1)
+
     def startServer(self):
         if isMacOS():
             multiprocessing.set_start_method("spawn", True)
@@ -105,6 +154,9 @@ class BAPSicleServer:
         self.state.update("server_version", config.VERSION)
         self.state.update("server_build", build_commit)
 
+        channel_count = self.state.state["num_channels"]
+        self.player = [None] * channel_count
+
         for channel in range(self.state.state["num_channels"]):
 
             self.player_to_q.append(multiprocessing.Queue())
@@ -113,39 +165,6 @@ class BAPSicleServer:
             self.websocket_to_q.append(multiprocessing.Queue())
             self.controller_to_q.append(multiprocessing.Queue())
 
-            # TODO Replace state with individual read-only StateManagers or something nicer?
-
-            self.player.append(
-                multiprocessing.Process(
-                    target=player.Player,
-                    args=(channel, self.player_to_q[-1], self.player_from_q[-1], self.state)
-                )
-            )
-            self.player[channel].start()
-
-        self.player_handler = multiprocessing.Process(
-            target=PlayerHandler,
-            args=(self.player_from_q, self.websocket_to_q, self.ui_to_q, self.controller_to_q),
-        )
-        self.player_handler.start()
-
-        # Note, state here will become a copy in the process.
-        # It will not update, and callbacks will not work :/
-        self.websockets_server = multiprocessing.Process(
-            target=WebsocketServer, args=(self.player_to_q, self.websocket_to_q, self.state)
-        )
-        self.websockets_server.start()
-
-        self.controller_handler = multiprocessing.Process(
-            target=MattchBox, args=(self.player_to_q, self.controller_to_q, self.state)
-        )
-        self.controller_handler.start()
-
-        self.webserver = multiprocessing.Process(
-            target=WebServer, args=(self.player_to_q, self.ui_to_q, self.state)
-        )
-        self.webserver.start()
-
         print("Welcome to BAPSicle Server version: {}, build: {}.".format(config.VERSION, build_commit))
         print("The Server UI is available at http://{}:{}".format(self.state.state["host"], self.state.state["port"]))
 

web_server.py

@@ -19,7 +19,7 @@ from time import sleep
 import json
 import os
 
-from helpers.os_environment import isBundelled, isMacOS, resolve_local_file_path
+from helpers.os_environment import resolve_local_file_path
 from helpers.logging_manager import LoggingManager
 from helpers.device_manager import DeviceManager
 from helpers.state_manager import StateManager