Improve handling of dead processes

This commit is contained in:
Matthew Stratford 2021-09-04 15:16:44 +01:00
parent cade8ecce8
commit 8a66ad86e6
2 changed files with 11 additions and 6 deletions

View file

@ -14,3 +14,4 @@ pyserial==3.5
requests==2.26.0
Jinja2==3.0.1
pydub==0.25.1
psutil

View file

@ -20,6 +20,8 @@ import time
from typing import Any, Optional
import json
from setproctitle import setproctitle
import psutil
from helpers.os_environment import isBundelled, isMacOS
if not isMacOS():
@ -111,7 +113,9 @@ class BAPSicleServer:
while not terminator.terminate and self.state.get()["running_state"] == "running":
for channel in range(self.state.get()["num_channels"]):
if not self.player[channel] or not self.player[channel].is_alive():
# Use pid_exists to confirm process is actually still running. Python may not report is_alive() correctly (especially over system sleeps etc.)
# https://medium.com/pipedrive-engineering/encountering-some-python-trickery-683bd5f66750
if not self.player[channel] or not self.player[channel].is_alive() or not psutil.pid_exists(self.player[channel].pid):
log_function("Player {} not running, (re)starting.".format(channel))
self.player[channel] = multiprocessing.Process(
target=player.Player,
@ -119,7 +123,7 @@ class BAPSicleServer:
)
self.player[channel].start()
if not self.player_handler or not self.player_handler.is_alive():
if not self.player_handler or not self.player_handler.is_alive() or not psutil.pid_exists(self.player_handler.pid):
log_function("Player Handler not running, (re)starting.")
self.player_handler = multiprocessing.Process(
target=PlayerHandler,
@ -127,7 +131,7 @@ class BAPSicleServer:
)
self.player_handler.start()
if not self.file_manager or not self.file_manager.is_alive():
if not self.file_manager or not self.file_manager.is_alive() or not psutil.pid_exists(self.file_manager.pid):
log_function("File Manager not running, (re)starting.")
self.file_manager = multiprocessing.Process(
target=FileManager,
@ -135,21 +139,21 @@ class BAPSicleServer:
)
self.file_manager.start()
if not self.websockets_server or not self.websockets_server.is_alive():
if not self.websockets_server or not self.websockets_server.is_alive() or not psutil.pid_exists(self.websockets_server.pid):
log_function("Websocket Server not running, (re)starting.")
self.websockets_server = multiprocessing.Process(
target=WebsocketServer, args=(self.player_to_q, self.websocket_to_q, self.state)
)
self.websockets_server.start()
if not self.webserver or not self.webserver.is_alive():
if not self.webserver or not self.webserver.is_alive() or not psutil.pid_exists(self.webserver.pid):
log_function("Webserver not running, (re)starting.")
self.webserver = multiprocessing.Process(
target=WebServer, args=(self.player_to_q, self.ui_to_q, self.state)
)
self.webserver.start()
if not self.controller_handler or not self.controller_handler.is_alive():
if not self.controller_handler or not self.controller_handler.is_alive() or not psutil.pid_exists(self.controller_handler.pid):
log_function("Controller Handler not running, (re)starting.")
self.controller_handler = multiprocessing.Process(
target=MattchBox, args=(self.player_to_q, self.controller_to_q, self.state)