This commit is contained in:
Matthew Stratford 2021-09-22 19:49:24 +01:00
parent b8c6f087c6
commit a172d03f0e
7 changed files with 237 additions and 221 deletions

View file

@ -2,17 +2,19 @@ from helpers.alert_manager import AlertProvider
from package import BETA
from baps_types.alert import WARNING, Alert
# Dummy alert provider for testing basics like UI without needing to actually cause errors.
class DummyAlertProvider(AlertProvider):
def get_alerts(self):
if BETA:
return [Alert(
{
"start_time": -1,
"id": "test",
"title": "BAPSicle is in Debug Mode",
"description": "This is a test alert. It will not appear on production builds.",
"module": "Test",
"severity": WARNING
}
)]
def get_alerts(self):
if BETA:
return [Alert(
{
"start_time": -1,
"id": "test",
"title": "BAPSicle is in Debug Mode",
"description": "This is a test alert. It will not appear on production builds.",
"module": "Test",
"severity": WARNING
}
)]

View file

@ -7,88 +7,90 @@ from helpers.alert_manager import AlertProvider
from baps_types.alert import CRITICAL, WARNING, Alert
from baps_types.happytime import happytime
MODULE = "Player" # This should match the log file, so the UI will link to the logs page.
MODULE = "Player" # This should match the log file, so the UI will link to the logs page.
class PlayerAlertProvider(AlertProvider):
_server_state: Dict[str, Any]
_states: List[Optional[Dict[str,Any]]] = []
_player_count: int
_server_state: Dict[str, Any]
_states: List[Optional[Dict[str, Any]]] = []
_player_count: int
def __init__(self):
# Player count only changes after server restart, may as well just load this once.
with open(resolve_external_file_path("state/BAPSicleServer.json")) as file:
self._server_state = json.loads(file.read())
def __init__(self):
# Player count only changes after server restart, may as well just load this once.
with open(resolve_external_file_path("state/BAPSicleServer.json")) as file:
self._server_state = json.loads(file.read())
self._player_count = int(self._server_state["num_channels"])
self._states = [None] * self._player_count
self._player_count = int(self._server_state["num_channels"])
self._states = [None] * self._player_count
# To simplify monitoring (and allow detection of things going super weird), we are going to read from the state file to work out the alerts.
def get_alerts(self):
for channel in range(self._player_count):
with open(resolve_external_file_path("state/Player{}.json".format(channel))) as file:
self._states[channel] = json.loads(file.read())
# To simplify monitoring (and allow detection of things going super
# weird), we are going to read from the state file to work out the alerts.
def get_alerts(self):
for channel in range(self._player_count):
with open(resolve_external_file_path("state/Player{}.json".format(channel))) as file:
self._states[channel] = json.loads(file.read())
funcs = [self._channel_count, self._initialised, self._start_time]
funcs = [self._channel_count, self._initialised, self._start_time]
alerts: List[Alert] = []
alerts: List[Alert] = []
for func in funcs:
func_alerts = func()
if func_alerts:
alerts.extend(func_alerts)
for func in funcs:
func_alerts = func()
if func_alerts:
alerts.extend(func_alerts)
return alerts
return alerts
def _channel_count(self):
if self._player_count <= 0:
return [Alert({
"start_time": -1, # Now
"id": "no_channels",
"title": "There are no players configured.",
"description": "The number of channels configured is {}. Please set to at least 1 on the 'Server Config' page.".format(self._player_count),
"module": MODULE+"Handler",
"severity": CRITICAL
})]
def _channel_count(self):
if self._player_count <= 0:
return [Alert({
"start_time": -1, # Now
"id": "no_channels",
"title": "There are no players configured.",
"description": "The number of channels configured is {}. Please set to at least 1 on the 'Server Config' page.".format(self._player_count),
"module": MODULE+"Handler",
"severity": CRITICAL
})]
def _initialised(self):
alerts: List[Alert] = []
for channel in range(self._player_count):
if self._states[channel] and not self._states[channel]["initialised"]:
alerts.append(Alert({
"start_time": -1, # Now
"id": "player_{}_not_initialised".format(channel),
"title": "Player {} is not initialised.".format(channel),
"description": "This typically means the player channel was not able find the configured sound output on the system. Please check the 'Player Config' and Player logs to determine the cause.",
"module": MODULE+str(channel),
"severity": CRITICAL
}))
return alerts
def _initialised(self):
alerts: List[Alert] = []
for channel in range(self._player_count):
if self._states[channel] and not self._states[channel]["initialised"]:
alerts.append(Alert({
"start_time": -1, # Now
"id": "player_{}_not_initialised".format(channel),
"title": "Player {} is not initialised.".format(channel),
"description": "This typically means the player channel was not able find the configured sound output on the system. Please check the 'Player Config' and Player logs to determine the cause.",
"module": MODULE+str(channel),
"severity": CRITICAL
}))
return alerts
def _start_time(self):
server_start_time = self._server_state["start_time"]
server_start_time = datetime.fromtimestamp(server_start_time)
delta = timedelta(
seconds=30,
)
def _start_time(self):
server_start_time = self._server_state["start_time"]
server_start_time = datetime.fromtimestamp(server_start_time)
delta = timedelta(
seconds=30,
)
alerts: List[Alert] = []
for channel in range(self._player_count):
start_time = self._states[channel]["start_time"]
start_time = datetime.fromtimestamp(start_time)
if (start_time > server_start_time + delta):
alerts.append(Alert({
"start_time": -1,
"id": "player_{}_restarted".format(channel),
"title": "Player {} restarted after the server started.".format(channel),
"description":
"""Player {} last restarted at {}, after the server first started at {}, suggesting a failure.
alerts: List[Alert] = []
for channel in range(self._player_count):
start_time = self._states[channel]["start_time"]
start_time = datetime.fromtimestamp(start_time)
if (start_time > server_start_time + delta):
alerts.append(Alert({
"start_time": -1,
"id": "player_{}_restarted".format(channel),
"title": "Player {} restarted after the server started.".format(channel),
"description":
"""Player {} last restarted at {}, after the server first started at {}, suggesting a failure.
This likely means there was an unhandled exception in the player code, causing the server to restart the player.
Please check player logs to investigate the cause. Please restart the server to clear this warning."""
.format(channel, happytime(start_time), happytime(server_start_time)),
"module": MODULE+str(channel),
"severity": WARNING
}))
return alerts
.format(channel, happytime(start_time), happytime(server_start_time)),
"module": MODULE+str(channel),
"severity": WARNING
}))
return alerts

View file

@ -7,67 +7,69 @@ from helpers.alert_manager import AlertProvider
from baps_types.alert import CRITICAL, WARNING, Alert
from baps_types.happytime import happytime
MODULE = "BAPSicleServer" # This should match the log file, so the UI will link to the logs page.
MODULE = "BAPSicleServer" # This should match the log file, so the UI will link to the logs page.
class ServerAlertProvider(AlertProvider):
_state: Dict[str, Any]
# To simplify monitoring (and allow detection of things going super weird), we are going to read from the state file to work out the alerts.
def get_alerts(self):
with open(resolve_external_file_path("state/BAPSicleServer.json")) as file:
self._state = json.loads(file.read())
_state: Dict[str, Any]
# To simplify monitoring (and allow detection of things going super
# weird), we are going to read from the state file to work out the alerts.
def get_alerts(self):
with open(resolve_external_file_path("state/BAPSicleServer.json")) as file:
self._state = json.loads(file.read())
funcs = [self._api_key, self._start_time]
funcs = [self._api_key, self._start_time]
alerts: List[Alert] = []
alerts: List[Alert] = []
for func in funcs:
func_alerts = func()
if func_alerts:
alerts.extend(func_alerts)
for func in funcs:
func_alerts = func()
if func_alerts:
alerts.extend(func_alerts)
return alerts
return alerts
def _api_key(self):
if not self._state["myradio_api_key"]:
return [Alert({
"start_time": -1, # Now
"id": "api_key_missing",
"title": "MyRadio API Key is not configured.",
"description": "This means you will be unable to load show plans, audio items, or tracklist. Please set one on the 'Server Config' page.",
"module": MODULE,
"severity": CRITICAL
})]
def _api_key(self):
if not self._state["myradio_api_key"]:
return [Alert({
"start_time": -1, # Now
"id": "api_key_missing",
"title": "MyRadio API Key is not configured.",
"description": "This means you will be unable to load show plans, audio items, or tracklist. Please set one on the 'Server Config' page.",
"module": MODULE,
"severity": CRITICAL
})]
if len(self._state["myradio_api_key"]) < 10:
return [Alert({
"start_time": -1,
"id": "api_key_missing",
"title": "MyRadio API Key seems incorrect.",
"description": "The API key is less than 10 characters, it's probably not a valid one. If it is valid, it shouldn't be.",
"module": MODULE,
"severity": WARNING
})]
if len(self._state["myradio_api_key"]) < 10:
return [Alert({
"start_time": -1,
"id": "api_key_missing",
"title": "MyRadio API Key seems incorrect.",
"description": "The API key is less than 10 characters, it's probably not a valid one. If it is valid, it shouldn't be.",
"module": MODULE,
"severity": WARNING
})]
def _start_time(self):
start_time = self._state["start_time"]
start_time = datetime.fromtimestamp(start_time)
delta = timedelta(
days=1,
)
if (start_time + delta > datetime.now()):
return [Alert({
"start_time": -1,
"id": "server_restarted",
"title": "BAPSicle restarted recently.",
"description":
"""The BAPSicle server restarted at {}, less than a day ago.
def _start_time(self):
start_time = self._state["start_time"]
start_time = datetime.fromtimestamp(start_time)
delta = timedelta(
days=1,
)
if (start_time + delta > datetime.now()):
return [Alert({
"start_time": -1,
"id": "server_restarted",
"title": "BAPSicle restarted recently.",
"description":
"""The BAPSicle server restarted at {}, less than a day ago.
It may have been automatically restarted by the OS.
If this is not expected, please check logs to investigate why BAPSicle restarted/crashed."""
.format(happytime(start_time)),
"module": MODULE,
"severity": WARNING
})]
.format(happytime(start_time)),
"module": MODULE,
"severity": WARNING
})]

View file

@ -4,6 +4,7 @@ from datetime import datetime
CRITICAL = "Critical"
WARNING = "Warning"
class Alert:
start_time: datetime
last_time: datetime
@ -14,56 +15,54 @@ class Alert:
module: str
severity: str
@property
def ui_class(self) -> str:
if self.severity == CRITICAL:
return "danger"
if self.severity == WARNING:
return "warning"
return "info"
if self.severity == CRITICAL:
return "danger"
if self.severity == WARNING:
return "warning"
return "info"
# This alert has happened again.
def reoccured(self):
self.last_time = datetime.now()
self.end_time = None
self.last_time = datetime.now()
self.end_time = None
# This alert has finished, just update end time and keep last_time.
def cleared(self):
self.end_time = datetime.now()
self.end_time = datetime.now()
@property
def __dict__(self):
attrs = ["start_time", "last_time", "end_time", "id", "title", "description", "module", "severity"]
out = {}
for attr in attrs:
out[attr] = self.__getattribute__(attr)
out[attr] = self.__getattribute__(attr)
return out
def __init__(self, new_data: Dict[str,Any]):
required_vars = [
"start_time", # Just in case an alert wants to show starting earlier than it is reported.
"id",
"title",
"description",
"module",
"severity"
]
def __init__(self, new_data: Dict[str, Any]):
required_vars = [
"start_time", # Just in case an alert wants to show starting earlier than it is reported.
"id",
"title",
"description",
"module",
"severity"
]
for key in required_vars:
if key not in new_data.keys():
raise KeyError("Key {} is missing from data to create Alert.".format(key))
for key in required_vars:
if key not in new_data.keys():
raise KeyError("Key {} is missing from data to create Alert.".format(key))
#if type(new_data[key]) != type(getattr(self,key)):
# raise TypeError("Key {} has type {}, was expecting {}.".format(key, type(new_data[key]), type(getattr(self,key))))
# if type(new_data[key]) != type(getattr(self,key)):
# raise TypeError("Key {} has type {}, was expecting {}.".format(key, type(new_data[key]), type(getattr(self,key))))
# Account for if the creator didn't want to set a custom time.
if key == "start_time" and new_data[key] == -1:
new_data[key] = datetime.now()
# Account for if the creator didn't want to set a custom time.
if key == "start_time" and new_data[key] == -1:
new_data[key] = datetime.now()
setattr(self,key,new_data[key])
setattr(self, key, new_data[key])
self.last_time = self.start_time
self.end_time = None
self.last_time = self.start_time
self.end_time = None

View file

@ -1,3 +1,5 @@
from datetime import datetime
def happytime(date: datetime):
return date.strftime("%Y-%m-%d %H:%M:%S")

View file

@ -1,14 +1,15 @@
from typing import Any, List, Optional
#Magic for importing alert providers from alerts directory.
# Magic for importing alert providers from alerts directory.
from pkgutil import iter_modules
from importlib import import_module
from inspect import getmembers,isclass
from inspect import getmembers, isclass
from sys import modules
from baps_types.alert import CRITICAL, Alert
import alerts
def iter_namespace(ns_pkg):
# Specifying the second argument (prefix) to iter_modules makes the
# returned name an absolute name instead of a relative one. This allows
@ -19,75 +20,78 @@ def iter_namespace(ns_pkg):
class AlertProvider():
def __init__(self):
return None
def __init__(self):
return None
def get_alerts(self):
return []
def get_alerts(self):
return []
class AlertManager():
_alerts: List[Alert]
_providers: List[AlertProvider] = []
_alerts: List[Alert]
_providers: List[AlertProvider] = []
def __init__(self):
self._alerts = []
def __init__(self):
self._alerts = []
# Find all the alert providers from the /alerts/ directory.
providers = {
name: import_module(name)
for _, name, _
in iter_namespace(alerts)
}
# Find all the alert providers from the /alerts/ directory.
providers = {
name: import_module(name)
for _, name, _
in iter_namespace(alerts)
}
for provider in providers:
classes: List[Any] = [mem[1] for mem in getmembers(modules[provider], isclass) if mem[1].__module__ == modules[provider].__name__]
for provider in providers:
classes: List[Any] = [
mem[1] for mem in getmembers(
modules[provider],
isclass) if mem[1].__module__ == modules[provider].__name__]
if (len(classes) != 1):
print(classes)
raise Exception("Can't import plugin " + provider + " because it doesn't have 1 class.")
if (len(classes) != 1):
print(classes)
raise Exception("Can't import plugin " + provider + " because it doesn't have 1 class.")
self._providers.append(classes[0]())
self._providers.append(classes[0]())
print("Discovered alert providers: ", self._providers)
print("Discovered alert providers: ", self._providers)
def poll_alerts(self):
def poll_alerts(self):
# Poll modules for any alerts.
new_alerts: List[Optional[Alert]] = []
for provider in self._providers:
provider_alerts = provider.get_alerts()
if provider_alerts:
new_alerts.extend(provider_alerts)
# Poll modules for any alerts.
new_alerts: List[Optional[Alert]] = []
for provider in self._providers:
provider_alerts = provider.get_alerts()
if provider_alerts:
new_alerts.extend(provider_alerts)
# Here we replace new firing alerts with older ones, to keep any context.
# (This doesn't do anything yet really, for future use.)
for existing in self._alerts:
found = False
for new in new_alerts:
# given we're removing alerts, got to skip any we removed.
if not new:
continue
# Here we replace new firing alerts with older ones, to keep any context.
# (This doesn't do anything yet really, for future use.)
for existing in self._alerts:
found = False
for new in new_alerts:
# given we're removing alerts, got to skip any we removed.
if not new:
continue
if existing.id == new.id:
# Alert is continuing. Replace it with the old one.
index = new_alerts.index(new)
existing.reoccured()
new_alerts[index] = None # We're going to merge the existing and new, so clear the new one out.
found = True
break
if not found:
# The existing alert is gone, mark it as ended.
existing.cleared()
if existing.id == new.id:
# Alert is continuing. Replace it with the old one.
index = new_alerts.index(new)
existing.reoccured()
new_alerts[index] = None # We're going to merge the existing and new, so clear the new one out.
found = True
break
if found == False:
# The existing alert is gone, mark it as ended.
existing.cleared()
self._alerts.extend([value for value in new_alerts if value]) # Remove any nulled out new alerts
self._alerts.extend([value for value in new_alerts if value]) # Remove any nulled out new alerts
@property
def alerts_current(self):
self.poll_alerts()
return [alert for alert in self._alerts if not alert.end_time]
@property
def alerts_current(self):
self.poll_alerts()
return [alert for alert in self._alerts if not alert.end_time]
@property
def alerts_previous(self):
self.poll_alerts()
return [alert for alert in self._alerts if alert.end_time]
@property
def alerts_previous(self):
self.poll_alerts()
return [alert for alert in self._alerts if alert.end_time]

View file

@ -99,6 +99,7 @@ def render_template(file, data, status=200):
def _filter_happytime(date):
return happytime(date)
env.filters["happytime"] = _filter_happytime
logger: LoggingManager
@ -114,14 +115,17 @@ player_from_q: List[Queue] = []
@app.exception(NotFound)
def page_not_found(request, e: Any):
data = {"ui_page": "404", "ui_title": "404", "code": 404, "title": "Page Not Found", "message": "Looks like you fell off the tip of the iceberg." }
data = {"ui_page": "404", "ui_title": "404", "code": 404, "title": "Page Not Found",
"message": "Looks like you fell off the tip of the iceberg."}
return render_template("error.html", data=data, status=404)
@app.exception(Exception, ServerError)
def server_error(request, e: Exception):
data = {"ui_page": "500", "ui_title": "500", "code": 500, "title": "Something went very wrong!", "message": "Looks like the server fell over. Try viewing the WebServer logs for more details." }
# Future use.
def error_page(code=500, ui_title="500", title="Something went very wrong!",
message="Looks like the server fell over. Try viewing the WebServer logs for more details."):
data = {"ui_page": ui_title, "ui_title": ui_title, "code": code, "title": title, "message": message}
return render_template("error.html", data=data, status=500)
@app.route("/")
def ui_index(request):
config = server_state.get()
@ -148,6 +152,7 @@ def ui_status(request):
"ui_page": "status", "ui_title": "Status"}
return render_template("status.html", data=data)
@app.route("/alerts")
def ui_alerts(request):
data = {