Merge pull request #29 from UniversityRadioYork/mstratford/alerts

Add support for alerting.

alerts/dummy.py

@@ -0,0 +1,20 @@
+from helpers.alert_manager import AlertProvider
+from package import BETA
+from baps_types.alert import WARNING, Alert
+# Dummy alert provider for testing basics like UI without needing to actually cause errors.
+
+
+class DummyAlertProvider(AlertProvider):
+
+    def get_alerts(self):
+        if BETA:
+            return [Alert(
+                {
+                    "start_time": -1,
+                    "id": "test",
+                    "title": "BAPSicle is in Debug Mode",
+                    "description": "This is a test alert. It will not appear on production builds.",
+                    "module": "Test",
+                    "severity": WARNING
+                }
+            )]

alerts/player.py

@@ -0,0 +1,99 @@
+# Any alerts produced by the player.py instances.
+import json
+from typing import Any, Dict, List, Optional
+from datetime import datetime, timedelta
+from helpers.os_environment import resolve_external_file_path
+from helpers.alert_manager import AlertProvider
+from baps_types.alert import CRITICAL, WARNING, Alert
+from baps_types.happytime import happytime
+
+MODULE = "Player"  # This should match the log file, so the UI will link to the logs page.
+
+
+class PlayerAlertProvider(AlertProvider):
+
+    _server_state: Dict[str, Any]
+    _states: List[Optional[Dict[str, Any]]] = []
+    _player_count: int
+
+    def __init__(self):
+        # Player count only changes after server restart, may as well just load this once.
+        with open(resolve_external_file_path("state/BAPSicleServer.json")) as file:
+            self._server_state = json.loads(file.read())
+
+        self._player_count = int(self._server_state["num_channels"])
+        self._states = [None] * self._player_count
+
+    # To simplify monitoring (and allow detection of things going super
+    # weird), we are going to read from the state file to work out the alerts.
+    def get_alerts(self):
+        for channel in range(self._player_count):
+            with open(resolve_external_file_path("state/Player{}.json".format(channel))) as file:
+                self._states[channel] = json.loads(file.read())
+
+        funcs = [self._channel_count, self._initialised, self._start_time]
+
+        alerts: List[Alert] = []
+
+        for func in funcs:
+            func_alerts = func()
+            if func_alerts:
+                alerts.extend(func_alerts)
+
+        return alerts
+
+    def _channel_count(self):
+        if self._player_count <= 0:
+            return [Alert({
+                "start_time": -1,  # Now
+                "id": "no_channels",
+                "title": "There are no players configured.",
+                "description": "The number of channels configured is {}. \
+                  Please set to at least 1 on the 'Server Config' page."
+                .format(self._player_count),
+                "module": MODULE+"Handler",
+                "severity": CRITICAL
+            })]
+
+    def _initialised(self):
+        alerts: List[Alert] = []
+        for channel in range(self._player_count):
+            if self._states[channel] and not self._states[channel]["initialised"]:
+                alerts.append(Alert({
+                    "start_time": -1,  # Now
+                    "id": "player_{}_not_initialised".format(channel),
+                    "title": "Player {} is not initialised.".format(channel),
+                    "description": "This typically means the player channel was not able find the configured sound output \
+                    on the system. Please check the 'Player Config' and Player logs to determine the cause.",
+                    "module": MODULE+str(channel),
+                    "severity": CRITICAL
+                }))
+        return alerts
+
+    def _start_time(self):
+        server_start_time = self._server_state["start_time"]
+        server_start_time = datetime.fromtimestamp(server_start_time)
+        delta = timedelta(
+            seconds=30,
+        )
+
+        alerts: List[Alert] = []
+        for channel in range(self._player_count):
+            start_time = self._states[channel]["start_time"]
+            start_time = datetime.fromtimestamp(start_time)
+            if (start_time > server_start_time + delta):
+                alerts.append(Alert({
+                    "start_time": -1,
+                    "id": "player_{}_restarted".format(channel),
+                    "title": "Player {} restarted after the server started.".format(channel),
+                    "description":
+                    """Player {} last restarted at {}, after the server first started at {}, suggesting a failure.
+
+This likely means there was an unhandled exception in the player code, causing the server to restart the player.
+
+Please check player logs to investigate the cause. Please restart the server to clear this warning."""
+                    .format(channel, happytime(start_time), happytime(server_start_time)),
+                    "module": MODULE+str(channel),
+                    "severity": WARNING
+                }))
+        return alerts

alerts/server.py

@@ -0,0 +1,77 @@
+# Any alerts produced by the server.py layer. This likely means BIG issues.
+import json
+from typing import Any, Dict, List
+from datetime import datetime, timedelta
+from helpers.os_environment import resolve_external_file_path
+from helpers.alert_manager import AlertProvider
+from baps_types.alert import CRITICAL, WARNING, Alert
+from baps_types.happytime import happytime
+
+MODULE = "BAPSicleServer"  # This should match the log file, so the UI will link to the logs page.
+
+
+class ServerAlertProvider(AlertProvider):
+
+    _state: Dict[str, Any]
+    # To simplify monitoring (and allow detection of things going super
+    # weird), we are going to read from the state file to work out the alerts.
+
+    def get_alerts(self):
+        with open(resolve_external_file_path("state/BAPSicleServer.json")) as file:
+            self._state = json.loads(file.read())
+
+        funcs = [self._api_key, self._start_time]
+
+        alerts: List[Alert] = []
+
+        for func in funcs:
+            func_alerts = func()
+            if func_alerts:
+                alerts.extend(func_alerts)
+
+        return alerts
+
+    def _api_key(self):
+        if not self._state["myradio_api_key"]:
+            return [Alert({
+                "start_time": -1,  # Now
+                "id": "api_key_missing",
+                "title": "MyRadio API Key is not configured.",
+                "description": "This means you will be unable to load show plans, audio items, or tracklist. \
+                    Please set one on the 'Server Config' page.",
+                "module": MODULE,
+                "severity": CRITICAL
+            })]
+
+        if len(self._state["myradio_api_key"]) < 10:
+            return [Alert({
+                "start_time": -1,
+                "id": "api_key_missing",
+                "title": "MyRadio API Key seems incorrect.",
+                "description": "The API key is less than 10 characters, it's probably not a valid one. \
+                    If it is valid, it shouldn't be.",
+                "module": MODULE,
+                "severity": WARNING
+            })]
+
+    def _start_time(self):
+        start_time = self._state["start_time"]
+        start_time = datetime.fromtimestamp(start_time)
+        delta = timedelta(
+            days=1,
+        )
+        if (start_time + delta > datetime.now()):
+            return [Alert({
+                "start_time": -1,
+                "id": "server_restarted",
+                "title": "BAPSicle restarted recently.",
+                "description":
+                """The BAPSicle server restarted at {}, less than a day ago.
+
+It may have been automatically restarted by the OS.
+
+If this is not expected, please check logs to investigate why BAPSicle restarted/crashed."""
+                .format(happytime(start_time)),
+                "module": MODULE,
+                "severity": WARNING
+            })]

baps_types/alert.py

@@ -0,0 +1,69 @@
+from typing import Any, Dict, Optional
+from datetime import datetime
+
+CRITICAL = "Critical"
+WARNING = "Warning"
+
+
+class Alert:
+    start_time: datetime
+    last_time: datetime
+    end_time: Optional[datetime]
+    id: str
+    title: str
+    description: str
+    module: str
+    severity: str
+
+    @property
+    def ui_class(self) -> str:
+        if self.severity == CRITICAL:
+            return "danger"
+        if self.severity == WARNING:
+            return "warning"
+        return "info"
+
+    # This alert has happened again.
+    def reoccured(self):
+        self.last_time = datetime.now()
+        self.end_time = None
+
+    # This alert has finished, just update end time and keep last_time.
+    def cleared(self):
+        self.end_time = datetime.now()
+
+    @property
+    def __dict__(self):
+        attrs = ["start_time", "last_time", "end_time", "id", "title", "description", "module", "severity"]
+        out = {}
+        for attr in attrs:
+            out[attr] = self.__getattribute__(attr)
+
+        return out
+
+    def __init__(self, new_data: Dict[str, Any]):
+        required_vars = [
+            "start_time",  # Just in case an alert wants to show starting earlier than it is reported.
+            "id",
+            "title",
+            "description",
+            "module",
+            "severity"
+        ]
+
+        for key in required_vars:
+            if key not in new_data.keys():
+                raise KeyError("Key {} is missing from data to create Alert.".format(key))
+
+            # if type(new_data[key]) != type(getattr(self,key)):
+            #  raise TypeError("Key {} has type {}, was expecting {}."
+            #   .format(key, type(new_data[key]), type(getattr(self,key))))
+
+            # Account for if the creator didn't want to set a custom time.
+            if key == "start_time" and new_data[key] == -1:
+                new_data[key] = datetime.now()
+
+            setattr(self, key, new_data[key])
+
+        self.last_time = self.start_time
+        self.end_time = None

baps_types/happytime.py

@@ -0,0 +1,5 @@
+from datetime import datetime
+
+
+def happytime(date: datetime):
+    return date.strftime("%Y-%m-%d %H:%M:%S")

helpers/alert_manager.py

@@ -0,0 +1,97 @@
+from typing import Any, List, Optional
+
+# Magic for importing alert providers from alerts directory.
+from pkgutil import iter_modules
+from importlib import import_module
+from inspect import getmembers, isclass
+from sys import modules
+
+from baps_types.alert import Alert
+import alerts
+
+
+def iter_namespace(ns_pkg):
+    # Specifying the second argument (prefix) to iter_modules makes the
+    # returned name an absolute name instead of a relative one. This allows
+    # import_module to work without having to do additional modification to
+    # the name.
+    return iter_modules(ns_pkg.__path__, ns_pkg.__name__ + ".")
+
+
+class AlertProvider():
+
+    def __init__(self):
+        return None
+
+    def get_alerts(self):
+        return []
+
+
+class AlertManager():
+    _alerts: List[Alert]
+    _providers: List[AlertProvider] = []
+
+    def __init__(self):
+        self._alerts = []
+
+        # Find all the alert providers from the /alerts/ directory.
+        providers = {
+            name: import_module(name)
+            for _, name, _
+            in iter_namespace(alerts)
+        }
+
+        for provider in providers:
+            classes: List[Any] = [
+                mem[1] for mem in getmembers(
+                    modules[provider],
+                    isclass) if mem[1].__module__ == modules[provider].__name__]
+
+            if (len(classes) != 1):
+                print(classes)
+                raise Exception("Can't import plugin " + provider + " because it doesn't have 1 class.")
+
+            self._providers.append(classes[0]())
+
+        print("Discovered alert providers: ", self._providers)
+
+    def poll_alerts(self):
+
+        # Poll modules for any alerts.
+        new_alerts: List[Optional[Alert]] = []
+        for provider in self._providers:
+            provider_alerts = provider.get_alerts()
+            if provider_alerts:
+                new_alerts.extend(provider_alerts)
+
+        # Here we replace new firing alerts with older ones, to keep any context.
+        # (This doesn't do much yet really, just remembering the start_time)
+        for existing in self._alerts:
+            found = False
+            for new in new_alerts:
+                # given we're removing alerts, got to skip any we removed.
+                if not new:
+                    continue
+
+                if existing.id == new.id:
+                    # Alert is continuing. Replace it with the old one.
+                    index = new_alerts.index(new)
+                    existing.reoccured()
+                    new_alerts[index] = None  # We're going to merge the existing and new, so clear the new one out.
+                    found = True
+                    break
+            if not found:
+                # The existing alert is gone, mark it as ended.
+                existing.cleared()
+
+        self._alerts.extend([value for value in new_alerts if value])  # Remove any nulled out new alerts
+
+    @property
+    def alerts_current(self):
+        self.poll_alerts()
+        return [alert for alert in self._alerts if not alert.end_time]
+
+    @property
+    def alerts_previous(self):
+        self.poll_alerts()
+        return [alert for alert in self._alerts if alert.end_time]

package.json

@@ -1,7 +1,7 @@
 {
   "name": "bapsicle",
   "nice_name": "BAPSicle",
-  "version": "3.0.0",
+  "version": "3.1.0",
   "description": "BAPS3, the third generation of University Radio York's Broadcast and Presenting Suite. This package includes the Server (BAPSicle) and Presenter (WebStudio)",
   "main": "index.js",
   "directories": {
@@ -13,7 +13,7 @@
     "presenter-make": "npm run presenter-install && (rm -r presenter-build || true) && cd presenter && yarn build-baps && cp -r build ../presenter-build && cd ../ && npm install",
     "test": "echo \"Error: no test specified\" && exit 1",
     "presenter-start": "cd presenter && yarn start-baps",
-    "lint": "autopep8 -r -a -a --ignore E402,E226,E24,W50,W690 --max-line-length 127 --in-place --exclude=\"*node_modules*,*venv/*,presenter/*\" . "
+    "lint": "./venv/bin/autopep8 -r -a -a --ignore E402,E226,E24,W50,W690 --max-line-length 127 --in-place --exclude=\"*node_modules*,*venv/*,presenter/*\" . && ./venv/bin/flake8 . --exclude=\"*node_modules*,*venv/*,presenter/*\" --count --ignore=E402,E226,E24,W50,W690 --max-complexity=25 --max-line-length=127 --statistics"
   },
   "repository": {
     "type": "git",

player.py

@@ -35,6 +35,7 @@ from pygame import mixer
 from mutagen.mp3 import MP3
 from syncer import sync
 from threading import Timer
+from datetime import datetime
 
 from helpers.normalisation import get_normalised_filename_if_available
 from helpers.myradio_api import MyRadioAPI
@@ -969,7 +970,7 @@ class Player:
         self.out_q = out_q
 
         self.logger = LoggingManager(
-            "Player" + str(channel), debug=package.build_beta)
+            "Player" + str(channel), debug=package.BETA)
 
         self.api = MyRadioAPI(self.logger, server_state)
 
@@ -980,6 +981,8 @@ class Player:
             self.__rate_limited_params,
         )
 
+        self.state.update("start_time", datetime.now().timestamp())
+
         self.state.add_callback(self._send_status)
 
         self.state.update("channel", channel)

server.py

@@ -12,6 +12,7 @@
     Date:
         October, November 2020
 """
+from datetime import datetime
 from file_manager import FileManager
 import multiprocessing
 from multiprocessing.queues import Queue
@@ -224,6 +225,7 @@ class BAPSicleServer:
         )
 
         self.state.update("running_state", "running")
+        self.state.update("start_time", datetime.now().timestamp())
 
         print("Launching BAPSicle...")
 

ui-templates/404.html

@@ -1,9 +0,0 @@
-{% extends 'base.html' %}
-{% block content_inner %}
-<div class="text-center">
-  <div class="error error-big mx-auto" data-text="404">404</div>
-  <p class="lead text-gray-800 mb-5">Page Not Found</p>
-  <p class="text-gray-900 mb-0">Looks like you fell off the tip of the iceberg.</p>
-  <a href="/">← Escape Back Home</a>
-</div>
-{% endblock %}

ui-templates/alerts.html

@@ -0,0 +1,14 @@
+{% extends 'base.html' %}
+{% from 'parts/alert_list.html' import alert_list %}
+{% block head %}
+<meta http-equiv="refresh" content="15;url=/alerts" />
+{% endblock %}
+{% block content_inner %}
+  {% if data %}
+  <h3 class="h4">Current Alerts: {{ data.alerts_current | length }}</h3>
+  {{ alert_list(data.alerts_current) }}
+  <hr>
+  <h3 class="h4">Previous Alerts: {{ data.alerts_previous | length }}</h3>
+  {{ alert_list(data.alerts_previous) }}
+  {% endif %}
+{% endblock %}

ui-templates/config_player.html

@@ -9,21 +9,25 @@
 <h4 class="h5">Currently Selected</h4>
 <p>
 {% for channel in data.channels %}
-  <strong>Player {{channel.channel}}:</strong>
-  {% if channel.output %}
-    {{channel.output}}
+  <strong>Player {{loop.index0}}:</strong>
+  {% if channel %}
+    {% if channel.output %}
+      {{channel.output}}
+    {% else %}
+      Default Audio Device
+    {% endif %}
+    {% if not channel.initialised %}
+      <code> - <strong>ERROR:</strong> Player did not start, potentially configured sound output is missing? Check <a href="/alerts">Alerts</a>.</code>
+    {% endif %}
   {% else %}
-    Default Audio Device
-  {% endif %}
-  {% if not channel.initialised %}
-    <code> - <strong>ERROR:</strong> Player did not start, potentially missing sound output?</code>
+    <code> - <strong>ERROR:</strong> Player did not respond, likely it is dead :/ Check <a href="/alerts">Alerts</a>.</code>
   {% endif %}
   <br>
 {% endfor %}
 <br/>
 <code>
 Set for:
-{% for channel in data.channels %}
+{% for channel in data.channels if channel %}
   <a href="/player/{{channel.channel}}/output/None">Player {{channel.channel}}</a> /
 {% endfor %}
 Default Audio Output
@@ -37,7 +41,9 @@ Default Audio Output
   {% if host_api.usable %}
   Set for:
     {% for channel in data.channels %}
-      {% if channel.output == output.name %}
+      {% if not channel %}
+      Player {{loop.index0}}
+      {% elif channel.output == output.name %}
       <strong>Player {{channel.channel}}</strong>
       {% else %}
       <a href="/player/{{channel.channel}}/output/{{output.name}}">Player {{channel.channel}}</a>

ui-templates/error.html

@@ -0,0 +1,9 @@
+{% extends 'base.html' %}
+{% block content_inner %}
+<div class="text-center">
+  <div class="error error-big mx-auto" data-text="{{data.code}}">{{data.code}}</div>
+  <p class="lead text-gray-800 mb-5">{{data.title}}</p>
+  <p class="text-gray-900 mb-0">{{data.message}}</p>
+  <a href="/">← Escape Back Home</a>
+</div>
+{% endblock %}

ui-templates/index.html

@@ -1,4 +1,7 @@
 {% extends 'base.html' %}
+{% block head %}
+<meta http-equiv="refresh" content="60;url=/" />
+{% endblock %}
 {% block content %}
 
   <div class="card-body p-0">
@@ -16,6 +19,23 @@
           <a href="/presenter/" class="btn btn-primary btn-user btn-block">
             Open BAPS Presenter
           </a>
+
+          <hr>
+          {% if data.alert_count > 0 %}
+            <div class="alert alert-danger" role="alert">
+              <h4 class="alert-heading">Something's up! <a href="/alerts" class="btn btn-sm btn-danger float-right"><span class="badge badge-light mr-1">{{data.alert_count}}</span>View Alerts</a></h4>
+              <p>BAPSicle is having some issues. Please review the alerts page.
+          {% else %}
+          <div class="alert alert-success" role="alert">
+            <h4 class="alert-heading">We're all good!</h4>
+            <p>BAPSicle seems to be running well. If this doesn't seem to be the case, try restarting or taking a closer look at the logs.</p>
+          {% endif %}
+            <hr>
+            <a href="/logs" class="btn btn-success">Logs</a>
+            <a href="/restart" class="btn btn-info">Restart</a>
+            <a href="/quit" class="btn btn-danger">Quit</a>
+          </div>
+
           <hr>
           <p>Version: {{data.server_version}} - Build: {{data.server_build}} - Branch: {{data.server_branch}}</p>
           <p>Server Name: {{data.server_name}}</p>

ui-templates/parts/alert_list.html

@@ -0,0 +1,27 @@
+{% macro alert_list(alerts) %}
+  {% for alert in alerts %}
+  <div class="card alert-{{ alert.ui_class }}">
+    <div class="card-header" id="headingOne">
+      <span class="badge badge-{{ alert.ui_class}}">{{ alert.severity }}</span>
+      <h4 class="h5 mb-0 mt-1">{{ alert.title }}</h4>
+      <span class="badge badge-primary">Since {{ alert.start_time | happytime }}</span>
+      <span class="badge badge-secondary">Last Seen {{ alert.last_time | happytime }}</span>
+      {% if alert.end_time %}
+      <span class="badge badge-success">Ended {{ alert.end_time | happytime }}</span>
+      {% endif %}
+    </div>
+
+    <div id="collapseOne" class="collapse show" aria-labelledby="headingOne" data-parent="#accordionExample">
+      <div class="card-body pb-0">
+        <strong>Module: </strong><a href="/logs/{{ alert.module }}" title="Click for logs">{{ alert.module }}</a>
+        {% autoescape false %}
+        <p>{{ alert.description | replace("\n\n", "</p><p>") | replace("\n", "<br/>")}}</p>
+        {% endautoescape %}
+      </div>
+    </div>
+  </div>
+  {% endfor %}
+  {% if not alerts %}
+  <strong>No alerts here.</strong>
+  {% endif %}
+{% endmacro %}

web_server.py

@@ -17,7 +17,6 @@ import json
 import os
 
 from helpers.os_environment import (
-    isBundelled,
     resolve_external_file_path,
     resolve_local_file_path,
 )
@@ -27,6 +26,9 @@ from helpers.state_manager import StateManager
 from helpers.the_terminator import Terminator
 from helpers.normalisation import get_normalised_filename_if_available
 from helpers.myradio_api import MyRadioAPI
+from helpers.alert_manager import AlertManager
+import package
+from baps_types.happytime import happytime
 
 env = Environment(
     loader=FileSystemLoader("%s/ui-templates/" % os.path.dirname(__file__)),
@@ -94,9 +96,16 @@ def render_template(file, data, status=200):
     return html(html_content, status=status)
 
 
+def _filter_happytime(date):
+    return happytime(date)
+
+
+env.filters["happytime"] = _filter_happytime
+
 logger: LoggingManager
 server_state: StateManager
 api: MyRadioAPI
+alerts: AlertManager
 
 player_to_q: List[Queue] = []
 player_from_q: List[Queue] = []
@@ -106,8 +115,16 @@ player_from_q: List[Queue] = []
 
 @app.exception(NotFound)
 def page_not_found(request, e: Any):
-    data = {"ui_page": "404", "ui_title": "404"}
-    return render_template("404.html", data=data, status=404)
+    data = {"ui_page": "404", "ui_title": "404", "code": 404, "title": "Page Not Found",
+            "message": "Looks like you fell off the tip of the iceberg."}
+    return render_template("error.html", data=data, status=404)
+
+
+# Future use.
+def error_page(code=500, ui_title="500", title="Something went very wrong!",
+               message="Looks like the server fell over. Try viewing the WebServer logs for more details."):
+    data = {"ui_page": ui_title, "ui_title": ui_title, "code": code, "title": title, "message": message}
+    return render_template("error.html", data=data, status=500)
 
 
 @app.route("/")
@@ -116,6 +133,7 @@ def ui_index(request):
     data = {
         "ui_page": "index",
         "ui_title": "",
+        "alert_count": len(alerts.alerts_current),
         "server_version": config["server_version"],
         "server_build": config["server_build"],
         "server_name": config["server_name"],
@@ -136,6 +154,17 @@ def ui_status(request):
     return render_template("status.html", data=data)
 
 
+@app.route("/alerts")
+def ui_alerts(request):
+    data = {
+        "alerts_current": alerts.alerts_current,
+        "alerts_previous": alerts.alerts_previous,
+        "ui_page": "alerts",
+        "ui_title": "Alerts"
+    }
+    return render_template("alerts.html", data=data)
+
+
 @app.route("/config/player")
 def ui_config_player(request):
     channel_states = []
@@ -217,7 +246,12 @@ def ui_logs_render(request, path):
     page = int(page)
     assert page >= 1
 
-    log_file = open(resolve_external_file_path("/logs/{}.log").format(path))
+    try:
+        log_file = open(resolve_external_file_path("/logs/{}.log").format(path))
+    except FileNotFoundError:
+        abort(404)
+        return
+
     data = {
         "logs": log_file.read().splitlines()[
             -300 * page:(-300 * (page - 1) if page > 1 else None)
@@ -479,13 +513,14 @@ def restart(request):
 # Don't use reloader, it causes Nested Processes!
 def WebServer(player_to: List[Queue], player_from: List[Queue], state: StateManager):
 
-    global player_to_q, player_from_q, server_state, api, app
+    global player_to_q, player_from_q, server_state, api, app, alerts
     player_to_q = player_to
     player_from_q = player_from
     server_state = state
 
     logger = LoggingManager("WebServer")
     api = MyRadioAPI(logger, state)
+    alerts = AlertManager()
 
     process_title = "Web Server"
     setproctitle(process_title)
@@ -498,9 +533,9 @@ def WebServer(player_to: List[Queue], player_from: List[Queue], state: StateMana
                 app.run(
                     host=server_state.get()["host"],
                     port=server_state.get()["port"],
-                    debug=(not isBundelled()),
                     auto_reload=False,
-                    access_log=(not isBundelled()),
+                    debug=not package.BETA,
+                    access_log=not package.BETA,
                 )
             )
         except Exception: