FrogPilot features - openpilot crashed alert

selfdrive/controls/controlsd.py

@@ -6,6 +6,7 @@ import threading
 from typing import SupportsFloat
 
 import cereal.messaging as messaging
+import openpilot.selfdrive.sentry as sentry
 
 from cereal import car, custom, log
 from cereal.visionipc import VisionIpcClient, VisionStreamType
@@ -69,6 +70,8 @@ class Controls:
     # FrogPilot variables
     self.frogpilot_toggles = FrogPilotVariables.toggles
 
+    self.openpilot_crashed_triggered = False
+
     self.card = CarD(CI)
 
     self.params = Params()
@@ -892,6 +895,10 @@ class Controls:
     if self.block_user:
       self.events.add(EventName.blockUser)
 
+    if os.path.isfile(os.path.join(sentry.CRASHES_DIR, 'error.txt')) and not self.openpilot_crashed_triggered:
+      self.events.add(EventName.openpilotCrashed)
+      self.openpilot_crashed_triggered = True
+
   def update_frogpilot_variables(self, CS):
     self.driving_gear = CS.gearShifter not in (GearShifter.neutral, GearShifter.park, GearShifter.reverse, GearShifter.unknown)
 

selfdrive/controls/lib/events.py

@@ -965,6 +965,14 @@ EVENTS: dict[int, dict[str, Alert | AlertCallbackType]] = {
     ET.NO_ENTRY: NoEntryAlert("Please don't use the 'Development' branch!"),
   },
 
+  EventName.openpilotCrashed: {
+    ET.PERMANENT: Alert(
+      "openpilot crashed",
+      "Please post the error log in the FrogPilot Discord!",
+      AlertStatus.normal, AlertSize.mid,
+      Priority.HIGHEST, VisualAlert.none, AudibleAlert.none, 10.),
+  },
+
   EventName.pedalInterceptorNoBrake: {
     ET.WARNING: Alert(
       "Braking Unavailable",

selfdrive/manager/manager.py

@@ -407,6 +407,11 @@ def manager_thread() -> None:
 
     if started and not started_prev:
       params.clear_all(ParamKeyType.CLEAR_ON_ONROAD_TRANSITION)
+
+      error_log = os.path.join(sentry.CRASHES_DIR, 'error.txt')
+      if os.path.isfile(error_log):
+        os.remove(error_log)
+
     elif not started and started_prev:
       params.clear_all(ParamKeyType.CLEAR_ON_OFFROAD_TRANSITION)
       params_memory.clear_all(ParamKeyType.CLEAR_ON_OFFROAD_TRANSITION)

selfdrive/sentry.py

@@ -1,7 +1,9 @@
 """Install exception handler for process crash."""
+import os
 import sentry_sdk
 import socket
 import time
+import traceback
 import urllib.request
 import urllib.error
 
@@ -14,6 +16,7 @@ from openpilot.system.hardware import HARDWARE, PC
 from openpilot.common.swaglog import cloudlog
 from openpilot.system.version import get_commit, get_short_branch, get_origin, get_version
 
+CRASHES_DIR = "/data/community/crashes/"
 
 class SentryProject(Enum):
   # python project
@@ -145,6 +148,17 @@ def capture_fingerprint(candidate, params, blocked=False):
 
 
 def capture_exception(*args, **kwargs) -> None:
+  exc_text = traceback.format_exc()
+
+  phrases_to_check = [
+    "To overwrite it, set 'overwrite' to True.",
+    "device reports readiness to read but returned no data",
+  ]
+
+  if any(phrase in exc_text for phrase in phrases_to_check):
+    return
+
+  save_exception(exc_text)
   cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1))
 
   FrogPilot = "frogai" in get_origin().lower()
@@ -159,6 +173,26 @@ def capture_exception(*args, **kwargs) -> None:
     cloudlog.exception("sentry exception")
 
 
+def save_exception(exc_text: str) -> None:
+  if not os.path.exists(CRASHES_DIR):
+    os.makedirs(CRASHES_DIR)
+
+  files = [
+    os.path.join(CRASHES_DIR, datetime.now().strftime('%Y-%m-%d--%H-%M-%S.log')),
+    os.path.join(CRASHES_DIR, 'error.txt')
+  ]
+
+  for file in files:
+    with open(file, 'w') as f:
+      if file.endswith("error.txt"):
+        lines = exc_text.splitlines()[-10:]
+        f.write("\n".join(lines))
+      else:
+        f.write(exc_text)
+
+  print('Logged current crash to {}'.format(files))
+
+
 def set_tag(key: str, value: str) -> None:
   sentry_sdk.set_tag(key, value)
 

selfdrive/ui/ui.h

@@ -78,7 +78,12 @@ struct Alert {
       const int controls_missing = (nanos_since_boot() - sm.rcv_time("controlsState")) / 1e9;
 
       // Handle controls timeout
-      if (controls_frame < started_frame) {
+      if (std::ifstream("/data/community/crashes/error.txt")) {
+        alert = {"openpilot crashed", "Please post the error log in the FrogPilot Discord!",
+                 "controlsWaiting", cereal::ControlsState::AlertSize::MID,
+                 cereal::ControlsState::AlertStatus::NORMAL,
+                 AudibleAlert::NONE};
+      } else if (controls_frame < started_frame) {
         // car is started, but controlsState hasn't been seen at all
         alert = {"openpilot Unavailable", "Waiting for controls to start",
                  "controlsWaiting", cereal::ControlsState::AlertSize::MID,