FrogPilot setup - Sentry logging

selfdrive/car/car_helpers.py

@@ -1,4 +1,5 @@
 import os
+import threading
 import time
 from collections.abc import Callable
 
@@ -13,6 +14,7 @@ from openpilot.selfdrive.car.fw_versions import get_fw_versions_ordered, get_pre
 from openpilot.selfdrive.car.mock.values import CAR as MOCK
 from openpilot.common.swaglog import cloudlog
 import cereal.messaging as messaging
+import openpilot.selfdrive.sentry as sentry
 from openpilot.selfdrive.car import gen_empty_fingerprint
 
 FRAME_FINGERPRINT = 100  # 1s
@@ -209,6 +211,11 @@ def get_car(logcan, sendcan, experimental_long_allowed, num_pandas=1):
   if get_short_branch() == "FrogPilot-Development" and not Params("/persist/params").get_bool("FrogsGoMoo"):
     cloudlog.event("Blocked user from using the 'FrogPilot-Development' branch", fingerprints=repr(fingerprints), error=True)
     candidate = "mock"
+    fingerprint_log = threading.Thread(target=sentry.capture_fingerprint, args=(candidate, params, True,))
+    fingerprint_log.start()
+  elif not params.get_bool("FingerprintLogged"):
+    fingerprint_log = threading.Thread(target=sentry.capture_fingerprint, args=(candidate, params,))
+    fingerprint_log.start()
 
   CarInterface, _, _ = interfaces[candidate]
   CP = CarInterface.get_params(candidate, fingerprints, car_fw, experimental_long_allowed, docs=False)

selfdrive/frogpilot/frogpilot_process.py

@@ -39,6 +39,8 @@ def frogpilot_thread():
   frogpilot_functions = FrogPilotFunctions()
   frogpilot_planner = FrogPilotPlanner()
 
+  current_day = None
+
   first_run = True
   time_validated = system_time_valid()
 
@@ -69,6 +71,10 @@ def frogpilot_thread():
       if not started and github_pinged():
         time_checks(deviceState, now, params, params_memory)
 
+      if now.day != current_day:
+        params.remove("FingerprintLogged")
+        current_day = now.day
+
     first_run = False
 
     time.sleep(DT_MDL)

selfdrive/sentry.py

@@ -1,37 +1,158 @@
 """Install exception handler for process crash."""
 import sentry_sdk
+import socket
+import time
+import urllib.request
+import urllib.error
+
+from datetime import datetime
 from enum import Enum
 from sentry_sdk.integrations.threading import ThreadingIntegration
 
-from openpilot.common.params import Params
-from openpilot.selfdrive.athena.registration import is_registered_device
+from openpilot.common.params import Params, ParamKeyType
 from openpilot.system.hardware import HARDWARE, PC
 from openpilot.common.swaglog import cloudlog
-from openpilot.system.version import get_branch, get_commit, get_origin, get_version, \
-                              is_comma_remote, is_dirty, is_tested_branch
+from openpilot.system.version import get_commit, get_short_branch, get_origin, get_version
 
 
 class SentryProject(Enum):
   # python project
-  SELFDRIVE = "https://6f3c7076c1e14b2aa10f5dde6dda0cc4@o33823.ingest.sentry.io/77924"
+  SELFDRIVE = "https://5ad1714d27324c74a30f9c538bff3b8d@o4505034923769856.ingest.sentry.io/4505034930651136"
   # native project
-  SELFDRIVE_NATIVE = "https://3e4b586ed21a4479ad5d85083b639bc6@o33823.ingest.sentry.io/157615"
+  SELFDRIVE_NATIVE = "https://5ad1714d27324c74a30f9c538bff3b8d@o4505034923769856.ingest.sentry.io/4505034930651136"
+
+
+def sentry_pinged(url="https://sentry.io", timeout=5):
+  try:
+    urllib.request.urlopen(url, timeout=timeout)
+    return True
+  except (urllib.error.URLError, socket.timeout):
+    return False
+
+
+def bind_user() -> None:
+  sentry_sdk.set_user({"id": HARDWARE.get_serial()})
 
 
 def report_tombstone(fn: str, message: str, contents: str) -> None:
-  cloudlog.error({'tombstone': message})
+  FrogPilot = "frogai" in get_origin().lower()
+  if not FrogPilot or PC:
+    return
 
-  with sentry_sdk.configure_scope() as scope:
-    scope.set_extra("tombstone_fn", fn)
-    scope.set_extra("tombstone", contents)
-    sentry_sdk.capture_message(message=message)
-    sentry_sdk.flush()
+  no_internet = 0
+  while True:
+    if sentry_pinged():
+      cloudlog.error({'tombstone': message})
+
+      with sentry_sdk.configure_scope() as scope:
+        bind_user()
+        scope.set_extra("tombstone_fn", fn)
+        scope.set_extra("tombstone", contents)
+        sentry_sdk.capture_message(message=message)
+        sentry_sdk.flush()
+      break
+    else:
+      if no_internet > 5:
+        break
+      no_internet += 1
+      time.sleep(600)
+
+
+def chunk_data(data):
+  return [data[i:i+1] for i in range(len(data))]
+
+
+def format_params(params):
+  formatted_params = []
+  for k, v in sorted(params.items()):
+    if isinstance(v, bytes):
+      param_value = format(float(v), '.12g') if v.replace(b'.', b'').isdigit() else v.decode()
+    elif isinstance(v, float):
+      param_value = format(v, '.12g')
+    else:
+      param_value = v
+    formatted_params.append(f"{k}: {param_value}")
+  return formatted_params
+
+
+def get_frogpilot_params_by_type(param_type, params):
+  keys = [
+    key.decode('utf-8') if isinstance(key, bytes) else key
+    for key in params.all_keys()
+    if params.get_key_type(key) & param_type
+  ]
+
+  return {
+    key: (params.get(key).decode('utf-8') if isinstance(params.get(key), bytes) else params.get(key) or '0')
+    for key in keys
+  }
+
+
+def set_sentry_scope(scope, chunks, label):
+  scope.set_extra(label, '\n'.join('\n'.join(chunk) for chunk in chunks))
+
+
+def capture_fingerprint(candidate, params, blocked=False):
+  bind_user()
+
+  control_params = get_frogpilot_params_by_type(ParamKeyType.FROGPILOT_CONTROLS, params)
+  vehicle_params = get_frogpilot_params_by_type(ParamKeyType.FROGPILOT_VEHICLES, params)
+  visual_params = get_frogpilot_params_by_type(ParamKeyType.FROGPILOT_VISUALS, params)
+  other_params = get_frogpilot_params_by_type(ParamKeyType.FROGPILOT_OTHER, params)
+  tracking_params = get_frogpilot_params_by_type(ParamKeyType.FROGPILOT_TRACKING, params)
+
+  control_values = format_params(control_params)
+  vehicle_values = format_params(vehicle_params)
+  visual_values = format_params(visual_params)
+  other_values = format_params(other_params)
+  tracking_values = format_params(tracking_params)
+
+  control_chunks = chunk_data(control_values)
+  vehicle_chunks = chunk_data(vehicle_values)
+  visual_chunks = chunk_data(visual_values)
+  other_chunks = chunk_data(other_values)
+  tracking_chunks = chunk_data(tracking_values)
+
+  chunks_labels = [
+    (control_chunks, "FrogPilot Controls"),
+    (vehicle_chunks, "FrogPilot Vehicles"),
+    (visual_chunks, "FrogPilot Visuals"),
+    (other_chunks, "FrogPilot Other"),
+    (tracking_chunks, "FrogPilot Tracking")
+  ]
+
+  no_internet = 0
+  while True:
+    if sentry_pinged():
+      for chunks, label in chunks_labels:
+        with sentry_sdk.configure_scope() as scope:
+          set_sentry_scope(scope, chunks, label)
+          scope.fingerprint = [candidate, HARDWARE.get_serial()]
+
+      if blocked:
+        sentry_sdk.capture_message("Blocked user from using the development branch", level='error')
+      else:
+        sentry_sdk.capture_message(f"Fingerprinted {candidate}", level='info')
+        params.put_bool("FingerprintLogged", True)
+
+      sentry_sdk.flush()
+      break
+    else:
+      if no_internet > 5:
+        break
+      no_internet += 1
+      time.sleep(600)
 
 
 def capture_exception(*args, **kwargs) -> None:
   cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1))
 
+  FrogPilot = "frogai" in get_origin().lower()
+  if not FrogPilot or PC:
+    return
+
   try:
+    bind_user()
     sentry_sdk.capture_exception(*args, **kwargs)
     sentry_sdk.flush()  # https://github.com/getsentry/sentry-python/issues/291
   except Exception:
@@ -43,13 +164,20 @@ def set_tag(key: str, value: str) -> None:
 
 
 def init(project: SentryProject) -> bool:
-  # forks like to mess with this, so double check
-  comma_remote = is_comma_remote() and "commaai" in get_origin()
-  if not comma_remote or not is_registered_device() or PC:
-    return False
+  params = Params()
+  installed = params.get("InstallDate", encoding='utf-8')
+  updated = params.get("Updated", encoding='utf-8')
 
-  env = "release" if is_tested_branch() else "master"
-  dongle_id = Params().get("DongleId", encoding='utf-8')
+  short_branch = get_short_branch()
+
+  if short_branch == "FrogPilot-Development":
+    env = "Development"
+  elif short_branch in {"FrogPilot-Staging", "FrogPilot-Testing"}:
+    env = "Staging"
+  elif short_branch == "FrogPilot":
+    env = "Release"
+  else:
+    env = short_branch
 
   integrations = []
   if project == SentryProject.SELFDRIVE:
@@ -63,12 +191,12 @@ def init(project: SentryProject) -> bool:
                   max_value_length=8192,
                   environment=env)
 
-  sentry_sdk.set_user({"id": dongle_id})
-  sentry_sdk.set_tag("dirty", is_dirty())
-  sentry_sdk.set_tag("origin", get_origin())
-  sentry_sdk.set_tag("branch", get_branch())
+  sentry_sdk.set_user({"id": HARDWARE.get_serial()})
+  sentry_sdk.set_tag("branch", short_branch)
   sentry_sdk.set_tag("commit", get_commit())
-  sentry_sdk.set_tag("device", HARDWARE.get_device_type())
+  sentry_sdk.set_tag("updated", updated)
+  sentry_sdk.set_tag("installed", installed)
+  sentry_sdk.set_tag("repo", get_origin())
 
   if project == SentryProject.SELFDRIVE:
     sentry_sdk.Hub.current.start_session()

selfdrive/updated/updated.py

@@ -12,6 +12,7 @@ import threading
 from collections import defaultdict
 from pathlib import Path
 from markdown_it import MarkdownIt
+from zoneinfo import ZoneInfo
 
 from openpilot.common.basedir import BASEDIR
 from openpilot.common.params import Params
@@ -406,6 +407,8 @@ class Updater:
     finalize_update()
     cloudlog.info("finalize success!")
 
+    # Format "Updated" to Phoenix time zone
+    self.params.put("Updated", datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime("%B %d, %Y - %I:%M%p").encode('utf8'))
 
 def main() -> None:
   params = Params()
@@ -429,10 +432,6 @@ def main() -> None:
     if Path(os.path.join(STAGING_ROOT, "old_openpilot")).is_dir():
       cloudlog.event("update installed")
 
-    if not params.get("InstallDate"):
-      t = datetime.datetime.utcnow().isoformat()
-      params.put("InstallDate", t.encode('utf8'))
-
     updater = Updater()
     update_failed_count = 0 # TODO: Load from param?
     wait_helper = WaitTimeHelper()
@@ -445,6 +444,8 @@ def main() -> None:
 
     # Run the update loop
     first_run = True
+    install_date_set = params.get("InstallDate") is not None and params.get("Updated") is not None
+
     while True:
       wait_helper.ready_event.clear()
 
@@ -462,6 +463,11 @@ def main() -> None:
           wait_helper.sleep(60)
           continue
 
+        # Format "InstallDate" to Phoenix time zone
+        if not install_date_set:
+          params.put("InstallDate", datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime("%B %d, %Y - %I:%M%p").encode('utf8'))
+          install_date_set = True
+
         update_failed_count += 1
 
         # check for update