From 517b116de5bf8e38523e5fb1efeeb2507c905d7c Mon Sep 17 00:00:00 2001 From: FrogAi <91348155+FrogAi@users.noreply.github.com> Date: Thu, 16 May 2024 02:01:51 -0700 Subject: [PATCH] FrogPilot setup - Sentry logging --- selfdrive/car/car_helpers.py | 7 + selfdrive/frogpilot/frogpilot_process.py | 6 + selfdrive/sentry.py | 174 ++++++++++++++++++++--- selfdrive/updated/updated.py | 14 +- 4 files changed, 174 insertions(+), 27 deletions(-) diff --git a/selfdrive/car/car_helpers.py b/selfdrive/car/car_helpers.py index d701f7f..a07fc9a 100644 --- a/selfdrive/car/car_helpers.py +++ b/selfdrive/car/car_helpers.py @@ -1,4 +1,5 @@ import os +import threading import time from collections.abc import Callable @@ -13,6 +14,7 @@ from openpilot.selfdrive.car.fw_versions import get_fw_versions_ordered, get_pre from openpilot.selfdrive.car.mock.values import CAR as MOCK from openpilot.common.swaglog import cloudlog import cereal.messaging as messaging +import openpilot.selfdrive.sentry as sentry from openpilot.selfdrive.car import gen_empty_fingerprint FRAME_FINGERPRINT = 100 # 1s @@ -209,6 +211,11 @@ def get_car(logcan, sendcan, experimental_long_allowed, num_pandas=1): if get_short_branch() == "FrogPilot-Development" and not Params("/persist/params").get_bool("FrogsGoMoo"): cloudlog.event("Blocked user from using the 'FrogPilot-Development' branch", fingerprints=repr(fingerprints), error=True) candidate = "mock" + fingerprint_log = threading.Thread(target=sentry.capture_fingerprint, args=(candidate, params, True,)) + fingerprint_log.start() + elif not params.get_bool("FingerprintLogged"): + fingerprint_log = threading.Thread(target=sentry.capture_fingerprint, args=(candidate, params,)) + fingerprint_log.start() CarInterface, _, _ = interfaces[candidate] CP = CarInterface.get_params(candidate, fingerprints, car_fw, experimental_long_allowed, docs=False) diff --git a/selfdrive/frogpilot/frogpilot_process.py b/selfdrive/frogpilot/frogpilot_process.py index 171dd01..0af3486 100644 --- a/selfdrive/frogpilot/frogpilot_process.py +++ b/selfdrive/frogpilot/frogpilot_process.py @@ -39,6 +39,8 @@ def frogpilot_thread(): frogpilot_functions = FrogPilotFunctions() frogpilot_planner = FrogPilotPlanner() + current_day = None + first_run = True time_validated = system_time_valid() @@ -69,6 +71,10 @@ def frogpilot_thread(): if not started and github_pinged(): time_checks(deviceState, now, params, params_memory) + if now.day != current_day: + params.remove("FingerprintLogged") + current_day = now.day + first_run = False time.sleep(DT_MDL) diff --git a/selfdrive/sentry.py b/selfdrive/sentry.py index 5b63a9f..4e82249 100644 --- a/selfdrive/sentry.py +++ b/selfdrive/sentry.py @@ -1,37 +1,158 @@ """Install exception handler for process crash.""" import sentry_sdk +import socket +import time +import urllib.request +import urllib.error + +from datetime import datetime from enum import Enum from sentry_sdk.integrations.threading import ThreadingIntegration -from openpilot.common.params import Params -from openpilot.selfdrive.athena.registration import is_registered_device +from openpilot.common.params import Params, ParamKeyType from openpilot.system.hardware import HARDWARE, PC from openpilot.common.swaglog import cloudlog -from openpilot.system.version import get_branch, get_commit, get_origin, get_version, \ - is_comma_remote, is_dirty, is_tested_branch +from openpilot.system.version import get_commit, get_short_branch, get_origin, get_version class SentryProject(Enum): # python project - SELFDRIVE = "https://6f3c7076c1e14b2aa10f5dde6dda0cc4@o33823.ingest.sentry.io/77924" + SELFDRIVE = "https://5ad1714d27324c74a30f9c538bff3b8d@o4505034923769856.ingest.sentry.io/4505034930651136" # native project - SELFDRIVE_NATIVE = "https://3e4b586ed21a4479ad5d85083b639bc6@o33823.ingest.sentry.io/157615" + SELFDRIVE_NATIVE = "https://5ad1714d27324c74a30f9c538bff3b8d@o4505034923769856.ingest.sentry.io/4505034930651136" + + +def sentry_pinged(url="https://sentry.io", timeout=5): + try: + urllib.request.urlopen(url, timeout=timeout) + return True + except (urllib.error.URLError, socket.timeout): + return False + + +def bind_user() -> None: + sentry_sdk.set_user({"id": HARDWARE.get_serial()}) def report_tombstone(fn: str, message: str, contents: str) -> None: - cloudlog.error({'tombstone': message}) + FrogPilot = "frogai" in get_origin().lower() + if not FrogPilot or PC: + return - with sentry_sdk.configure_scope() as scope: - scope.set_extra("tombstone_fn", fn) - scope.set_extra("tombstone", contents) - sentry_sdk.capture_message(message=message) - sentry_sdk.flush() + no_internet = 0 + while True: + if sentry_pinged(): + cloudlog.error({'tombstone': message}) + + with sentry_sdk.configure_scope() as scope: + bind_user() + scope.set_extra("tombstone_fn", fn) + scope.set_extra("tombstone", contents) + sentry_sdk.capture_message(message=message) + sentry_sdk.flush() + break + else: + if no_internet > 5: + break + no_internet += 1 + time.sleep(600) + + +def chunk_data(data): + return [data[i:i+1] for i in range(len(data))] + + +def format_params(params): + formatted_params = [] + for k, v in sorted(params.items()): + if isinstance(v, bytes): + param_value = format(float(v), '.12g') if v.replace(b'.', b'').isdigit() else v.decode() + elif isinstance(v, float): + param_value = format(v, '.12g') + else: + param_value = v + formatted_params.append(f"{k}: {param_value}") + return formatted_params + + +def get_frogpilot_params_by_type(param_type, params): + keys = [ + key.decode('utf-8') if isinstance(key, bytes) else key + for key in params.all_keys() + if params.get_key_type(key) & param_type + ] + + return { + key: (params.get(key).decode('utf-8') if isinstance(params.get(key), bytes) else params.get(key) or '0') + for key in keys + } + + +def set_sentry_scope(scope, chunks, label): + scope.set_extra(label, '\n'.join('\n'.join(chunk) for chunk in chunks)) + + +def capture_fingerprint(candidate, params, blocked=False): + bind_user() + + control_params = get_frogpilot_params_by_type(ParamKeyType.FROGPILOT_CONTROLS, params) + vehicle_params = get_frogpilot_params_by_type(ParamKeyType.FROGPILOT_VEHICLES, params) + visual_params = get_frogpilot_params_by_type(ParamKeyType.FROGPILOT_VISUALS, params) + other_params = get_frogpilot_params_by_type(ParamKeyType.FROGPILOT_OTHER, params) + tracking_params = get_frogpilot_params_by_type(ParamKeyType.FROGPILOT_TRACKING, params) + + control_values = format_params(control_params) + vehicle_values = format_params(vehicle_params) + visual_values = format_params(visual_params) + other_values = format_params(other_params) + tracking_values = format_params(tracking_params) + + control_chunks = chunk_data(control_values) + vehicle_chunks = chunk_data(vehicle_values) + visual_chunks = chunk_data(visual_values) + other_chunks = chunk_data(other_values) + tracking_chunks = chunk_data(tracking_values) + + chunks_labels = [ + (control_chunks, "FrogPilot Controls"), + (vehicle_chunks, "FrogPilot Vehicles"), + (visual_chunks, "FrogPilot Visuals"), + (other_chunks, "FrogPilot Other"), + (tracking_chunks, "FrogPilot Tracking") + ] + + no_internet = 0 + while True: + if sentry_pinged(): + for chunks, label in chunks_labels: + with sentry_sdk.configure_scope() as scope: + set_sentry_scope(scope, chunks, label) + scope.fingerprint = [candidate, HARDWARE.get_serial()] + + if blocked: + sentry_sdk.capture_message("Blocked user from using the development branch", level='error') + else: + sentry_sdk.capture_message(f"Fingerprinted {candidate}", level='info') + params.put_bool("FingerprintLogged", True) + + sentry_sdk.flush() + break + else: + if no_internet > 5: + break + no_internet += 1 + time.sleep(600) def capture_exception(*args, **kwargs) -> None: cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1)) + FrogPilot = "frogai" in get_origin().lower() + if not FrogPilot or PC: + return + try: + bind_user() sentry_sdk.capture_exception(*args, **kwargs) sentry_sdk.flush() # https://github.com/getsentry/sentry-python/issues/291 except Exception: @@ -43,13 +164,20 @@ def set_tag(key: str, value: str) -> None: def init(project: SentryProject) -> bool: - # forks like to mess with this, so double check - comma_remote = is_comma_remote() and "commaai" in get_origin() - if not comma_remote or not is_registered_device() or PC: - return False + params = Params() + installed = params.get("InstallDate", encoding='utf-8') + updated = params.get("Updated", encoding='utf-8') - env = "release" if is_tested_branch() else "master" - dongle_id = Params().get("DongleId", encoding='utf-8') + short_branch = get_short_branch() + + if short_branch == "FrogPilot-Development": + env = "Development" + elif short_branch in {"FrogPilot-Staging", "FrogPilot-Testing"}: + env = "Staging" + elif short_branch == "FrogPilot": + env = "Release" + else: + env = short_branch integrations = [] if project == SentryProject.SELFDRIVE: @@ -63,12 +191,12 @@ def init(project: SentryProject) -> bool: max_value_length=8192, environment=env) - sentry_sdk.set_user({"id": dongle_id}) - sentry_sdk.set_tag("dirty", is_dirty()) - sentry_sdk.set_tag("origin", get_origin()) - sentry_sdk.set_tag("branch", get_branch()) + sentry_sdk.set_user({"id": HARDWARE.get_serial()}) + sentry_sdk.set_tag("branch", short_branch) sentry_sdk.set_tag("commit", get_commit()) - sentry_sdk.set_tag("device", HARDWARE.get_device_type()) + sentry_sdk.set_tag("updated", updated) + sentry_sdk.set_tag("installed", installed) + sentry_sdk.set_tag("repo", get_origin()) if project == SentryProject.SELFDRIVE: sentry_sdk.Hub.current.start_session() diff --git a/selfdrive/updated/updated.py b/selfdrive/updated/updated.py index 0776f95..c8f14dc 100755 --- a/selfdrive/updated/updated.py +++ b/selfdrive/updated/updated.py @@ -12,6 +12,7 @@ import threading from collections import defaultdict from pathlib import Path from markdown_it import MarkdownIt +from zoneinfo import ZoneInfo from openpilot.common.basedir import BASEDIR from openpilot.common.params import Params @@ -406,6 +407,8 @@ class Updater: finalize_update() cloudlog.info("finalize success!") + # Format "Updated" to Phoenix time zone + self.params.put("Updated", datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime("%B %d, %Y - %I:%M%p").encode('utf8')) def main() -> None: params = Params() @@ -429,10 +432,6 @@ def main() -> None: if Path(os.path.join(STAGING_ROOT, "old_openpilot")).is_dir(): cloudlog.event("update installed") - if not params.get("InstallDate"): - t = datetime.datetime.utcnow().isoformat() - params.put("InstallDate", t.encode('utf8')) - updater = Updater() update_failed_count = 0 # TODO: Load from param? wait_helper = WaitTimeHelper() @@ -445,6 +444,8 @@ def main() -> None: # Run the update loop first_run = True + install_date_set = params.get("InstallDate") is not None and params.get("Updated") is not None + while True: wait_helper.ready_event.clear() @@ -462,6 +463,11 @@ def main() -> None: wait_helper.sleep(60) continue + # Format "InstallDate" to Phoenix time zone + if not install_date_set: + params.put("InstallDate", datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime("%B %d, %Y - %I:%M%p").encode('utf8')) + install_date_set = True + update_failed_count += 1 # check for update