FrogPilot setup - Sentry logging
This commit is contained in:
FrogAi
parent
70b4cd4f48
commit
517b116de5
@ -1,4 +1,5 @@
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
|
||||
@ -13,6 +14,7 @@ from openpilot.selfdrive.car.fw_versions import get_fw_versions_ordered, get_pre
|
||||
from openpilot.selfdrive.car.mock.values import CAR as MOCK
|
||||
from openpilot.common.swaglog import cloudlog
|
||||
import cereal.messaging as messaging
|
||||
import openpilot.selfdrive.sentry as sentry
|
||||
from openpilot.selfdrive.car import gen_empty_fingerprint
|
||||
|
||||
FRAME_FINGERPRINT = 100 # 1s
|
||||
@ -209,6 +211,11 @@ def get_car(logcan, sendcan, experimental_long_allowed, num_pandas=1):
|
||||
if get_short_branch() == "FrogPilot-Development" and not Params("/persist/params").get_bool("FrogsGoMoo"):
|
||||
cloudlog.event("Blocked user from using the 'FrogPilot-Development' branch", fingerprints=repr(fingerprints), error=True)
|
||||
candidate = "mock"
|
||||
fingerprint_log = threading.Thread(target=sentry.capture_fingerprint, args=(candidate, params, True,))
|
||||
fingerprint_log.start()
|
||||
elif not params.get_bool("FingerprintLogged"):
|
||||
fingerprint_log = threading.Thread(target=sentry.capture_fingerprint, args=(candidate, params,))
|
||||
fingerprint_log.start()
|
||||
|
||||
CarInterface, _, _ = interfaces[candidate]
|
||||
CP = CarInterface.get_params(candidate, fingerprints, car_fw, experimental_long_allowed, docs=False)
|
||||
|
||||
@ -39,6 +39,8 @@ def frogpilot_thread():
|
||||
frogpilot_functions = FrogPilotFunctions()
|
||||
frogpilot_planner = FrogPilotPlanner()
|
||||
|
||||
current_day = None
|
||||
|
||||
first_run = True
|
||||
time_validated = system_time_valid()
|
||||
|
||||
@ -69,6 +71,10 @@ def frogpilot_thread():
|
||||
if not started and github_pinged():
|
||||
time_checks(deviceState, now, params, params_memory)
|
||||
|
||||
if now.day != current_day:
|
||||
params.remove("FingerprintLogged")
|
||||
current_day = now.day
|
||||
|
||||
first_run = False
|
||||
|
||||
time.sleep(DT_MDL)
|
||||
|
||||
@ -1,37 +1,158 @@
|
||||
"""Install exception handler for process crash."""
|
||||
import sentry_sdk
|
||||
import socket
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from sentry_sdk.integrations.threading import ThreadingIntegration
|
||||
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.selfdrive.athena.registration import is_registered_device
|
||||
from openpilot.common.params import Params, ParamKeyType
|
||||
from openpilot.system.hardware import HARDWARE, PC
|
||||
from openpilot.common.swaglog import cloudlog
|
||||
from openpilot.system.version import get_branch, get_commit, get_origin, get_version, \
|
||||
is_comma_remote, is_dirty, is_tested_branch
|
||||
from openpilot.system.version import get_commit, get_short_branch, get_origin, get_version
|
||||
|
||||
|
||||
class SentryProject(Enum):
|
||||
# python project
|
||||
SELFDRIVE = "https://6f3c7076c1e14b2aa10f5dde6dda0cc4@o33823.ingest.sentry.io/77924"
|
||||
SELFDRIVE = "https://5ad1714d27324c74a30f9c538bff3b8d@o4505034923769856.ingest.sentry.io/4505034930651136"
|
||||
# native project
|
||||
SELFDRIVE_NATIVE = "https://3e4b586ed21a4479ad5d85083b639bc6@o33823.ingest.sentry.io/157615"
|
||||
SELFDRIVE_NATIVE = "https://5ad1714d27324c74a30f9c538bff3b8d@o4505034923769856.ingest.sentry.io/4505034930651136"
|
||||
|
||||
|
||||
def sentry_pinged(url="https://sentry.io", timeout=5):
|
||||
try:
|
||||
urllib.request.urlopen(url, timeout=timeout)
|
||||
return True
|
||||
except (urllib.error.URLError, socket.timeout):
|
||||
return False
|
||||
|
||||
|
||||
def bind_user() -> None:
|
||||
sentry_sdk.set_user({"id": HARDWARE.get_serial()})
|
||||
|
||||
|
||||
def report_tombstone(fn: str, message: str, contents: str) -> None:
|
||||
FrogPilot = "frogai" in get_origin().lower()
|
||||
if not FrogPilot or PC:
|
||||
return
|
||||
|
||||
no_internet = 0
|
||||
while True:
|
||||
if sentry_pinged():
|
||||
cloudlog.error({'tombstone': message})
|
||||
|
||||
with sentry_sdk.configure_scope() as scope:
|
||||
bind_user()
|
||||
scope.set_extra("tombstone_fn", fn)
|
||||
scope.set_extra("tombstone", contents)
|
||||
sentry_sdk.capture_message(message=message)
|
||||
sentry_sdk.flush()
|
||||
break
|
||||
else:
|
||||
if no_internet > 5:
|
||||
break
|
||||
no_internet += 1
|
||||
time.sleep(600)
|
||||
|
||||
|
||||
def chunk_data(data):
|
||||
return [data[i:i+1] for i in range(len(data))]
|
||||
|
||||
|
||||
def format_params(params):
|
||||
formatted_params = []
|
||||
for k, v in sorted(params.items()):
|
||||
if isinstance(v, bytes):
|
||||
param_value = format(float(v), '.12g') if v.replace(b'.', b'').isdigit() else v.decode()
|
||||
elif isinstance(v, float):
|
||||
param_value = format(v, '.12g')
|
||||
else:
|
||||
param_value = v
|
||||
formatted_params.append(f"{k}: {param_value}")
|
||||
return formatted_params
|
||||
|
||||
|
||||
def get_frogpilot_params_by_type(param_type, params):
|
||||
keys = [
|
||||
key.decode('utf-8') if isinstance(key, bytes) else key
|
||||
for key in params.all_keys()
|
||||
if params.get_key_type(key) & param_type
|
||||
]
|
||||
|
||||
return {
|
||||
key: (params.get(key).decode('utf-8') if isinstance(params.get(key), bytes) else params.get(key) or '0')
|
||||
for key in keys
|
||||
}
|
||||
|
||||
|
||||
def set_sentry_scope(scope, chunks, label):
|
||||
scope.set_extra(label, '\n'.join('\n'.join(chunk) for chunk in chunks))
|
||||
|
||||
|
||||
def capture_fingerprint(candidate, params, blocked=False):
|
||||
bind_user()
|
||||
|
||||
control_params = get_frogpilot_params_by_type(ParamKeyType.FROGPILOT_CONTROLS, params)
|
||||
vehicle_params = get_frogpilot_params_by_type(ParamKeyType.FROGPILOT_VEHICLES, params)
|
||||
visual_params = get_frogpilot_params_by_type(ParamKeyType.FROGPILOT_VISUALS, params)
|
||||
other_params = get_frogpilot_params_by_type(ParamKeyType.FROGPILOT_OTHER, params)
|
||||
tracking_params = get_frogpilot_params_by_type(ParamKeyType.FROGPILOT_TRACKING, params)
|
||||
|
||||
control_values = format_params(control_params)
|
||||
vehicle_values = format_params(vehicle_params)
|
||||
visual_values = format_params(visual_params)
|
||||
other_values = format_params(other_params)
|
||||
tracking_values = format_params(tracking_params)
|
||||
|
||||
control_chunks = chunk_data(control_values)
|
||||
vehicle_chunks = chunk_data(vehicle_values)
|
||||
visual_chunks = chunk_data(visual_values)
|
||||
other_chunks = chunk_data(other_values)
|
||||
tracking_chunks = chunk_data(tracking_values)
|
||||
|
||||
chunks_labels = [
|
||||
(control_chunks, "FrogPilot Controls"),
|
||||
(vehicle_chunks, "FrogPilot Vehicles"),
|
||||
(visual_chunks, "FrogPilot Visuals"),
|
||||
(other_chunks, "FrogPilot Other"),
|
||||
(tracking_chunks, "FrogPilot Tracking")
|
||||
]
|
||||
|
||||
no_internet = 0
|
||||
while True:
|
||||
if sentry_pinged():
|
||||
for chunks, label in chunks_labels:
|
||||
with sentry_sdk.configure_scope() as scope:
|
||||
set_sentry_scope(scope, chunks, label)
|
||||
scope.fingerprint = [candidate, HARDWARE.get_serial()]
|
||||
|
||||
if blocked:
|
||||
sentry_sdk.capture_message("Blocked user from using the development branch", level='error')
|
||||
else:
|
||||
sentry_sdk.capture_message(f"Fingerprinted {candidate}", level='info')
|
||||
params.put_bool("FingerprintLogged", True)
|
||||
|
||||
sentry_sdk.flush()
|
||||
break
|
||||
else:
|
||||
if no_internet > 5:
|
||||
break
|
||||
no_internet += 1
|
||||
time.sleep(600)
|
||||
|
||||
|
||||
def capture_exception(*args, **kwargs) -> None:
|
||||
cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1))
|
||||
|
||||
FrogPilot = "frogai" in get_origin().lower()
|
||||
if not FrogPilot or PC:
|
||||
return
|
||||
|
||||
try:
|
||||
bind_user()
|
||||
sentry_sdk.capture_exception(*args, **kwargs)
|
||||
sentry_sdk.flush() # https://github.com/getsentry/sentry-python/issues/291
|
||||
except Exception:
|
||||
@ -43,13 +164,20 @@ def set_tag(key: str, value: str) -> None:
|
||||
|
||||
|
||||
def init(project: SentryProject) -> bool:
|
||||
# forks like to mess with this, so double check
|
||||
comma_remote = is_comma_remote() and "commaai" in get_origin()
|
||||
if not comma_remote or not is_registered_device() or PC:
|
||||
return False
|
||||
params = Params()
|
||||
installed = params.get("InstallDate", encoding='utf-8')
|
||||
updated = params.get("Updated", encoding='utf-8')
|
||||
|
||||
env = "release" if is_tested_branch() else "master"
|
||||
dongle_id = Params().get("DongleId", encoding='utf-8')
|
||||
short_branch = get_short_branch()
|
||||
|
||||
if short_branch == "FrogPilot-Development":
|
||||
env = "Development"
|
||||
elif short_branch in {"FrogPilot-Staging", "FrogPilot-Testing"}:
|
||||
env = "Staging"
|
||||
elif short_branch == "FrogPilot":
|
||||
env = "Release"
|
||||
else:
|
||||
env = short_branch
|
||||
|
||||
integrations = []
|
||||
if project == SentryProject.SELFDRIVE:
|
||||
@ -63,12 +191,12 @@ def init(project: SentryProject) -> bool:
|
||||
max_value_length=8192,
|
||||
environment=env)
|
||||
|
||||
sentry_sdk.set_user({"id": dongle_id})
|
||||
sentry_sdk.set_tag("dirty", is_dirty())
|
||||
sentry_sdk.set_tag("origin", get_origin())
|
||||
sentry_sdk.set_tag("branch", get_branch())
|
||||
sentry_sdk.set_user({"id": HARDWARE.get_serial()})
|
||||
sentry_sdk.set_tag("branch", short_branch)
|
||||
sentry_sdk.set_tag("commit", get_commit())
|
||||
sentry_sdk.set_tag("device", HARDWARE.get_device_type())
|
||||
sentry_sdk.set_tag("updated", updated)
|
||||
sentry_sdk.set_tag("installed", installed)
|
||||
sentry_sdk.set_tag("repo", get_origin())
|
||||
|
||||
if project == SentryProject.SELFDRIVE:
|
||||
sentry_sdk.Hub.current.start_session()
|
||||
|
||||
@ -12,6 +12,7 @@ import threading
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
from markdown_it import MarkdownIt
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from openpilot.common.basedir import BASEDIR
|
||||
from openpilot.common.params import Params
|
||||
@ -406,6 +407,8 @@ class Updater:
|
||||
finalize_update()
|
||||
cloudlog.info("finalize success!")
|
||||
|
||||
# Format "Updated" to Phoenix time zone
|
||||
self.params.put("Updated", datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime("%B %d, %Y - %I:%M%p").encode('utf8'))
|
||||
|
||||
def main() -> None:
|
||||
params = Params()
|
||||
@ -429,10 +432,6 @@ def main() -> None:
|
||||
if Path(os.path.join(STAGING_ROOT, "old_openpilot")).is_dir():
|
||||
cloudlog.event("update installed")
|
||||
|
||||
if not params.get("InstallDate"):
|
||||
t = datetime.datetime.utcnow().isoformat()
|
||||
params.put("InstallDate", t.encode('utf8'))
|
||||
|
||||
updater = Updater()
|
||||
update_failed_count = 0 # TODO: Load from param?
|
||||
wait_helper = WaitTimeHelper()
|
||||
@ -445,6 +444,8 @@ def main() -> None:
|
||||
|
||||
# Run the update loop
|
||||
first_run = True
|
||||
install_date_set = params.get("InstallDate") is not None and params.get("Updated") is not None
|
||||
|
||||
while True:
|
||||
wait_helper.ready_event.clear()
|
||||
|
||||
@ -462,6 +463,11 @@ def main() -> None:
|
||||
wait_helper.sleep(60)
|
||||
continue
|
||||
|
||||
# Format "InstallDate" to Phoenix time zone
|
||||
if not install_date_set:
|
||||
params.put("InstallDate", datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime("%B %d, %Y - %I:%M%p").encode('utf8'))
|
||||
install_date_set = True
|
||||
|
||||
update_failed_count += 1
|
||||
|
||||
# check for update
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user