diff --git a/release/files_common b/release/files_common index 9ed0675..eb7b613 100644 --- a/release/files_common +++ b/release/files_common @@ -571,5 +571,6 @@ selfdrive/frogpilot/controls/lib/conditional_experimental_mode.py selfdrive/frogpilot/controls/lib/frogpilot_functions.py selfdrive/frogpilot/controls/lib/frogpilot_variables.py selfdrive/frogpilot/controls/lib/map_turn_speed_controller.py +selfdrive/frogpilot/controls/lib/model_manager.py selfdrive/frogpilot/fleet_manager/fleet_manager.py selfdrive/frogpilot/navigation/mapd.py diff --git a/selfdrive/controls/controlsd.py b/selfdrive/controls/controlsd.py index 651c026..61794d8 100644 --- a/selfdrive/controls/controlsd.py +++ b/selfdrive/controls/controlsd.py @@ -107,6 +107,8 @@ class Controls: ignore = self.sensor_packets + ['testJoystick'] if SIMULATION: ignore += ['driverCameraState', 'managerState'] + if self.frogpilot_toggles.radarless_model: + ignore += ['radarState'] self.sm = messaging.SubMaster(['deviceState', 'pandaStates', 'peripheralState', 'modelV2', 'liveCalibration', 'carOutput', 'driverMonitoringState', 'longitudinalPlan', 'liveLocationKalman', 'managerState', 'liveParameters', 'radarState', 'liveTorqueParameters', @@ -341,8 +343,9 @@ class Controls: self.events.add(EventName.cameraFrameRate) if not REPLAY and self.rk.lagging: self.events.add(EventName.controlsdLagging) - if len(self.sm['radarState'].radarErrors) or (not self.rk.lagging and not self.sm.all_checks(['radarState'])): - self.events.add(EventName.radarFault) + if not self.frogpilot_toggles.radarless_model: + if len(self.sm['radarState'].radarErrors) or (not self.rk.lagging and not self.sm.all_checks(['radarState'])): + self.events.add(EventName.radarFault) if not self.sm.valid['pandaStates']: self.events.add(EventName.usbError) if CS.canTimeout: diff --git a/selfdrive/controls/lib/longitudinal_mpc_lib/long_mpc.py b/selfdrive/controls/lib/longitudinal_mpc_lib/long_mpc.py index 39ae82b..ed426fa 100644 --- a/selfdrive/controls/lib/longitudinal_mpc_lib/long_mpc.py +++ b/selfdrive/controls/lib/longitudinal_mpc_lib/long_mpc.py @@ -8,7 +8,6 @@ from openpilot.common.swaglog import cloudlog # WARNING: imports outside of constants will not trigger a rebuild from openpilot.selfdrive.modeld.constants import index_function from openpilot.selfdrive.car.interfaces import ACCEL_MIN -from openpilot.selfdrive.controls.radard import _LEAD_ACCEL_TAU if __name__ == '__main__': # generating code from openpilot.third_party.acados.acados_template import AcadosModel, AcadosOcp, AcadosOcpSolver @@ -44,6 +43,8 @@ CRASH_DISTANCE = .25 LEAD_DANGER_FACTOR = 0.75 LIMIT_COST = 1e6 ACADOS_SOLVER_TYPE = 'SQP_RTI' +# Default lead acceleration decay set to 50% at 1s +LEAD_ACCEL_TAU = 1.5 # Fewer timestamps don't hurt performance and lead to @@ -335,7 +336,7 @@ class LongitudinalMpc: x_lead = 50.0 v_lead = v_ego + 10.0 a_lead = 0.0 - a_lead_tau = _LEAD_ACCEL_TAU + a_lead_tau = LEAD_ACCEL_TAU # MPC will not converge if immediate crash is expected # Clip lead distance to what is still possible to brake for @@ -352,13 +353,13 @@ class LongitudinalMpc: self.cruise_min_a = min_a self.max_a = max_a - def update(self, radarstate, v_cruise, x, v, a, j, t_follow, trafficModeActive, frogpilot_toggles, personality=log.LongitudinalPersonality.standard): + def update(self, lead_one, lead_two, v_cruise, x, v, a, j, t_follow, trafficModeActive, frogpilot_toggles, personality=log.LongitudinalPersonality.standard): v_ego = self.x0[1] - self.status = radarstate.leadOne.status or radarstate.leadTwo.status + self.status = lead_one.status or lead_two.status increased_distance = frogpilot_toggles.increased_stopping_distance + min(CITY_SPEED_LIMIT - v_ego, 0) if not trafficModeActive else 0 - lead_xv_0 = self.process_lead(radarstate.leadOne, increased_distance) - lead_xv_1 = self.process_lead(radarstate.leadTwo) + lead_xv_0 = self.process_lead(lead_one, increased_distance) + lead_xv_1 = self.process_lead(lead_two) # To estimate a safe distance from a moving lead, we calculate how much stopping # distance that lead needs as a minimum. We can add that to the current distance @@ -417,8 +418,8 @@ class LongitudinalMpc: self.params[:,4] = t_follow self.run() - if (np.any(lead_xv_0[FCW_IDXS,0] - self.x_sol[FCW_IDXS,0] < CRASH_DISTANCE) and - radarstate.leadOne.modelProb > 0.9): + lead_probability = lead_one.prob if frogpilot_toggles.radarless_model else lead_one.modelProb + if (np.any(lead_xv_0[FCW_IDXS,0] - self.x_sol[FCW_IDXS,0] < CRASH_DISTANCE) and lead_probability > 0.9): self.crash_cnt += 1 else: self.crash_cnt = 0 diff --git a/selfdrive/controls/lib/longitudinal_planner.py b/selfdrive/controls/lib/longitudinal_planner.py index ab61b0b..56c7a4f 100755 --- a/selfdrive/controls/lib/longitudinal_planner.py +++ b/selfdrive/controls/lib/longitudinal_planner.py @@ -6,14 +6,15 @@ from openpilot.common.numpy_fast import clip, interp import cereal.messaging as messaging from openpilot.common.conversions import Conversions as CV from openpilot.common.filter_simple import FirstOrderFilter +from openpilot.common.simple_kalman import KF1D from openpilot.common.realtime import DT_MDL +from openpilot.common.swaglog import cloudlog from openpilot.selfdrive.modeld.constants import ModelConstants from openpilot.selfdrive.car.interfaces import ACCEL_MIN, ACCEL_MAX from openpilot.selfdrive.controls.lib.longcontrol import LongCtrlState from openpilot.selfdrive.controls.lib.longitudinal_mpc_lib.long_mpc import LongitudinalMpc -from openpilot.selfdrive.controls.lib.longitudinal_mpc_lib.long_mpc import T_IDXS as T_IDXS_MPC +from openpilot.selfdrive.controls.lib.longitudinal_mpc_lib.long_mpc import T_IDXS as T_IDXS_MPC, LEAD_ACCEL_TAU from openpilot.selfdrive.controls.lib.drive_helpers import V_CRUISE_MAX, CONTROL_N, get_speed_error -from openpilot.common.swaglog import cloudlog LON_MPC_STEP = 0.2 # first step is 0.2s A_CRUISE_MIN = -1.2 @@ -24,6 +25,9 @@ A_CRUISE_MAX_BP = [0., 10.0, 25., 40.] _A_TOTAL_MAX_V = [1.7, 3.2] _A_TOTAL_MAX_BP = [20., 40.] +# Kalman filter states enum +LEAD_KALMAN_SPEED, LEAD_KALMAN_ACCEL = 0, 1 + def get_max_accel(v_ego): return interp(v_ego, A_CRUISE_MAX_BP, A_CRUISE_MAX_VALS) @@ -44,6 +48,72 @@ def limit_accel_in_turns(v_ego, angle_steers, a_target, CP): return [a_target[0], min(a_target[1], a_x_allowed)] +def lead_kf(v_lead: float, dt: float = 0.05): + # Lead Kalman Filter params, calculating K from A, C, Q, R requires the control library. + # hardcoding a lookup table to compute K for values of radar_ts between 0.01s and 0.2s + assert dt > .01 and dt < .2, "Radar time step must be between .01s and 0.2s" + A = [[1.0, dt], [0.0, 1.0]] + C = [1.0, 0.0] + #Q = np.matrix([[10., 0.0], [0.0, 100.]]) + #R = 1e3 + #K = np.matrix([[ 0.05705578], [ 0.03073241]]) + dts = [dt * 0.01 for dt in range(1, 21)] + K0 = [0.12287673, 0.14556536, 0.16522756, 0.18281627, 0.1988689, 0.21372394, + 0.22761098, 0.24069424, 0.253096, 0.26491023, 0.27621103, 0.28705801, + 0.29750003, 0.30757767, 0.31732515, 0.32677158, 0.33594201, 0.34485814, + 0.35353899, 0.36200124] + K1 = [0.29666309, 0.29330885, 0.29042818, 0.28787125, 0.28555364, 0.28342219, + 0.28144091, 0.27958406, 0.27783249, 0.27617149, 0.27458948, 0.27307714, + 0.27162685, 0.27023228, 0.26888809, 0.26758976, 0.26633338, 0.26511557, + 0.26393339, 0.26278425] + K = [[interp(dt, dts, K0)], [interp(dt, dts, K1)]] + + kf = KF1D([[v_lead], [0.0]], A, C, K) + return kf + + +class Lead: + def __init__(self): + self.dRel = 0.0 + self.yRel = 0.0 + self.vLead = 0.0 + self.aLead = 0.0 + self.vLeadK = 0.0 + self.aLeadK = 0.0 + self.aLeadTau = LEAD_ACCEL_TAU + self.prob = 0.0 + self.status = False + + self.kf: KF1D | None = None + + def reset(self): + self.status = False + self.kf = None + self.aLeadTau = LEAD_ACCEL_TAU + + def update(self, dRel: float, yRel: float, vLead: float, aLead: float, prob: float): + self.dRel = dRel + self.yRel = yRel + self.vLead = vLead + self.aLead = aLead + self.prob = prob + self.status = True + + if self.kf is None: + self.kf = lead_kf(self.vLead) + else: + self.kf.update(self.vLead) + + self.vLeadK = float(self.kf.x[LEAD_KALMAN_SPEED][0]) + self.aLeadK = float(self.kf.x[LEAD_KALMAN_ACCEL][0]) + + # Learn if constant acceleration + if abs(self.aLeadK) < 0.5: + self.aLeadTau = LEAD_ACCEL_TAU + else: + self.aLeadTau *= 0.9 + + class LongitudinalPlanner: def __init__(self, CP, init_v=0.0, init_a=0.0, dt=DT_MDL): self.CP = CP @@ -55,6 +125,9 @@ class LongitudinalPlanner: self.v_desired_filter = FirstOrderFilter(init_v, 2.0, self.dt) self.v_model_error = 0.0 + self.lead_one = Lead() + self.lead_two = Lead() + self.v_desired_trajectory = np.zeros(CONTROL_N) self.a_desired_trajectory = np.zeros(CONTROL_N) self.j_desired_trajectory = np.zeros(CONTROL_N) @@ -121,11 +194,25 @@ class LongitudinalPlanner: accel_limits_turns[0] = min(accel_limits_turns[0], self.a_desired + 0.05) accel_limits_turns[1] = max(accel_limits_turns[1], self.a_desired - 0.05) + if frogpilot_toggles.radarless_model: + model_leads = list(sm['modelV2'].leadsV3) + # TODO lead state should be invalidated if its different point than the previous one + lead_states = [self.lead_one, self.lead_two] + for index in range(len(lead_states)): + if len(model_leads) > index: + model_lead = model_leads[index] + lead_states[index].update(model_lead.x[0], model_lead.y[0], model_lead.v[0], model_lead.a[0], model_lead.prob) + else: + lead_states[index].reset() + else: + self.lead_one = sm['radarState'].leadOne + self.lead_two = sm['radarState'].leadTwo + self.mpc.set_weights(sm['frogpilotPlan'].accelerationJerk, sm['frogpilotPlan'].speedJerk, prev_accel_constraint, personality=sm['controlsState'].personality) self.mpc.set_accel_limits(accel_limits_turns[0], accel_limits_turns[1]) self.mpc.set_cur_state(self.v_desired_filter.x, self.a_desired) x, v, a, j = self.parse_model(sm['modelV2'], self.v_model_error, v_ego, frogpilot_toggles.taco_tune) - self.mpc.update(sm['radarState'], sm['frogpilotPlan'].vCruise, x, v, a, j, sm['frogpilotPlan'].tFollow, + self.mpc.update(self.lead_one, self.lead_two, sm['frogpilotPlan'].vCruise, x, v, a, j, sm['frogpilotPlan'].tFollow, sm['frogpilotCarControl'].trafficModeActive, frogpilot_toggles, personality=sm['controlsState'].personality) self.v_desired_trajectory_full = np.interp(ModelConstants.T_IDXS, T_IDXS_MPC, self.mpc.v_solution) @@ -157,7 +244,7 @@ class LongitudinalPlanner: longitudinalPlan.accels = self.a_desired_trajectory.tolist() longitudinalPlan.jerks = self.j_desired_trajectory.tolist() - longitudinalPlan.hasLead = sm['radarState'].leadOne.status + longitudinalPlan.hasLead = self.lead_one.status longitudinalPlan.longitudinalPlanSource = self.mpc.source longitudinalPlan.fcw = self.fcw diff --git a/selfdrive/controls/radard.py b/selfdrive/controls/radard.py index 4819d7b..5cf8dc4 100755 --- a/selfdrive/controls/radard.py +++ b/selfdrive/controls/radard.py @@ -195,6 +195,8 @@ def get_lead(v_ego: float, ready: bool, tracks: dict[int, Track], lead_msg: capn class RadarD: def __init__(self, radar_ts: float, delay: int = 0): + self.points: dict[int, tuple[float, float, float]] = {} + self.current_time = 0.0 self.tracks: dict[int, Track] = {} @@ -206,6 +208,7 @@ class RadarD: self.radar_state: capnp._DynamicStructBuilder | None = None self.radar_state_valid = False + self.radar_tracks_valid = False self.ready = False @@ -289,6 +292,31 @@ class RadarD: } pm.send('liveTracks', tracks_msg) + def update_radardless(self, rr: Optional[car.RadarData]): + radar_points = [] + radar_errors = [] + if rr is not None: + radar_points = rr.points + radar_errors = rr.errors + + self.radar_tracks_valid = len(radar_errors) == 0 + + self.points = {} + for pt in radar_points: + self.points[pt.trackId] = (pt.dRel, pt.yRel, pt.vRel) + + def publish_radardless(self): + tracks_msg = messaging.new_message('liveTracks', len(self.points)) + tracks_msg.valid = self.radar_tracks_valid + for index, tid in enumerate(sorted(self.points.keys())): + tracks_msg.liveTracks[index] = { + "trackId": tid, + "dRel": float(self.points[tid][0]) + RADAR_TO_CAMERA, + "yRel": -float(self.points[tid][1]), + "vRel": float(self.points[tid][2]), + } + + return tracks_msg # fuses camera and radar data for best lead detection def main(): @@ -306,26 +334,41 @@ def main(): # *** setup messaging can_sock = messaging.sub_sock('can') - sm = messaging.SubMaster(['modelV2', 'carState'], frequency=int(1./DT_CTRL)) - pm = messaging.PubMaster(['radarState', 'liveTracks']) + pub_sock = messaging.pub_sock('liveTracks') RI = RadarInterface(CP) + # TODO timing is different between cars, need a single time step for all cars + # TODO just take the fastest one for now, and keep resending same messages for slower radars rk = Ratekeeper(1.0 / CP.radarTimeStep, print_delay_threshold=None) RD = RadarD(CP.radarTimeStep, RI.delay) - while 1: - can_strings = messaging.drain_sock_raw(can_sock, wait_for_one=True) - rr = RI.update(can_strings) - sm.update(0) - if rr is None: - continue + if not FrogPilotVariables.toggles.radarless_model: + sm = messaging.SubMaster(['modelV2', 'carState'], frequency=int(1./DT_CTRL)) + pm = messaging.PubMaster(['radarState', 'liveTracks']) - RD.update(sm, rr) - RD.publish(pm, -rk.remaining*1000.0) + while True: + can_strings = messaging.drain_sock_raw(can_sock, wait_for_one=True) + rr = RI.update(can_strings) + sm.update(0) + if rr is None: + continue - rk.monitor_time() + RD.update(sm, rr) + RD.publish(pm, -rk.remaining*1000.0) + rk.monitor_time() + else: + while True: + can_strings = messaging.drain_sock_raw(can_sock, wait_for_one=True) + rr = RI.update(can_strings) + if rr is None: + continue + RD.update_radardless(rr) + msg = RD.publish_radardless() + pub_sock.send(msg.to_bytes()) + + rk.monitor_time() if __name__ == "__main__": main() diff --git a/selfdrive/frogpilot/controls/frogpilot_planner.py b/selfdrive/frogpilot/controls/frogpilot_planner.py index c644633..dadfe61 100644 --- a/selfdrive/frogpilot/controls/frogpilot_planner.py +++ b/selfdrive/frogpilot/controls/frogpilot_planner.py @@ -52,6 +52,7 @@ class FrogPilotPlanner: self.params_memory = Params("/dev/shm/params") self.cem = ConditionalExperimentalMode() + self.lead_one = Lead() self.mtsc = MapTurnSpeedController() self.slower_lead = False @@ -63,7 +64,15 @@ class FrogPilotPlanner: self.speed_jerk = 0 def update(self, carState, controlsState, frogpilotCarControl, frogpilotCarState, frogpilotNavigation, liveLocationKalman, modelData, radarState, frogpilot_toggles): - self.lead_one = radarState.leadOne + if frogpilot_toggles.radarless_model: + model_leads = list(modelData.leadsV3) + if len(model_leads) > 0: + model_lead = model_leads[0] + self.lead_one.update(model_lead.x[0], model_lead.y[0], model_lead.v[0], model_lead.a[0], model_lead.prob) + else: + self.lead_one.reset() + else: + self.lead_one = radarState.leadOne v_cruise_kph = min(controlsState.vCruise, V_CRUISE_UNSET) v_cruise = v_cruise_kph * CV.KPH_TO_MS diff --git a/selfdrive/frogpilot/controls/lib/frogpilot_functions.py b/selfdrive/frogpilot/controls/lib/frogpilot_functions.py index cc8a913..cf51135 100644 --- a/selfdrive/frogpilot/controls/lib/frogpilot_functions.py +++ b/selfdrive/frogpilot/controls/lib/frogpilot_functions.py @@ -14,6 +14,7 @@ from openpilot.system.hardware import HARDWARE from openpilot.system.version import get_short_branch, get_commit_date from openpilot.selfdrive.frogpilot.controls.lib.frogpilot_variables import PROBABILITY, THRESHOLD +from openpilot.selfdrive.frogpilot.controls.lib.model_manager import MODELS_PATH def calculate_lane_width(lane, current_lane, road_edge): lane_x, lane_y = np.array(lane.x), np.array(lane.y) @@ -176,6 +177,7 @@ class FrogPilotFunctions: remount_persist = ['sudo', 'mount', '-o', 'remount,rw', '/persist'] cls.run_cmd(remount_persist, "Successfully remounted /persist as read-write.", "Failed to remount /persist.") + os.makedirs(MODELS_PATH, exist_ok=True) os.makedirs("/persist/params", exist_ok=True) frogpilot_boot_logo = f'{BASEDIR}/selfdrive/frogpilot/assets/other_images/frogpilot_boot_logo.png' diff --git a/selfdrive/frogpilot/controls/lib/model_manager.py b/selfdrive/frogpilot/controls/lib/model_manager.py new file mode 100644 index 0000000..f516f42 --- /dev/null +++ b/selfdrive/frogpilot/controls/lib/model_manager.py @@ -0,0 +1,110 @@ +import http.client +import os +import socket +import urllib.error +import urllib.request + +from openpilot.common.params import Params +from openpilot.system.version import get_short_branch + +VERSION = 'v1' if get_short_branch() == "FrogPilot" else 'v2' +GITHUB_REPOSITORY_URL = 'https://raw.githubusercontent.com/FrogAi/FrogPilot-Resources/' +GITLAB_REPOSITORY_URL = 'https://gitlab.com/FrogAi/FrogPilot-Resources/-/raw/' + +DEFAULT_MODEL = "wd-40" +DEFAULT_MODEL_NAME = "WD40 (Default)" +MODELS_PATH = '/data/models' + +NAVIGATION_MODELS = {"certified-herbalist", "duck-amigo", "los-angeles", "recertified-herbalist"} +RADARLESS_MODELS = {"radical-turtle"} + +params = Params() +params_memory = Params("/dev/shm/params") + +def ping_url(url, timeout=5): + try: + urllib.request.urlopen(url, timeout=timeout) + return True + except (urllib.error.URLError, socket.timeout, http.client.RemoteDisconnected): + return False + +def determine_url(model): + if ping_url(GITHUB_REPOSITORY_URL): + return f"{GITHUB_REPOSITORY_URL}/Models/{model}.thneed" + else: + return f"{GITLAB_REPOSITORY_URL}/Models/{model}.thneed" + +def delete_deprecated_models(): + populate_models() + + available_models = params.get("AvailableModels", encoding='utf-8').split(',') + + current_model = params.get("Model", block=True, encoding='utf-8') + current_model_file = os.path.join(MODELS_PATH, f"{current_model}.thneed") + + if current_model not in available_models or not os.path.exists(current_model_file): + params.put("Model", DEFAULT_MODEL) + params.put("ModelName", DEFAULT_MODEL_NAME) + + for model_file in os.listdir(MODELS_PATH): + if model_file.endswith('.thneed') and model_file[:-7] not in available_models: + os.remove(os.path.join(MODELS_PATH, model_file)) + +def download_model(): + model = params_memory.get("ModelToDownload", encoding='utf-8') + model_path = os.path.join(MODELS_PATH, f"{model}.thneed") + + if os.path.exists(model_path): + print(f"Model {model} already exists, skipping download.") + return + + url = determine_url(model) + + for attempt in range(3): + try: + with urllib.request.urlopen(url) as f: + total_file_size = int(f.getheader('Content-Length')) + if total_file_size == 0: + raise ValueError("File is empty") + + with open(model_path, 'wb') as output: + current_file_size = 0 + for chunk in iter(lambda: f.read(8192), b''): + output.write(chunk) + current_file_size += len(chunk) + progress = (current_file_size / total_file_size) * 100 + params_memory.put_int("ModelDownloadProgress", int(progress)) + os.fsync(output) + + verify_download(model_path, total_file_size) + return + except Exception as e: + handle_download_error(model_path, attempt, e, url) + +def verify_download(model_path, total_file_size): + if os.path.getsize(model_path) == total_file_size: + print(f"Successfully downloaded the model!") + else: + raise Exception("Downloaded file size does not match expected size.") + +def handle_download_error(model_path, attempt, exception, url): + print(f"Attempt {attempt + 1} failed with error: {exception}. Retrying...") + if os.path.exists(model_path): + os.remove(model_path) + if attempt == 2: + print(f"Failed to download the model after 3 attempts from {url}") + +def populate_models(): + url = f"{GITHUB_REPOSITORY_URL}Versions/model_names_{VERSION}.txt" if ping_url(GITHUB_REPOSITORY_URL) else f"{GITLAB_REPOSITORY_URL}Versions/model_names_{VERSION}.txt" + try: + with urllib.request.urlopen(url) as response: + model_info = [line.decode('utf-8').strip().split(' - ') for line in response.readlines()] + update_params(model_info) + except Exception as e: + print(f"Failed to update models list. Error: {e}") + +def update_params(model_info): + available_models = ','.join(model[0] for model in model_info) + params.put("AvailableModels", available_models) + params.put("AvailableModelsNames", ','.join(model[1] for model in model_info)) + print("Models list updated successfully.") diff --git a/selfdrive/frogpilot/frogpilot_process.py b/selfdrive/frogpilot/frogpilot_process.py index 307d1e1..0b60096 100644 --- a/selfdrive/frogpilot/frogpilot_process.py +++ b/selfdrive/frogpilot/frogpilot_process.py @@ -17,6 +17,7 @@ from openpilot.system.hardware import HARDWARE from openpilot.selfdrive.frogpilot.controls.frogpilot_planner import FrogPilotPlanner from openpilot.selfdrive.frogpilot.controls.lib.frogpilot_functions import FrogPilotFunctions from openpilot.selfdrive.frogpilot.controls.lib.frogpilot_variables import FrogPilotVariables +from openpilot.selfdrive.frogpilot.controls.lib.model_manager import DEFAULT_MODEL, DEFAULT_MODEL_NAME, download_model, populate_models WIFI = log.DeviceState.NetworkType.wifi @@ -40,6 +41,8 @@ def automatic_update_check(params): os.system("pkill -SIGUSR1 -f selfdrive.updated.updated") def time_checks(automatic_updates, deviceState, maps_downloaded, now, params, params_memory): + populate_models() + screen_off = deviceState.screenBrightnessPercent == 0 wifi_connection = deviceState.networkType == WIFI @@ -82,6 +85,7 @@ def frogpilot_thread(frogpilot_toggles): first_run = True maps_downloaded = os.path.exists('/data/media/0/osm/offline') or params.get("MapsSelected") is None + model_list_empty = params.get("AvailableModelsNames", encoding='utf-8') is None time_validated = system_time_valid() pm = messaging.PubMaster(['frogpilotPlan']) @@ -102,9 +106,16 @@ def frogpilot_thread(frogpilot_toggles): sm['frogpilotNavigation'], sm['liveLocationKalman'], sm['modelV2'], sm['radarState'], frogpilot_toggles) frogpilot_planner.publish(sm, pm, frogpilot_toggles) + if params_memory.get("ModelToDownload", encoding='utf-8') is not None: + download_model() + if FrogPilotVariables.toggles_updated: FrogPilotVariables.update_frogpilot_params(started) + if not frogpilot_toggles.model_selector: + params.put("Model", DEFAULT_MODEL) + params.put("ModelName", DEFAULT_MODEL_NAME) + if not started and time_validated: frogpilot_functions.backup_toggles() @@ -117,6 +128,7 @@ def frogpilot_thread(frogpilot_toggles): if (not started or not maps_downloaded) and github_pinged(): time_checks(frogpilot_toggles.automatic_updates, deviceState, maps_downloaded, now, params, params_memory) maps_downloaded = os.path.exists('/data/media/0/osm/offline') or params.get("OSMDownloadProgress") is not None or params.get("MapsSelected") is None + model_list_empty = params.get("AvailableModelsNames", encoding='utf-8') is None if now.day != current_day: params.remove("FingerprintLogged") diff --git a/selfdrive/manager/manager.py b/selfdrive/manager/manager.py old mode 100644 new mode 100755 index 9ed8523..f3e8627 --- a/selfdrive/manager/manager.py +++ b/selfdrive/manager/manager.py @@ -25,6 +25,7 @@ from openpilot.system.version import is_dirty, get_commit, get_version, get_orig is_tested_branch, is_release_branch, get_commit_date from openpilot.selfdrive.frogpilot.controls.lib.frogpilot_functions import FrogPilotFunctions +from openpilot.selfdrive.frogpilot.controls.lib.model_manager import DEFAULT_MODEL, DEFAULT_MODEL_NAME, delete_deprecated_models def frogpilot_boot_functions(frogpilot_functions): @@ -33,6 +34,12 @@ def frogpilot_boot_functions(frogpilot_functions): print("Waiting for system time to become valid...") time.sleep(1) + try: + delete_deprecated_models() + except subprocess.CalledProcessError as e: + print(f"Failed to delete deprecated models. Error: {e}") + return + try: frogpilot_functions.backup_frogpilot() except subprocess.CalledProcessError as e: diff --git a/selfdrive/modeld/constants.py b/selfdrive/modeld/constants.py index dda1ff5..3ff4712 100644 --- a/selfdrive/modeld/constants.py +++ b/selfdrive/modeld/constants.py @@ -24,6 +24,7 @@ class ModelConstants: LAT_PLANNER_STATE_LEN = 4 LATERAL_CONTROL_PARAMS_LEN = 2 PREV_DESIRED_CURV_LEN = 1 + RADAR_TRACKS_LEN = 64 # model outputs constants FCW_THRESHOLDS_5MS2 = np.array([.05, .05, .15, .15, .15], dtype=np.float32) @@ -42,6 +43,7 @@ class ModelConstants: DESIRE_PRED_WIDTH = 8 LAT_PLANNER_SOLUTION_WIDTH = 4 DESIRED_CURV_WIDTH = 1 + RADAR_TRACKS_WIDTH = 3 NUM_LANE_LINES = 4 NUM_ROAD_EDGES = 2 diff --git a/selfdrive/modeld/modeld.py b/selfdrive/modeld/modeld.py index 1a52489..659ade4 100755 --- a/selfdrive/modeld/modeld.py +++ b/selfdrive/modeld/modeld.py @@ -25,12 +25,18 @@ from openpilot.selfdrive.modeld.constants import ModelConstants from openpilot.selfdrive.modeld.models.commonmodel_pyx import ModelFrame, CLContext from openpilot.selfdrive.frogpilot.controls.lib.frogpilot_variables import FrogPilotVariables +from openpilot.selfdrive.frogpilot.controls.lib.model_manager import DEFAULT_MODEL, MODELS_PATH, NAVIGATION_MODELS, RADARLESS_MODELS PROCESS_NAME = "selfdrive.modeld.modeld" SEND_RAW_PRED = os.getenv('SEND_RAW_PRED') +MODEL_NAME = Params().get("Model", block=True, encoding='utf-8') + +DISABLE_NAV = MODEL_NAME not in NAVIGATION_MODELS +DISABLE_RADAR = MODEL_NAME in RADARLESS_MODELS + MODEL_PATHS = { - ModelRunner.THNEED: Path(__file__).parent / 'models/supercombo.thneed', + ModelRunner.THNEED: Path(__file__).parent / ('models/supercombo.thneed' if MODEL_NAME == DEFAULT_MODEL else f'{MODELS_PATH}/{MODEL_NAME}.thneed'), ModelRunner.ONNX: Path(__file__).parent / 'models/supercombo.onnx'} METADATA_PATH = Path(__file__).parent / 'models/supercombo_metadata.pkl' @@ -61,9 +67,10 @@ class ModelState: 'traffic_convention': np.zeros(ModelConstants.TRAFFIC_CONVENTION_LEN, dtype=np.float32), 'lateral_control_params': np.zeros(ModelConstants.LATERAL_CONTROL_PARAMS_LEN, dtype=np.float32), 'prev_desired_curv': np.zeros(ModelConstants.PREV_DESIRED_CURV_LEN * (ModelConstants.HISTORY_BUFFER_LEN+1), dtype=np.float32), - 'nav_features': np.zeros(ModelConstants.NAV_FEATURE_LEN, dtype=np.float32), - 'nav_instructions': np.zeros(ModelConstants.NAV_INSTRUCTION_LEN, dtype=np.float32), + **({'nav_features': np.zeros(ModelConstants.NAV_FEATURE_LEN, dtype=np.float32), + 'nav_instructions': np.zeros(ModelConstants.NAV_INSTRUCTION_LEN, dtype=np.float32)} if not DISABLE_NAV else {}), 'features_buffer': np.zeros(ModelConstants.HISTORY_BUFFER_LEN * ModelConstants.FEATURE_LEN, dtype=np.float32), + **({'radar_tracks': np.zeros(ModelConstants.RADAR_TRACKS_LEN * ModelConstants.RADAR_TRACKS_WIDTH, dtype=np.float32)} if DISABLE_RADAR else {}), } with open(METADATA_PATH, 'rb') as f: @@ -96,8 +103,11 @@ class ModelState: self.inputs['traffic_convention'][:] = inputs['traffic_convention'] self.inputs['lateral_control_params'][:] = inputs['lateral_control_params'] - self.inputs['nav_features'][:] = inputs['nav_features'] - self.inputs['nav_instructions'][:] = inputs['nav_instructions'] + if not DISABLE_NAV: + self.inputs['nav_features'][:] = inputs['nav_features'] + self.inputs['nav_instructions'][:] = inputs['nav_instructions'] + if DISABLE_RADAR: + self.inputs['radar_tracks'][:] = inputs['radar_tracks'] # if getCLBuffer is not None, frame will be None self.model.setInputBuffer("input_imgs", self.frame.prepare(buf, transform.flatten(), self.model.getCLBuffer("input_imgs"))) @@ -156,7 +166,7 @@ def main(demo=False, frogpilot_toggles=None): # messaging pm = PubMaster(["modelV2", "cameraOdometry"]) - sm = SubMaster(["deviceState", "carState", "roadCameraState", "liveCalibration", "driverMonitoringState", "navModel", "navInstruction", "carControl", "frogpilotPlan"]) + sm = SubMaster(["deviceState", "carState", "roadCameraState", "liveCalibration", "driverMonitoringState", "navModel", "navInstruction", "carControl", "liveTracks", "frogpilotPlan"]) publish_state = PublishState() params = Params() @@ -245,7 +255,7 @@ def main(demo=False, frogpilot_toggles=None): # Enable/disable nav features timestamp_llk = sm["navModel"].locationMonoTime nav_valid = sm.valid["navModel"] # and (nanos_since_boot() - timestamp_llk < 1e9) - nav_enabled = nav_valid and params.get_bool("ExperimentalMode") + nav_enabled = nav_valid and not DISABLE_NAV if not nav_enabled: nav_features[:] = 0 @@ -266,6 +276,14 @@ def main(demo=False, frogpilot_toggles=None): if 0 <= distance_idx < 50: nav_instructions[distance_idx*3 + direction_idx] = 1 + radar_tracks = np.zeros(ModelConstants.RADAR_TRACKS_LEN * ModelConstants.RADAR_TRACKS_WIDTH, dtype=np.float32) + if sm.updated["liveTracks"]: + for i, track in enumerate(sm["liveTracks"]): + if i >= ModelConstants.RADAR_TRACKS_LEN: + break + vec_index = i * ModelConstants.RADAR_TRACKS_WIDTH + radar_tracks[vec_index:vec_index+ModelConstants.RADAR_TRACKS_WIDTH] = [track.dRel, track.yRel, track.vRel] + # tracked dropped frames vipc_dropped_frames = max(0, meta_main.frame_id - last_vipc_frame_id - 1) frames_dropped = frame_dropped_filter.update(min(vipc_dropped_frames, 10)) @@ -283,8 +301,9 @@ def main(demo=False, frogpilot_toggles=None): 'desire': vec_desire, 'traffic_convention': traffic_convention, 'lateral_control_params': lateral_control_params, - 'nav_features': nav_features, - 'nav_instructions': nav_instructions} + **({'nav_features': nav_features, 'nav_instructions': nav_instructions} if not DISABLE_NAV else {}), + **({'radar_tracks': radar_tracks,} if DISABLE_RADAR else {}), + } mt1 = time.perf_counter() model_output = model.run(buf_main, buf_extra, model_transform_main, model_transform_extra, inputs, prepare_only) diff --git a/selfdrive/ui/qt/home.cc b/selfdrive/ui/qt/home.cc index e711e73..4b04357 100644 --- a/selfdrive/ui/qt/home.cc +++ b/selfdrive/ui/qt/home.cc @@ -239,8 +239,10 @@ void OffroadHome::hideEvent(QHideEvent *event) { } void OffroadHome::refresh() { + QString model = QString::fromStdString(params.get("ModelName")); + date->setText(QLocale(uiState()->language.mid(5)).toString(QDateTime::currentDateTime(), "dddd, MMMM d")); - version->setText(getBrand() + " v" + getVersion().left(14).trimmed()); + version->setText(getBrand() + " v" + getVersion().left(14).trimmed() + " - " + model); bool updateAvailable = update_widget->refresh(); int alerts = alerts_widget->refresh(); diff --git a/selfdrive/ui/qt/onroad.cc b/selfdrive/ui/qt/onroad.cc index 387e030..33314ad 100644 --- a/selfdrive/ui/qt/onroad.cc +++ b/selfdrive/ui/qt/onroad.cc @@ -623,13 +623,13 @@ void AnnotatedCameraWidget::drawDriverState(QPainter &painter, const UIState *s) painter.restore(); } -void AnnotatedCameraWidget::drawLead(QPainter &painter, const cereal::RadarState::LeadData::Reader &lead_data, const QPointF &vd) { +void AnnotatedCameraWidget::drawLead(QPainter &painter, const cereal::ModelDataV2::LeadDataV3::Reader &lead_data, const QPointF &vd, const float v_ego) { painter.save(); const float speedBuff = 10.; const float leadBuff = 40.; - const float d_rel = lead_data.getDRel(); - const float v_rel = lead_data.getVRel(); + const float d_rel = lead_data.getX()[0]; + const float v_rel = lead_data.getV()[0] - v_ego; float fillAlpha = 0; if (d_rel < leadBuff) { @@ -664,6 +664,7 @@ void AnnotatedCameraWidget::paintGL() { SubMaster &sm = *(s->sm); const double start_draw_t = millis_since_boot(); const cereal::ModelDataV2::Reader &model = sm["modelV2"].getModelV2(); + const float v_ego = sm["carState"].getCarState().getVEgo(); // draw camera frame { @@ -685,7 +686,6 @@ void AnnotatedCameraWidget::paintGL() { // Wide or narrow cam dependent on speed bool has_wide_cam = available_streams.count(VISION_STREAM_WIDE_ROAD); if (has_wide_cam) { - float v_ego = sm["carState"].getCarState().getVEgo(); if ((v_ego < 10) || available_streams.size() == 1) { wide_cam_requested = true; } else if (v_ego > 15) { @@ -716,16 +716,16 @@ void AnnotatedCameraWidget::paintGL() { update_model(s, model, sm["uiPlan"].getUiPlan()); drawLaneLines(painter, s); - if (s->scene.longitudinal_control && sm.rcv_frame("radarState") > s->scene.started_frame) { - auto radar_state = sm["radarState"].getRadarState(); - update_leads(s, radar_state, model.getPosition()); - auto lead_one = radar_state.getLeadOne(); - auto lead_two = radar_state.getLeadTwo(); - if (lead_one.getStatus()) { - drawLead(painter, lead_one, s->scene.lead_vertices[0]); - } - if (lead_two.getStatus() && (std::abs(lead_one.getDRel() - lead_two.getDRel()) > 3.0)) { - drawLead(painter, lead_two, s->scene.lead_vertices[1]); + if (s->scene.longitudinal_control && sm.rcv_frame("modelV2") > s->scene.started_frame) { + update_leads(s, model); + float prev_drel = -1; + for (int i = 0; i < model.getLeadsV3().size() && i < 2; i++) { + const auto &lead = model.getLeadsV3()[i]; + auto lead_drel = lead.getX()[0]; + if (lead.getProb() > scene.lead_detection_threshold && (prev_drel < 0 || std::abs(lead_drel - prev_drel) > 3.0)) { + drawLead(painter, lead, s->scene.lead_vertices[i], v_ego); + } + prev_drel = lead_drel; } } } diff --git a/selfdrive/ui/qt/onroad.h b/selfdrive/ui/qt/onroad.h index efa8423..efcfd8b 100644 --- a/selfdrive/ui/qt/onroad.h +++ b/selfdrive/ui/qt/onroad.h @@ -174,7 +174,7 @@ protected: void showEvent(QShowEvent *event) override; void updateFrameMat() override; void drawLaneLines(QPainter &painter, const UIState *s); - void drawLead(QPainter &painter, const cereal::RadarState::LeadData::Reader &lead_data, const QPointF &vd); + void drawLead(QPainter &painter, const cereal::ModelDataV2::LeadDataV3::Reader &lead_data, const QPointF &vd, const float v_ego); void drawHud(QPainter &p); void drawDriverState(QPainter &painter, const UIState *s); inline QColor redColor(int alpha = 255) { return QColor(201, 34, 49, alpha); } diff --git a/selfdrive/ui/ui.cc b/selfdrive/ui/ui.cc index c9ca0d9..0d552ea 100644 --- a/selfdrive/ui/ui.cc +++ b/selfdrive/ui/ui.cc @@ -44,12 +44,15 @@ int get_path_length_idx(const cereal::XYZTData::Reader &line, const float path_h return max_idx; } -void update_leads(UIState *s, const cereal::RadarState::Reader &radar_state, const cereal::XYZTData::Reader &line) { - for (int i = 0; i < 2; ++i) { - auto lead_data = (i == 0) ? radar_state.getLeadOne() : radar_state.getLeadTwo(); - if (lead_data.getStatus()) { - float z = line.getZ()[get_path_length_idx(line, lead_data.getDRel())]; - calib_frame_to_full_frame(s, lead_data.getDRel(), -lead_data.getYRel(), z + 1.22, &s->scene.lead_vertices[i]); +void update_leads(UIState *s, const cereal::ModelDataV2::Reader &model_data) { + const cereal::XYZTData::Reader &line = model_data.getPosition(); + for (int i = 0; i < model_data.getLeadsV3().size() && i < 2; ++i) { + const auto &lead = model_data.getLeadsV3()[i]; + if (lead.getProb() > s->scene.lead_detection_threshold) { + float d_rel = lead.getX()[0]; + float y_rel = lead.getY()[0]; + float z = line.getZ()[get_path_length_idx(line, d_rel)]; + calib_frame_to_full_frame(s, d_rel, y_rel, z + 1.22, &s->scene.lead_vertices[i]); } } } @@ -108,10 +111,13 @@ void update_model(UIState *s, } // update path - auto lead_one = (*s->sm)["radarState"].getRadarState().getLeadOne(); - if (lead_one.getStatus()) { - const float lead_d = lead_one.getDRel() * 2.; - max_distance = std::clamp((float)(lead_d - fmin(lead_d * 0.35, 10.)), 0.0f, max_distance); + auto lead_count = model.getLeadsV3().size(); + if (lead_count > 0) { + auto lead_one = model.getLeadsV3()[0]; + if (lead_one.getProb() > scene.lead_detection_threshold) { + const float lead_d = lead_one.getX()[0] * 2.; + max_distance = std::clamp((float)(lead_d - fmin(lead_d * 0.35, 10.)), 0.0f, max_distance); + } } max_idx = get_path_length_idx(plan_position, max_distance); update_line_data(s, plan_position, 0.9, 1.22, &scene.track_vertices, max_idx, false); @@ -278,6 +284,10 @@ void ui_update_frogpilot_params(UIState *s) { scene.use_kaofui_icons = scene.onroad_distance_button && params.getBool("KaofuiIcons"); scene.experimental_mode_via_screen = scene.longitudinal_control && params.getBool("ExperimentalModeActivation") && params.getBool("ExperimentalModeViaTap"); + + bool longitudinal_tune = scene.longitudinal_control && params.getBool("LongitudinalTune"); + bool radarless_model = params.get("Model") == "radical-turtle"; + scene.lead_detection_threshold = longitudinal_tune && !radarless_model ? params.getInt("LeadDetectionThreshold") / 100.0f : 0.5; } void UIState::updateStatus() { diff --git a/selfdrive/ui/ui.h b/selfdrive/ui/ui.h index ee5b80a..81234ab 100644 --- a/selfdrive/ui/ui.h +++ b/selfdrive/ui/ui.h @@ -205,6 +205,7 @@ typedef struct UIScene { bool use_kaofui_icons; float adjusted_cruise; + float lead_detection_threshold; int alert_size; int conditional_speed; @@ -303,7 +304,7 @@ void update_model(UIState *s, const cereal::ModelDataV2::Reader &model, const cereal::UiPlan::Reader &plan); void update_dmonitoring(UIState *s, const cereal::DriverStateV2::Reader &driverstate, float dm_fade_state, bool is_rhd); -void update_leads(UIState *s, const cereal::RadarState::Reader &radar_state, const cereal::XYZTData::Reader &line); +void update_leads(UIState *s, const cereal::ModelDataV2::Reader &model_data); void update_line_data(const UIState *s, const cereal::XYZTData::Reader &line, float y_off, float z_off, QPolygonF *pvd, int max_idx, bool allow_invert);