test_models: fuzz test panda and CarState (commaai#30443)

* pre-hypothesis

* some hypothesis junk

* this kinda works but is really slow due to counter check

* choose addrs from fingerprint

* stash

* honda nidec brake pressed mismatches fixed

* bump panda

* stash

* tesla: use DI_torque2 (panda msg)

* run

* run

* ah this honda mismatch too

* no more multi can msgs

* clean up, remove old file

* add todo

* prob can remove urandom

* stash, huge examples

* fix pq standstill mismatch

* yuge

* yup there's a leak somewhere

* try to find leak

* skip dashcam (pq and tesla)

* PR comments

* bump

* draft stash

* fix alt brake hondas

* bump

* bump

* bump

* some clean up

* minor clean up

* more clean up

* stash

* fix honda bug

* more

* 100 examples

* revert tesla

* no memory leak any more?

* bring back tests with skips

* parameterize max_examples

* skip interceptor

* is jenkins on my branch?

* ooh that's fast

* 50 is not bad for GH CI

* 300 might be better with rest of test_models

* no more detection

* bump

* need CS_prev to catch bugs where openpilot changes and panda doesn't (eg. not setting interceptor safety mode)

* need to simplify all this

* need a warm up first, since some signals are 1 by default (toyota's gas_released!=1)

* changes

* set honda safety param

* set toyota safety param

* bump panda

* clean up honda

* rm interceptor

* thought interleaving addrs might help, but we can fine tune later

* Revert "thought interleaving addrs might help, but we can fine tune later"

This reverts commit 1533013.

* get size from dict

* what

* add nocapture marker

* clean up

* try to raise logging level

* need to run last as pytest_runtest_call, since it starts capturing

* get capman conditionally

* mark

* type fingerprint

* should use gen_empty_fingerprint

* no longer needed

* draft

* no longer need gc

* clean that up

* test everything!

* more clean up

* more

* no point

* fix that

* fix errors

* bump

* nice even 300 examples for 300 segs

* final bump :fingers_crossed:

* better import order

* remove debugging prints

* warm up kinda works

* Revert "warm up kinda works"

This reverts commit 7fc77b0.

* random seed

* revert

* strat

strat

* add expl comment

* cmt

* check controls allowed

* Revert "check controls allowed"

This reverts commit e82a0e5.

* not unittests

* run tests!

* run tests 2!

* run tests 3!

* seed unused

* revert

* add shrink phase, and remove health check suppression

* hello

* oncemore

* Update selfdrive/car/tests/test_models.py

Jenkinsfile

@@ -268,7 +268,7 @@ node {
       'car tests': {
         pcStage("car tests") {
           sh label: "build", script: "selfdrive/manager/build.py"
-          sh label: "run car tests", script: "cd selfdrive/car/tests && MAX_EXAMPLES=100 INTERNAL_SEG_CNT=250 FILEREADER_CACHE=1 \
+          sh label: "run car tests", script: "cd selfdrive/car/tests && MAX_EXAMPLES=300 INTERNAL_SEG_CNT=300 FILEREADER_CACHE=1 \
               INTERNAL_SEG_LIST=selfdrive/car/tests/test_models_segs.txt pytest test_models.py test_car_interfaces.py"
         }
       },

conftest.py

@@ -12,6 +12,17 @@ def pytest_sessionstart(session):
     session.config.option.randomly_reorganize = False
 
 
+@pytest.hookimpl(hookwrapper=True, trylast=True)
+def pytest_runtest_call(item):
+  # ensure we run as a hook after capturemanager's
+  if item.get_closest_marker("nocapture") is not None:
+    capmanager = item.config.pluginmanager.getplugin("capturemanager")
+    with capmanager.global_and_fixture_disabled():
+      yield
+  else:
+    yield
+
+
 @pytest.fixture(scope="function", autouse=True)
 def openpilot_function_fixture():
   starting_env = dict(os.environ)
@@ -58,7 +69,8 @@ def pytest_collection_modifyitems(config, items):
 
 @pytest.hookimpl(trylast=True)
 def pytest_configure(config):
-  config_line = (
-    "xdist_group_class_property: group tests by a property of the class that contains them"
-  )
+  config_line = "xdist_group_class_property: group tests by a property of the class that contains them"
+  config.addinivalue_line("markers", config_line)
+
+  config_line = "nocapture: don't capture test output"
   config.addinivalue_line("markers", config_line)

selfdrive/car/tests/test_models.py

@@ -6,10 +6,12 @@
 import random
 import unittest
 from collections import defaultdict, Counter
+import hypothesis.strategies as st
+from hypothesis import Phase, given, settings
 from typing import List, Optional, Tuple
 from parameterized import parameterized_class
 
-from cereal import log, car
+from cereal import messaging, log, car
 from openpilot.common.basedir import BASEDIR
 from openpilot.common.params import Params
 from openpilot.common.realtime import DT_CTRL
@@ -33,6 +35,7 @@
 JOB_ID = int(os.environ.get("JOB_ID", "0"))
 INTERNAL_SEG_LIST = os.environ.get("INTERNAL_SEG_LIST", "")
 INTERNAL_SEG_CNT = int(os.environ.get("INTERNAL_SEG_CNT", "0"))
+MAX_EXAMPLES = int(os.environ.get("MAX_EXAMPLES", "50"))
 
 
 def get_test_cases() -> List[Tuple[str, Optional[CarTestRoute]]]:
@@ -67,6 +70,7 @@ class TestCarModelBase(unittest.TestCase):
   ci: bool = True
 
   can_msgs: List[capnp.lib.capnp._DynamicStructReader]
+  fingerprint: dict[int, dict[int, int]]
   elm_frame: Optional[int]
   car_safety_mode_frame: Optional[int]
 
@@ -105,15 +109,15 @@ def setUpClass(cls):
       can_msgs = []
       cls.elm_frame = None
       cls.car_safety_mode_frame = None
-      fingerprint = gen_empty_fingerprint()
+      cls.fingerprint = gen_empty_fingerprint()
       experimental_long = False
       for msg in lr:
         if msg.which() == "can":
           can_msgs.append(msg)
           if len(can_msgs) <= FRAME_FINGERPRINT:
             for m in msg.can:
               if m.src < 64:
-                fingerprint[m.src][m.address] = len(m.dat)
+                cls.fingerprint[m.src][m.address] = len(m.dat)
 
         elif msg.which() == "carParams":
           car_fw = msg.carParams.carFw
@@ -149,7 +153,7 @@ def setUpClass(cls):
     cls.can_msgs = sorted(can_msgs, key=lambda msg: msg.logMonoTime)
 
     cls.CarInterface, cls.CarController, cls.CarState = interfaces[cls.car_model]
-    cls.CP = cls.CarInterface.get_params(cls.car_model, fingerprint, car_fw, experimental_long, docs=False)
+    cls.CP = cls.CarInterface.get_params(cls.car_model, cls.fingerprint, car_fw, experimental_long, docs=False)
     assert cls.CP
     assert cls.CP.carFingerprint == cls.car_model
 
@@ -297,6 +301,73 @@ def test_car_controller(car_control):
     CC = car.CarControl.new_message(cruiseControl={'resume': True})
     test_car_controller(CC)
 
+  # Skip stdout/stderr capture with pytest, causes elevated memory usage
+  @pytest.mark.nocapture
+  @settings(max_examples=MAX_EXAMPLES, deadline=None,
+            phases=(Phase.reuse, Phase.generate, Phase.shrink))
+  @given(data=st.data())
+  def test_panda_safety_carstate_fuzzy(self, data):
+    """
+      For each example, pick a random CAN message on the bus and fuzz its data,
+      checking for panda state mismatches.
+    """
+
+    if self.CP.dashcamOnly:
+      self.skipTest("no need to check panda safety for dashcamOnly")
+
+    valid_addrs = [(addr, bus, size) for bus, addrs in self.fingerprint.items() for addr, size in addrs.items()]
+    address, bus, size = data.draw(st.sampled_from(valid_addrs))
+
+    msg_strategy = st.binary(min_size=size, max_size=size)
+    msgs = data.draw(st.lists(msg_strategy, min_size=20))
+
+    CC = car.CarControl.new_message()
+
+    for dat in msgs:
+      # due to panda updating state selectively, only edges are expected to match
+      # TODO: warm up CarState with real CAN messages to check edge of both sources
+      #  (eg. toyota's gasPressed is the inverse of a signal being set)
+      prev_panda_gas = self.safety.get_gas_pressed_prev()
+      prev_panda_brake = self.safety.get_brake_pressed_prev()
+      prev_panda_regen_braking = self.safety.get_regen_braking_prev()
+      prev_panda_vehicle_moving = self.safety.get_vehicle_moving()
+      prev_panda_cruise_engaged = self.safety.get_cruise_engaged_prev()
+      prev_panda_acc_main_on = self.safety.get_acc_main_on()
+
+      to_send = libpanda_py.make_CANPacket(address, bus, dat)
+      self.safety.safety_rx_hook(to_send)
+
+      can = messaging.new_message('can', 1)
+      can.can = [log.CanData(address=address, dat=dat, src=bus)]
+
+      CS = self.CI.update(CC, (can.to_bytes(),))
+
+      if self.safety.get_gas_pressed_prev() != prev_panda_gas:
+        self.assertEqual(CS.gasPressed, self.safety.get_gas_pressed_prev())
+
+      if self.safety.get_brake_pressed_prev() != prev_panda_brake:
+        # TODO: remove this exception once this mismatch is resolved
+        brake_pressed = CS.brakePressed
+        if CS.brakePressed and not self.safety.get_brake_pressed_prev():
+          if self.CP.carFingerprint in (HONDA.PILOT, HONDA.RIDGELINE) and CS.brake > 0.05:
+            brake_pressed = False
+
+        self.assertEqual(brake_pressed, self.safety.get_brake_pressed_prev())
+
+      if self.safety.get_regen_braking_prev() != prev_panda_regen_braking:
+        self.assertEqual(CS.regenBraking, self.safety.get_regen_braking_prev())
+
+      if self.safety.get_vehicle_moving() != prev_panda_vehicle_moving:
+        self.assertEqual(not CS.standstill, self.safety.get_vehicle_moving())
+
+      if not (self.CP.carName == "honda" and self.CP.carFingerprint not in HONDA_BOSCH):
+        if self.safety.get_cruise_engaged_prev() != prev_panda_cruise_engaged:
+          self.assertEqual(CS.cruiseState.enabled, self.safety.get_cruise_engaged_prev())
+
+      if self.CP.carName == "honda":
+        if self.safety.get_acc_main_on() != prev_panda_acc_main_on:
+          self.assertEqual(CS.cruiseState.available, self.safety.get_acc_main_on())
+
   def test_panda_safety_carstate(self):
     """
       Assert that panda safety matches openpilot's carState