move_detect/tests/test_daemon.py
유창욱 752afc3f00 feat(edge): heartbeat telemetry + monotonic timers + queue/camera robustness
- Split clocks: detector timers/cooldowns run on a MONOTONIC clock (immune to an
  NTP step on an RTC-less Pi); wall time is used only for the HMAC X-Timestamp
  and Event.occurred_at. Fixes false IMMOBILITY / dropped tracks after a clock jump.
- Heartbeat is time-based (30s, predictable regardless of load) and now carries
  temp/throttle/queue_pending/config_version telemetry.
- A dead (no-frame) camera counts as erroring instead of a healthy frame, so a
  silent monitoring outage no longer looks green.
- Offline queue: isolate one undecodable row instead of wedging every event behind
  it; classify 401 (stale clock) as transient, never a poison-drop.
- New edge/health.py: vcgencmd temp/throttle parsing, None off-Pi (no fake 0).
2026-07-25 19:20:23 +09:00

172 lines
7.6 KiB
Python

import pytest
from edge.daemon import run_loop
class _Runtime:
def __init__(self): self.steps = 0; self.applied = None
def step(self, now): self.steps += 1; return []
def apply_zones(self, zones): self.applied = zones
class _Client:
def __init__(self): self.configs = 0; self.beats = 0; self.metrics = []
def fetch_config(self): self.configs += 1; return {"zones": [], "thresholds": {}}
def send_heartbeat(self, m): self.beats += 1; self.metrics.append(m); return True
class _Queue:
def __init__(self, pending=0): self.flushes = 0; self._pending = pending
def flush(self): self.flushes += 1; return 0
def pending(self): return self._pending
def test_loop_steps_and_periodics():
rt, cl, q = _Runtime(), _Client(), _Queue()
ticks = iter(range(1000))
counters = run_loop(rt, cl, q, clock=lambda: float(next(ticks)),
sleep=lambda s: None, iterations=10,
sync_every=5, heartbeat_every=5, flush_every=5)
assert rt.steps == 10
assert cl.configs == 2 and cl.beats == 2 and q.flushes == 2 # at frames 5 and 10
assert rt.applied == [] # zones applied from fetched config
def test_loop_survives_step_exception():
class Boom(_Runtime):
def step(self, now):
self.steps += 1
if self.steps == 2: raise RuntimeError("frame glitch")
return []
rt = Boom()
counters = run_loop(rt, _Client(), _Queue(), clock=lambda: 0.0,
sleep=lambda s: None, iterations=3,
sync_every=99, heartbeat_every=99, flush_every=99)
assert rt.steps == 3 # kept going past the exception
assert counters["errors"] == 1
def test_loop_logs_swallowed_step_exception_rate_limited():
# A permanently-broken pipeline must not present as a silent green unit: the swallowed
# exception has to be logged. But it must be rate-limited — a camera failing every frame
# cannot spam the journal thousands of times a second.
logs = []
class AlwaysBoom(_Runtime):
camera_id = "cam-x"
def step(self, now):
self.steps += 1
raise RuntimeError("perma-broken")
rt = AlwaysBoom()
counters = run_loop(rt, _Client(), _Queue(), clock=lambda: 0.0,
sleep=lambda s: None, iterations=5,
sync_every=99, heartbeat_every=99, flush_every=99,
log=lambda msg, exc=None: logs.append(msg))
assert counters["errors"] == 5 # every step raised
assert len(logs) == 1 # logged once, not five times (rate-limited)
assert "cam-x" in logs[0] # names the failing camera
def test_heartbeat_reports_real_fps_and_camera_health():
# A hardcoded fps=0.0 makes the server's device-health columns fiction; it can't tell a
# live device from a dead one. The heartbeat must carry the measured fps and per-camera
# health so a degraded device is visible remotely.
rt, cl, q = _Runtime(), _Client(), _Queue()
ticks = iter(float(t) for t in range(1000))
run_loop(rt, cl, q, clock=lambda: next(ticks), sleep=lambda s: None,
iterations=10, sync_every=99, heartbeat_every=10, flush_every=99)
assert len(cl.metrics) == 1
m = cl.metrics[0]
# 10 frames over the elapsed wall time between the first tick (t=0) and the beat (t=9).
assert m["fps"] == pytest.approx(10 / 9, rel=0.05)
assert m["cameras_total"] == 1
assert m["cameras_erroring"] == 0
def test_heartbeat_flags_a_failing_camera():
class AlwaysBoom(_Runtime):
camera_id = "cam-x"
def step(self, now):
self.steps += 1
raise RuntimeError("dead")
cl = _Client()
ticks = iter(float(t) for t in range(1000))
run_loop(AlwaysBoom(), cl, _Queue(), clock=lambda: next(ticks), sleep=lambda s: None,
iterations=10, sync_every=99, heartbeat_every=10, flush_every=99,
log=lambda *a, **k: None)
m = cl.metrics[0]
assert m["cameras_erroring"] == 1 # the server can now see the camera is down
class _NoFrameRuntime:
# step() returns None on a tick with no frame (dead/unplugged camera), a list otherwise.
camera_id = "cam-dead"
def __init__(self): self.steps = 0
def step(self, now): self.steps += 1; return None
def test_no_frame_counts_as_erroring_not_a_healthy_frame():
# A camera that returns no frame every tick (unplugged USB) must NOT tick frames++ and must
# surface as erroring — otherwise a monitoring outage looks perfectly green (the whole point
# of run_once returning None instead of []).
rt, cl = _NoFrameRuntime(), _Client()
ticks = iter(float(t) for t in range(1000))
counters = run_loop(rt, cl, _Queue(), clock=lambda: next(ticks), sleep=lambda s: None,
iterations=10, sync_every=99, heartbeat_every=10, flush_every=99,
log=lambda *a, **k: None)
assert rt.steps == 10
assert counters["frames"] == 0 # no frame was ever counted as healthy
m = cl.metrics[0]
assert m["fps"] == 0.0
assert m["cameras_erroring"] == 1 # the dead camera is visible to the server
def test_heartbeat_reports_queue_depth_and_echoes_config_version():
class _VersionClient(_Client):
def fetch_config(self):
self.configs += 1
return {"version": 7, "zones": [], "thresholds": {}}
rt, cl, q = _Runtime(), _VersionClient(), _Queue(pending=4)
ticks = iter(float(t) for t in range(1000))
# sync at frame 5 (captures version 7) BEFORE the heartbeat at frame 10.
run_loop(rt, cl, q, clock=lambda: next(ticks), sleep=lambda s: None,
iterations=10, sync_every=5, heartbeat_every=10, flush_every=99)
m = cl.metrics[0]
assert m["queue_pending"] == 4 # transmitted so the dashboard can show backlog
assert m["config_version"] == 7 # echoed so config-drift detection actually works
def test_heartbeat_omits_config_version_until_a_config_is_fetched():
# Before any sync, the device does not know its version — it must OMIT config_version rather
# than send a fake 0 (which made the server's drift indicator fire on every device forever).
rt, cl, q = _Runtime(), _Client(), _Queue()
ticks = iter(float(t) for t in range(1000))
run_loop(rt, cl, q, clock=lambda: next(ticks), sleep=lambda s: None,
iterations=10, sync_every=99, heartbeat_every=10, flush_every=99)
assert "config_version" not in cl.metrics[0]
def test_time_based_heartbeat_fires_on_elapsed_wall_time_not_iteration_count():
# Production uses heartbeat_interval_s so the cadence is predictable no matter how slow a tick
# is. With the clock advancing 10s/tick and a 30s interval, a beat fires every ~3 ticks.
rt, cl, q = _Runtime(), _Client(), _Queue()
ticks = iter(float(t * 10) for t in range(1000)) # t = 0,10,20,30,...
run_loop(rt, cl, q, clock=lambda: next(ticks), sleep=lambda s: None,
iterations=10, sync_every=99, heartbeat_every=None, flush_every=99,
heartbeat_interval_s=30)
# ticks at 0,10,20,30,...,90. Beats when now-last_beat >= 30: at 30, 60, 90 -> 3 beats.
assert cl.beats == 3
def test_loop_survives_flush_exception():
class BoomQueue(_Queue):
def flush(self):
self.flushes += 1
raise RuntimeError("db locked")
rt = _Runtime()
counters = run_loop(rt, _Client(), BoomQueue(), clock=lambda: 0.0,
sleep=lambda s: None, iterations=5,
sync_every=99, heartbeat_every=99, flush_every=2)
assert rt.steps == 5 # loop kept running past the flush exceptions
assert counters["errors"] >= 1