fix(edge): daemon periodic ops (sync/heartbeat/flush) can't crash the loop

Wrap sync_config, send_heartbeat, and queue.flush operations in separate
try/except to ensure daemon survives periodic operation exceptions. Critical
for unattended 24/7 pilot devices where a locked database or disk error would
otherwise kill the entire loop. Per-frame step() handling unchanged.
This commit is contained in:
유창욱 2026-07-12 11:07:27 +09:00
parent 6f54706476
commit 581f337021
2 changed files with 28 additions and 12 deletions

View file

@ -13,18 +13,21 @@ def run_loop(runtime, client, queue, *, clock, sleep, iterations=None,
counters["frames"] += 1 counters["frames"] += 1
except Exception: except Exception:
counters["errors"] += 1 counters["errors"] += 1
if sync_every and i % sync_every == 0: try:
cfg = client.fetch_config() if sync_every and i % sync_every == 0:
if cfg is not None: cfg = client.fetch_config()
from edge.zones import zones_from_config if cfg is not None:
runtime.apply_zones(zones_from_config(cfg)) from edge.zones import zones_from_config
counters["syncs"] += 1 runtime.apply_zones(zones_from_config(cfg))
if heartbeat_every and i % heartbeat_every == 0: counters["syncs"] += 1
if client.send_heartbeat({"fps": 0.0, "frames": counters["frames"]}): if heartbeat_every and i % heartbeat_every == 0:
counters["beats"] += 1 if client.send_heartbeat({"fps": 0.0, "frames": counters["frames"]}):
if flush_every and i % flush_every == 0: counters["beats"] += 1
queue.flush() if flush_every and i % flush_every == 0:
counters["flushes"] += 1 queue.flush()
counters["flushes"] += 1
except Exception:
counters["errors"] += 1
if frame_interval: if frame_interval:
sleep(frame_interval) sleep(frame_interval)
return counters return counters

View file

@ -41,3 +41,16 @@ def test_loop_survives_step_exception():
sync_every=99, heartbeat_every=99, flush_every=99) sync_every=99, heartbeat_every=99, flush_every=99)
assert rt.steps == 3 # kept going past the exception assert rt.steps == 3 # kept going past the exception
assert counters["errors"] == 1 assert counters["errors"] == 1
def test_loop_survives_flush_exception():
class BoomQueue(_Queue):
def flush(self):
self.flushes += 1
raise RuntimeError("db locked")
rt = _Runtime()
counters = run_loop(rt, _Client(), BoomQueue(), clock=lambda: 0.0,
sleep=lambda s: None, iterations=5,
sync_every=99, heartbeat_every=99, flush_every=2)
assert rt.steps == 5 # loop kept running past the flush exceptions
assert counters["errors"] >= 1