fix(edge): daemon periodic ops (sync/heartbeat/flush) can't crash the loop

Wrap sync_config, send_heartbeat, and queue.flush operations in separate
try/except to ensure daemon survives periodic operation exceptions. Critical
for unattended 24/7 pilot devices where a locked database or disk error would
otherwise kill the entire loop. Per-frame step() handling unchanged.
This commit is contained in:
유창욱 2026-07-12 11:07:27 +09:00
parent 6f54706476
commit 581f337021
2 changed files with 28 additions and 12 deletions

View file

@ -13,6 +13,7 @@ def run_loop(runtime, client, queue, *, clock, sleep, iterations=None,
counters["frames"] += 1 counters["frames"] += 1
except Exception: except Exception:
counters["errors"] += 1 counters["errors"] += 1
try:
if sync_every and i % sync_every == 0: if sync_every and i % sync_every == 0:
cfg = client.fetch_config() cfg = client.fetch_config()
if cfg is not None: if cfg is not None:
@ -25,6 +26,8 @@ def run_loop(runtime, client, queue, *, clock, sleep, iterations=None,
if flush_every and i % flush_every == 0: if flush_every and i % flush_every == 0:
queue.flush() queue.flush()
counters["flushes"] += 1 counters["flushes"] += 1
except Exception:
counters["errors"] += 1
if frame_interval: if frame_interval:
sleep(frame_interval) sleep(frame_interval)
return counters return counters

View file

@ -41,3 +41,16 @@ def test_loop_survives_step_exception():
sync_every=99, heartbeat_every=99, flush_every=99) sync_every=99, heartbeat_every=99, flush_every=99)
assert rt.steps == 3 # kept going past the exception assert rt.steps == 3 # kept going past the exception
assert counters["errors"] == 1 assert counters["errors"] == 1
def test_loop_survives_flush_exception():
class BoomQueue(_Queue):
def flush(self):
self.flushes += 1
raise RuntimeError("db locked")
rt = _Runtime()
counters = run_loop(rt, _Client(), BoomQueue(), clock=lambda: 0.0,
sleep=lambda s: None, iterations=5,
sync_every=99, heartbeat_every=99, flush_every=2)
assert rt.steps == 5 # loop kept running past the flush exceptions
assert counters["errors"] >= 1