fix(edge): daemon periodic ops (sync/heartbeat/flush) can't crash the loop
Wrap sync_config, send_heartbeat, and queue.flush operations in separate try/except to ensure daemon survives periodic operation exceptions. Critical for unattended 24/7 pilot devices where a locked database or disk error would otherwise kill the entire loop. Per-frame step() handling unchanged.
This commit is contained in:
parent
6f54706476
commit
581f337021
2 changed files with 28 additions and 12 deletions
|
|
@ -13,6 +13,7 @@ def run_loop(runtime, client, queue, *, clock, sleep, iterations=None,
|
||||||
counters["frames"] += 1
|
counters["frames"] += 1
|
||||||
except Exception:
|
except Exception:
|
||||||
counters["errors"] += 1
|
counters["errors"] += 1
|
||||||
|
try:
|
||||||
if sync_every and i % sync_every == 0:
|
if sync_every and i % sync_every == 0:
|
||||||
cfg = client.fetch_config()
|
cfg = client.fetch_config()
|
||||||
if cfg is not None:
|
if cfg is not None:
|
||||||
|
|
@ -25,6 +26,8 @@ def run_loop(runtime, client, queue, *, clock, sleep, iterations=None,
|
||||||
if flush_every and i % flush_every == 0:
|
if flush_every and i % flush_every == 0:
|
||||||
queue.flush()
|
queue.flush()
|
||||||
counters["flushes"] += 1
|
counters["flushes"] += 1
|
||||||
|
except Exception:
|
||||||
|
counters["errors"] += 1
|
||||||
if frame_interval:
|
if frame_interval:
|
||||||
sleep(frame_interval)
|
sleep(frame_interval)
|
||||||
return counters
|
return counters
|
||||||
|
|
|
||||||
|
|
@ -41,3 +41,16 @@ def test_loop_survives_step_exception():
|
||||||
sync_every=99, heartbeat_every=99, flush_every=99)
|
sync_every=99, heartbeat_every=99, flush_every=99)
|
||||||
assert rt.steps == 3 # kept going past the exception
|
assert rt.steps == 3 # kept going past the exception
|
||||||
assert counters["errors"] == 1
|
assert counters["errors"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_loop_survives_flush_exception():
|
||||||
|
class BoomQueue(_Queue):
|
||||||
|
def flush(self):
|
||||||
|
self.flushes += 1
|
||||||
|
raise RuntimeError("db locked")
|
||||||
|
rt = _Runtime()
|
||||||
|
counters = run_loop(rt, _Client(), BoomQueue(), clock=lambda: 0.0,
|
||||||
|
sleep=lambda s: None, iterations=5,
|
||||||
|
sync_every=99, heartbeat_every=99, flush_every=2)
|
||||||
|
assert rt.steps == 5 # loop kept running past the flush exceptions
|
||||||
|
assert counters["errors"] >= 1
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue