Two independent adversarial reviews (codex CLI + a 6-dimension verification workflow) of the delivery-hardening branch surfaced ~40 findings; this closes the confirmed correctness/security/deploy blockers plus a codex re-review pass. edge: - geometry: track_displacement per-second threshold recalibrated (0.02->0.25, env-tunable via MOTION_THR, validated finite/positive); confidence-filter keypoints with a bbox-center fallback so hallucinated points don't fake motion - fall: low-confidence frames no longer cancel an active FALLING/FALLEN incident (freeze + reset recovery timer so recovery needs a fresh confident streak); baseline height only seeded from trusted frames - queue: PostClient.send() returns a tri-state SendResult; only definite 4xx count toward the poison drop cap, so a network outage (incl. 408/425/429/5xx) never deletes queued safety events; legacy rows persist their minted uuid - config: reject placeholder / http / short DEVICE_SECRET at startup - zones: warn+skip legacy pixel-space polygons that would silently match nothing cloud: - zone endpoints wired into P3-2b site-scope (cross-site read/write/delete closed) + role gate on list; catch narrowed so DB faults surface as 500 - snapshot purge keeps the DB row when unlink() fails (no untrackable PIPA orphan) - ingest dedup path re-stores missing snapshots (+ short-write guard); UNIQUE (event_id, content_hash) makes recovery idempotent under retry/race - event list cursor carries full-precision occurred_at (no float rounding at the page boundary); string keyset end-to-end - SiteScope '' fails closed (only NULL is global); dashboard fetches full page cap - runnable migration (site/event_uuid + backfill ordering + indexes); rate-limit reap fails open deploy: - offline bundle hashes EVERY file (fatal on missing/failed/injected; models in SBOM; arch in tarball name; REQUIRE_SIGNATURE option) - installer verifies before any mutation, Hailo arch preflight before install, atomic src swap, backend reconcile preserves a working operator choice - systemd DeviceAllow=char-dma_heap (the /dev/dma_heap/ dir form was inert) Verified: edge pytest 165 passed / 2 skipped; cloud phpunit 109 passed / 4 skipped; deploy scripts bash -n clean. Reviewed by codex twice (findings + fixes) with the second-pass findings folded in.
148 lines
6 KiB
Python
148 lines
6 KiB
Python
import json
|
|
import sqlite3
|
|
|
|
from edge.events import Event, EventType, Severity
|
|
from edge.queue import OfflineQueue
|
|
from edge.alert import SendResult
|
|
|
|
def _event(n=1):
|
|
return Event(EventType.FALL, Severity.HIGH, "cam-01", n, None, float(n), 3.0, {}, None)
|
|
|
|
class _Client:
|
|
def __init__(self, fail=False):
|
|
self.fail = fail
|
|
self.sent = []
|
|
def send(self, event, snapshots):
|
|
if self.fail:
|
|
return False
|
|
self.sent.append(event.track_id)
|
|
return True
|
|
|
|
def test_try_send_success_not_queued(tmp_path):
|
|
q = OfflineQueue(str(tmp_path / "q.db"), _Client())
|
|
assert q.try_send(_event(1), [b"x"]) is True
|
|
assert q.pending() == 0
|
|
|
|
def test_failure_is_queued(tmp_path):
|
|
q = OfflineQueue(str(tmp_path / "q.db"), _Client(fail=True))
|
|
assert q.try_send(_event(1), [b"x"]) is False
|
|
assert q.pending() == 1
|
|
|
|
def test_flush_resends_when_recovered(tmp_path):
|
|
client = _Client(fail=True)
|
|
q = OfflineQueue(str(tmp_path / "q.db"), client)
|
|
q.try_send(_event(1), [b"x"])
|
|
q.try_send(_event(2), [b"y"])
|
|
assert q.pending() == 2
|
|
client.fail = False
|
|
assert q.flush() == 2
|
|
assert q.pending() == 0
|
|
assert client.sent == [1, 2]
|
|
|
|
|
|
class _PoisonClient:
|
|
"""Rejects one specific event forever with a definite 4xx the cloud will never accept."""
|
|
def __init__(self, poison_id):
|
|
self.poison = poison_id
|
|
self.sent = []
|
|
def send(self, event, snapshots):
|
|
if event.track_id == self.poison:
|
|
return SendResult.REJECTED
|
|
self.sent.append(event.track_id)
|
|
return SendResult.DELIVERED
|
|
|
|
|
|
def test_queue_is_bounded_and_keeps_newest(tmp_path):
|
|
# An extended outage (or a permanent misconfig) must not fill the SD card. The queue caps
|
|
# itself, evicting the OLDEST — a fresh fall alert matters more than a stale one.
|
|
q = OfflineQueue(str(tmp_path / "q.db"), _Client(fail=True), max_rows=5)
|
|
for n in range(20):
|
|
q.try_send(_event(n), [b"x"])
|
|
assert q.pending() == 5
|
|
q.client.fail = False
|
|
q.flush()
|
|
assert q.client.sent == [15, 16, 17, 18, 19] # the newest 5 survived
|
|
|
|
|
|
def test_event_uuid_survives_the_queue_roundtrip(tmp_path):
|
|
# A re-sent event (failed once -> queued -> flushed) must carry the SAME event_uuid so the
|
|
# cloud dedups it instead of storing a duplicate alert. This is the whole point of the key.
|
|
class _CaptureClient:
|
|
def __init__(self):
|
|
self.fail = True
|
|
self.uuids = []
|
|
def send(self, event, snapshots):
|
|
self.uuids.append(event.event_uuid)
|
|
return not self.fail
|
|
client = _CaptureClient()
|
|
q = OfflineQueue(str(tmp_path / "q.db"), client)
|
|
ev = _event(1)
|
|
q.try_send(ev, [b"x"]) # fails -> queued (1st send attempt)
|
|
client.fail = False
|
|
assert q.flush() == 1 # re-sent from the queue (2nd send attempt)
|
|
assert len(client.uuids) == 2
|
|
assert client.uuids[0] == client.uuids[1] == ev.event_uuid
|
|
|
|
|
|
def test_poison_head_is_dropped_and_unblocks_the_queue(tmp_path):
|
|
# A permanently-rejected event at the head must not wedge the whole queue forever; after
|
|
# max_send_attempts it is dropped so the events behind it can drain.
|
|
client = _PoisonClient(99)
|
|
q = OfflineQueue(str(tmp_path / "q.db"), client, max_send_attempts=3)
|
|
q.enqueue(_event(99), [b"x"]) # poison, at the head
|
|
q.enqueue(_event(2), [b"y"]) # good, behind it
|
|
for _ in range(3):
|
|
q.flush()
|
|
assert client.sent == [2] # good event delivered
|
|
assert q.pending() == 0 # poison dropped, queue drained
|
|
|
|
|
|
def test_sustained_outage_never_drops_events(tmp_path):
|
|
# A transient outage (network down / 5xx) must NEVER poison-drop queued events, no matter how
|
|
# long it lasts — a real fall alert has to survive an overnight outage. Only a definite 4xx
|
|
# rejection counts toward the drop cap (contrast test_poison_head_is_dropped above).
|
|
class _Outage:
|
|
up = False
|
|
sent = []
|
|
def send(self, event, snapshots):
|
|
if not _Outage.up:
|
|
return SendResult.TRANSIENT
|
|
_Outage.sent.append(event.track_id)
|
|
return SendResult.DELIVERED
|
|
client = _Outage()
|
|
q = OfflineQueue(str(tmp_path / "q.db"), client, max_send_attempts=3)
|
|
q.enqueue(_event(1), [b"x"])
|
|
for _ in range(10): # far beyond max_send_attempts
|
|
q.flush()
|
|
assert q.pending() == 1 # still queued — a transient failure is never poison-dropped
|
|
_Outage.up = True
|
|
q.flush()
|
|
assert q.pending() == 0 and _Outage.sent == [1] # drains once connectivity returns
|
|
|
|
|
|
def test_legacy_row_without_uuid_reuses_minted_uuid_across_retries(tmp_path):
|
|
# A row queued before event_uuid existed has none stored. On first flush a fresh uuid is
|
|
# minted; it MUST be persisted so every retry reuses the SAME key — else each failed-then-
|
|
# retried delivery sends a different uuid and the cloud stores duplicates (the P1-9 bug).
|
|
db = str(tmp_path / "q.db")
|
|
con = sqlite3.connect(db)
|
|
con.execute("CREATE TABLE queue (id INTEGER PRIMARY KEY AUTOINCREMENT, payload TEXT, "
|
|
"snaps TEXT, attempts INTEGER DEFAULT 0)")
|
|
legacy = {"event_type": "fall", "severity": "high", "camera_id": "cam-01", "track_id": 1,
|
|
"zone_id": None, "occurred_at": 100.0, "duration_s": 3.0, "metrics": {}, "spatial": None}
|
|
con.execute("INSERT INTO queue (payload, snaps) VALUES (?, ?)",
|
|
(json.dumps(legacy), json.dumps([])))
|
|
con.commit(); con.close()
|
|
|
|
class _Capture:
|
|
up = False
|
|
uuids = []
|
|
def send(self, event, snapshots):
|
|
_Capture.uuids.append(event.event_uuid)
|
|
return SendResult.DELIVERED if _Capture.up else SendResult.TRANSIENT
|
|
q = OfflineQueue(db, _Capture())
|
|
q.flush() # transient -> row stays; uuid minted and persisted
|
|
_Capture.up = True
|
|
q.flush() # delivered
|
|
assert len(_Capture.uuids) == 2
|
|
assert _Capture.uuids[0] == _Capture.uuids[1] # same key reused across retries
|