fe6a144754
- Heartbeat reports the CONFIGURED camera count, not just the started ones: a camera that failed to construct at boot (wrong index / unseated ribbon) now shows as total/erroring instead of vanishing, so the server can see it is dark. - GHOST emits without an engine cooldown key (ghost.update already fires once per never-reused track id) — removes a permanent _last_emit key that forget() could never reclaim (a slow 24/7 leak). - Malformed zone rows are logged loudly instead of silently swallowed, matching the pixel-space branch's "don't fail dark" — a dead zone is no longer discovered only after a worker enters it.
239 lines
12 KiB
Python
239 lines
12 KiB
Python
import os
|
|
|
|
|
|
# Rate-limit for the swallowed-exception log: a camera failing every frame must be VISIBLE
|
|
# (not a silent green unit) but must not spam the journal. Log the 1st of a run of consecutive
|
|
# failures, then every LOG_EVERY-th, and again after any recovery.
|
|
_LOG_EVERY = 300
|
|
|
|
|
|
def _default_log(msg, exc=None): # pragma: no cover (thin stderr wrapper)
|
|
import sys
|
|
import traceback
|
|
print(msg, file=sys.stderr)
|
|
if exc is not None:
|
|
traceback.print_exception(type(exc), exc, exc.__traceback__, file=sys.stderr)
|
|
|
|
|
|
def run_loop(runtime, client, queue, *, clock, sleep, iterations=None,
|
|
sync_every=150, heartbeat_every=900, flush_every=300,
|
|
frame_interval=0.0, zones_for=None, log=None,
|
|
heartbeat_interval_s=None, health_fn=None, configured_cameras=None):
|
|
# `runtime` may be a single PipelineRuntime or a list of them (multi-camera). All
|
|
# cameras share this loop, one shared clock read per tick, and the periodic
|
|
# sync/heartbeat/flush. A per-camera step exception is counted and rate-limit-logged,
|
|
# never fatal — but it is no longer SILENT (a permanently broken pipeline used to look
|
|
# perfectly green).
|
|
runtimes = list(runtime) if isinstance(runtime, (list, tuple)) else [runtime]
|
|
if zones_for is None:
|
|
from edge.zones import zones_from_config
|
|
zones_for = lambda cfg, rt: zones_from_config(cfg)
|
|
if log is None:
|
|
log = _default_log
|
|
if health_fn is None:
|
|
from edge.health import read_health
|
|
health_fn = read_health
|
|
counters = {"frames": 0, "syncs": 0, "beats": 0, "flushes": 0, "errors": 0}
|
|
streak = {} # id(rt)/"periodic" -> consecutive-error count, for rate-limited logging
|
|
|
|
def note(key, label, exc=None):
|
|
counters["errors"] += 1
|
|
n = streak.get(key, 0) + 1
|
|
streak[key] = n
|
|
if n == 1 or n % _LOG_EVERY == 0:
|
|
if exc is not None:
|
|
log(f"{label}: {type(exc).__name__}: {exc} (consecutive #{n})", exc)
|
|
else:
|
|
log(f"{label} (consecutive #{n})", None)
|
|
|
|
beat_anchor = None # (wall_time, frames) at the previous heartbeat, for measured fps
|
|
last_beat = None # wall time of the last heartbeat, for the time-based cadence
|
|
reported_version = None # last config version the server told us; echoed in heartbeats
|
|
i = 0
|
|
while iterations is None or i < iterations:
|
|
i += 1
|
|
now = clock() # one timestamp per tick, shared across cameras
|
|
if beat_anchor is None:
|
|
beat_anchor = (now, 0)
|
|
if last_beat is None:
|
|
last_beat = now
|
|
for rt in runtimes:
|
|
try:
|
|
produced = rt.step(now)
|
|
except Exception as e:
|
|
cam = getattr(rt, "camera_id", "?")
|
|
note(id(rt), f"camera {cam} step failed", e)
|
|
continue
|
|
if produced is None:
|
|
# step() returned no frame (unplugged/browned-out camera). Count it toward the
|
|
# erroring streak instead of as a healthy frame — otherwise a dead camera ticks
|
|
# frames++ forever and reports fps~=target with cameras_erroring=0 (a silent
|
|
# monitoring outage that looks green). A single dropped frame self-clears on the
|
|
# next good frame, so only SUSTAINED starvation shows as erroring.
|
|
cam = getattr(rt, "camera_id", "?")
|
|
note(id(rt), f"camera {cam} produced no frame")
|
|
else:
|
|
counters["frames"] += 1
|
|
streak.pop(id(rt), None) # recovered — reset the streak
|
|
try:
|
|
if sync_every and i % sync_every == 0:
|
|
cfg = client.fetch_config()
|
|
if cfg is not None:
|
|
for rt in runtimes:
|
|
rt.apply_zones(zones_for(cfg, rt))
|
|
# Remember the config version so heartbeats can echo it back — the server's
|
|
# config-drift indicator is meaningless until the device reports what it runs.
|
|
if isinstance(cfg, dict) and cfg.get("version") is not None:
|
|
reported_version = int(cfg["version"])
|
|
counters["syncs"] += 1
|
|
# Heartbeat cadence: time-based when heartbeat_interval_s is set (production), so the
|
|
# interval is predictable regardless of per-tick cost (CPU MoveNet, multi-cam, thermal
|
|
# throttle can push a tick past 100ms) — an iteration count cannot guarantee a wall-clock
|
|
# cadence, and the server marks a device offline on ELAPSED time. Falls back to the
|
|
# iteration count when unset (used by the deterministic unit tests).
|
|
if heartbeat_interval_s:
|
|
do_beat = (now - last_beat) >= heartbeat_interval_s
|
|
else:
|
|
do_beat = bool(heartbeat_every) and (i % heartbeat_every == 0)
|
|
if do_beat:
|
|
# Measured fps over the window since the last beat, and per-camera health, so
|
|
# the server can tell a live device from a degraded/dead one (was hardcoded 0.0
|
|
# + a frames counter that ticked even when no frame was read).
|
|
t0, f0 = beat_anchor
|
|
elapsed = now - t0
|
|
fps = (counters["frames"] - f0) / elapsed if elapsed > 0 else 0.0
|
|
# cameras_total is the CONFIGURED count, not just the started one: a camera that
|
|
# failed to construct at boot (wrong /dev index, unseated ribbon) is skipped by
|
|
# build_runtimes and would otherwise vanish from the heartbeat entirely — the
|
|
# server could never see that a configured camera is missing. Count the not-started
|
|
# cameras as erroring so a 4-cam device that lost one reports 4 total / 1 erroring.
|
|
total = configured_cameras if configured_cameras is not None else len(runtimes)
|
|
not_started = max(0, total - len(runtimes))
|
|
erroring = not_started + sum(1 for rt in runtimes if id(rt) in streak)
|
|
metrics = {"fps": round(fps, 2), "frames": counters["frames"],
|
|
"cameras_total": total, "cameras_erroring": erroring,
|
|
"errors": counters["errors"], "queue_pending": queue.pending()}
|
|
# Pi thermal telemetry (None off-Pi or on failure → field omitted, never a fake 0).
|
|
health = health_fn()
|
|
if health.get("temp") is not None:
|
|
metrics["temp"] = health["temp"]
|
|
if health.get("throttle") is not None:
|
|
metrics["throttle"] = health["throttle"]
|
|
if reported_version is not None:
|
|
metrics["config_version"] = reported_version
|
|
if client.send_heartbeat(metrics):
|
|
counters["beats"] += 1
|
|
beat_anchor = (now, counters["frames"])
|
|
last_beat = now
|
|
if flush_every and i % flush_every == 0:
|
|
queue.flush()
|
|
counters["flushes"] += 1
|
|
streak.pop("periodic", None)
|
|
except Exception as e:
|
|
note("periodic", "periodic op (sync/heartbeat/flush) failed", e)
|
|
if frame_interval:
|
|
sleep(frame_interval)
|
|
return counters
|
|
|
|
|
|
def parse_cameras(env, default_camera_id):
|
|
"""CAMERAS='backend:device:camera_id,...' -> [(backend, device, camera_id), ...].
|
|
When CAMERAS is unset, fall back to a single camera from
|
|
CAMERA_BACKEND + {UVC,CSI}_DEVICE (using default_camera_id)."""
|
|
raw = (env.get("CAMERAS") or "").strip()
|
|
if raw:
|
|
cams = []
|
|
for spec in raw.split(","):
|
|
spec = spec.strip()
|
|
if not spec:
|
|
continue
|
|
parts = [p.strip() for p in spec.split(":")]
|
|
if len(parts) != 3:
|
|
raise ValueError(f"bad CAMERAS entry {spec!r} (want backend:device:camera_id)")
|
|
cams.append((parts[0].lower(), parts[1], parts[2]))
|
|
if not cams:
|
|
raise ValueError("CAMERAS is set but has no valid entries") # never monitor zero cameras
|
|
return cams
|
|
backend = (env.get("CAMERA_BACKEND") or "uvc").lower()
|
|
device = env.get("CSI_DEVICE" if backend == "csi" else "UVC_DEVICE") or "0"
|
|
return [(backend, device, default_camera_id)]
|
|
|
|
|
|
def build_runtimes(cameras, make_source, make_runtime, log=None):
|
|
"""Build one (source -> runtime) per camera. A camera that fails to CONSTRUCT
|
|
(e.g. an unseated/absent CSI camera whose Picamera2(...) raises) is logged and
|
|
SKIPPED so it never crash-loops the whole daemon and the other cameras still run.
|
|
Raises if no camera comes up (never run monitoring nothing)."""
|
|
if log is None:
|
|
import sys
|
|
log = lambda m: print(m, file=sys.stderr)
|
|
runtimes = []
|
|
for backend, device, cam in cameras:
|
|
try:
|
|
runtimes.append(make_runtime(cam, make_source(backend, device)))
|
|
except Exception as e:
|
|
log(f"camera {cam} ({backend}:{device}) failed to start, skipping: {e}")
|
|
if not runtimes:
|
|
raise RuntimeError("no cameras could be started")
|
|
return runtimes
|
|
|
|
|
|
def main() -> None: # pragma: no cover (integration entrypoint, exercised on device)
|
|
import time
|
|
from dataclasses import replace
|
|
from edge.config import (frame_interval_from_env, load_config, queue_db_path,
|
|
require_upload_config)
|
|
from edge.serverclient import ServerClient
|
|
from edge.alert import PostClient
|
|
from edge.queue import OfflineQueue
|
|
from edge.runtime import PipelineRuntime
|
|
from edge.zones import zones_from_config
|
|
|
|
def build_source(backend, device):
|
|
if backend == "csi":
|
|
from edge.adapters.picamera_source import PiCameraSource
|
|
return PiCameraSource(int(device))
|
|
from edge.adapters.uvc_source import UvcSource
|
|
return UvcSource(int(device))
|
|
|
|
def build_pose():
|
|
# POSE_BACKEND: "hailo" (Pi5 + Hailo-8, needs HAILO_HEF) or CPU "movenet"
|
|
# (needs MOVENET_ONNX). One shared adapter serves every camera (the Hailo
|
|
# device / onnxruntime session is reused per frame, round-robin across cams).
|
|
if os.environ.get("POSE_BACKEND", "movenet").lower() == "hailo":
|
|
from edge.adapters.hailo_pose import HailoPoseAdapter
|
|
return HailoPoseAdapter(os.environ["HAILO_HEF"])
|
|
from edge.adapters.movenet_pose import MoveNetPoseAdapter
|
|
return MoveNetPoseAdapter(os.environ["MOVENET_ONNX"])
|
|
|
|
cfg = load_config(dict(os.environ))
|
|
require_upload_config(cfg) # fail loud now, not silently upload nothing forever
|
|
nonce = lambda: os.urandom(8).hex()
|
|
# Two clocks, deliberately: the loop/detector timers run on a MONOTONIC clock so an NTP step
|
|
# (a Pi 5 has no RTC battery; it boots on stale fake-hwclock time until chrony corrects it)
|
|
# cannot instantly satisfy immobile_s, expire every track, or reset an immobility countdown.
|
|
# Wall time is used only where a real timestamp is required: the HMAC X-Timestamp headers,
|
|
# which the server validates against its own wall clock, and Event.occurred_at.
|
|
mono = time.monotonic
|
|
wall = time.time
|
|
base = cfg.cafe24_url.rstrip("/")
|
|
client = ServerClient(base, cfg.device_id, cfg.secret, nonce, wall)
|
|
post = PostClient(base + "/api/events", cfg.device_id, cfg.secret, nonce, wall)
|
|
queue = OfflineQueue(queue_db_path(dict(os.environ)), post)
|
|
|
|
pose = build_pose() # shared across cameras
|
|
|
|
def make_runtime(cam, source):
|
|
# Per-camera config so events are stamped with that camera's id.
|
|
cam_cfg = replace(cfg, camera_id=cam, engine=replace(cfg.engine, camera_id=cam))
|
|
return PipelineRuntime(cam_cfg, source, pose, queue, wall_clock=wall)
|
|
|
|
# A missing/unseated camera is skipped (logged), not fatal — the others still run.
|
|
cameras = parse_cameras(dict(os.environ), cfg.camera_id)
|
|
runtimes = build_runtimes(cameras, build_source, make_runtime)
|
|
|
|
run_loop(runtimes, client, queue, clock=mono, sleep=time.sleep, iterations=None,
|
|
frame_interval=frame_interval_from_env(dict(os.environ)),
|
|
heartbeat_interval_s=30, # predictable cadence; server marks offline after 120s (4x)
|
|
configured_cameras=len(cameras), # so a camera that failed to start stays visible
|
|
zones_for=lambda c, rt: zones_from_config(c, rt.camera_id))
|