feat(deploy): probe every camera before enabling the service
build_runtimes() logs-and-skips a camera it cannot open, so a wrong /dev/video index or an unseated ribbon produced a green systemd unit that silently monitored one camera less. - edge/probe.py: opens each camera in CAMERAS, retries while the sensor settles, reports frame geometry, optionally saves preview PNGs, and exits non-zero with a backend-specific hint (v4l2-ctl / rpicam-hello) - setup_online.sh: new step [8/9] runs it against the installed package and the CAMERAS line actually written to edge.env, and refuses to enable the unit on failure. SKIP_CAMERA_CHECK=1 opts out. Tests: 174 pass, 2 skipped (7 new).
This commit is contained in:
유창욱
parent
f062667a65
commit
5444f19d45
3 changed files with 250 additions and 15 deletions
|
|
@ -30,12 +30,12 @@ STATE=/var/lib/move-detect
|
|||
ETC=/etc/move-detect
|
||||
SVC_USER=movedetect
|
||||
|
||||
echo ">> [1/8] system packages (CSI camera, venv; Hailo runtime best-effort)"
|
||||
echo ">> [1/9] system packages (CSI camera, venv; Hailo runtime best-effort)"
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
# A freshly-booted Pi runs apt-daily / apt-daily-upgrade / unattended-upgrades, which hold
|
||||
# /var/lib/apt/lists/lock and /var/lib/dpkg/lock-frontend for the first few minutes —
|
||||
# exactly when someone runs a one-command installer. A bare apt-get then exits 100 with
|
||||
# "E: Unable to lock directory /var/lib/apt/lists/" and `set -e` (line 12) kills us here.
|
||||
# "E: Unable to lock directory /var/lib/apt/lists/" and `set -e` (set -e above) kills us here.
|
||||
# -o DPkg::Lock::Timeout is not enough on its own: apt applies it to the dpkg locks, but
|
||||
# `apt-get update`'s lists lock is taken with a single non-blocking fcntl(F_SETLK) and no
|
||||
# retry. That wait has to live here, in the shell.
|
||||
|
|
@ -51,7 +51,7 @@ apt_retry() {
|
|||
# renders as "%s 디렉터리를 잠글 수 없습니다", the grep below misses it, and lock
|
||||
# contention gets misread as a real error — fail-fast, zero retries, exactly the bug
|
||||
# this function exists to fix. Pinning it per-call (not script-wide) keeps pip/venv and
|
||||
# fetch_movenet at [4/8]-[5/8] on the system locale.
|
||||
# fetch_movenet at [4/9]-[5/9] on the system locale.
|
||||
LC_ALL=C apt-get -o DPkg::Lock::Timeout=60 "$@" 2>&1 | tee "$log" || rc=${PIPESTATUS[0]}
|
||||
if [ "$rc" -eq 0 ]; then rm -f "$log"; return 0; fi
|
||||
if ! grep -qE 'Could not get lock|Unable to lock|Unable to acquire the dpkg' "$log"; then
|
||||
|
|
@ -73,13 +73,13 @@ apt_retry install -y python3-picamera2 python3-venv python3-pip
|
|||
# Best-effort so a Pi without it still installs and runs on CPU MoveNet.
|
||||
apt_retry install -y hailo-all || echo " WARN: hailo-all not installed — Hailo accel unavailable, CPU MoveNet will be used."
|
||||
|
||||
echo ">> [2/8] service account + device groups"
|
||||
echo ">> [2/9] service account + device groups"
|
||||
id -u "$SVC_USER" >/dev/null 2>&1 || useradd --system --no-create-home --shell /usr/sbin/nologin "$SVC_USER"
|
||||
for g in video render plugdev hailo; do
|
||||
getent group "$g" >/dev/null 2>&1 && usermod -aG "$g" "$SVC_USER" || true
|
||||
done
|
||||
|
||||
echo ">> [3/8] source + dirs under $PREFIX"
|
||||
echo ">> [3/9] source + dirs under $PREFIX"
|
||||
install -d -m 0755 "$PREFIX" "$PREFIX/src" "$PREFIX/models" "$PREFIX/docs"
|
||||
rm -rf "$PREFIX/src/edge" # full replace on re-run (no nested edge/edge)
|
||||
cp -r "$REPO_ROOT/edge" "$PREFIX/src/edge"
|
||||
|
|
@ -87,7 +87,7 @@ cp "$REPO_ROOT/pyproject.toml" "$PREFIX/src/" 2>/dev/null || true
|
|||
find "$PREFIX/src" -name __pycache__ -type d -prune -exec rm -rf {} +
|
||||
cp "$REPO_ROOT/docs/pilot-runbook.md" "$PREFIX/docs/" 2>/dev/null || true
|
||||
|
||||
echo ">> [4/8] venv (+ system site-packages) + pip runtime deps"
|
||||
echo ">> [4/9] venv (+ system site-packages) + pip runtime deps"
|
||||
python3 -m venv --system-site-packages "$PREFIX/venv"
|
||||
"$PREFIX/venv/bin/pip" install --upgrade pip >/dev/null
|
||||
# MANDATORY for every backend — do not soften these to `|| echo WARN`:
|
||||
|
|
@ -106,12 +106,12 @@ python3 -m venv --system-site-packages "$PREFIX/venv"
|
|||
# MoveNetPoseAdapter.__init__, and the Hailo path never imports the module at all. It
|
||||
# publishes NO sdist and NO 32-bit-ARM wheel, so pip can legitimately find zero candidates
|
||||
# and exit non-zero in seconds — which under `set -euo pipefail` used to kill this installer
|
||||
# on a Hailo-8 Pi that never needed it. Best-effort, like hailo-all at [1/8]; [6/8] decides
|
||||
# on a Hailo-8 Pi that never needed it. Best-effort, like hailo-all at [1/9]; [6/9] decides
|
||||
# whether a backend actually exists.
|
||||
"$PREFIX/venv/bin/pip" install onnxruntime \
|
||||
|| echo " WARN: onnxruntime did not install — CPU MoveNet unavailable (Hailo unaffected)."
|
||||
|
||||
echo ">> [5/8] MoveNet CPU model"
|
||||
echo ">> [5/9] MoveNet CPU model"
|
||||
# Best-effort for the same reason: this model is inert without onnxruntime, and a download
|
||||
# hiccup must not kill an install whose backend is Hailo. fetch_movenet is pure stdlib, so it
|
||||
# runs fine with onnxruntime absent.
|
||||
|
|
@ -119,7 +119,7 @@ echo ">> [5/8] MoveNet CPU model"
|
|||
|| echo " WARN: MoveNet ONNX download failed — CPU MoveNet unavailable (Hailo unaffected)."
|
||||
cp "$REPO_ROOT/models/movenet_lightning.onnx" "$PREFIX/models/" 2>/dev/null || true
|
||||
|
||||
echo ">> [6/8] pose backend"
|
||||
echo ">> [6/9] pose backend"
|
||||
HEF_DEST="$PREFIX/models/yolov8s_pose.hef"
|
||||
ONNX_DEST="$PREFIX/models/movenet_lightning.onnx"
|
||||
if [ -n "${HAILO_HEF:-}" ]; then
|
||||
|
|
@ -133,7 +133,7 @@ if [ -n "${HAILO_HEF:-}" ]; then
|
|||
else
|
||||
# No HEF supplied: fetch the one matching THIS board. Nothing in this repo could previously
|
||||
# produce a .hef — MoveNet had fetch_movenet.py, Hailo had nothing and no doc named a
|
||||
# source, so [6/8] demanded a file the operator had no way to obtain.
|
||||
# source, so [6/9] demanded a file the operator had no way to obtain.
|
||||
# fetch_hailo_hef refuses to guess: it reads the arch from `hailortcli fw-control identify`
|
||||
# and exits non-zero when no board answers — exactly the "no Hailo here" case that must fall
|
||||
# through to MoveNet rather than install an h8 HEF that dies at load with
|
||||
|
|
@ -152,7 +152,7 @@ else
|
|||
fi
|
||||
|
||||
# Probe what can ACTUALLY run, in the venv the service will use. A HEF on disk is not enough
|
||||
# (hailo_platform comes from apt hailo-all, best-effort at [1/8]), and a pip exit code is not
|
||||
# (hailo_platform comes from apt hailo-all, best-effort at [1/9]), and a pip exit code is not
|
||||
# enough either (a wheel can install and still fail to load its .so). Both probes must sit
|
||||
# inside `if` conditions — a bare `a && b` list that fails is fatal under set -e.
|
||||
pyhas() { "$PREFIX/venv/bin/python" -c "import $1" >/dev/null 2>&1; }
|
||||
|
|
@ -206,13 +206,13 @@ else
|
|||
exit 1
|
||||
fi
|
||||
|
||||
echo ">> [7/8] config -> $ETC/edge.env"
|
||||
echo ">> [7/9] config -> $ETC/edge.env"
|
||||
install -d -m 0750 "$STATE"; chown "$SVC_USER":"$SVC_USER" "$STATE"
|
||||
install -d -m 0750 "$ETC"
|
||||
if [ -f "$ETC/edge.env" ]; then
|
||||
# Re-running IS the normal Hailo flow: install -> reboot so the driver creates /dev/hailo0
|
||||
# -> re-run. So [6/8] can legitimately reach a different verdict than last time. Leaving the
|
||||
# file wholly unchanged meant run #1's POSE_BACKEND=movenet survived while [6/8] printed
|
||||
# -> re-run. So [6/9] can legitimately reach a different verdict than last time. Leaving the
|
||||
# file wholly unchanged meant run #1's POSE_BACKEND=movenet survived while [6/9] printed
|
||||
# "POSE_BACKEND=hailo" — and with onnxruntime absent that is a ModuleNotFoundError crash
|
||||
# loop the installer just called a success.
|
||||
# Reconcile ONLY the backend keys, in place: DEVICE_SECRET and the operator's other answers
|
||||
|
|
@ -274,7 +274,31 @@ EOF
|
|||
echo " -> wrote $ETC/edge.env"
|
||||
fi
|
||||
|
||||
echo ">> [8/8] systemd service"
|
||||
echo ">> [8/9] camera check (open every camera in CAMERAS and grab a frame)"
|
||||
# Do this BEFORE enabling the unit. build_runtimes() logs-and-skips a camera it cannot open,
|
||||
# so a wrong /dev/video index or an unseated ribbon would otherwise produce a green unit that
|
||||
# silently monitors one camera less. Fail here, while someone is still at the Pi.
|
||||
# SKIP_CAMERA_CHECK=1 for a headless/bench install with no cameras attached yet.
|
||||
if [ "${SKIP_CAMERA_CHECK:-0}" = "1" ]; then
|
||||
echo " -> skipped (SKIP_CAMERA_CHECK=1)"
|
||||
else
|
||||
# Probe exactly what the unit will run with: the CAMERAS line as written to edge.env, and
|
||||
# the installed copy of the package ($PREFIX/src, not the repo checkout).
|
||||
probe_cams="$(sed -n 's/^CAMERAS=//p' "$ETC/edge.env" | head -1)"
|
||||
if ( cd "$PREFIX/src" && CAMERAS="$probe_cams" "$PREFIX/venv/bin/python" -m edge.probe \
|
||||
--save "$STATE/probe" ); then
|
||||
echo " -> every camera delivered a frame (previews in $STATE/probe)"
|
||||
chown -R "$SVC_USER":"$SVC_USER" "$STATE/probe" 2>/dev/null || true
|
||||
else
|
||||
echo "" >&2
|
||||
echo "ERROR: camera check failed — refusing to enable the service." >&2
|
||||
echo " Fix CAMERAS in $ETC/edge.env and re-run, or install with SKIP_CAMERA_CHECK=1" >&2
|
||||
echo " to enable the service anyway (it will monitor only the cameras that work)." >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
echo ">> [9/9] systemd service"
|
||||
install -m 0644 "$REPO_ROOT/deploy/systemd/move-detect-edge.service" /etc/systemd/system/
|
||||
systemctl daemon-reload
|
||||
|
||||
|
|
|
|||
133
edge/probe.py
Normal file
133
edge/probe.py
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
"""Camera probe: open every configured camera, grab a frame, report what came back.
|
||||
|
||||
Run by deploy/setup_online.sh BEFORE the service is enabled, so a miswired ribbon or a
|
||||
USB camera that enumerated on a different /dev/video index fails during the install --
|
||||
while an operator is still standing at the Pi -- instead of becoming a Restart=always
|
||||
unit that monitors one camera less and looks perfectly healthy on the dashboard.
|
||||
|
||||
Standalone too:
|
||||
/opt/move-detect/venv/bin/python -m edge.probe # uses CAMERAS from env
|
||||
CAMERAS=uvc:2:cam-02 python -m edge.probe --save /tmp/probe # write preview PNGs
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
|
||||
from edge.daemon import parse_cameras
|
||||
|
||||
# A camera can open and still hand back nothing for the first few reads while the sensor
|
||||
# settles (UVC auto-exposure, CSI pipeline start). Retry rather than fail a healthy camera.
|
||||
_READ_ATTEMPTS = 8
|
||||
_READ_DELAY_S = 0.25
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProbeResult:
|
||||
camera_id: str
|
||||
backend: str
|
||||
device: str
|
||||
ok: bool
|
||||
detail: str
|
||||
|
||||
|
||||
def _describe(frame) -> str:
|
||||
shape = getattr(frame, "shape", None)
|
||||
if not shape or len(shape) < 2:
|
||||
return "frame of unknown shape"
|
||||
h, w = shape[0], shape[1]
|
||||
ch = shape[2] if len(shape) > 2 else 1
|
||||
return f"{w}x{h}, {ch} channel(s)"
|
||||
|
||||
|
||||
def _default_build_source(backend: str, device: str):
|
||||
"""Same construction the daemon uses (edge/daemon.py build_source)."""
|
||||
if backend == "csi":
|
||||
from edge.adapters.picamera_source import PiCameraSource
|
||||
return PiCameraSource(int(device))
|
||||
from edge.adapters.uvc_source import UvcSource
|
||||
return UvcSource(int(device))
|
||||
|
||||
|
||||
def probe_one(backend: str, device: str, camera_id: str, *, build_source=None,
|
||||
save_dir: str | None = None) -> ProbeResult:
|
||||
"""Open one camera, read a frame, optionally save a preview PNG."""
|
||||
build_source = build_source or _default_build_source
|
||||
try:
|
||||
source = build_source(backend, device)
|
||||
except Exception as exc: # device missing, wrong index, picamera2 not installed
|
||||
return ProbeResult(camera_id, backend, device, False, f"open failed: {exc}")
|
||||
|
||||
try:
|
||||
for attempt in range(_READ_ATTEMPTS):
|
||||
ok, frame = source.read()
|
||||
if ok and frame is not None:
|
||||
detail = _describe(frame)
|
||||
if save_dir:
|
||||
detail += f"; {_save_preview(frame, save_dir, camera_id)}"
|
||||
return ProbeResult(camera_id, backend, device, True, detail)
|
||||
if attempt < _READ_ATTEMPTS - 1:
|
||||
time.sleep(_READ_DELAY_S)
|
||||
return ProbeResult(camera_id, backend, device, False,
|
||||
f"opened but returned no frame in {_READ_ATTEMPTS} reads")
|
||||
finally:
|
||||
close = getattr(source, "close", None) or getattr(source, "release", None)
|
||||
if callable(close):
|
||||
try:
|
||||
close()
|
||||
except Exception:
|
||||
pass # a probe must never fail the install on a teardown hiccup
|
||||
|
||||
|
||||
def _save_preview(frame, save_dir: str, camera_id: str) -> str:
|
||||
try:
|
||||
import cv2
|
||||
os.makedirs(save_dir, exist_ok=True)
|
||||
path = os.path.join(save_dir, f"{camera_id}.png")
|
||||
cv2.imwrite(path, frame)
|
||||
return f"preview {path}"
|
||||
except Exception as exc:
|
||||
return f"preview failed: {exc}"
|
||||
|
||||
|
||||
def probe_all(env=None, *, build_source=None, save_dir: str | None = None) -> list[ProbeResult]:
|
||||
env = dict(os.environ if env is None else env)
|
||||
cams = parse_cameras(env, env.get("DEVICE_ID", "cam-01"))
|
||||
return [probe_one(be, dev, cid, build_source=build_source, save_dir=save_dir)
|
||||
for be, dev, cid in cams]
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
ap = argparse.ArgumentParser(description="probe every camera in CAMERAS")
|
||||
ap.add_argument("--save", metavar="DIR", help="write a preview PNG per camera")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
try:
|
||||
results = probe_all(save_dir=args.save)
|
||||
except ValueError as exc: # malformed CAMERAS -- a config error, not a camera fault
|
||||
print(f"CAMERAS is invalid: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
for r in results:
|
||||
mark = "OK " if r.ok else "FAIL"
|
||||
print(f" [{mark}] {r.camera_id} {r.backend}:{r.device} -- {r.detail}")
|
||||
|
||||
failed = [r for r in results if not r.ok]
|
||||
if failed:
|
||||
print(f"{len(failed)}/{len(results)} camera(s) failed.", file=sys.stderr)
|
||||
if any(r.backend == "uvc" for r in failed):
|
||||
print(" USB cameras enumerate in boot order -- check `v4l2-ctl --list-devices` "
|
||||
"and fix the index in CAMERAS.", file=sys.stderr)
|
||||
if any(r.backend == "csi" for r in failed):
|
||||
print(" CSI: check the ribbon seating/orientation and `rpicam-hello --list-cameras`.",
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
raise SystemExit(main())
|
||||
78
tests/test_probe.py
Normal file
78
tests/test_probe.py
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
"""Camera probe (edge/probe.py) — all with fakes, no hardware."""
|
||||
|
||||
import pytest
|
||||
|
||||
from edge import probe
|
||||
|
||||
|
||||
class FakeFrame:
|
||||
def __init__(self, shape):
|
||||
self.shape = shape
|
||||
|
||||
|
||||
class FakeSource:
|
||||
"""Yields `blanks` empty reads, then frames. Records that close() was called."""
|
||||
|
||||
def __init__(self, blanks=0, shape=(480, 640, 3)):
|
||||
self._blanks = blanks
|
||||
self._shape = shape
|
||||
self.closed = False
|
||||
|
||||
def read(self):
|
||||
if self._blanks > 0:
|
||||
self._blanks -= 1
|
||||
return False, None
|
||||
return True, FakeFrame(self._shape)
|
||||
|
||||
def close(self):
|
||||
self.closed = True
|
||||
|
||||
|
||||
def test_probe_reports_frame_geometry():
|
||||
r = probe.probe_one("uvc", "0", "cam-02", build_source=lambda b, d: FakeSource())
|
||||
assert r.ok
|
||||
assert "640x480" in r.detail and "3 channel" in r.detail
|
||||
|
||||
|
||||
def test_probe_retries_while_the_sensor_settles():
|
||||
# A UVC camera commonly returns nothing for the first reads; that is not a failure.
|
||||
src = FakeSource(blanks=probe._READ_ATTEMPTS - 1)
|
||||
r = probe.probe_one("uvc", "0", "cam-02", build_source=lambda b, d: src)
|
||||
assert r.ok
|
||||
|
||||
|
||||
def test_probe_fails_when_no_frame_ever_arrives(monkeypatch):
|
||||
monkeypatch.setattr(probe, "_READ_DELAY_S", 0)
|
||||
src = FakeSource(blanks=probe._READ_ATTEMPTS)
|
||||
r = probe.probe_one("csi", "0", "cam-01", build_source=lambda b, d: src)
|
||||
assert not r.ok
|
||||
assert "no frame" in r.detail
|
||||
|
||||
|
||||
def test_probe_fails_when_the_device_cannot_be_opened():
|
||||
def boom(backend, device):
|
||||
raise RuntimeError("UVC device 0 failed to open")
|
||||
|
||||
r = probe.probe_one("uvc", "0", "cam-02", build_source=boom)
|
||||
assert not r.ok
|
||||
assert "open failed" in r.detail
|
||||
|
||||
|
||||
def test_probe_closes_the_source_even_on_failure(monkeypatch):
|
||||
monkeypatch.setattr(probe, "_READ_DELAY_S", 0)
|
||||
src = FakeSource(blanks=probe._READ_ATTEMPTS)
|
||||
probe.probe_one("uvc", "0", "cam-02", build_source=lambda b, d: src)
|
||||
assert src.closed, "a probe that holds the device open would block the daemon start"
|
||||
|
||||
|
||||
def test_probe_all_covers_every_configured_camera():
|
||||
env = {"CAMERAS": "csi:0:cam-01,uvc:0:cam-02"}
|
||||
results = probe.probe_all(env, build_source=lambda b, d: FakeSource())
|
||||
assert [r.camera_id for r in results] == ["cam-01", "cam-02"]
|
||||
assert [r.backend for r in results] == ["csi", "uvc"]
|
||||
assert all(r.ok for r in results)
|
||||
|
||||
|
||||
def test_probe_all_rejects_a_malformed_cameras_value():
|
||||
with pytest.raises(ValueError):
|
||||
probe.probe_all({"CAMERAS": "csi:0"}, build_source=lambda b, d: FakeSource())
|
||||
Loading…
Reference in a new issue