From 5444f19d45f11a4869f772ec9d7659e171e097d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=9C=A0=EC=B0=BD=EC=9A=B1?= Date: Mon, 20 Jul 2026 23:22:20 +0900 Subject: [PATCH] feat(deploy): probe every camera before enabling the service build_runtimes() logs-and-skips a camera it cannot open, so a wrong /dev/video index or an unseated ribbon produced a green systemd unit that silently monitored one camera less. - edge/probe.py: opens each camera in CAMERAS, retries while the sensor settles, reports frame geometry, optionally saves preview PNGs, and exits non-zero with a backend-specific hint (v4l2-ctl / rpicam-hello) - setup_online.sh: new step [8/9] runs it against the installed package and the CAMERAS line actually written to edge.env, and refuses to enable the unit on failure. SKIP_CAMERA_CHECK=1 opts out. Tests: 174 pass, 2 skipped (7 new). --- deploy/setup_online.sh | 54 ++++++++++++----- edge/probe.py | 133 +++++++++++++++++++++++++++++++++++++++++ tests/test_probe.py | 78 ++++++++++++++++++++++++ 3 files changed, 250 insertions(+), 15 deletions(-) create mode 100644 edge/probe.py create mode 100644 tests/test_probe.py diff --git a/deploy/setup_online.sh b/deploy/setup_online.sh index 6b42893..c7579c1 100755 --- a/deploy/setup_online.sh +++ b/deploy/setup_online.sh @@ -30,12 +30,12 @@ STATE=/var/lib/move-detect ETC=/etc/move-detect SVC_USER=movedetect -echo ">> [1/8] system packages (CSI camera, venv; Hailo runtime best-effort)" +echo ">> [1/9] system packages (CSI camera, venv; Hailo runtime best-effort)" export DEBIAN_FRONTEND=noninteractive # A freshly-booted Pi runs apt-daily / apt-daily-upgrade / unattended-upgrades, which hold # /var/lib/apt/lists/lock and /var/lib/dpkg/lock-frontend for the first few minutes — # exactly when someone runs a one-command installer. A bare apt-get then exits 100 with -# "E: Unable to lock directory /var/lib/apt/lists/" and `set -e` (line 12) kills us here. +# "E: Unable to lock directory /var/lib/apt/lists/" and `set -e` (set -e above) kills us here. # -o DPkg::Lock::Timeout is not enough on its own: apt applies it to the dpkg locks, but # `apt-get update`'s lists lock is taken with a single non-blocking fcntl(F_SETLK) and no # retry. That wait has to live here, in the shell. @@ -51,7 +51,7 @@ apt_retry() { # renders as "%s 디렉터리를 잠글 수 없습니다", the grep below misses it, and lock # contention gets misread as a real error — fail-fast, zero retries, exactly the bug # this function exists to fix. Pinning it per-call (not script-wide) keeps pip/venv and - # fetch_movenet at [4/8]-[5/8] on the system locale. + # fetch_movenet at [4/9]-[5/9] on the system locale. LC_ALL=C apt-get -o DPkg::Lock::Timeout=60 "$@" 2>&1 | tee "$log" || rc=${PIPESTATUS[0]} if [ "$rc" -eq 0 ]; then rm -f "$log"; return 0; fi if ! grep -qE 'Could not get lock|Unable to lock|Unable to acquire the dpkg' "$log"; then @@ -73,13 +73,13 @@ apt_retry install -y python3-picamera2 python3-venv python3-pip # Best-effort so a Pi without it still installs and runs on CPU MoveNet. apt_retry install -y hailo-all || echo " WARN: hailo-all not installed — Hailo accel unavailable, CPU MoveNet will be used." -echo ">> [2/8] service account + device groups" +echo ">> [2/9] service account + device groups" id -u "$SVC_USER" >/dev/null 2>&1 || useradd --system --no-create-home --shell /usr/sbin/nologin "$SVC_USER" for g in video render plugdev hailo; do getent group "$g" >/dev/null 2>&1 && usermod -aG "$g" "$SVC_USER" || true done -echo ">> [3/8] source + dirs under $PREFIX" +echo ">> [3/9] source + dirs under $PREFIX" install -d -m 0755 "$PREFIX" "$PREFIX/src" "$PREFIX/models" "$PREFIX/docs" rm -rf "$PREFIX/src/edge" # full replace on re-run (no nested edge/edge) cp -r "$REPO_ROOT/edge" "$PREFIX/src/edge" @@ -87,7 +87,7 @@ cp "$REPO_ROOT/pyproject.toml" "$PREFIX/src/" 2>/dev/null || true find "$PREFIX/src" -name __pycache__ -type d -prune -exec rm -rf {} + cp "$REPO_ROOT/docs/pilot-runbook.md" "$PREFIX/docs/" 2>/dev/null || true -echo ">> [4/8] venv (+ system site-packages) + pip runtime deps" +echo ">> [4/9] venv (+ system site-packages) + pip runtime deps" python3 -m venv --system-site-packages "$PREFIX/venv" "$PREFIX/venv/bin/pip" install --upgrade pip >/dev/null # MANDATORY for every backend — do not soften these to `|| echo WARN`: @@ -106,12 +106,12 @@ python3 -m venv --system-site-packages "$PREFIX/venv" # MoveNetPoseAdapter.__init__, and the Hailo path never imports the module at all. It # publishes NO sdist and NO 32-bit-ARM wheel, so pip can legitimately find zero candidates # and exit non-zero in seconds — which under `set -euo pipefail` used to kill this installer -# on a Hailo-8 Pi that never needed it. Best-effort, like hailo-all at [1/8]; [6/8] decides +# on a Hailo-8 Pi that never needed it. Best-effort, like hailo-all at [1/9]; [6/9] decides # whether a backend actually exists. "$PREFIX/venv/bin/pip" install onnxruntime \ || echo " WARN: onnxruntime did not install — CPU MoveNet unavailable (Hailo unaffected)." -echo ">> [5/8] MoveNet CPU model" +echo ">> [5/9] MoveNet CPU model" # Best-effort for the same reason: this model is inert without onnxruntime, and a download # hiccup must not kill an install whose backend is Hailo. fetch_movenet is pure stdlib, so it # runs fine with onnxruntime absent. @@ -119,7 +119,7 @@ echo ">> [5/8] MoveNet CPU model" || echo " WARN: MoveNet ONNX download failed — CPU MoveNet unavailable (Hailo unaffected)." cp "$REPO_ROOT/models/movenet_lightning.onnx" "$PREFIX/models/" 2>/dev/null || true -echo ">> [6/8] pose backend" +echo ">> [6/9] pose backend" HEF_DEST="$PREFIX/models/yolov8s_pose.hef" ONNX_DEST="$PREFIX/models/movenet_lightning.onnx" if [ -n "${HAILO_HEF:-}" ]; then @@ -133,7 +133,7 @@ if [ -n "${HAILO_HEF:-}" ]; then else # No HEF supplied: fetch the one matching THIS board. Nothing in this repo could previously # produce a .hef — MoveNet had fetch_movenet.py, Hailo had nothing and no doc named a - # source, so [6/8] demanded a file the operator had no way to obtain. + # source, so [6/9] demanded a file the operator had no way to obtain. # fetch_hailo_hef refuses to guess: it reads the arch from `hailortcli fw-control identify` # and exits non-zero when no board answers — exactly the "no Hailo here" case that must fall # through to MoveNet rather than install an h8 HEF that dies at load with @@ -152,7 +152,7 @@ else fi # Probe what can ACTUALLY run, in the venv the service will use. A HEF on disk is not enough -# (hailo_platform comes from apt hailo-all, best-effort at [1/8]), and a pip exit code is not +# (hailo_platform comes from apt hailo-all, best-effort at [1/9]), and a pip exit code is not # enough either (a wheel can install and still fail to load its .so). Both probes must sit # inside `if` conditions — a bare `a && b` list that fails is fatal under set -e. pyhas() { "$PREFIX/venv/bin/python" -c "import $1" >/dev/null 2>&1; } @@ -206,13 +206,13 @@ else exit 1 fi -echo ">> [7/8] config -> $ETC/edge.env" +echo ">> [7/9] config -> $ETC/edge.env" install -d -m 0750 "$STATE"; chown "$SVC_USER":"$SVC_USER" "$STATE" install -d -m 0750 "$ETC" if [ -f "$ETC/edge.env" ]; then # Re-running IS the normal Hailo flow: install -> reboot so the driver creates /dev/hailo0 - # -> re-run. So [6/8] can legitimately reach a different verdict than last time. Leaving the - # file wholly unchanged meant run #1's POSE_BACKEND=movenet survived while [6/8] printed + # -> re-run. So [6/9] can legitimately reach a different verdict than last time. Leaving the + # file wholly unchanged meant run #1's POSE_BACKEND=movenet survived while [6/9] printed # "POSE_BACKEND=hailo" — and with onnxruntime absent that is a ModuleNotFoundError crash # loop the installer just called a success. # Reconcile ONLY the backend keys, in place: DEVICE_SECRET and the operator's other answers @@ -274,7 +274,31 @@ EOF echo " -> wrote $ETC/edge.env" fi -echo ">> [8/8] systemd service" +echo ">> [8/9] camera check (open every camera in CAMERAS and grab a frame)" +# Do this BEFORE enabling the unit. build_runtimes() logs-and-skips a camera it cannot open, +# so a wrong /dev/video index or an unseated ribbon would otherwise produce a green unit that +# silently monitors one camera less. Fail here, while someone is still at the Pi. +# SKIP_CAMERA_CHECK=1 for a headless/bench install with no cameras attached yet. +if [ "${SKIP_CAMERA_CHECK:-0}" = "1" ]; then + echo " -> skipped (SKIP_CAMERA_CHECK=1)" +else + # Probe exactly what the unit will run with: the CAMERAS line as written to edge.env, and + # the installed copy of the package ($PREFIX/src, not the repo checkout). + probe_cams="$(sed -n 's/^CAMERAS=//p' "$ETC/edge.env" | head -1)" + if ( cd "$PREFIX/src" && CAMERAS="$probe_cams" "$PREFIX/venv/bin/python" -m edge.probe \ + --save "$STATE/probe" ); then + echo " -> every camera delivered a frame (previews in $STATE/probe)" + chown -R "$SVC_USER":"$SVC_USER" "$STATE/probe" 2>/dev/null || true + else + echo "" >&2 + echo "ERROR: camera check failed — refusing to enable the service." >&2 + echo " Fix CAMERAS in $ETC/edge.env and re-run, or install with SKIP_CAMERA_CHECK=1" >&2 + echo " to enable the service anyway (it will monitor only the cameras that work)." >&2 + exit 1 + fi +fi + +echo ">> [9/9] systemd service" install -m 0644 "$REPO_ROOT/deploy/systemd/move-detect-edge.service" /etc/systemd/system/ systemctl daemon-reload diff --git a/edge/probe.py b/edge/probe.py new file mode 100644 index 0000000..b058426 --- /dev/null +++ b/edge/probe.py @@ -0,0 +1,133 @@ +"""Camera probe: open every configured camera, grab a frame, report what came back. + +Run by deploy/setup_online.sh BEFORE the service is enabled, so a miswired ribbon or a +USB camera that enumerated on a different /dev/video index fails during the install -- +while an operator is still standing at the Pi -- instead of becoming a Restart=always +unit that monitors one camera less and looks perfectly healthy on the dashboard. + +Standalone too: + /opt/move-detect/venv/bin/python -m edge.probe # uses CAMERAS from env + CAMERAS=uvc:2:cam-02 python -m edge.probe --save /tmp/probe # write preview PNGs +""" + +from __future__ import annotations + +import argparse +import os +import sys +import time +from dataclasses import dataclass + +from edge.daemon import parse_cameras + +# A camera can open and still hand back nothing for the first few reads while the sensor +# settles (UVC auto-exposure, CSI pipeline start). Retry rather than fail a healthy camera. +_READ_ATTEMPTS = 8 +_READ_DELAY_S = 0.25 + + +@dataclass(frozen=True) +class ProbeResult: + camera_id: str + backend: str + device: str + ok: bool + detail: str + + +def _describe(frame) -> str: + shape = getattr(frame, "shape", None) + if not shape or len(shape) < 2: + return "frame of unknown shape" + h, w = shape[0], shape[1] + ch = shape[2] if len(shape) > 2 else 1 + return f"{w}x{h}, {ch} channel(s)" + + +def _default_build_source(backend: str, device: str): + """Same construction the daemon uses (edge/daemon.py build_source).""" + if backend == "csi": + from edge.adapters.picamera_source import PiCameraSource + return PiCameraSource(int(device)) + from edge.adapters.uvc_source import UvcSource + return UvcSource(int(device)) + + +def probe_one(backend: str, device: str, camera_id: str, *, build_source=None, + save_dir: str | None = None) -> ProbeResult: + """Open one camera, read a frame, optionally save a preview PNG.""" + build_source = build_source or _default_build_source + try: + source = build_source(backend, device) + except Exception as exc: # device missing, wrong index, picamera2 not installed + return ProbeResult(camera_id, backend, device, False, f"open failed: {exc}") + + try: + for attempt in range(_READ_ATTEMPTS): + ok, frame = source.read() + if ok and frame is not None: + detail = _describe(frame) + if save_dir: + detail += f"; {_save_preview(frame, save_dir, camera_id)}" + return ProbeResult(camera_id, backend, device, True, detail) + if attempt < _READ_ATTEMPTS - 1: + time.sleep(_READ_DELAY_S) + return ProbeResult(camera_id, backend, device, False, + f"opened but returned no frame in {_READ_ATTEMPTS} reads") + finally: + close = getattr(source, "close", None) or getattr(source, "release", None) + if callable(close): + try: + close() + except Exception: + pass # a probe must never fail the install on a teardown hiccup + + +def _save_preview(frame, save_dir: str, camera_id: str) -> str: + try: + import cv2 + os.makedirs(save_dir, exist_ok=True) + path = os.path.join(save_dir, f"{camera_id}.png") + cv2.imwrite(path, frame) + return f"preview {path}" + except Exception as exc: + return f"preview failed: {exc}" + + +def probe_all(env=None, *, build_source=None, save_dir: str | None = None) -> list[ProbeResult]: + env = dict(os.environ if env is None else env) + cams = parse_cameras(env, env.get("DEVICE_ID", "cam-01")) + return [probe_one(be, dev, cid, build_source=build_source, save_dir=save_dir) + for be, dev, cid in cams] + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(description="probe every camera in CAMERAS") + ap.add_argument("--save", metavar="DIR", help="write a preview PNG per camera") + args = ap.parse_args(argv) + + try: + results = probe_all(save_dir=args.save) + except ValueError as exc: # malformed CAMERAS -- a config error, not a camera fault + print(f"CAMERAS is invalid: {exc}", file=sys.stderr) + return 2 + + for r in results: + mark = "OK " if r.ok else "FAIL" + print(f" [{mark}] {r.camera_id} {r.backend}:{r.device} -- {r.detail}") + + failed = [r for r in results if not r.ok] + if failed: + print(f"{len(failed)}/{len(results)} camera(s) failed.", file=sys.stderr) + if any(r.backend == "uvc" for r in failed): + print(" USB cameras enumerate in boot order -- check `v4l2-ctl --list-devices` " + "and fix the index in CAMERAS.", file=sys.stderr) + if any(r.backend == "csi" for r in failed): + print(" CSI: check the ribbon seating/orientation and `rpicam-hello --list-cameras`.", + file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/tests/test_probe.py b/tests/test_probe.py new file mode 100644 index 0000000..3568788 --- /dev/null +++ b/tests/test_probe.py @@ -0,0 +1,78 @@ +"""Camera probe (edge/probe.py) — all with fakes, no hardware.""" + +import pytest + +from edge import probe + + +class FakeFrame: + def __init__(self, shape): + self.shape = shape + + +class FakeSource: + """Yields `blanks` empty reads, then frames. Records that close() was called.""" + + def __init__(self, blanks=0, shape=(480, 640, 3)): + self._blanks = blanks + self._shape = shape + self.closed = False + + def read(self): + if self._blanks > 0: + self._blanks -= 1 + return False, None + return True, FakeFrame(self._shape) + + def close(self): + self.closed = True + + +def test_probe_reports_frame_geometry(): + r = probe.probe_one("uvc", "0", "cam-02", build_source=lambda b, d: FakeSource()) + assert r.ok + assert "640x480" in r.detail and "3 channel" in r.detail + + +def test_probe_retries_while_the_sensor_settles(): + # A UVC camera commonly returns nothing for the first reads; that is not a failure. + src = FakeSource(blanks=probe._READ_ATTEMPTS - 1) + r = probe.probe_one("uvc", "0", "cam-02", build_source=lambda b, d: src) + assert r.ok + + +def test_probe_fails_when_no_frame_ever_arrives(monkeypatch): + monkeypatch.setattr(probe, "_READ_DELAY_S", 0) + src = FakeSource(blanks=probe._READ_ATTEMPTS) + r = probe.probe_one("csi", "0", "cam-01", build_source=lambda b, d: src) + assert not r.ok + assert "no frame" in r.detail + + +def test_probe_fails_when_the_device_cannot_be_opened(): + def boom(backend, device): + raise RuntimeError("UVC device 0 failed to open") + + r = probe.probe_one("uvc", "0", "cam-02", build_source=boom) + assert not r.ok + assert "open failed" in r.detail + + +def test_probe_closes_the_source_even_on_failure(monkeypatch): + monkeypatch.setattr(probe, "_READ_DELAY_S", 0) + src = FakeSource(blanks=probe._READ_ATTEMPTS) + probe.probe_one("uvc", "0", "cam-02", build_source=lambda b, d: src) + assert src.closed, "a probe that holds the device open would block the daemon start" + + +def test_probe_all_covers_every_configured_camera(): + env = {"CAMERAS": "csi:0:cam-01,uvc:0:cam-02"} + results = probe.probe_all(env, build_source=lambda b, d: FakeSource()) + assert [r.camera_id for r in results] == ["cam-01", "cam-02"] + assert [r.backend for r in results] == ["csi", "uvc"] + assert all(r.ok for r in results) + + +def test_probe_all_rejects_a_malformed_cameras_value(): + with pytest.raises(ValueError): + probe.probe_all({"CAMERAS": "csi:0"}, build_source=lambda b, d: FakeSource())