949c6d05c3
The offline bundle shipped unpinned pip deps with no digests, no SBOM, and no signature — an air-gap target can't re-resolve from PyPI, so a truncated or tampered wheel would install silently. build_offline_bundle.sh now emits, from the resolved wheelhouse: - SHA256SUMS — sha256 of every wheel + model - requirements.lock — exact name==version pins for a reproducible rebuild - sbom.json — CycloneDX 1.5 components (name, version, pypi purl, sha256) - SHA256SUMS.asc — optional detached GPG signature when SIGN_KEY is set MANIFEST advertises the integrity set and whether it is signed. install_offline.sh verifies `sha256sum -c SHA256SUMS` (and the signature, if present) BEFORE any pip install, hard-failing on mismatch; older bundles without the file pass with a warning. The generation logic was verified end-to-end on a synthetic stage: requirements.lock/SHA256SUMS correct, `sha256sum -c` passes, sbom.json is valid CycloneDX JSON. Both scripts pass `bash -n`. Tests (test_deploy_artifacts.py): pinned integrity manifest is emitted (+ optional SIGN_KEY), installer verifies integrity before install. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019Xmn8KZWqiQUguEMRtn6rW
175 lines
9.1 KiB
Bash
Executable file
175 lines
9.1 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Build a fully self-contained, offline-installable bundle for the move-detect
|
|
# edge daemon (air-gapped / 폐쇄망 target — see rules/common/build-deployment.md).
|
|
#
|
|
# Run this on a BUILD machine that HAS internet. The resulting tarball carries
|
|
# every runtime wheel + the pose model + the edge source + the systemd unit, so
|
|
# the air-gapped Pi installs with ZERO network fetches (see install_offline.sh).
|
|
#
|
|
# RELIABILITY NOTE: onnxruntime and opencv publish per-architecture wheels. The
|
|
# most reliable build is a NATIVE one — run this ON an aarch64 host (a Pi 5, or
|
|
# an arm64 CI runner) so pip downloads wheels for the exact target. Cross-downloading
|
|
# aarch64 wheels from an x86 box (--cross) is best-effort and can miss a wheel whose
|
|
# tags don't match; if it fails, build natively.
|
|
#
|
|
# Usage:
|
|
# deploy/bundle/build_offline_bundle.sh # native (host arch)
|
|
# deploy/bundle/build_offline_bundle.sh --cross # cross to aarch64/cp311
|
|
# TARGET_ARCH=aarch64 PYVER=3.11 deploy/bundle/build_offline_bundle.sh --cross
|
|
set -euo pipefail
|
|
|
|
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
OUT_DIR="${OUT_DIR:-$REPO_ROOT/dist}"
|
|
STAGE="$OUT_DIR/move-detect-edge-bundle"
|
|
WHEELHOUSE="$STAGE/wheelhouse"
|
|
PYVER="${PYVER:-3.11}" # Pi OS Bookworm ships CPython 3.11
|
|
PYABI="cp${PYVER//./}" # cp311
|
|
TARGET_ARCH="${TARGET_ARCH:-aarch64}"
|
|
HAILO_ARCH="${HAILO_ARCH:-hailo8}" # 26 TOPS AI HAT+ (Hailo-8); set hailo8l for the 13 TOPS AI Kit
|
|
# Mandatory for EVERY backend (cv2/numpy/requests are on the unconditional startup chain).
|
|
# onnxruntime is the CPU-MoveNet fallback only and publishes no 32-bit-ARM wheel / no sdist, so
|
|
# it is downloaded best-effort — a Hailo bundle need not carry it (install_offline treats it optional).
|
|
RUNTIME_PKGS=(numpy requests opencv-python-headless)
|
|
OPTIONAL_PKGS=(onnxruntime)
|
|
|
|
MODE="native"
|
|
[ "${1:-}" = "--cross" ] && MODE="cross"
|
|
|
|
echo ">> build mode : $MODE"
|
|
echo ">> python target : $PYVER ($PYABI)"
|
|
echo ">> target arch : $TARGET_ARCH"
|
|
echo ">> hailo arch : $HAILO_ARCH"
|
|
echo ">> output : $OUT_DIR"
|
|
|
|
rm -rf "$STAGE"
|
|
mkdir -p "$WHEELHOUSE" "$STAGE/src" "$STAGE/models" "$STAGE/deploy/systemd" "$STAGE/docs"
|
|
|
|
echo ">> [1/6] downloading runtime wheels -> wheelhouse/"
|
|
if [ "$MODE" = "cross" ]; then
|
|
# Best-effort cross download. Try the common manylinux aarch64 tags.
|
|
python3 -m pip download \
|
|
--only-binary=:all: \
|
|
--python-version "$PYVER" --implementation cp --abi "$PYABI" \
|
|
--platform "manylinux2014_${TARGET_ARCH}" \
|
|
--platform "manylinux_2_28_${TARGET_ARCH}" \
|
|
-d "$WHEELHOUSE" "${RUNTIME_PKGS[@]}"
|
|
# CPU fallback wheel, best-effort — onnxruntime may have no matching aarch64/$PYABI tag.
|
|
python3 -m pip download \
|
|
--only-binary=:all: \
|
|
--python-version "$PYVER" --implementation cp --abi "$PYABI" \
|
|
--platform "manylinux2014_${TARGET_ARCH}" \
|
|
--platform "manylinux_2_28_${TARGET_ARCH}" \
|
|
-d "$WHEELHOUSE" "${OPTIONAL_PKGS[@]}" \
|
|
|| echo " -> onnxruntime aarch64/$PYABI wheel unavailable; Hailo-only bundle (no CPU MoveNet)."
|
|
else
|
|
# Native: wheels for THIS host's interpreter/arch (run on the Pi / arm64 CI).
|
|
# Guard the ABI: numpy/onnxruntime ship version-specific cpXY wheels, so a build
|
|
# host whose python3 != target PYVER produces a wheelhouse the Pi's cp311 venv
|
|
# cannot install offline. Fail loudly instead of shipping a broken bundle.
|
|
HOST_PYVER="$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')"
|
|
if [ "$HOST_PYVER" != "$PYVER" ]; then
|
|
echo "ERROR: native build host python3 is $HOST_PYVER but target PYVER is $PYVER." >&2
|
|
echo " Run with python$PYVER (e.g. 'PYVER=$HOST_PYVER ...' only if the Pi runs $HOST_PYVER)," >&2
|
|
echo " or use --cross to pin aarch64/$PYABI wheels explicitly." >&2
|
|
exit 1
|
|
fi
|
|
# Guard the ARCH too: a native build on an x86_64 workstation would download x86_64 wheels
|
|
# while the tarball is still named/declared aarch64 (line 27/86/92) — a bundle that installs
|
|
# then fails to import on the Pi. numpy/opencv/onnxruntime are per-arch, so this is fatal.
|
|
HOST_ARCH="$(uname -m)"
|
|
if [ "$HOST_ARCH" != "$TARGET_ARCH" ]; then
|
|
echo "ERROR: native build host arch is $HOST_ARCH but TARGET_ARCH is $TARGET_ARCH." >&2
|
|
echo " Build natively ON an $TARGET_ARCH host (a Pi 5 / arm64 CI), or use --cross." >&2
|
|
exit 1
|
|
fi
|
|
python3 -m pip download -d "$WHEELHOUSE" "${RUNTIME_PKGS[@]}"
|
|
# CPU fallback wheel, best-effort — a Hailo bundle is complete without it.
|
|
python3 -m pip download -d "$WHEELHOUSE" "${OPTIONAL_PKGS[@]}" \
|
|
|| echo " -> onnxruntime unavailable for this host; Hailo-only bundle (no CPU MoveNet)."
|
|
fi
|
|
|
|
echo ">> [2/6] fetching MoveNet ONNX model (optional CPU fallback; build-machine only)"
|
|
# Optional: the MoveNet source may be unreachable, and a Hailo-8 delivery runs off the HEF. Do
|
|
# not abort the whole bundle over the CPU fallback — stage it if we can, otherwise carry on.
|
|
if ( cd "$REPO_ROOT" && python3 -m edge.tools.fetch_movenet ); then
|
|
cp "$REPO_ROOT/models/movenet_lightning.onnx" "$STAGE/models/"
|
|
else
|
|
echo " -> MoveNet ONNX unavailable; Hailo-only bundle (CPU fallback not included)."
|
|
fi
|
|
|
|
echo ">> [3/6] fetching Hailo $HAILO_ARCH yolov8s_pose HEF (build-machine only)"
|
|
# Mandatory for a Hailo delivery: this is the primary pose backend. fetch_hailo_hef verifies the
|
|
# download against a pinned sha256 and refuses a mismatch, so a bad/tampered HEF aborts the build.
|
|
( cd "$REPO_ROOT" && HAILO_ARCH="$HAILO_ARCH" python3 -m edge.tools.fetch_hailo_hef --arch "$HAILO_ARCH" )
|
|
cp "$REPO_ROOT/models/yolov8s_pose.hef" "$STAGE/models/"
|
|
|
|
echo ">> [4/6] staging edge source (package + pyproject)"
|
|
cp -r "$REPO_ROOT/edge" "$STAGE/src/edge"
|
|
cp "$REPO_ROOT/pyproject.toml" "$STAGE/src/"
|
|
# do not ship compiled caches
|
|
find "$STAGE/src" -name '__pycache__' -type d -prune -exec rm -rf {} +
|
|
|
|
echo ">> [5/6] staging deploy assets + install script"
|
|
cp "$REPO_ROOT/deploy/systemd/move-detect-edge.service" "$STAGE/deploy/systemd/"
|
|
cp "$REPO_ROOT/deploy/systemd/move-detect-edge.env.example" "$STAGE/deploy/systemd/"
|
|
cp "$REPO_ROOT/deploy/bundle/install_offline.sh" "$STAGE/"
|
|
cp "$REPO_ROOT/docs/pilot-runbook.md" "$STAGE/docs/" 2>/dev/null || true
|
|
chmod +x "$STAGE/install_offline.sh"
|
|
|
|
# Supply-chain integrity (air-gap: the target can't reach PyPI to re-resolve, so pin + hash here).
|
|
# SHA256SUMS — sha256 of every wheel + model; install_offline verifies before installing.
|
|
# requirements.lock — exact name==version of every bundled wheel, for a reproducible rebuild.
|
|
# sbom.json — CycloneDX component list (name, version, pypi purl, sha256).
|
|
echo ">> generating integrity manifest (SHA256SUMS, requirements.lock, sbom.json)"
|
|
( cd "$STAGE" && sha256sum wheelhouse/*.whl models/* > SHA256SUMS )
|
|
|
|
: > "$STAGE/requirements.lock"
|
|
{
|
|
printf '{\n "bomFormat": "CycloneDX",\n "specVersion": "1.5",\n "version": 1,\n "components": [\n'
|
|
sep=""
|
|
for whl in "$WHEELHOUSE"/*.whl; do
|
|
[ -e "$whl" ] || continue
|
|
base="$(basename "$whl")"
|
|
name="$(echo "$base" | cut -d- -f1)" # wheel: {name}-{version}-{pyabi}-...-{plat}.whl
|
|
ver="$(echo "$base" | cut -d- -f2)"
|
|
sum="$(sha256sum "$whl" | cut -d' ' -f1)"
|
|
echo "${name}==${ver}" >> "$STAGE/requirements.lock"
|
|
printf '%s {"type":"library","name":"%s","version":"%s","purl":"pkg:pypi/%s@%s","hashes":[{"alg":"SHA-256","content":"%s"}]}' \
|
|
"$sep" "$name" "$ver" "$name" "$ver" "$sum"
|
|
sep=$',\n'
|
|
done
|
|
printf '\n ]\n}\n'
|
|
} > "$STAGE/sbom.json"
|
|
sort -u -o "$STAGE/requirements.lock" "$STAGE/requirements.lock"
|
|
|
|
# Optional detached GPG signature over the integrity file. The air-gap target verifies via
|
|
# SHA256SUMS regardless; a signature adds provenance when a signing key exists on the build box.
|
|
if [ -n "${SIGN_KEY:-}" ]; then
|
|
gpg --local-user "$SIGN_KEY" --armor --detach-sign --output "$STAGE/SHA256SUMS.asc" "$STAGE/SHA256SUMS"
|
|
echo ">> signed SHA256SUMS with key $SIGN_KEY -> SHA256SUMS.asc"
|
|
else
|
|
echo ">> NOTE: SHA256SUMS is unsigned (set SIGN_KEY=<gpg-id> to add a detached signature)."
|
|
fi
|
|
|
|
# Which pose backends this bundle can actually run, so the operator knows before it crosses the
|
|
# air gap. Hailo needs the HEF (present) + system hailo_platform on the Pi; MoveNet needs the ONNX.
|
|
HAS_ONNX=no; [ -f "$STAGE/models/movenet_lightning.onnx" ] && HAS_ONNX=yes
|
|
{
|
|
echo "bundle: move-detect-edge"
|
|
echo "built_mode: $MODE"
|
|
echo "python: $PYVER ($PYABI)"
|
|
echo "target_arch: $TARGET_ARCH"
|
|
echo "hailo_arch: $HAILO_ARCH"
|
|
echo "pose_backends: hailo (yolov8s_pose.hef)$([ "$HAS_ONNX" = yes ] && echo ', movenet (movenet_lightning.onnx)')"
|
|
echo "integrity: SHA256SUMS (verified on install), requirements.lock, sbom.json$([ -f "$STAGE/SHA256SUMS.asc" ] && echo ', SHA256SUMS.asc (signed)')"
|
|
echo "models:"
|
|
ls -1 "$STAGE/models"
|
|
echo "wheels:"
|
|
ls -1 "$WHEELHOUSE"
|
|
} > "$STAGE/MANIFEST.txt"
|
|
|
|
echo ">> [6/6] packaging tarball"
|
|
TARBALL="$OUT_DIR/move-detect-edge-bundle-${TARGET_ARCH}-${PYABI}.tar.gz"
|
|
tar -C "$OUT_DIR" -czf "$TARBALL" "move-detect-edge-bundle"
|
|
echo ">> DONE: $TARBALL"
|
|
echo ">> transfer this single file across the air gap, then run install_offline.sh on the Pi."
|