move_detect/server/tests/DeviceHealthWatchTest.php

<?php
declare(strict_types=1);
namespace App\Tests;

use App\Alert\DeviceHealthWatch;
use App\Tests\Support\{InMemoryEventRepository, InMemoryDeviceRepository, InMemoryDeviceMetricRepository};
use PHPUnit\Framework\TestCase;

/**
 * The two events nobody produced.
 *
 * These tests pin the behaviour that makes them trustworthy rather than noisy: a fault must be
 * filed ONCE per outage (not once per cron tick), a slow node must be degraded for a sustained
 * period before it counts, and a node that has never been installed must stay silent.
 */
final class DeviceHealthWatchTest extends TestCase
{
    private const NOW = 1_700_000_000;

    private function devices(array $rows): InMemoryDeviceRepository
    {
        return new InMemoryDeviceRepository([], [], [], $rows);
    }

    private function device(string $id, ?int $heartbeat, ?string $site = 'site-a'): array
    {
        return ['device_id' => $id, 'name' => $id, 'site' => $site,
                'last_fps' => null, 'last_temp' => null, 'last_throttle' => null,
                'config_version' => 1, 'reported_config_version' => 1,
                'last_heartbeat_at' => $heartbeat];
    }

    /** @param array<int,array> $points [tsOffsetFromNow => metric fields] */
    private function metrics(string $deviceId, array $points): InMemoryDeviceMetricRepository
    {
        $repo = new InMemoryDeviceMetricRepository();
        foreach ($points as $offset => $fields) {
            $repo->append($deviceId, $fields, self::NOW - $offset);
        }
        return $repo;
    }

    // --- device_fault ---------------------------------------------------------------------

    public function testSilentDeviceFilesDeviceFault(): void
    {
        $events = new InMemoryEventRepository();
        $watch = new DeviceHealthWatch(
            $events,
            $this->devices([$this->device('pi-01', self::NOW - 600)]),
            new InMemoryDeviceMetricRepository()
        );

        $filed = $watch->run(self::NOW);

        self::assertCount(1, $filed);
        self::assertSame('device_fault', $filed[0]['event_type']);
        $row = $events->find($filed[0]['event_id']);
        self::assertSame('warning', $row['severity']);
        self::assertSame(DeviceHealthWatch::DEVICE_CAMERA, $row['camera_id']);
        self::assertSame('site-a', $row['site']);
        self::assertSame(600, $row['metrics']['silent_s']);
    }

    public function testHealthyHeartbeatFilesNothing(): void
    {
        $events = new InMemoryEventRepository();
        $watch = new DeviceHealthWatch(
            $events,
            $this->devices([$this->device('pi-01', self::NOW - 20)]),
            $this->metrics('pi-01', [20 => ['fps' => 12.0, 'cameras_erroring' => 0]])
        );

        self::assertSame([], $watch->run(self::NOW));
    }

    public function testOutageFilesOnceAcrossRepeatedRuns(): void
    {
        // The cron runs every minute. An hour-long outage must leave ONE event, not sixty.
        $events = new InMemoryEventRepository();
        $devices = $this->devices([$this->device('pi-01', self::NOW - 600)]);
        $watch = new DeviceHealthWatch($events, $devices, new InMemoryDeviceMetricRepository());

        $first = $watch->run(self::NOW);
        $second = $watch->run(self::NOW + 60);
        $third = $watch->run(self::NOW + 120);

        self::assertCount(1, $first);
        self::assertSame([], $second);
        self::assertSame([], $third);
    }

    public function testRecoveredThenFailedAgainIsASecondEvent(): void
    {
        $events = new InMemoryEventRepository();
        $devices = $this->devices([$this->device('pi-01', self::NOW - 600)]);
        $watch = new DeviceHealthWatch($events, $devices, new InMemoryDeviceMetricRepository());
        self::assertCount(1, $watch->run(self::NOW));

        // The node came back (new heartbeat) and then went silent again.
        $devices->statusRows = [$this->device('pi-01', self::NOW + 1000)];
        self::assertCount(1, $watch->run(self::NOW + 2000));
    }

    public function testNeverInstalledDeviceIsNotAFault(): void
    {
        // A provisioned row that has never reported is an uninstalled node. Alarming here would
        // make every new site's first day a wall of red.
        $events = new InMemoryEventRepository();
        $watch = new DeviceHealthWatch(
            $events,
            $this->devices([$this->device('pi-new', null), $this->device('pi-zero', 0)]),
            new InMemoryDeviceMetricRepository()
        );

        self::assertSame([], $watch->run(self::NOW));
    }

    public function testSilentDeviceDoesNotAlsoFileDegraded(): void
    {
        // A node that is gone must not be reported twice for one failure.
        $events = new InMemoryEventRepository();
        $watch = new DeviceHealthWatch(
            $events,
            $this->devices([$this->device('pi-01', self::NOW - 600)]),
            $this->metrics('pi-01', [900 => ['fps' => 1.0], 700 => ['fps' => 1.0]])
        );

        $filed = $watch->run(self::NOW);
        self::assertCount(1, $filed);
        self::assertSame('device_fault', $filed[0]['event_type']);
    }

    // --- monitoring_degraded --------------------------------------------------------------

    public function testSustainedLowFpsFilesDegraded(): void
    {
        $events = new InMemoryEventRepository();
        $watch = new DeviceHealthWatch(
            $events,
            $this->devices([$this->device('pi-01', self::NOW - 10)]),
            $this->metrics('pi-01', [
                800 => ['fps' => 12.0],   // healthy anchor
                600 => ['fps' => 3.0],
                300 => ['fps' => 3.0],
                10  => ['fps' => 3.0],
            ])
        );

        $filed = $watch->run(self::NOW);

        self::assertCount(1, $filed);
        self::assertSame('monitoring_degraded', $filed[0]['event_type']);
        $row = $events->find($filed[0]['event_id']);
        self::assertSame(['low_fps'], $row['metrics']['reasons']);
        self::assertSame(590, $row['metrics']['degraded_s']);
    }

    public function testBriefDipDoesNotFileDegraded(): void
    {
        // One slow heartbeat is a hiccup. Requiring a sustained run is what separates this event
        // from noise the operator learns to ignore.
        $events = new InMemoryEventRepository();
        $watch = new DeviceHealthWatch(
            $events,
            $this->devices([$this->device('pi-01', self::NOW - 10)]),
            $this->metrics('pi-01', [
                200 => ['fps' => 12.0],
                100 => ['fps' => 12.0],
                10  => ['fps' => 3.0],
            ])
        );

        self::assertSame([], $watch->run(self::NOW));
    }

    public function testRecoveredFpsFilesNothingEvenAfterALongBadRun(): void
    {
        // The latest heartbeat is what decides. A node that already recovered must not alarm.
        $events = new InMemoryEventRepository();
        $watch = new DeviceHealthWatch(
            $events,
            $this->devices([$this->device('pi-01', self::NOW - 10)]),
            $this->metrics('pi-01', [
                900 => ['fps' => 2.0],
                600 => ['fps' => 2.0],
                10  => ['fps' => 12.0],
            ])
        );

        self::assertSame([], $watch->run(self::NOW));
    }

    public function testErroringCameraIsDegradedEvenAtFullFps(): void
    {
        $events = new InMemoryEventRepository();
        $watch = new DeviceHealthWatch(
            $events,
            $this->devices([$this->device('pi-01', self::NOW - 10)]),
            $this->metrics('pi-01', [
                800 => ['fps' => 12.0, 'cameras_total' => 2, 'cameras_erroring' => 0],
                600 => ['fps' => 12.0, 'cameras_total' => 2, 'cameras_erroring' => 1],
                10  => ['fps' => 12.0, 'cameras_total' => 2, 'cameras_erroring' => 1],
            ])
        );

        $filed = $watch->run(self::NOW);

        self::assertCount(1, $filed);
        $row = $events->find($filed[0]['event_id']);
        self::assertSame(['cameras_erroring'], $row['metrics']['reasons']);
        self::assertSame(1, $row['metrics']['cameras_erroring']);
    }

    public function testThermalThrottleCountsAsDegraded(): void
    {
        $events = new InMemoryEventRepository();
        $watch = new DeviceHealthWatch(
            $events,
            $this->devices([$this->device('pi-01', self::NOW - 10)]),
            $this->metrics('pi-01', [
                800 => ['fps' => 12.0, 'throttle' => 0],
                600 => ['fps' => 12.0, 'throttle' => 1],
                10  => ['fps' => 12.0, 'throttle' => 1],
            ])
        );

        $filed = $watch->run(self::NOW);

        self::assertCount(1, $filed);
        self::assertSame(['thermal_throttle'],
            $events->find($filed[0]['event_id'])['metrics']['reasons']);
    }

    public function testOngoingDegradationFilesOncePerRunNotPerTick(): void
    {
        $events = new InMemoryEventRepository();
        $watch = new DeviceHealthWatch(
            $events,
            $this->devices([$this->device('pi-01', self::NOW - 10)]),
            $this->metrics('pi-01', [
                800 => ['fps' => 12.0],
                600 => ['fps' => 3.0],
                10  => ['fps' => 3.0],
            ])
        );

        self::assertCount(1, $watch->run(self::NOW));
        self::assertSame([], $watch->run(self::NOW + 60));
        self::assertSame([], $watch->run(self::NOW + 120));
    }

    public function testMissingFpsIsNotTreatedAsZero(): void
    {
        // A node that cannot read a metric reports NULL. Coercing that to 0 would invent an
        // outage out of a missing reading.
        $events = new InMemoryEventRepository();
        $watch = new DeviceHealthWatch(
            $events,
            $this->devices([$this->device('pi-01', self::NOW - 10)]),
            $this->metrics('pi-01', [
                600 => ['fps' => null, 'cameras_erroring' => 0],
                10  => ['fps' => null, 'cameras_erroring' => 0],
            ])
        );

        self::assertSame([], $watch->run(self::NOW));
    }

    public function testNoHeartbeatSeriesFilesNothingWhileDeviceIsFresh(): void
    {
        // Fresh heartbeat but no metric rows yet (just installed): nothing to judge.
        $events = new InMemoryEventRepository();
        $watch = new DeviceHealthWatch(
            $events,
            $this->devices([$this->device('pi-01', self::NOW - 5)]),
            new InMemoryDeviceMetricRepository()
        );

        self::assertSame([], $watch->run(self::NOW));
    }

    public function testEachDeviceIsJudgedIndependently(): void
    {
        $events = new InMemoryEventRepository();
        $metrics = new InMemoryDeviceMetricRepository();
        $metrics->append('pi-ok', ['fps' => 12.0], self::NOW - 10);
        $metrics->append('pi-slow', ['fps' => 12.0], self::NOW - 800);
        $metrics->append('pi-slow', ['fps' => 2.0], self::NOW - 600);
        $metrics->append('pi-slow', ['fps' => 2.0], self::NOW - 10);

        $watch = new DeviceHealthWatch($events, $this->devices([
            $this->device('pi-ok', self::NOW - 10),
            $this->device('pi-slow', self::NOW - 10),
            $this->device('pi-dead', self::NOW - 3600),
        ]), $metrics);

        $filed = $watch->run(self::NOW);
        $byDevice = [];
        foreach ($filed as $f) $byDevice[$f['device_id']] = $f['event_type'];

        self::assertSame(['pi-slow' => 'monitoring_degraded', 'pi-dead' => 'device_fault'], $byDevice);
    }

    // --- 간헐 고장(플래핑) -------------------------------------------------------------
    //
    // 아래 네 건이 이 클래스에서 가장 중요한 테스트다. edge/daemon.py 는 성공 프레임 하나만
    // 들어와도 streak 를 지워 cameras_erroring 를 0 으로 되돌린다 — 리본 접촉 불량이나 USB
    // 브라운아웃이면 30초 하트비트마다 1↔0 이 오간다. 정상 표본 하나로 저하 구간을 끊는
    // 구현은 이 입력에서 (a) 두 시간을 고장 나 있어도 아무것도 못 올리고 (b) 동시에 같은
    // 고장을 수십 건으로 쏟아낸다. 기존 테스트가 전부 단조 시퀀스라 그 버그가 CI를 통과했다.

    /** @param array<int,bool> $pattern [초 전 => 저하 여부] */
    private function flapping(string $deviceId, array $pattern): InMemoryDeviceMetricRepository
    {
        $repo = new InMemoryDeviceMetricRepository();
        foreach ($pattern as $offset => $bad) {
            $repo->append($deviceId, ['fps' => $bad ? 2.0 : 12.0], self::NOW - $offset);
        }
        return $repo;
    }

    public function testFlappingCameraStillFilesDegraded(): void
    {
        // 30초 간격 40분: 대체로 저하인데 중간중간 정상 프레임이 한 번씩 낀다.
        $pattern = [];
        for ($off = 2400; $off >= 0; $off -= 30) {
            $pattern[$off] = !(($off / 30) % 4 === 0);   // 4비트에 한 번 정상
        }
        $pattern[0] = true;                              // 지금은 저하

        $events = new InMemoryEventRepository();
        $watch = new DeviceHealthWatch(
            $events,
            $this->devices([$this->device('pi-01', self::NOW - 10)]),
            $this->flapping('pi-01', $pattern)
        );

        $filed = $watch->run(self::NOW);
        self::assertCount(1, $filed, '깜빡이는 정상 표본에 저하 구간이 끊겨 아무것도 올라오지 않았다');
        self::assertGreaterThanOrEqual(
            300, $events->find($filed[0]['event_id'])['metrics']['degraded_s']);
    }

    public function testFlappingDoesNotStorm(): void
    {
        // 6시간 내내 간헐 고장이 이어지고 하트비트도 계속 들어오는 상황을 그대로 재현한다.
        // 고정 표본으로 now 만 밀면 낡은 행으로 판정하게 되어 실제와 다른 것을 재는 셈이다.
        $events = new InMemoryEventRepository();
        $devices = $this->devices([$this->device('pi-01', self::NOW)]);
        $repo = new InMemoryDeviceMetricRepository();

        $total = 0;
        for ($t = 0; $t <= 6 * 3600; $t += 30) {
            $now = self::NOW + $t;
            // 4비트에 한 번 정상이 끼는 저하 — edge/daemon.py 의 streak 리셋과 같은 양상
            $repo->append('pi-01', ['fps' => (($t / 30) % 4 === 0) ? 12.0 : 2.0], $now);
            $devices->statusRows = [$this->device('pi-01', $now)];
            if ($t % 60 === 0) {                 // 크론은 1분 주기
                $total += count((new DeviceHealthWatch($events, $devices, $repo))->run($now));
            }
        }

        self::assertGreaterThanOrEqual(1, $total, '6시간 내내 고장인데 한 건도 올라오지 않았다');
        // 재고지 주기가 1시간이므로 6~7건이 상한이다. 수정 전 구현은 같은 입력에서 수십 건을 냈다.
        self::assertLessThanOrEqual(7, $total, "6시간 동안 $total 건 — 폭주");
    }

    public function testSustainedRecoveryStartsANewRun(): void
    {
        // 긴 정상 구간(유예 초과)은 진짜 회복이다 — 그 이전의 저하는 현재 구간에 포함되지 않는다.
        $events = new InMemoryEventRepository();
        $watch = new DeviceHealthWatch(
            $events,
            $this->devices([$this->device('pi-01', self::NOW - 10)]),
            $this->metrics('pi-01', [
                3000 => ['fps' => 2.0],   // 옛 저하
                2700 => ['fps' => 2.0],
                2400 => ['fps' => 12.0],  // 정상이 600초 지속 = 회복
                2100 => ['fps' => 12.0],
                1800 => ['fps' => 12.0],
                120  => ['fps' => 2.0],   // 새 저하 — 아직 120초뿐
                0    => ['fps' => 2.0],
            ])
        );
        // 새 구간은 sustain(300)에 못 미치므로 아무것도 올라오지 않아야 한다.
        self::assertSame([], $watch->run(self::NOW));
    }

    public function testDeadNodeIsRaisedAgainInTheNextWindow(): void
    {
        // 한 달째 죽은 노드가 첫 시각 이후 영원히 조용해지면 안 된다.
        $events = new InMemoryEventRepository();
        $devices = $this->devices([$this->device('pi-01', self::NOW - 600)]);
        $watch = new DeviceHealthWatch($events, $devices, new InMemoryDeviceMetricRepository());

        self::assertCount(1, $watch->run(self::NOW));
        self::assertSame([], $watch->run(self::NOW + 600));        // 같은 주기 — 조용
        self::assertCount(1, $watch->run(self::NOW + 3600));       // 다음 주기 — 한 번 더
    }

    public function testFutureTimestampedMetricIsIgnored(): void
    {
        // NTP 보정이나 수동 백필로 들어온 미래 표본 하나가 "최신"으로 고정되면, 실제로 멈춘
        // 노드가 정상으로 보이거나 없던 장애가 sustain 을 건너뛴다.
        $events = new InMemoryEventRepository();
        $repo = new InMemoryDeviceMetricRepository();
        $repo->append('pi-01', ['fps' => 12.0], self::NOW + 86400);   // 미래·정상
        foreach ([900, 600, 300, 0] as $off) {
            $repo->append('pi-01', ['fps' => 1.0], self::NOW - $off); // 실제로는 계속 저하
        }
        $watch = new DeviceHealthWatch(
            $events, $this->devices([$this->device('pi-01', self::NOW - 10)]), $repo);

        $filed = $watch->run(self::NOW);
        self::assertCount(1, $filed, '미래 표본이 최신으로 잡혀 실제 저하를 가렸다');
        self::assertSame('monitoring_degraded', $filed[0]['event_type']);
    }

    public function testUuidIsStableAndDistinctPerKindAndDevice(): void
    {
        $a = DeviceHealthWatch::uuidFor('device_fault', 'pi-01', 100);
        self::assertSame($a, DeviceHealthWatch::uuidFor('device_fault', 'pi-01', 100));
        self::assertSame(32, strlen($a));
        self::assertNotSame($a, DeviceHealthWatch::uuidFor('monitoring_degraded', 'pi-01', 100));
        self::assertNotSame($a, DeviceHealthWatch::uuidFor('device_fault', 'pi-02', 100));
        self::assertNotSame($a, DeviceHealthWatch::uuidFor('device_fault', 'pi-01', 101));
    }
}