$id, 'name' => $id, 'site' => $site, 'last_fps' => null, 'last_temp' => null, 'last_throttle' => null, 'config_version' => 1, 'reported_config_version' => 1, 'last_heartbeat_at' => $heartbeat]; } /** @param array $points [tsOffsetFromNow => metric fields] */ private function metrics(string $deviceId, array $points): InMemoryDeviceMetricRepository { $repo = new InMemoryDeviceMetricRepository(); foreach ($points as $offset => $fields) { $repo->append($deviceId, $fields, self::NOW - $offset); } return $repo; } // --- device_fault --------------------------------------------------------------------- public function testSilentDeviceFilesDeviceFault(): void { $events = new InMemoryEventRepository(); $watch = new DeviceHealthWatch( $events, $this->devices([$this->device('pi-01', self::NOW - 600)]), new InMemoryDeviceMetricRepository() ); $filed = $watch->run(self::NOW); self::assertCount(1, $filed); self::assertSame('device_fault', $filed[0]['event_type']); $row = $events->find($filed[0]['event_id']); self::assertSame('warning', $row['severity']); self::assertSame(DeviceHealthWatch::DEVICE_CAMERA, $row['camera_id']); self::assertSame('site-a', $row['site']); self::assertSame(600, $row['metrics']['silent_s']); } public function testHealthyHeartbeatFilesNothing(): void { $events = new InMemoryEventRepository(); $watch = new DeviceHealthWatch( $events, $this->devices([$this->device('pi-01', self::NOW - 20)]), $this->metrics('pi-01', [20 => ['fps' => 12.0, 'cameras_erroring' => 0]]) ); self::assertSame([], $watch->run(self::NOW)); } public function testOutageFilesOnceAcrossRepeatedRuns(): void { // The cron runs every minute. An hour-long outage must leave ONE event, not sixty. $events = new InMemoryEventRepository(); $devices = $this->devices([$this->device('pi-01', self::NOW - 600)]); $watch = new DeviceHealthWatch($events, $devices, new InMemoryDeviceMetricRepository()); $first = $watch->run(self::NOW); $second = $watch->run(self::NOW + 60); $third = $watch->run(self::NOW + 120); self::assertCount(1, $first); self::assertSame([], $second); self::assertSame([], $third); } public function testRecoveredThenFailedAgainIsASecondEvent(): void { $events = new InMemoryEventRepository(); $devices = $this->devices([$this->device('pi-01', self::NOW - 600)]); $watch = new DeviceHealthWatch($events, $devices, new InMemoryDeviceMetricRepository()); self::assertCount(1, $watch->run(self::NOW)); // The node came back (new heartbeat) and then went silent again. $devices->statusRows = [$this->device('pi-01', self::NOW + 1000)]; self::assertCount(1, $watch->run(self::NOW + 2000)); } public function testNeverInstalledDeviceIsNotAFault(): void { // A provisioned row that has never reported is an uninstalled node. Alarming here would // make every new site's first day a wall of red. $events = new InMemoryEventRepository(); $watch = new DeviceHealthWatch( $events, $this->devices([$this->device('pi-new', null), $this->device('pi-zero', 0)]), new InMemoryDeviceMetricRepository() ); self::assertSame([], $watch->run(self::NOW)); } public function testSilentDeviceDoesNotAlsoFileDegraded(): void { // A node that is gone must not be reported twice for one failure. $events = new InMemoryEventRepository(); $watch = new DeviceHealthWatch( $events, $this->devices([$this->device('pi-01', self::NOW - 600)]), $this->metrics('pi-01', [900 => ['fps' => 1.0], 700 => ['fps' => 1.0]]) ); $filed = $watch->run(self::NOW); self::assertCount(1, $filed); self::assertSame('device_fault', $filed[0]['event_type']); } // --- monitoring_degraded -------------------------------------------------------------- public function testSustainedLowFpsFilesDegraded(): void { $events = new InMemoryEventRepository(); $watch = new DeviceHealthWatch( $events, $this->devices([$this->device('pi-01', self::NOW - 10)]), $this->metrics('pi-01', [ 800 => ['fps' => 12.0], // healthy anchor 600 => ['fps' => 3.0], 300 => ['fps' => 3.0], 10 => ['fps' => 3.0], ]) ); $filed = $watch->run(self::NOW); self::assertCount(1, $filed); self::assertSame('monitoring_degraded', $filed[0]['event_type']); $row = $events->find($filed[0]['event_id']); self::assertSame(['low_fps'], $row['metrics']['reasons']); self::assertSame(590, $row['metrics']['degraded_s']); } public function testBriefDipDoesNotFileDegraded(): void { // One slow heartbeat is a hiccup. Requiring a sustained run is what separates this event // from noise the operator learns to ignore. $events = new InMemoryEventRepository(); $watch = new DeviceHealthWatch( $events, $this->devices([$this->device('pi-01', self::NOW - 10)]), $this->metrics('pi-01', [ 200 => ['fps' => 12.0], 100 => ['fps' => 12.0], 10 => ['fps' => 3.0], ]) ); self::assertSame([], $watch->run(self::NOW)); } public function testRecoveredFpsFilesNothingEvenAfterALongBadRun(): void { // The latest heartbeat is what decides. A node that already recovered must not alarm. $events = new InMemoryEventRepository(); $watch = new DeviceHealthWatch( $events, $this->devices([$this->device('pi-01', self::NOW - 10)]), $this->metrics('pi-01', [ 900 => ['fps' => 2.0], 600 => ['fps' => 2.0], 10 => ['fps' => 12.0], ]) ); self::assertSame([], $watch->run(self::NOW)); } public function testErroringCameraIsDegradedEvenAtFullFps(): void { $events = new InMemoryEventRepository(); $watch = new DeviceHealthWatch( $events, $this->devices([$this->device('pi-01', self::NOW - 10)]), $this->metrics('pi-01', [ 800 => ['fps' => 12.0, 'cameras_total' => 2, 'cameras_erroring' => 0], 600 => ['fps' => 12.0, 'cameras_total' => 2, 'cameras_erroring' => 1], 10 => ['fps' => 12.0, 'cameras_total' => 2, 'cameras_erroring' => 1], ]) ); $filed = $watch->run(self::NOW); self::assertCount(1, $filed); $row = $events->find($filed[0]['event_id']); self::assertSame(['cameras_erroring'], $row['metrics']['reasons']); self::assertSame(1, $row['metrics']['cameras_erroring']); } public function testThermalThrottleCountsAsDegraded(): void { $events = new InMemoryEventRepository(); $watch = new DeviceHealthWatch( $events, $this->devices([$this->device('pi-01', self::NOW - 10)]), $this->metrics('pi-01', [ 800 => ['fps' => 12.0, 'throttle' => 0], 600 => ['fps' => 12.0, 'throttle' => 1], 10 => ['fps' => 12.0, 'throttle' => 1], ]) ); $filed = $watch->run(self::NOW); self::assertCount(1, $filed); self::assertSame(['thermal_throttle'], $events->find($filed[0]['event_id'])['metrics']['reasons']); } public function testOngoingDegradationFilesOncePerRunNotPerTick(): void { $events = new InMemoryEventRepository(); $watch = new DeviceHealthWatch( $events, $this->devices([$this->device('pi-01', self::NOW - 10)]), $this->metrics('pi-01', [ 800 => ['fps' => 12.0], 600 => ['fps' => 3.0], 10 => ['fps' => 3.0], ]) ); self::assertCount(1, $watch->run(self::NOW)); self::assertSame([], $watch->run(self::NOW + 60)); self::assertSame([], $watch->run(self::NOW + 120)); } public function testMissingFpsIsNotTreatedAsZero(): void { // A node that cannot read a metric reports NULL. Coercing that to 0 would invent an // outage out of a missing reading. $events = new InMemoryEventRepository(); $watch = new DeviceHealthWatch( $events, $this->devices([$this->device('pi-01', self::NOW - 10)]), $this->metrics('pi-01', [ 600 => ['fps' => null, 'cameras_erroring' => 0], 10 => ['fps' => null, 'cameras_erroring' => 0], ]) ); self::assertSame([], $watch->run(self::NOW)); } public function testNoHeartbeatSeriesFilesNothingWhileDeviceIsFresh(): void { // Fresh heartbeat but no metric rows yet (just installed): nothing to judge. $events = new InMemoryEventRepository(); $watch = new DeviceHealthWatch( $events, $this->devices([$this->device('pi-01', self::NOW - 5)]), new InMemoryDeviceMetricRepository() ); self::assertSame([], $watch->run(self::NOW)); } public function testEachDeviceIsJudgedIndependently(): void { $events = new InMemoryEventRepository(); $metrics = new InMemoryDeviceMetricRepository(); $metrics->append('pi-ok', ['fps' => 12.0], self::NOW - 10); $metrics->append('pi-slow', ['fps' => 12.0], self::NOW - 800); $metrics->append('pi-slow', ['fps' => 2.0], self::NOW - 600); $metrics->append('pi-slow', ['fps' => 2.0], self::NOW - 10); $watch = new DeviceHealthWatch($events, $this->devices([ $this->device('pi-ok', self::NOW - 10), $this->device('pi-slow', self::NOW - 10), $this->device('pi-dead', self::NOW - 3600), ]), $metrics); $filed = $watch->run(self::NOW); $byDevice = []; foreach ($filed as $f) $byDevice[$f['device_id']] = $f['event_type']; self::assertSame(['pi-slow' => 'monitoring_degraded', 'pi-dead' => 'device_fault'], $byDevice); } // --- 간헐 고장(플래핑) ------------------------------------------------------------- // // 아래 네 건이 이 클래스에서 가장 중요한 테스트다. edge/daemon.py 는 성공 프레임 하나만 // 들어와도 streak 를 지워 cameras_erroring 를 0 으로 되돌린다 — 리본 접촉 불량이나 USB // 브라운아웃이면 30초 하트비트마다 1↔0 이 오간다. 정상 표본 하나로 저하 구간을 끊는 // 구현은 이 입력에서 (a) 두 시간을 고장 나 있어도 아무것도 못 올리고 (b) 동시에 같은 // 고장을 수십 건으로 쏟아낸다. 기존 테스트가 전부 단조 시퀀스라 그 버그가 CI를 통과했다. /** @param array $pattern [초 전 => 저하 여부] */ private function flapping(string $deviceId, array $pattern): InMemoryDeviceMetricRepository { $repo = new InMemoryDeviceMetricRepository(); foreach ($pattern as $offset => $bad) { $repo->append($deviceId, ['fps' => $bad ? 2.0 : 12.0], self::NOW - $offset); } return $repo; } public function testFlappingCameraStillFilesDegraded(): void { // 30초 간격 40분: 대체로 저하인데 중간중간 정상 프레임이 한 번씩 낀다. $pattern = []; for ($off = 2400; $off >= 0; $off -= 30) { $pattern[$off] = !(($off / 30) % 4 === 0); // 4비트에 한 번 정상 } $pattern[0] = true; // 지금은 저하 $events = new InMemoryEventRepository(); $watch = new DeviceHealthWatch( $events, $this->devices([$this->device('pi-01', self::NOW - 10)]), $this->flapping('pi-01', $pattern) ); $filed = $watch->run(self::NOW); self::assertCount(1, $filed, '깜빡이는 정상 표본에 저하 구간이 끊겨 아무것도 올라오지 않았다'); self::assertGreaterThanOrEqual( 300, $events->find($filed[0]['event_id'])['metrics']['degraded_s']); } public function testFlappingDoesNotStorm(): void { // 6시간 내내 간헐 고장이 이어지고 하트비트도 계속 들어오는 상황을 그대로 재현한다. // 고정 표본으로 now 만 밀면 낡은 행으로 판정하게 되어 실제와 다른 것을 재는 셈이다. $events = new InMemoryEventRepository(); $devices = $this->devices([$this->device('pi-01', self::NOW)]); $repo = new InMemoryDeviceMetricRepository(); $total = 0; for ($t = 0; $t <= 6 * 3600; $t += 30) { $now = self::NOW + $t; // 4비트에 한 번 정상이 끼는 저하 — edge/daemon.py 의 streak 리셋과 같은 양상 $repo->append('pi-01', ['fps' => (($t / 30) % 4 === 0) ? 12.0 : 2.0], $now); $devices->statusRows = [$this->device('pi-01', $now)]; if ($t % 60 === 0) { // 크론은 1분 주기 $total += count((new DeviceHealthWatch($events, $devices, $repo))->run($now)); } } self::assertGreaterThanOrEqual(1, $total, '6시간 내내 고장인데 한 건도 올라오지 않았다'); // 재고지 주기가 1시간이므로 6~7건이 상한이다. 수정 전 구현은 같은 입력에서 수십 건을 냈다. self::assertLessThanOrEqual(7, $total, "6시간 동안 $total 건 — 폭주"); } public function testSustainedRecoveryStartsANewRun(): void { // 긴 정상 구간(유예 초과)은 진짜 회복이다 — 그 이전의 저하는 현재 구간에 포함되지 않는다. $events = new InMemoryEventRepository(); $watch = new DeviceHealthWatch( $events, $this->devices([$this->device('pi-01', self::NOW - 10)]), $this->metrics('pi-01', [ 3000 => ['fps' => 2.0], // 옛 저하 2700 => ['fps' => 2.0], 2400 => ['fps' => 12.0], // 정상이 600초 지속 = 회복 2100 => ['fps' => 12.0], 1800 => ['fps' => 12.0], 120 => ['fps' => 2.0], // 새 저하 — 아직 120초뿐 0 => ['fps' => 2.0], ]) ); // 새 구간은 sustain(300)에 못 미치므로 아무것도 올라오지 않아야 한다. self::assertSame([], $watch->run(self::NOW)); } public function testDeadNodeIsRaisedAgainInTheNextWindow(): void { // 한 달째 죽은 노드가 첫 시각 이후 영원히 조용해지면 안 된다. $events = new InMemoryEventRepository(); $devices = $this->devices([$this->device('pi-01', self::NOW - 600)]); $watch = new DeviceHealthWatch($events, $devices, new InMemoryDeviceMetricRepository()); self::assertCount(1, $watch->run(self::NOW)); self::assertSame([], $watch->run(self::NOW + 600)); // 같은 주기 — 조용 self::assertCount(1, $watch->run(self::NOW + 3600)); // 다음 주기 — 한 번 더 } public function testFutureTimestampedMetricIsIgnored(): void { // NTP 보정이나 수동 백필로 들어온 미래 표본 하나가 "최신"으로 고정되면, 실제로 멈춘 // 노드가 정상으로 보이거나 없던 장애가 sustain 을 건너뛴다. $events = new InMemoryEventRepository(); $repo = new InMemoryDeviceMetricRepository(); $repo->append('pi-01', ['fps' => 12.0], self::NOW + 86400); // 미래·정상 foreach ([900, 600, 300, 0] as $off) { $repo->append('pi-01', ['fps' => 1.0], self::NOW - $off); // 실제로는 계속 저하 } $watch = new DeviceHealthWatch( $events, $this->devices([$this->device('pi-01', self::NOW - 10)]), $repo); $filed = $watch->run(self::NOW); self::assertCount(1, $filed, '미래 표본이 최신으로 잡혀 실제 저하를 가렸다'); self::assertSame('monitoring_degraded', $filed[0]['event_type']); } public function testUuidIsStableAndDistinctPerKindAndDevice(): void { $a = DeviceHealthWatch::uuidFor('device_fault', 'pi-01', 100); self::assertSame($a, DeviceHealthWatch::uuidFor('device_fault', 'pi-01', 100)); self::assertSame(32, strlen($a)); self::assertNotSame($a, DeviceHealthWatch::uuidFor('monitoring_degraded', 'pi-01', 100)); self::assertNotSame($a, DeviceHealthWatch::uuidFor('device_fault', 'pi-02', 100)); self::assertNotSame($a, DeviceHealthWatch::uuidFor('device_fault', 'pi-01', 101)); } }