feat(workspace): canvas overlay 疊 bbox + WS 即時推論面板（前端塊2）
推論工作區前端塊2：canvas overlay 疊在 MJPEG <img> 上、接 WS inference:<deviceId> 推的 raw InferenceResult、即時繪 bbox+label+信心度； 右側 Inference panel 從 Phase 1 佔位換成 WS 即時結果清單 + FPS/延遲。 照 edge-ai-platform POC 移植（唯讀參考）。 - camera-overlay.tsx：normalized bbox→canvas 像素換算、label 框 - inference-panel.tsx：指標 + 清單 + aria-live 無障礙 - inference-store.ts：fps/avgLatency、MAX_RESULTS=100 上限 - use-inference-stream.ts：用既有 useWebSocket（same-origin cookie、無 token URL） - workspace-client：overlay 塞 CameraFeed slot、訂閱 WS（僅 isRunning && isOnline） canvas 顏色讀設計 token（--chart-*/--background，getComputedStyle 快取、 主題切換才重讀）、跟隨深淺主題、不裸色值。 Reviewer 2 輪通過（0C/0M）。修正： - M1 overlay 座標錯位：feedSize 初值 null、未拿真實顯示尺寸不繪（MJPEG 非 4:3 不錯位） - M2 效能 jank：palette 快取 useState + MutationObserver 主題切換才重讀 （不再每 frame getComputedStyle） - m2 label fallback 色改實際色（canvas 不認 currentColor） - m3 isValidBBox guard：壞 payload 略過不 crash +6 新測試（M1 640×360→height=360、M2 呼叫數不增、m3 NaN/undefined 略過）。 tsc/eslint clean、塊2 範圍 37 test 綠。（全套 11 failed 全屬 conversion-store 既有 time-based flaky、未觸碰） Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
dc6ca211ae
commit
0010cc35c3
@ -0,0 +1,142 @@
|
||||
/**
|
||||
* WorkspaceClient 測試（塊 2-review M1：overlay 未就緒不繪製）
|
||||
*
|
||||
* 聚焦 M1：CameraFeed `<img>` 是 height:auto（實際高依 MJPEG 比例），overlay 必須等
|
||||
* ResizeObserver 回報**真實顯示尺寸**後才繪製；在那之前不可用寫死猜測換算 → 否則 bbox 錯位。
|
||||
*
|
||||
* 策略：
|
||||
* - 用真實 device-store（setState 灌一台 online 裝置）+ 真實 inference-store
|
||||
* - mock `@/lib/api`（start 回 streamUrl）、mock `@/hooks/use-inference-stream`（no-op）
|
||||
* - 用可控 ResizeObserver：預設「不」觸發 callback（模擬尺寸尚未回報）；
|
||||
* 另一測試手動觸發 callback 模擬尺寸就緒
|
||||
*/
|
||||
import { act, fireEvent, render, screen, waitFor } from "@testing-library/react";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { LocaleProvider } from "@/lib/i18n/context";
|
||||
import { useDeviceStore } from "@/stores/device-store";
|
||||
import { useInferenceStore } from "@/stores/inference-store";
|
||||
|
||||
// mock api：start 回 streamUrl；stop 直接 resolve
|
||||
const post = vi.fn();
|
||||
vi.mock("@/lib/api", async () => {
|
||||
const actual = await vi.importActual<typeof import("@/lib/api")>("@/lib/api");
|
||||
return { ...actual, api: { ...actual.api, post: (...a: unknown[]) => post(...a) } };
|
||||
});
|
||||
|
||||
// mock WS hook（避免真連線）
|
||||
vi.mock("@/hooks/use-inference-stream", () => ({
|
||||
useInferenceStream: vi.fn(),
|
||||
}));
|
||||
|
||||
// 可控 ResizeObserver：把 callback 存起來，測試決定何時觸發
|
||||
let roCallbacks: ResizeObserverCallback[] = [];
|
||||
class ControllableRO {
|
||||
cb: ResizeObserverCallback;
|
||||
constructor(cb: ResizeObserverCallback) {
|
||||
this.cb = cb;
|
||||
roCallbacks.push(cb);
|
||||
}
|
||||
observe() {}
|
||||
unobserve() {}
|
||||
disconnect() {}
|
||||
}
|
||||
|
||||
import { WorkspaceClient } from "./workspace-client";
|
||||
|
||||
function seedOnlineDevice() {
|
||||
useDeviceStore.setState({
|
||||
selectedDevice: {
|
||||
id: "dev-1",
|
||||
name: "KL520",
|
||||
remoteStatus: "online",
|
||||
lastSeenAt: null,
|
||||
} as ReturnType<typeof useDeviceStore.getState>["selectedDevice"],
|
||||
isLoading: false,
|
||||
});
|
||||
}
|
||||
|
||||
function renderClient() {
|
||||
return render(
|
||||
<LocaleProvider>
|
||||
<WorkspaceClient deviceId="dev-1" />
|
||||
</LocaleProvider>,
|
||||
);
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
post.mockReset();
|
||||
roCallbacks = [];
|
||||
(globalThis as { ResizeObserver?: unknown }).ResizeObserver = ControllableRO;
|
||||
useInferenceStore.setState({
|
||||
result: null,
|
||||
results: [],
|
||||
fps: 0,
|
||||
avgLatency: 0,
|
||||
batchResults: {},
|
||||
confidenceThreshold: 0.5,
|
||||
});
|
||||
seedOnlineDevice();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
describe("WorkspaceClient — overlay 尺寸就緒閘門（M1）", () => {
|
||||
it("start 後、尺寸尚未回報前不繪製 overlay", async () => {
|
||||
post.mockResolvedValue({ streamUrl: "/api/camera/stream", sourceType: "camera" });
|
||||
// 先放一筆偵測結果（模擬第一筆結果早於 ResizeObserver 回報）
|
||||
useInferenceStore.setState({
|
||||
result: {
|
||||
taskType: "detection",
|
||||
timestamp: Date.now(),
|
||||
latencyMs: 10,
|
||||
detections: [
|
||||
{ label: "x", confidence: 0.9, bbox: { x: 0, y: 0, width: 0.5, height: 0.5 } },
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
renderClient();
|
||||
fireEvent.click(screen.getByText("開始推論"));
|
||||
|
||||
// CameraFeed 應已顯示（有 img），但 overlay 尚未繪製（feedSize 為 null）
|
||||
await waitFor(() => {
|
||||
expect(screen.getByTestId("camera-feed-img")).toBeInTheDocument();
|
||||
});
|
||||
expect(screen.queryByTestId("camera-overlay")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("ResizeObserver 回報真實尺寸後才繪製 overlay", async () => {
|
||||
post.mockResolvedValue({ streamUrl: "/api/camera/stream", sourceType: "camera" });
|
||||
renderClient();
|
||||
fireEvent.click(screen.getByText("開始推論"));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByTestId("camera-feed-img")).toBeInTheDocument();
|
||||
});
|
||||
// 尚未回報 → 無 overlay
|
||||
expect(screen.queryByTestId("camera-overlay")).not.toBeInTheDocument();
|
||||
|
||||
// 模擬 ResizeObserver 回報真實顯示尺寸（非 4:3，例如 640×360）
|
||||
act(() => {
|
||||
const img = screen.getByTestId("camera-feed-img");
|
||||
roCallbacks.forEach((cb) =>
|
||||
cb(
|
||||
[
|
||||
{ target: img, contentRect: { width: 640, height: 360 } } as unknown as ResizeObserverEntry,
|
||||
],
|
||||
{} as ResizeObserver,
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByTestId("camera-overlay")).toBeInTheDocument();
|
||||
});
|
||||
// canvas 尺寸用回報的真實值（360，非寫死 480）
|
||||
const canvas = screen.getByTestId("camera-overlay") as HTMLCanvasElement;
|
||||
expect(canvas.getAttribute("height")).toBe("360");
|
||||
});
|
||||
});
|
||||
@ -28,14 +28,18 @@ import { ArrowLeft, WifiOff } from "lucide-react";
|
||||
|
||||
import { RemoteDeviceBadge } from "@/components/cloud/remote-device-badge";
|
||||
import { CameraFeed } from "@/components/workspace/camera-feed";
|
||||
import { CameraOverlay } from "@/components/workspace/camera-overlay";
|
||||
import { InferencePanel } from "@/components/workspace/inference-panel";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
|
||||
import { Skeleton } from "@/components/ui/skeleton";
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||
import { useInferenceStream } from "@/hooks/use-inference-stream";
|
||||
import { api, ApiError } from "@/lib/api";
|
||||
import { buildStreamUrl } from "@/lib/camera";
|
||||
import { useT } from "@/lib/i18n/context";
|
||||
import { useDeviceStore } from "@/stores/device-store";
|
||||
import { useInferenceStore } from "@/stores/inference-store";
|
||||
import type { MediaUploadResponse } from "@/types/camera";
|
||||
import { toast } from "sonner";
|
||||
|
||||
@ -52,6 +56,12 @@ export function WorkspaceClient({ deviceId }: WorkspaceClientProps) {
|
||||
const [busy, setBusy] = useState(false);
|
||||
// 塊 1：Camera MJPEG 串流 URL（start 成功後由後端回傳的 streamUrl 組出）
|
||||
const [streamUrl, setStreamUrl] = useState("");
|
||||
// 塊 2：CameraFeed `<img>` 實際 render 尺寸（供 overlay 換算 normalized bbox → px）。
|
||||
// review M1：初值為 null（尚未從 ResizeObserver 拿到真實顯示尺寸）。overlay 在拿到
|
||||
// 真實尺寸前不繪製，避免用寫死猜測（640×480）換算 → MJPEG 非 4:3 時 bbox 框錯位。
|
||||
const [feedSize, setFeedSize] = useState<{ w: number; h: number } | null>(null);
|
||||
|
||||
const resetInference = useInferenceStore((s) => s.reset);
|
||||
|
||||
useEffect(() => {
|
||||
if (deviceId) void fetchDevice(deviceId);
|
||||
@ -61,8 +71,24 @@ export function WorkspaceClient({ deviceId }: WorkspaceClientProps) {
|
||||
const clearStream = useCallback(() => {
|
||||
setStreamUrl("");
|
||||
setIsRunning(false);
|
||||
setFeedSize(null); // 重置尺寸，下次 start 重新等 ResizeObserver 回報真實值
|
||||
resetInference();
|
||||
}, [resetInference]);
|
||||
|
||||
const handleDimensionsChange = useCallback((w: number, h: number) => {
|
||||
setFeedSize({ w, h });
|
||||
}, []);
|
||||
|
||||
// 塊 2：裝置線上性（hooks 必須無條件呼叫，故在此處先算，早退前）
|
||||
const isOnline = selectedDevice?.remoteStatus === "online";
|
||||
|
||||
// 塊 2：訂閱 WS `inference:<deviceId>`（只在推論中且線上時連線）
|
||||
useInferenceStream(deviceId, isRunning && isOnline);
|
||||
|
||||
// 塊 2：最新結果的偵測框（供 overlay 繪製）
|
||||
const liveResult = useInferenceStore((s) => s.result);
|
||||
const confidenceThreshold = useInferenceStore((s) => s.confidenceThreshold);
|
||||
|
||||
async function handleStart() {
|
||||
setBusy(true);
|
||||
try {
|
||||
@ -112,7 +138,6 @@ export function WorkspaceClient({ deviceId }: WorkspaceClientProps) {
|
||||
|
||||
// 裝置尚未載入（雛形：selectedDevice 可能為 null）— 簡易占位
|
||||
const device = selectedDevice;
|
||||
const isOnline = device?.remoteStatus === "online";
|
||||
// 裝置掉線時不顯示串流（雲端版對離線極敏感；避免 <img> 卡在已死的 MJPEG 連線）。
|
||||
// 用 render 期衍生而非 effect + setState，避免 cascading render。
|
||||
const effectiveStreamUrl = isOnline ? streamUrl : "";
|
||||
@ -173,6 +198,19 @@ export function WorkspaceClient({ deviceId }: WorkspaceClientProps) {
|
||||
streamUrl={effectiveStreamUrl}
|
||||
sourceType="camera"
|
||||
width={640}
|
||||
onDimensionsChange={handleDimensionsChange}
|
||||
overlay={
|
||||
// M1：feedSize 未就緒（ResizeObserver 尚未回報真實顯示尺寸）前不繪製 overlay，
|
||||
// 避免用寫死猜測換算 bbox → 框錯位。
|
||||
feedSize ? (
|
||||
<CameraOverlay
|
||||
detections={liveResult?.detections ?? []}
|
||||
width={feedSize.w}
|
||||
height={feedSize.h}
|
||||
confidenceThreshold={confidenceThreshold}
|
||||
/>
|
||||
) : undefined
|
||||
}
|
||||
/>
|
||||
) : (
|
||||
<div className="text-center">
|
||||
@ -190,10 +228,8 @@ export function WorkspaceClient({ deviceId }: WorkspaceClientProps) {
|
||||
</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<p className="text-muted-foreground text-sm">
|
||||
{/* 塊 2 補：接 WS `inference:<deviceId>` → ClassificationResult + PerformanceMetrics + overlay */}
|
||||
{t("workspace.inference.panelPending")}
|
||||
</p>
|
||||
{/* 塊 2：接 WS `inference:<deviceId>` → 效能指標 + 結果清單 */}
|
||||
<InferencePanel isRunning={isRunning && isOnline} />
|
||||
</CardContent>
|
||||
</Card>
|
||||
</div>
|
||||
|
||||
@ -0,0 +1,198 @@
|
||||
/**
|
||||
* CameraOverlay 單元測試（塊 2 + 塊 2-review 修正）
|
||||
*
|
||||
* jsdom 無 canvas 2D 實作 → mock getContext 回傳 spy ctx，斷言繪製呼叫。
|
||||
* （jsdom 的 getComputedStyle 對 CSS 自訂變數回空字串 → palette 走 fallback，測試不受影響。）
|
||||
*
|
||||
* 驗證：
|
||||
* - render <canvas>（aria-hidden，語意交給 InferencePanel）
|
||||
* - 每個通過門檻的 detection 觸發 strokeRect + fillText
|
||||
* - 低於 confidenceThreshold 的 detection 不繪製
|
||||
* - normalized bbox 依 width/height 換算 px
|
||||
* - m2：label 文字 fillStyle 為實際色字串（非 "currentColor"）
|
||||
* - m3：缺 bbox / 非數字欄位的壞 detection 被略過（不 throw、不繪製）
|
||||
* - M2：getComputedStyle 不隨 detections 更新而每次重讀（palette 快取）
|
||||
*/
|
||||
import { render, screen } from "@testing-library/react";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import type { DetectionResult } from "@/types/inference";
|
||||
|
||||
import { CameraOverlay } from "./camera-overlay";
|
||||
|
||||
interface CtxSpy {
|
||||
clearRect: ReturnType<typeof vi.fn>;
|
||||
strokeRect: ReturnType<typeof vi.fn>;
|
||||
fillRect: ReturnType<typeof vi.fn>;
|
||||
fillText: ReturnType<typeof vi.fn>;
|
||||
measureText: ReturnType<typeof vi.fn>;
|
||||
strokeStyle: string;
|
||||
fillStyle: string;
|
||||
lineWidth: number;
|
||||
font: string;
|
||||
}
|
||||
|
||||
let ctx: CtxSpy;
|
||||
/** 記錄每次 fillText 呼叫時當下的 fillStyle 值（驗證 m2 label 色）。 */
|
||||
let fillTextStyles: string[];
|
||||
|
||||
beforeEach(() => {
|
||||
fillTextStyles = [];
|
||||
let fillStyleValue = "";
|
||||
const base = {
|
||||
clearRect: vi.fn(),
|
||||
strokeRect: vi.fn(),
|
||||
fillRect: vi.fn(),
|
||||
measureText: vi.fn(() => ({ width: 40 })),
|
||||
strokeStyle: "",
|
||||
lineWidth: 0,
|
||||
font: "",
|
||||
};
|
||||
ctx = {
|
||||
...base,
|
||||
fillText: vi.fn(() => {
|
||||
fillTextStyles.push(fillStyleValue);
|
||||
}),
|
||||
get fillStyle() {
|
||||
return fillStyleValue;
|
||||
},
|
||||
set fillStyle(v: string) {
|
||||
fillStyleValue = v;
|
||||
},
|
||||
} as unknown as CtxSpy;
|
||||
|
||||
vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue(
|
||||
ctx as unknown as CanvasRenderingContext2D,
|
||||
);
|
||||
});
|
||||
|
||||
function det(overrides: Partial<DetectionResult> = {}): DetectionResult {
|
||||
return {
|
||||
label: "obj",
|
||||
confidence: 0.9,
|
||||
bbox: { x: 0.1, y: 0.2, width: 0.3, height: 0.4 },
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("<CameraOverlay />", () => {
|
||||
it("render aria-hidden canvas", () => {
|
||||
render(
|
||||
<CameraOverlay detections={[]} width={640} height={480} confidenceThreshold={0.5} />,
|
||||
);
|
||||
const canvas = screen.getByTestId("camera-overlay");
|
||||
expect(canvas).toBeInTheDocument();
|
||||
expect(canvas).toHaveAttribute("aria-hidden", "true");
|
||||
});
|
||||
|
||||
it("每個通過門檻的 detection 觸發 strokeRect + fillText", () => {
|
||||
render(
|
||||
<CameraOverlay
|
||||
detections={[det(), det({ label: "b" })]}
|
||||
width={640}
|
||||
height={480}
|
||||
confidenceThreshold={0.5}
|
||||
/>,
|
||||
);
|
||||
expect(ctx.strokeRect).toHaveBeenCalledTimes(2);
|
||||
expect(ctx.fillText).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("低於門檻的 detection 不繪製", () => {
|
||||
render(
|
||||
<CameraOverlay
|
||||
detections={[det({ confidence: 0.3 })]}
|
||||
width={640}
|
||||
height={480}
|
||||
confidenceThreshold={0.5}
|
||||
/>,
|
||||
);
|
||||
expect(ctx.strokeRect).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("normalized bbox 依尺寸換算 px", () => {
|
||||
render(
|
||||
<CameraOverlay
|
||||
detections={[det({ bbox: { x: 0.1, y: 0.2, width: 0.3, height: 0.4 } })]}
|
||||
width={1000}
|
||||
height={500}
|
||||
confidenceThreshold={0.5}
|
||||
/>,
|
||||
);
|
||||
// x*1000=100, y*500=100, w*1000=300, h*500=200
|
||||
expect(ctx.strokeRect).toHaveBeenCalledWith(100, 100, 300, 200);
|
||||
});
|
||||
|
||||
it("繪製前先 clearRect 清畫布", () => {
|
||||
render(
|
||||
<CameraOverlay detections={[det()]} width={640} height={480} confidenceThreshold={0.5} />,
|
||||
);
|
||||
expect(ctx.clearRect).toHaveBeenCalledWith(0, 0, 640, 480);
|
||||
});
|
||||
|
||||
it("m2：label 文字 fillStyle 為實際色字串，非 currentColor", () => {
|
||||
render(
|
||||
<CameraOverlay detections={[det()]} width={640} height={480} confidenceThreshold={0.5} />,
|
||||
);
|
||||
expect(fillTextStyles).toHaveLength(1);
|
||||
expect(fillTextStyles[0]).not.toBe("");
|
||||
expect(fillTextStyles[0]).not.toBe("currentColor");
|
||||
});
|
||||
|
||||
it("m3：缺 bbox 的壞 detection 被略過、不 throw", () => {
|
||||
const bad = { label: "bad", confidence: 0.9 } as unknown as DetectionResult;
|
||||
expect(() =>
|
||||
render(
|
||||
<CameraOverlay
|
||||
detections={[bad, det()]}
|
||||
width={640}
|
||||
height={480}
|
||||
confidenceThreshold={0.5}
|
||||
/>,
|
||||
),
|
||||
).not.toThrow();
|
||||
// 只有合法那筆被繪製
|
||||
expect(ctx.strokeRect).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("m3：bbox 欄位非數字（NaN/undefined）被略過", () => {
|
||||
const bad = det({
|
||||
bbox: { x: NaN, y: 0.2, width: 0.3, height: 0.4 },
|
||||
});
|
||||
const bad2 = det({
|
||||
bbox: { x: 0.1, y: 0.2, width: undefined as unknown as number, height: 0.4 },
|
||||
});
|
||||
render(
|
||||
<CameraOverlay
|
||||
detections={[bad, bad2]}
|
||||
width={640}
|
||||
height={480}
|
||||
confidenceThreshold={0.5}
|
||||
/>,
|
||||
);
|
||||
expect(ctx.strokeRect).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("M2：detections 更新時 getComputedStyle 不每次重讀（palette 已快取）", () => {
|
||||
const spy = vi.spyOn(window, "getComputedStyle");
|
||||
const { rerender } = render(
|
||||
<CameraOverlay detections={[det()]} width={640} height={480} confidenceThreshold={0.5} />,
|
||||
);
|
||||
const afterMount = spy.mock.calls.length;
|
||||
|
||||
// 模擬多筆 WS 結果進來（只換 detections）
|
||||
for (let i = 0; i < 5; i++) {
|
||||
rerender(
|
||||
<CameraOverlay
|
||||
detections={[det({ label: `x${i}` })]}
|
||||
width={640}
|
||||
height={480}
|
||||
confidenceThreshold={0.5}
|
||||
/>,
|
||||
);
|
||||
}
|
||||
// detections 更新不應再呼叫 getComputedStyle（palette 快取在 state，只掛載/主題切換時讀）
|
||||
expect(spy.mock.calls.length).toBe(afterMount);
|
||||
spy.mockRestore();
|
||||
});
|
||||
});
|
||||
162
visionA-frontend/src/components/workspace/camera-overlay.tsx
Normal file
162
visionA-frontend/src/components/workspace/camera-overlay.tsx
Normal file
@ -0,0 +1,162 @@
|
||||
"use client";
|
||||
|
||||
/**
|
||||
* CameraOverlay — 疊在 MJPEG `<img>` 上的偵測結果 canvas（塊 2）
|
||||
*
|
||||
* 移植自 POC edge-ai-platform/.../camera/camera-overlay.tsx（唯讀參考、非搬 code）。
|
||||
*
|
||||
* 職責：
|
||||
* - 依 detections（normalized bbox 0~1）換算成 render 尺寸的 px 繪製邊界框
|
||||
* - 每個框附 label + 信心度百分比
|
||||
* - 依 confidenceThreshold 過濾低信心度框
|
||||
*
|
||||
* 設計 token（不裸色值）：
|
||||
* canvas 2D API 無法直接吃 CSS 變數，故用 getComputedStyle 讀取設計系統的
|
||||
* `--chart-1`..`--chart-5`（框色）與 `--background`（label 文字色）token。
|
||||
* ⚠️ 效能（塊 2 review M2）：**不在每 frame 讀** getComputedStyle（forced reflow、30fps 下會 jank）。
|
||||
* 改為掛載時讀一次快取進 state，只在主題切換（`<html>` 的 `.dark` class 變動，由 next-themes
|
||||
* 切換，見 theme-provider.tsx）時透過 MutationObserver 重讀。仍符合「讀設計 token 不裸色值」。
|
||||
*/
|
||||
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
|
||||
import type { DetectionResult } from "@/types/inference";
|
||||
|
||||
interface CameraOverlayProps {
|
||||
detections: DetectionResult[];
|
||||
/** `<img>` 實際 render 寬（px，由 CameraFeed 的 ResizeObserver 回報） */
|
||||
width: number;
|
||||
/** `<img>` 實際 render 高（px） */
|
||||
height: number;
|
||||
confidenceThreshold: number;
|
||||
}
|
||||
|
||||
/** 設計系統 chart token 名稱（globals.css）。canvas 繪圖時讀其 computed 值。 */
|
||||
const CHART_TOKENS = [
|
||||
"--chart-1",
|
||||
"--chart-2",
|
||||
"--chart-3",
|
||||
"--chart-4",
|
||||
"--chart-5",
|
||||
] as const;
|
||||
|
||||
/**
|
||||
* canvas 認得的最終 fallback 色（m2）：`--background` token 讀不到時用。
|
||||
* canvas 2D **不認 `currentColor`**（會沿用前一個 fillStyle → label 與框同色看不見），
|
||||
* 故 fallback 必須是實際色字串。淺色主題背景近白，取 `#ffffff` 當保底（僅在 token 讀失敗才會用到）。
|
||||
*/
|
||||
const LABEL_TEXT_FALLBACK = "#ffffff";
|
||||
|
||||
interface Palette {
|
||||
/** 框色循環調色盤 */
|
||||
colors: string[];
|
||||
/** label 文字色（主題背景色） */
|
||||
labelText: string;
|
||||
}
|
||||
|
||||
/** 從設計 token 讀出調色盤（只在掛載 / 主題切換時呼叫，非每 frame）。 */
|
||||
function readPalette(): Palette {
|
||||
if (typeof window === "undefined") {
|
||||
return { colors: [LABEL_TEXT_FALLBACK], labelText: LABEL_TEXT_FALLBACK };
|
||||
}
|
||||
const style = getComputedStyle(document.documentElement);
|
||||
const colors = CHART_TOKENS.map((tok) => style.getPropertyValue(tok).trim()).filter(
|
||||
Boolean,
|
||||
);
|
||||
const bg = style.getPropertyValue("--background").trim();
|
||||
return {
|
||||
colors: colors.length > 0 ? colors : [LABEL_TEXT_FALLBACK],
|
||||
labelText: bg || LABEL_TEXT_FALLBACK,
|
||||
};
|
||||
}
|
||||
|
||||
export function CameraOverlay({
|
||||
detections,
|
||||
width,
|
||||
height,
|
||||
confidenceThreshold,
|
||||
}: CameraOverlayProps) {
|
||||
const canvasRef = useRef<HTMLCanvasElement>(null);
|
||||
// M2：palette 快取進 state，掛載讀一次；主題切換時由 MutationObserver 重讀。
|
||||
const [palette, setPalette] = useState<Palette>(() => readPalette());
|
||||
|
||||
// 監聽 `<html>` class 變動（next-themes 切換 `.dark`）→ 重讀 palette。
|
||||
// 初始 palette 由 useState lazy initializer（readPalette）於掛載時讀一次（client-only 元件，
|
||||
// 只在 start 推論後才 render，故 initializer 已能拿到正確 computed 值，不需 effect 內再 setState）。
|
||||
useEffect(() => {
|
||||
if (typeof window === "undefined") return;
|
||||
const observer = new MutationObserver(() => setPalette(readPalette()));
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ["class"],
|
||||
});
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
const canvas = canvasRef.current;
|
||||
if (!canvas) return;
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) return;
|
||||
|
||||
ctx.clearRect(0, 0, width, height);
|
||||
|
||||
const { colors, labelText } = palette;
|
||||
const filtered = detections.filter(
|
||||
// m3：防禦壞 payload — 缺 bbox 或欄位非數字的 detection 直接略過，避免 det.bbox.x throw
|
||||
(d) => d.confidence >= confidenceThreshold && isValidBBox(d),
|
||||
);
|
||||
|
||||
filtered.forEach((det, i) => {
|
||||
const color = colors[i % colors.length];
|
||||
// normalized (0~1) → px
|
||||
const px = det.bbox.x * width;
|
||||
const py = det.bbox.y * height;
|
||||
const pw = det.bbox.width * width;
|
||||
const ph = det.bbox.height * height;
|
||||
|
||||
// 邊界框
|
||||
ctx.strokeStyle = color;
|
||||
ctx.lineWidth = 2;
|
||||
ctx.strokeRect(px, py, pw, ph);
|
||||
|
||||
// label 背景 + 文字
|
||||
const label = `${det.label} ${(det.confidence * 100).toFixed(0)}%`;
|
||||
ctx.font = "14px sans-serif";
|
||||
const textWidth = ctx.measureText(label).width;
|
||||
const labelH = 20;
|
||||
const labelY = py - labelH >= 0 ? py - labelH : py; // 避免框在最上緣時 label 出界
|
||||
ctx.fillStyle = color;
|
||||
ctx.fillRect(px, labelY, textWidth + 8, labelH);
|
||||
|
||||
// m2：文字色用實際色字串（快取的 --background），canvas 不認 currentColor
|
||||
ctx.fillStyle = labelText;
|
||||
ctx.fillText(label, px + 4, labelY + labelH - 5);
|
||||
});
|
||||
}, [detections, width, height, confidenceThreshold, palette]);
|
||||
|
||||
return (
|
||||
<canvas
|
||||
ref={canvasRef}
|
||||
width={width}
|
||||
height={height}
|
||||
style={{ width, height }}
|
||||
className="pointer-events-none absolute top-0 left-0"
|
||||
// 純視覺標註層，語意結果由右側 InferencePanel 提供給螢幕閱讀器
|
||||
aria-hidden="true"
|
||||
data-testid="camera-overlay"
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
/** m3：檢查 detection 的 bbox 四個欄位都是有限數字，過濾壞 payload。 */
|
||||
function isValidBBox(d: DetectionResult): boolean {
|
||||
const b = d?.bbox;
|
||||
return (
|
||||
!!b &&
|
||||
Number.isFinite(b.x) &&
|
||||
Number.isFinite(b.y) &&
|
||||
Number.isFinite(b.width) &&
|
||||
Number.isFinite(b.height)
|
||||
);
|
||||
}
|
||||
@ -0,0 +1,115 @@
|
||||
/**
|
||||
* InferencePanel 單元測試（塊 2）
|
||||
*
|
||||
* 驗證：
|
||||
* - isRunning=false → 顯示等待提示（idle）
|
||||
* - isRunning=true 但無結果 → 顯示 waitingResults
|
||||
* - 有 classifications → 顯示 label + 信心度百分比
|
||||
* - 有 detections → 顯示 label + 信心度
|
||||
* - 低於 confidenceThreshold 的結果被過濾
|
||||
* - 效能指標 fps / latency 呈現
|
||||
*/
|
||||
import { render, screen } from "@testing-library/react";
|
||||
import { beforeEach, describe, expect, it } from "vitest";
|
||||
|
||||
import { LocaleProvider } from "@/lib/i18n/context";
|
||||
import { useInferenceStore } from "@/stores/inference-store";
|
||||
|
||||
import { InferencePanel } from "./inference-panel";
|
||||
|
||||
function resetStore() {
|
||||
useInferenceStore.setState({
|
||||
result: null,
|
||||
results: [],
|
||||
fps: 0,
|
||||
avgLatency: 0,
|
||||
batchResults: {},
|
||||
confidenceThreshold: 0.5,
|
||||
});
|
||||
}
|
||||
|
||||
function renderPanel(isRunning: boolean) {
|
||||
return render(
|
||||
<LocaleProvider>
|
||||
<InferencePanel isRunning={isRunning} />
|
||||
</LocaleProvider>,
|
||||
);
|
||||
}
|
||||
|
||||
describe("<InferencePanel />", () => {
|
||||
beforeEach(resetStore);
|
||||
|
||||
it("未推論時顯示 idle 等待提示", () => {
|
||||
renderPanel(false);
|
||||
expect(screen.getByTestId("inference-panel-idle")).toBeInTheDocument();
|
||||
expect(screen.queryByTestId("inference-panel")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("推論中但無結果 → 顯示 waitingResults", () => {
|
||||
renderPanel(true);
|
||||
expect(screen.getByTestId("inference-panel")).toBeInTheDocument();
|
||||
expect(screen.getByText("等待第一筆結果…")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("顯示 classification 結果的 label + 信心度", () => {
|
||||
useInferenceStore.setState({
|
||||
result: {
|
||||
taskType: "classification",
|
||||
timestamp: Date.now(),
|
||||
latencyMs: 15,
|
||||
classifications: [
|
||||
{ label: "dog", confidence: 0.92 },
|
||||
{ label: "cat", confidence: 0.61 },
|
||||
],
|
||||
},
|
||||
fps: 12,
|
||||
avgLatency: 15,
|
||||
});
|
||||
renderPanel(true);
|
||||
expect(screen.getByText("dog")).toBeInTheDocument();
|
||||
expect(screen.getByText("92%")).toBeInTheDocument();
|
||||
expect(screen.getByText("cat")).toBeInTheDocument();
|
||||
expect(screen.getByText("61%")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("過濾低於 confidenceThreshold 的結果", () => {
|
||||
useInferenceStore.setState({
|
||||
confidenceThreshold: 0.7,
|
||||
result: {
|
||||
taskType: "classification",
|
||||
timestamp: Date.now(),
|
||||
latencyMs: 15,
|
||||
classifications: [
|
||||
{ label: "high", confidence: 0.8 },
|
||||
{ label: "low", confidence: 0.4 },
|
||||
],
|
||||
},
|
||||
});
|
||||
renderPanel(true);
|
||||
expect(screen.getByText("high")).toBeInTheDocument();
|
||||
expect(screen.queryByText("low")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("顯示 detection 結果", () => {
|
||||
useInferenceStore.setState({
|
||||
result: {
|
||||
taskType: "detection",
|
||||
timestamp: Date.now(),
|
||||
latencyMs: 20,
|
||||
detections: [
|
||||
{ label: "person", confidence: 0.88, bbox: { x: 0, y: 0, width: 0.3, height: 0.6 } },
|
||||
],
|
||||
},
|
||||
});
|
||||
renderPanel(true);
|
||||
expect(screen.getByText("person")).toBeInTheDocument();
|
||||
expect(screen.getByText("88%")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("呈現 fps 與 latency 指標", () => {
|
||||
useInferenceStore.setState({ fps: 24, avgLatency: 33.7 });
|
||||
renderPanel(true);
|
||||
expect(screen.getByTestId("metric-fps")).toHaveTextContent("24");
|
||||
expect(screen.getByTestId("metric-latency")).toHaveTextContent("34 ms");
|
||||
});
|
||||
});
|
||||
120
visionA-frontend/src/components/workspace/inference-panel.tsx
Normal file
120
visionA-frontend/src/components/workspace/inference-panel.tsx
Normal file
@ -0,0 +1,120 @@
|
||||
"use client";
|
||||
|
||||
/**
|
||||
* InferencePanel — 右側即時推論結果面板（塊 2）
|
||||
*
|
||||
* 取代 workspace-client 原本寫死的「— Phase 1」。接 inference-store 的 WS 結果，顯示：
|
||||
* - 效能指標：FPS + 平均延遲（ms）
|
||||
* - 結果清單：classification（label + 信心度）或 detection（label + 信心度）
|
||||
* - 信心度過濾：低於 confidenceThreshold 的不顯示（與 overlay 同步）
|
||||
*
|
||||
* 無障礙：
|
||||
* - 結果區 aria-live="polite" — 螢幕閱讀器在推論更新時朗讀最新結果
|
||||
* （overlay canvas 是 aria-hidden，語意資訊由此面板負責提供）
|
||||
* - 效能指標用 <dl> 語意結構
|
||||
*
|
||||
* 無結果時（尚未開始 / 剛啟動還沒回第一筆）顯示等待提示。
|
||||
*/
|
||||
|
||||
import { useInferenceStore } from "@/stores/inference-store";
|
||||
import { useT } from "@/lib/i18n/context";
|
||||
import { cn } from "@/lib/utils";
|
||||
|
||||
export interface InferencePanelProps {
|
||||
/** 是否正在推論（決定顯示等待提示或結果） */
|
||||
isRunning: boolean;
|
||||
}
|
||||
|
||||
export function InferencePanel({ isRunning }: InferencePanelProps) {
|
||||
const t = useT();
|
||||
const result = useInferenceStore((s) => s.result);
|
||||
const fps = useInferenceStore((s) => s.fps);
|
||||
const avgLatency = useInferenceStore((s) => s.avgLatency);
|
||||
const threshold = useInferenceStore((s) => s.confidenceThreshold);
|
||||
|
||||
if (!isRunning) {
|
||||
return (
|
||||
<p className="text-muted-foreground text-sm" data-testid="inference-panel-idle">
|
||||
{t("workspace.inference.panelPending")}
|
||||
</p>
|
||||
);
|
||||
}
|
||||
|
||||
const classifications = (result?.classifications ?? []).filter(
|
||||
(c) => c.confidence >= threshold,
|
||||
);
|
||||
const detections = (result?.detections ?? []).filter(
|
||||
(d) => d.confidence >= threshold,
|
||||
);
|
||||
const hasResults = classifications.length > 0 || detections.length > 0;
|
||||
|
||||
return (
|
||||
<div className="space-y-4" data-testid="inference-panel">
|
||||
{/* 效能指標 */}
|
||||
<dl className="grid grid-cols-2 gap-2 text-sm">
|
||||
<div className="bg-muted/50 rounded-md p-2">
|
||||
<dt className="text-muted-foreground text-xs">
|
||||
{t("workspace.inference.fps")}
|
||||
</dt>
|
||||
<dd className="font-mono text-base font-semibold" data-testid="metric-fps">
|
||||
{fps}
|
||||
</dd>
|
||||
</div>
|
||||
<div className="bg-muted/50 rounded-md p-2">
|
||||
<dt className="text-muted-foreground text-xs">
|
||||
{t("workspace.inference.latency")}
|
||||
</dt>
|
||||
<dd
|
||||
className="font-mono text-base font-semibold"
|
||||
data-testid="metric-latency"
|
||||
>
|
||||
{avgLatency.toFixed(0)} ms
|
||||
</dd>
|
||||
</div>
|
||||
</dl>
|
||||
|
||||
{/* 結果清單（aria-live 供螢幕閱讀器朗讀更新） */}
|
||||
<div aria-live="polite" data-testid="inference-results">
|
||||
{!hasResults ? (
|
||||
<p className="text-muted-foreground text-sm">
|
||||
{t("workspace.inference.waitingResults")}
|
||||
</p>
|
||||
) : (
|
||||
<ul className="space-y-1.5">
|
||||
{classifications.map((c, i) => (
|
||||
<li
|
||||
key={`cls-${i}-${c.label}`}
|
||||
className="flex items-center justify-between gap-2 text-sm"
|
||||
>
|
||||
<span className="truncate">{c.label}</span>
|
||||
<ConfidenceBadge value={c.confidence} />
|
||||
</li>
|
||||
))}
|
||||
{detections.map((d, i) => (
|
||||
<li
|
||||
key={`det-${i}-${d.label}`}
|
||||
className="flex items-center justify-between gap-2 text-sm"
|
||||
>
|
||||
<span className="truncate">{d.label}</span>
|
||||
<ConfidenceBadge value={d.confidence} />
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/** 信心度百分比標籤（用設計 token class，不裸色值）。 */
|
||||
function ConfidenceBadge({ value }: { value: number }) {
|
||||
return (
|
||||
<span
|
||||
className={cn(
|
||||
"bg-primary/10 text-primary rounded px-1.5 py-0.5 font-mono text-xs",
|
||||
)}
|
||||
>
|
||||
{(value * 100).toFixed(0)}%
|
||||
</span>
|
||||
);
|
||||
}
|
||||
96
visionA-frontend/src/hooks/use-inference-stream.test.tsx
Normal file
96
visionA-frontend/src/hooks/use-inference-stream.test.tsx
Normal file
@ -0,0 +1,96 @@
|
||||
/**
|
||||
* useInferenceStream 單元測試（塊 2）
|
||||
*
|
||||
* 策略：mock `@/hooks/use-websocket`，攔截 useWebSocket 的 (path, options)，
|
||||
* 直接呼叫 options.onMessage 模擬 WS 推播，斷言 inference-store 被正確更新。
|
||||
*
|
||||
* 驗證：
|
||||
* - path 帶 deviceId（`/ws/devices/:id/inference`）+ enabled 傳遞
|
||||
* - camera 單筆結果 → addResult
|
||||
* - batch 結果（有 imageIndex/totalImages）→ addBatchResult
|
||||
* - pipeline_complete / 非法形狀 → 忽略（不進 store）
|
||||
*/
|
||||
import { renderHook } from "@testing-library/react";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { useInferenceStore } from "@/stores/inference-store";
|
||||
|
||||
// 攔截 useWebSocket：把最後一次呼叫的參數存起來，測試手動觸發 onMessage
|
||||
let lastCall: { path: string; options: { enabled?: boolean; onMessage: (d: unknown) => void } } | null =
|
||||
null;
|
||||
|
||||
vi.mock("@/hooks/use-websocket", () => ({
|
||||
useWebSocket: (path: string, options: { enabled?: boolean; onMessage: (d: unknown) => void }) => {
|
||||
lastCall = { path, options };
|
||||
return { send: vi.fn(), close: vi.fn() };
|
||||
},
|
||||
}));
|
||||
|
||||
// 在 mock 宣告後 import 受測 hook
|
||||
import { useInferenceStream } from "./use-inference-stream";
|
||||
|
||||
function resetStore() {
|
||||
useInferenceStore.setState({
|
||||
result: null,
|
||||
results: [],
|
||||
fps: 0,
|
||||
avgLatency: 0,
|
||||
batchResults: {},
|
||||
confidenceThreshold: 0.5,
|
||||
});
|
||||
}
|
||||
|
||||
describe("useInferenceStream", () => {
|
||||
beforeEach(() => {
|
||||
lastCall = null;
|
||||
resetStore();
|
||||
});
|
||||
afterEach(() => vi.clearAllMocks());
|
||||
|
||||
it("以 deviceId 組出 WS path 並傳遞 enabled", () => {
|
||||
renderHook(() => useInferenceStream("dev-42", true));
|
||||
expect(lastCall?.path).toBe("/ws/devices/dev-42/inference");
|
||||
expect(lastCall?.options.enabled).toBe(true);
|
||||
});
|
||||
|
||||
it("deviceId 有特殊字元時 encode", () => {
|
||||
renderHook(() => useInferenceStream("a/b", false));
|
||||
expect(lastCall?.path).toBe("/ws/devices/a%2Fb/inference");
|
||||
expect(lastCall?.options.enabled).toBe(false);
|
||||
});
|
||||
|
||||
it("camera 單筆結果 → addResult", () => {
|
||||
renderHook(() => useInferenceStream("dev-1", true));
|
||||
lastCall!.options.onMessage({
|
||||
taskType: "detection",
|
||||
timestamp: Date.now(),
|
||||
latencyMs: 12,
|
||||
detections: [{ label: "cat", confidence: 0.9, bbox: { x: 0, y: 0, width: 0.5, height: 0.5 } }],
|
||||
});
|
||||
expect(useInferenceStore.getState().result?.detections?.[0].label).toBe("cat");
|
||||
});
|
||||
|
||||
it("batch 結果（有 imageIndex/totalImages）→ addBatchResult", () => {
|
||||
renderHook(() => useInferenceStream("dev-1", true));
|
||||
lastCall!.options.onMessage({
|
||||
taskType: "classification",
|
||||
timestamp: Date.now(),
|
||||
latencyMs: 5,
|
||||
imageIndex: 3,
|
||||
totalImages: 10,
|
||||
});
|
||||
expect(useInferenceStore.getState().batchResults[3]?.imageIndex).toBe(3);
|
||||
});
|
||||
|
||||
it("pipeline_complete 事件被忽略", () => {
|
||||
renderHook(() => useInferenceStream("dev-1", true));
|
||||
lastCall!.options.onMessage({ type: "pipeline_complete" });
|
||||
expect(useInferenceStore.getState().result).toBeNull();
|
||||
});
|
||||
|
||||
it("非法形狀（缺 timestamp/latencyMs）被丟棄", () => {
|
||||
renderHook(() => useInferenceStream("dev-1", true));
|
||||
lastCall!.options.onMessage({ taskType: "classification" });
|
||||
expect(useInferenceStore.getState().result).toBeNull();
|
||||
});
|
||||
});
|
||||
62
visionA-frontend/src/hooks/use-inference-stream.ts
Normal file
62
visionA-frontend/src/hooks/use-inference-stream.ts
Normal file
@ -0,0 +1,62 @@
|
||||
/**
|
||||
* useInferenceStream — 訂閱 WS `inference:<deviceId>` 推論結果（塊 2）
|
||||
*
|
||||
* 移植自 POC edge-ai-platform/frontend/src/hooks/use-inference-stream.ts（唯讀參考、非搬 code）。
|
||||
* 差異：改用 visionA 既有的 `useWebSocket(path, options)` 簽章（options 物件、非 POC 的 positional）。
|
||||
*
|
||||
* 資料流（見 .autoflow/04-architecture/camera-e2e-effort-estimate.md §0.1 / §8）：
|
||||
* local-tool pipeline → wsHub.BroadcastToRoom("inference:<deviceId>", raw InferenceResult)
|
||||
* → local agent WS pipe → api-server WS forward → 瀏覽器 WS（此 hook）→ inference-store
|
||||
*
|
||||
* Room 加入方式：路徑帶 deviceId（`/ws/devices/:id/inference`），由 server 端依路徑對應 room，
|
||||
* client 不需另送「join room」訊息（與 POC 一致；backend stub 為 stubs.go:72）。
|
||||
*
|
||||
* 認證：same-origin cookie（visiona_session HttpOnly）自動帶，**不放 token 到 URL**
|
||||
* （對齊 use-websocket.ts §10 安全決策 + security 對 token-in-URL 的 Critical 否決）。
|
||||
*
|
||||
* payload 為 raw `driver.InferenceResult` JSON（不含 envelope，contract 定案）。
|
||||
*/
|
||||
|
||||
"use client";
|
||||
|
||||
import { useWebSocket } from "@/hooks/use-websocket";
|
||||
import { useInferenceStore } from "@/stores/inference-store";
|
||||
import type { InferenceResult } from "@/types/inference";
|
||||
|
||||
/**
|
||||
* @param deviceId 目標裝置
|
||||
* @param enabled false 時不連線（例如尚未開始推論 / 裝置離線）
|
||||
*/
|
||||
export function useInferenceStream(deviceId: string, enabled = false): void {
|
||||
const addResult = useInferenceStore((s) => s.addResult);
|
||||
const addBatchResult = useInferenceStore((s) => s.addBatchResult);
|
||||
|
||||
useWebSocket(`/ws/devices/${encodeURIComponent(deviceId)}/inference`, {
|
||||
enabled,
|
||||
onMessage: (data) => {
|
||||
// pipeline 完成事件（batch/video 跑完）— 目前不需特別處理，忽略非結果訊息
|
||||
const msg = data as Record<string, unknown>;
|
||||
if (msg && typeof msg === "object" && msg.type === "pipeline_complete") {
|
||||
return;
|
||||
}
|
||||
|
||||
const result = data as InferenceResult;
|
||||
// 防呆：非預期形狀（缺 timestamp/latencyMs）直接丟棄，不污染 store
|
||||
if (
|
||||
typeof result?.timestamp !== "number" ||
|
||||
typeof result?.latencyMs !== "number"
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
// 批次結果（塊 3）— 有 imageIndex/totalImages 走批次分支
|
||||
if (result.imageIndex !== undefined && result.totalImages !== undefined) {
|
||||
addBatchResult(result);
|
||||
return;
|
||||
}
|
||||
|
||||
// camera / video 單筆結果
|
||||
addResult(result);
|
||||
},
|
||||
});
|
||||
}
|
||||
@ -281,6 +281,9 @@ export const en: Dictionary = {
|
||||
"workspace.inference.panelTitle": "Inference",
|
||||
"workspace.inference.panelPending":
|
||||
"Live results (labels, confidence, bounding boxes) will appear here once inference is running.",
|
||||
"workspace.inference.fps": "FPS",
|
||||
"workspace.inference.latency": "Latency",
|
||||
"workspace.inference.waitingResults": "Waiting for the first result…",
|
||||
"workspace.camera.idleHint":
|
||||
"Press \"Start inference\" to open the camera and begin streaming.",
|
||||
"workspace.camera.startNotWired":
|
||||
|
||||
@ -273,6 +273,9 @@ export const zhHant: Dictionary = {
|
||||
"workspace.inference.panelTitle": "推論結果",
|
||||
"workspace.inference.panelPending":
|
||||
"開始推論後，即時結果（標籤、信心度、邊界框）會顯示在這裡。",
|
||||
"workspace.inference.fps": "FPS",
|
||||
"workspace.inference.latency": "延遲",
|
||||
"workspace.inference.waitingResults": "等待第一筆結果…",
|
||||
"workspace.camera.idleHint": "按「開始推論」開啟攝影機並開始串流。",
|
||||
"workspace.camera.startNotWired": "後端攝影機 proxy 尚未接上。",
|
||||
"workspace.camera.noSource": "沒有輸入來源",
|
||||
|
||||
101
visionA-frontend/src/stores/inference-store.test.ts
Normal file
101
visionA-frontend/src/stores/inference-store.test.ts
Normal file
@ -0,0 +1,101 @@
|
||||
/**
|
||||
* inference-store 單元測試（塊 2）
|
||||
*
|
||||
* 驗證：
|
||||
* - addResult 更新 result / results / fps / avgLatency
|
||||
* - FPS 只計近 1 秒（用固定 timestamp + fake now）
|
||||
* - results 佇列上限 MAX_RESULTS（100）
|
||||
* - addBatchResult 依 imageIndex 寫入 batchResults；無 imageIndex 忽略
|
||||
* - setConfidenceThreshold / reset
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { useInferenceStore } from "./inference-store";
|
||||
import type { InferenceResult } from "@/types/inference";
|
||||
|
||||
function base(overrides: Partial<InferenceResult> = {}): InferenceResult {
|
||||
return {
|
||||
taskType: "classification",
|
||||
timestamp: Date.now(),
|
||||
latencyMs: 10,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function resetStore() {
|
||||
useInferenceStore.setState({
|
||||
result: null,
|
||||
results: [],
|
||||
fps: 0,
|
||||
avgLatency: 0,
|
||||
batchResults: {},
|
||||
confidenceThreshold: 0.5,
|
||||
});
|
||||
}
|
||||
|
||||
describe("inference-store", () => {
|
||||
beforeEach(resetStore);
|
||||
afterEach(() => vi.useRealTimers());
|
||||
|
||||
it("addResult 更新 result 並算 fps / avgLatency", () => {
|
||||
const now = 1_000_000;
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(now);
|
||||
|
||||
const s = useInferenceStore.getState();
|
||||
s.addResult(base({ timestamp: now - 100, latencyMs: 20 }));
|
||||
s.addResult(base({ timestamp: now - 50, latencyMs: 40 }));
|
||||
|
||||
const state = useInferenceStore.getState();
|
||||
expect(state.result?.latencyMs).toBe(40);
|
||||
expect(state.results.length).toBe(2);
|
||||
expect(state.fps).toBe(2); // 兩筆都在近 1 秒內
|
||||
expect(state.avgLatency).toBe(30); // (20+40)/2
|
||||
});
|
||||
|
||||
it("超過 1 秒的舊結果不計入 fps", () => {
|
||||
const now = 2_000_000;
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(now);
|
||||
|
||||
const s = useInferenceStore.getState();
|
||||
s.addResult(base({ timestamp: now - 5000 })); // 5 秒前
|
||||
s.addResult(base({ timestamp: now - 200 })); // 近期
|
||||
|
||||
expect(useInferenceStore.getState().fps).toBe(1);
|
||||
});
|
||||
|
||||
it("results 佇列上限 100", () => {
|
||||
const s = useInferenceStore.getState();
|
||||
for (let i = 0; i < 150; i++) {
|
||||
s.addResult(base({ timestamp: Date.now(), latencyMs: i }));
|
||||
}
|
||||
expect(useInferenceStore.getState().results.length).toBe(100);
|
||||
});
|
||||
|
||||
it("addBatchResult 依 imageIndex 寫入，缺 imageIndex 忽略", () => {
|
||||
const s = useInferenceStore.getState();
|
||||
s.addBatchResult(base({ imageIndex: 2, totalImages: 5, taskType: "detection" }));
|
||||
s.addBatchResult(base({ taskType: "detection" })); // 無 imageIndex → 忽略
|
||||
|
||||
const state = useInferenceStore.getState();
|
||||
expect(Object.keys(state.batchResults)).toEqual(["2"]);
|
||||
expect(state.batchResults[2]?.imageIndex).toBe(2);
|
||||
});
|
||||
|
||||
it("setConfidenceThreshold 更新門檻", () => {
|
||||
useInferenceStore.getState().setConfidenceThreshold(0.8);
|
||||
expect(useInferenceStore.getState().confidenceThreshold).toBe(0.8);
|
||||
});
|
||||
|
||||
it("reset 清空結果但保留 confidenceThreshold 的預設重置行為", () => {
|
||||
const s = useInferenceStore.getState();
|
||||
s.addResult(base());
|
||||
s.reset();
|
||||
const state = useInferenceStore.getState();
|
||||
expect(state.result).toBeNull();
|
||||
expect(state.results).toEqual([]);
|
||||
expect(state.fps).toBe(0);
|
||||
expect(state.batchResults).toEqual({});
|
||||
});
|
||||
});
|
||||
98
visionA-frontend/src/stores/inference-store.ts
Normal file
98
visionA-frontend/src/stores/inference-store.ts
Normal file
@ -0,0 +1,98 @@
|
||||
/**
|
||||
* Inference Store — visionA Cloud（塊 2）
|
||||
*
|
||||
* 對齊：
|
||||
* - local-tool `driver.InferenceResult`（WS `inference:<deviceId>` raw payload，不含 envelope）
|
||||
* - POC 範本 edge-ai-platform/frontend/src/stores/inference-store.ts（唯讀參考，非搬 code）
|
||||
* - .autoflow/04-architecture/camera-e2e-effort-estimate.md §4（overlay/面板由 WS 結果驅動）
|
||||
*
|
||||
* 職責（塊 2）：
|
||||
* - 保存 WS 推來的最新一筆推論結果 + 近期結果佇列（算 FPS / 平均延遲）
|
||||
* - confidenceThreshold：overlay 與面板共用的信心度過濾門檻
|
||||
* - batchResults：塊 3（批次）用；塊 2 已備欄位，camera 模式不寫入
|
||||
*
|
||||
* 設計原則：
|
||||
* - 純 client 狀態，不打 API（結果由 use-inference-stream WS hook 灌入）
|
||||
* - results 佇列上限 MAX_RESULTS，避免長時間串流記憶體無限成長
|
||||
*/
|
||||
|
||||
"use client";
|
||||
|
||||
import { create } from "zustand";
|
||||
|
||||
import type { InferenceResult } from "@/types/inference";
|
||||
|
||||
/** 近期結果佇列上限（防長串流記憶體膨脹；FPS 只看最近 1 秒故足夠）。 */
|
||||
const MAX_RESULTS = 100;
|
||||
|
||||
interface InferenceState {
|
||||
/** 最新一筆結果（overlay / 面板主要顯示） */
|
||||
result: InferenceResult | null;
|
||||
/** 近期結果佇列（算 FPS / 平均延遲用） */
|
||||
results: InferenceResult[];
|
||||
/** 每秒幀數（由近 1 秒結果數推算） */
|
||||
fps: number;
|
||||
/** 平均推論延遲（ms，近期結果平均） */
|
||||
avgLatency: number;
|
||||
/** 信心度過濾門檻（0~1），overlay 與面板共用 */
|
||||
confidenceThreshold: number;
|
||||
/** 批次結果（塊 3 用，key = imageIndex） */
|
||||
batchResults: Record<number, InferenceResult>;
|
||||
|
||||
/** 灌入一筆結果（WS onMessage 呼叫） */
|
||||
addResult: (result: InferenceResult) => void;
|
||||
/** 灌入一筆批次結果（塊 3 用） */
|
||||
addBatchResult: (result: InferenceResult) => void;
|
||||
/** 設定信心度門檻 */
|
||||
setConfidenceThreshold: (threshold: number) => void;
|
||||
/** 重置（stop 推論 / 切裝置時清空） */
|
||||
reset: () => void;
|
||||
}
|
||||
|
||||
const INITIAL: Pick<
|
||||
InferenceState,
|
||||
"result" | "results" | "fps" | "avgLatency" | "batchResults"
|
||||
> = {
|
||||
result: null,
|
||||
results: [],
|
||||
fps: 0,
|
||||
avgLatency: 0,
|
||||
batchResults: {},
|
||||
};
|
||||
|
||||
export const useInferenceStore = create<InferenceState>((set, get) => ({
|
||||
...INITIAL,
|
||||
confidenceThreshold: 0.5,
|
||||
|
||||
addResult: (result) => {
|
||||
const { results } = get();
|
||||
const next = [...results, result].slice(-MAX_RESULTS);
|
||||
|
||||
// FPS = 近 1 秒內的結果數（timestamp 為 local-tool 端 ms 時戳）
|
||||
const now = Date.now();
|
||||
const recent = next.filter((r) => now - r.timestamp < 1000);
|
||||
const fps = recent.length;
|
||||
|
||||
// 平均延遲取近期結果
|
||||
const avgLatency =
|
||||
recent.length > 0
|
||||
? recent.reduce((sum, r) => sum + r.latencyMs, 0) / recent.length
|
||||
: 0;
|
||||
|
||||
set({ result, results: next, fps, avgLatency });
|
||||
},
|
||||
|
||||
addBatchResult: (result) => {
|
||||
if (result.imageIndex === undefined) return;
|
||||
const { batchResults } = get();
|
||||
set({
|
||||
batchResults: { ...batchResults, [result.imageIndex]: result },
|
||||
result,
|
||||
});
|
||||
},
|
||||
|
||||
setConfidenceThreshold: (threshold) => set({ confidenceThreshold: threshold }),
|
||||
|
||||
reset: () =>
|
||||
set({ result: null, results: [], fps: 0, avgLatency: 0, batchResults: {} }),
|
||||
}));
|
||||
Loading…
x
Reference in New Issue
Block a user