From 744283bd28bab35fe91b49dd8c37758ef473213b Mon Sep 17 00:00:00 2001 From: jim800121chen Date: Wed, 22 Jul 2026 19:27:40 +0800 Subject: [PATCH] =?UTF-8?q?feat(build):=20=E5=AE=89=E8=A3=9D=E5=8C=85?= =?UTF-8?q?=E5=8F=AA=E6=89=93=E5=8C=85=E6=8C=87=E5=AE=9A=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E4=B8=A6=E6=94=B9=E7=94=A8=E4=B8=AD=E6=96=87=E5=90=8D=E7=A8=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 安裝包從 8 個 .nef 縮減為 2 個，models.json 保留完整定義以便日後 加回。 - Makefile 加 BUNDLED_NEFS 白名單，三平台共用 copy_bundled_data helper；用 POSIX find/cp 而非 rsync（Windows CI 的 Git Bash 沒有 rsync） - 白名單檔案不存在時 build 直接失敗，並在複製後驗證 models.json 存在且 .nef 數量相符 —— 避免產出「安裝後 0 個模型」卻回報成功 - FCOS Detection (KL520) 改名為「物件辨識」 - Tiny YOLOv3 (KL520) 改名為「人型監測」 （只改 name/description，id 不動以免影響既有設定與紀錄） - Repository 啟動時過濾 .nef 不存在的模型，否則使用者會看到未打包 的模型並在選取後拿到莫名錯誤 Co-Authored-By: Claude Opus 4.8 (1M context) --- local-tool/Makefile | 61 +++++- local-tool/server/data/models.json | 8 +- .../server/internal/model/repository.go | 57 +++++- .../server/internal/model/repository_test.go | 190 ++++++++++++++++++ 4 files changed, 308 insertions(+), 8 deletions(-) diff --git a/local-tool/Makefile b/local-tool/Makefile index 87f2ac2..a89e930 100644 --- a/local-tool/Makefile +++ b/local-tool/Makefile @@ -18,6 +18,61 @@ OS := $(shell uname -s | tr A-Z a-z) DIST := dist PAYLOAD := visiona-local/payload +# ── 打包用 .nef 白名單（M5-a）──────────────────────────────────────── +# +# server/data/ 底下有 8 個 .nef（KL520 五個、KL720 三個），但安裝包只帶 +# 白名單內的這幾個，其餘不進 payload，藉此縮小安裝檔體積。 +# +# models.json 不做任何過濾，7 個 model 定義全部照原樣複製。執行期由 +# server/internal/model/repository.go 的 NewRepository() 檢查每個 model 的 +# filePath 是否實際存在，不存在的直接不載入（見 M5-c）。因此使用者在 UI +# 只會看到白名單內的 model，未打包的不會出現、也不會選到後拿到莫名錯誤。 +# +# 未來要把某個 model 加回安裝包：把對應的 .nef 相對路徑加進下面這個變數即可， +# models.json 不用動。 +# +# 路徑相對於 server/data/。 +BUNDLED_NEFS := \ + nef/kl520/kl520_20004_fcos-drk53s_w512h512.nef \ + nef/kl520/kl520_tiny_yolo_v3.nef + +# copy_bundled_data：把 server/data/ 複製到 $(1)，但 nef/ 只帶 BUNDLED_NEFS 白名單。 +# $(1) = 目標 data 目錄（例：payload/darwin/data） +# +# 步驟：(a) 複製 server/data/ 下除了 nef/ 以外的所有東西（models.json 等） +# (b) 再逐一複製白名單內的 .nef +# (c) 白名單檔案不存在就直接 fail，避免安靜地產出缺 model 的安裝包 +# +# 只用 POSIX find / cp，不用 rsync —— Windows CI 跑在 Git Bash（windows-2022 + +# shell: bash），該環境沒有 rsync。 +define copy_bundled_data + @set -e; \ + echo "==> 複製 server/data → $(1)（.nef 白名單：$(words $(BUNDLED_NEFS)) 個）"; \ + if [ ! -d server/data ]; then echo "!! ERROR: server/data 不存在 !!"; exit 1; fi; \ + mkdir -p "$(1)"; \ + dest="$$(cd "$(1)" && pwd)"; \ + ( cd server/data && \ + find . -path ./nef -prune -o -type d -print | while read -r d; do mkdir -p "$$dest/$$d"; done && \ + find . -path ./nef -prune -o -type f -print | while read -r f; do cp "$$f" "$$dest/$$f"; done ); \ + if [ ! -f "$$dest/models.json" ]; then \ + echo "!! ERROR: models.json 沒有被複製到 $$dest !!"; exit 1; \ + fi; \ + for nef in $(BUNDLED_NEFS); do \ + if [ ! -f "server/data/$$nef" ]; then \ + echo "!! ERROR: BUNDLED_NEFS 列出的 server/data/$$nef 不存在 !!"; \ + exit 1; \ + fi; \ + mkdir -p "$$dest/$$(dirname $$nef)"; \ + cp "server/data/$$nef" "$$dest/$$nef"; \ + echo " + $$nef"; \ + done; \ + copied=$$(find "$$dest" -name '*.nef' | wc -l | tr -d ' '); \ + if [ "$$copied" != "$(words $(BUNDLED_NEFS))" ]; then \ + echo "!! ERROR: 預期 $(words $(BUNDLED_NEFS)) 個 .nef，實際 $$copied 個 !!"; exit 1; \ + fi; \ + echo " models.json + $$copied 個 .nef 已就位" +endef + .PHONY: help \ vendor-sync vendor-python vendor-wheels vendor-ffmpeg vendor-ffmpeg-macos-build \ vendor-python-windows vendor-wheels-windows vendor-ffmpeg-windows \ @@ -245,7 +300,7 @@ payload-macos: build-server vendor-python vendor-wheels vendor-ffmpeg ## 準備 cp vendor/ffmpeg/macos/ffprobe payload/darwin/bin/ cp vendor/ffmpeg/macos/COPYING.LGPLv3 payload/darwin/bin/ffmpeg-COPYING.LGPLv3 chmod +x payload/darwin/bin/ffmpeg payload/darwin/bin/ffprobe - cp -R server/data/* payload/darwin/data/ + $(call copy_bundled_data,payload/darwin/data) cp -R server/scripts/* payload/darwin/scripts/ cp vendor/python/darwin/python.tar.gz payload/darwin/python/ @cp vendor/wheels/darwin/*.whl payload/darwin/wheels/ 2>/dev/null || true @@ -360,7 +415,7 @@ payload-windows: build-server-windows vendor-python-windows vendor-wheels-window @# LGPL 授權條款（BtbN build 自帶 LICENSE.txt；COPYING.LGPLv3 不一定在壓縮檔內，失敗不致命） @cp vendor/ffmpeg/windows/LICENSE.txt payload/windows/bin/ffmpeg-LICENSE.txt 2>/dev/null || true @cp vendor/ffmpeg/windows/COPYING.LGPLv3 payload/windows/bin/ffmpeg-COPYING.LGPLv3 2>/dev/null || true - cp -R server/data/. payload/windows/data/ + $(call copy_bundled_data,payload/windows/data) cp -R server/scripts/. payload/windows/scripts/ cp vendor/python/windows/python.tar.gz payload/windows/python/ @cp vendor/wheels/windows/*.whl payload/windows/wheels/ 2>/dev/null || true @@ -444,7 +499,7 @@ payload-linux: build-server-linux vendor-python-linux vendor-wheels-linux vendor @cp vendor/ffmpeg/linux/ffmpeg payload/linux/bin/ 2>/dev/null && chmod +x payload/linux/bin/ffmpeg || echo "!! WARN: ffmpeg 缺失" @cp vendor/ffmpeg/linux/ffprobe payload/linux/bin/ 2>/dev/null && chmod +x payload/linux/bin/ffprobe || echo "!! WARN: ffprobe 缺失" @cp vendor/ffmpeg/linux/LICENSE.txt payload/linux/bin/ffmpeg-LICENSE.txt 2>/dev/null || true - @if [ -d server/data ]; then cp -R server/data/. payload/linux/data/; fi + $(call copy_bundled_data,payload/linux/data) @if [ -d server/scripts ]; then cp -R server/scripts/. payload/linux/scripts/; fi @if [ ! -f vendor/python/linux/python.tar.gz ]; then \ echo "!! ERROR: vendor/python/linux/python.tar.gz 不存在，vendor-python-linux 應該已先跑過 !!"; \ diff --git a/local-tool/server/data/models.json b/local-tool/server/data/models.json index 34c3416..c06f36f 100644 --- a/local-tool/server/data/models.json +++ b/local-tool/server/data/models.json @@ -40,8 +40,8 @@ }, { "id": "kl520-fcos-detection", - "name": "FCOS Detection (KL520)", - "description": "FCOS (Fully Convolutional One-Stage) object detection with DarkNet53s backbone, compiled for KL520. Anchor-free detection at 512x512.", + "name": "物件辨識", + "description": "通用物件偵測模型，可辨識人、車輛、動物等常見物件，適合一般場景的多物件偵測。", "thumbnail": "/images/models/fcos-det.png", "taskType": "object_detection", "categories": [ @@ -109,8 +109,8 @@ }, { "id": "kl520-tiny-yolov3", - "name": "Tiny YOLOv3 (KL520)", - "description": "Tiny YOLOv3 object detection model compiled for KL520. Compact and fast model for general-purpose multi-object detection on edge devices.", + "name": "人型監測", + "description": "輕量快速的人員偵測模型，適合即時監控場景，可在邊緣裝置上高速偵測畫面中的人員。", "thumbnail": "/images/models/tiny-yolov3.png", "taskType": "object_detection", "categories": [ diff --git a/local-tool/server/internal/model/repository.go b/local-tool/server/internal/model/repository.go index 37c7fae..f477091 100644 --- a/local-tool/server/internal/model/repository.go +++ b/local-tool/server/internal/model/repository.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "os" + "path/filepath" "strings" "sync" ) @@ -13,6 +14,15 @@ type Repository struct { mu sync.RWMutex } +// NewRepository 載入 models.json 的內建模型目錄。 +// +// models.json 會列出所有「產品支援」的模型定義，但安裝包不一定會帶上每個 +// 對應的 .nef（見 Makefile 的 BUNDLED_NEFS 白名單）。因此載入後會過濾掉 +// .nef 檔案實際不存在的 model —— 否則使用者會在 UI 看到選不了的模型，選下去 +// 才在 flash 階段拿到 "model file not found" 這種沒頭沒尾的錯誤。 +// +// 過濾只作用在 models.json 的內建模型。使用者上傳的自訂模型走 Add()， +// 路徑是絕對路徑且必定存在，不受影響。 func NewRepository(dataPath string) *Repository { r := &Repository{} data, err := os.ReadFile(dataPath) @@ -20,12 +30,57 @@ func NewRepository(dataPath string) *Repository { fmt.Printf("Warning: could not load models from %s: %v\n", dataPath, err) return r } - if err := json.Unmarshal(data, &r.models); err != nil { + var declared []Model + if err := json.Unmarshal(data, &declared); err != nil { fmt.Printf("Warning: could not parse models JSON: %v\n", err) + return r } + r.models = filterAvailableModels(declared, filepath.Dir(dataPath)) return r } +// filterAvailableModels 只保留 .nef 檔案實際存在的 model。 +// +// dataDir 是 models.json 所在的目錄（即 bundle 內的 data/），models.json 的 +// filePath 以它為基準解析。 +func filterAvailableModels(models []Model, dataDir string) []Model { + available := make([]Model, 0, len(models)) + for _, m := range models { + path := resolveBuiltInModelPath(m.FilePath, dataDir) + // 沒宣告 filePath 的 model 不做檔案檢查（沒有東西可以檢查），保留原行為。 + if path == "" { + available = append(available, m) + continue + } + if info, err := os.Stat(path); err != nil || info.IsDir() { + fmt.Printf("[INFO] Skipping model %q (%s): .nef not bundled at %s\n", m.ID, m.Name, path) + continue + } + available = append(available, m) + } + return available +} + +// resolveBuiltInModelPath 把 models.json 的 filePath 解析成實際的檔案路徑。 +// +// 規則與 flash.Service.StartFlash 一致： +// - 絕對路徑 → 原樣使用 +// - "data/nef/..." → 去掉 "data/" 前綴後接在 dataDir 之下 +// （因為 dataDir 本身就是那個 data/ 目錄，不去掉會變成 data/data/nef/...） +// - 其他相對路徑 → 直接接在 dataDir 之下 +func resolveBuiltInModelPath(filePath, dataDir string) string { + if filePath == "" { + return "" + } + if filepath.IsAbs(filePath) { + return filePath + } + if strings.HasPrefix(filePath, "data/") || strings.HasPrefix(filePath, "data\\") { + return filepath.Join(dataDir, filePath[len("data/"):]) + } + return filepath.Join(dataDir, filePath) +} + func (r *Repository) List(filter ModelFilter) ([]ModelSummary, int) { r.mu.RLock() defer r.mu.RUnlock() diff --git a/local-tool/server/internal/model/repository_test.go b/local-tool/server/internal/model/repository_test.go index ae20e59..34746ed 100644 --- a/local-tool/server/internal/model/repository_test.go +++ b/local-tool/server/internal/model/repository_test.go @@ -1,6 +1,9 @@ package model import ( + "encoding/json" + "os" + "path/filepath" "testing" ) @@ -98,6 +101,193 @@ func TestRepository_Add(t *testing.T) { } } +// writeModelsJSON 在 dir 底下建立 models.json，回傳它的路徑。 +func writeModelsJSON(t *testing.T, dir string, models []Model) string { + t.Helper() + data, err := json.MarshalIndent(models, "", " ") + if err != nil { + t.Fatalf("marshal models: %v", err) + } + path := filepath.Join(dir, "models.json") + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatalf("write models.json: %v", err) + } + return path +} + +// touchNef 在 dir 底下建立一個假的 .nef（內容不重要，只檢查存在性）。 +func touchNef(t *testing.T, dir, relPath string) { + t.Helper() + full := filepath.Join(dir, relPath) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatalf("mkdir for %s: %v", relPath, err) + } + if err := os.WriteFile(full, []byte("fake nef"), 0o644); err != nil { + t.Fatalf("write %s: %v", relPath, err) + } +} + +func TestResolveBuiltInModelPath(t *testing.T) { + dataDir := filepath.Join("/bundle", "data") + + tests := []struct { + name string + filePath string + want string + }{ + { + name: "strips data/ prefix so it does not become data/data/", + filePath: "data/nef/kl520/a.nef", + want: filepath.Join("/bundle", "data", "nef", "kl520", "a.nef"), + }, + { + name: "relative path without data/ prefix joins directly", + filePath: "nef/kl520/a.nef", + want: filepath.Join("/bundle", "data", "nef", "kl520", "a.nef"), + }, + { + name: "absolute path is used as-is", + filePath: filepath.Join("/custom", "models", "x", "model.nef"), + want: filepath.Join("/custom", "models", "x", "model.nef"), + }, + { + name: "empty file path stays empty", + filePath: "", + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := resolveBuiltInModelPath(tt.filePath, dataDir); got != tt.want { + t.Errorf("resolveBuiltInModelPath(%q) = %q, want %q", tt.filePath, got, tt.want) + } + }) + } +} + +func TestFilterAvailableModels(t *testing.T) { + dataDir := t.TempDir() + touchNef(t, dataDir, "nef/kl520/bundled.nef") + + absent := filepath.Join(dataDir, "nef", "kl520", "abs-missing.nef") + present := filepath.Join(dataDir, "nef", "kl520", "abs.nef") + touchNef(t, dataDir, "nef/kl520/abs.nef") + + // 目錄而非檔案：不該被當成可用的 model + if err := os.MkdirAll(filepath.Join(dataDir, "nef/kl520/dir.nef"), 0o755); err != nil { + t.Fatalf("mkdir dir.nef: %v", err) + } + + models := []Model{ + {ID: "bundled", FilePath: "data/nef/kl520/bundled.nef"}, + {ID: "not-bundled", FilePath: "data/nef/kl520/nope.nef"}, + {ID: "abs-present", FilePath: present}, + {ID: "abs-absent", FilePath: absent}, + {ID: "no-file-path"}, + {ID: "dir-not-file", FilePath: "data/nef/kl520/dir.nef"}, + } + + got := filterAvailableModels(models, dataDir) + + var gotIDs []string + for _, m := range got { + gotIDs = append(gotIDs, m.ID) + } + want := []string{"bundled", "abs-present", "no-file-path"} + + if len(gotIDs) != len(want) { + t.Fatalf("filterAvailableModels() = %v, want %v", gotIDs, want) + } + for i := range want { + if gotIDs[i] != want[i] { + t.Errorf("filterAvailableModels()[%d] = %q, want %q", i, gotIDs[i], want[i]) + } + } +} + +func TestNewRepository_FiltersUnbundledModels(t *testing.T) { + dataDir := t.TempDir() + touchNef(t, dataDir, "nef/kl520/kl520_20004_fcos-drk53s_w512h512.nef") + touchNef(t, dataDir, "nef/kl520/kl520_tiny_yolo_v3.nef") + + // 模擬正式情境：models.json 宣告 4 個 model，但只打包了其中 2 個 .nef + path := writeModelsJSON(t, dataDir, []Model{ + {ID: "kl520-fcos-detection", Name: "物件辨識", TaskType: "object_detection", + FilePath: "data/nef/kl520/kl520_20004_fcos-drk53s_w512h512.nef"}, + {ID: "kl520-tiny-yolov3", Name: "人型監測", TaskType: "object_detection", + FilePath: "data/nef/kl520/kl520_tiny_yolo_v3.nef"}, + {ID: "kl520-yolov5-detection", Name: "YOLOv5", TaskType: "object_detection", + FilePath: "data/nef/kl520/kl520_20005_yolov5-noupsample_w640h640.nef"}, + {ID: "kl720-resnet18-classification", Name: "ResNet18", TaskType: "classification", + FilePath: "data/nef/kl720/kl720_20001_resnet18_w224h224.nef"}, + }) + + repo := NewRepository(path) + + if repo.Count() != 2 { + t.Fatalf("Count() = %d, want 2 (only bundled .nef should load)", repo.Count()) + } + for _, id := range []string{"kl520-fcos-detection", "kl520-tiny-yolov3"} { + if _, err := repo.GetByID(id); err != nil { + t.Errorf("GetByID(%q) failed, expected it to be available: %v", id, err) + } + } + for _, id := range []string{"kl520-yolov5-detection", "kl720-resnet18-classification"} { + if _, err := repo.GetByID(id); err == nil { + t.Errorf("GetByID(%q) succeeded, expected it to be filtered out", id) + } + } + + // 過濾後的清單也不該出現在 List() + results, count := repo.List(ModelFilter{}) + if count != 2 || len(results) != 2 { + t.Errorf("List() = %d results (count %d), want 2", len(results), count) + } +} + +func TestNewRepository_CustomModelsUnaffectedByFilter(t *testing.T) { + dataDir := t.TempDir() + path := writeModelsJSON(t, dataDir, []Model{ + {ID: "built-in-missing", FilePath: "data/nef/kl520/missing.nef"}, + }) + + repo := NewRepository(path) + if repo.Count() != 0 { + t.Fatalf("Count() = %d, want 0 after filtering", repo.Count()) + } + + // 自訂模型走 Add()，不經過過濾 + repo.Add(Model{ID: "custom-1", IsCustom: true, FilePath: "/anywhere/model.nef"}) + if repo.Count() != 1 { + t.Errorf("Count() = %d after Add(), want 1", repo.Count()) + } + if _, err := repo.GetByID("custom-1"); err != nil { + t.Errorf("GetByID(custom-1) failed: %v", err) + } +} + +func TestNewRepository_MissingOrInvalidFile(t *testing.T) { + t.Run("missing models.json yields empty repo", func(t *testing.T) { + repo := NewRepository(filepath.Join(t.TempDir(), "nope.json")) + if repo.Count() != 0 { + t.Errorf("Count() = %d, want 0", repo.Count()) + } + }) + + t.Run("invalid JSON yields empty repo", func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "models.json") + if err := os.WriteFile(path, []byte("{not json"), 0o644); err != nil { + t.Fatalf("write: %v", err) + } + repo := NewRepository(path) + if repo.Count() != 0 { + t.Errorf("Count() = %d, want 0", repo.Count()) + } + }) +} + func TestRepository_Remove(t *testing.T) { repo := newTestRepo()