feat(build): 安裝包只打包指定模型並改用中文名稱
安裝包從 8 個 .nef 縮減為 2 個，models.json 保留完整定義以便日後 加回。 - Makefile 加 BUNDLED_NEFS 白名單，三平台共用 copy_bundled_data helper；用 POSIX find/cp 而非 rsync（Windows CI 的 Git Bash 沒有 rsync） - 白名單檔案不存在時 build 直接失敗，並在複製後驗證 models.json 存在且 .nef 數量相符 —— 避免產出「安裝後 0 個模型」卻回報成功 - FCOS Detection (KL520) 改名為「物件辨識」 - Tiny YOLOv3 (KL520) 改名為「人型監測」 （只改 name/description，id 不動以免影響既有設定與紀錄） - Repository 啟動時過濾 .nef 不存在的模型，否則使用者會看到未打包 的模型並在選取後拿到莫名錯誤 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
5c1c37d151
commit
744283bd28
@ -18,6 +18,61 @@ OS := $(shell uname -s | tr A-Z a-z)
|
||||
DIST := dist
|
||||
PAYLOAD := visiona-local/payload
|
||||
|
||||
# ── 打包用 .nef 白名單（M5-a）────────────────────────────────────────
|
||||
#
|
||||
# server/data/ 底下有 8 個 .nef（KL520 五個、KL720 三個），但安裝包只帶
|
||||
# 白名單內的這幾個，其餘不進 payload，藉此縮小安裝檔體積。
|
||||
#
|
||||
# models.json 不做任何過濾，7 個 model 定義全部照原樣複製。執行期由
|
||||
# server/internal/model/repository.go 的 NewRepository() 檢查每個 model 的
|
||||
# filePath 是否實際存在，不存在的直接不載入（見 M5-c）。因此使用者在 UI
|
||||
# 只會看到白名單內的 model，未打包的不會出現、也不會選到後拿到莫名錯誤。
|
||||
#
|
||||
# 未來要把某個 model 加回安裝包：把對應的 .nef 相對路徑加進下面這個變數即可，
|
||||
# models.json 不用動。
|
||||
#
|
||||
# 路徑相對於 server/data/。
|
||||
BUNDLED_NEFS := \
|
||||
nef/kl520/kl520_20004_fcos-drk53s_w512h512.nef \
|
||||
nef/kl520/kl520_tiny_yolo_v3.nef
|
||||
|
||||
# copy_bundled_data：把 server/data/ 複製到 $(1)，但 nef/ 只帶 BUNDLED_NEFS 白名單。
|
||||
# $(1) = 目標 data 目錄（例：payload/darwin/data）
|
||||
#
|
||||
# 步驟：(a) 複製 server/data/ 下除了 nef/ 以外的所有東西（models.json 等）
|
||||
# (b) 再逐一複製白名單內的 .nef
|
||||
# (c) 白名單檔案不存在就直接 fail，避免安靜地產出缺 model 的安裝包
|
||||
#
|
||||
# 只用 POSIX find / cp，不用 rsync —— Windows CI 跑在 Git Bash（windows-2022 +
|
||||
# shell: bash），該環境沒有 rsync。
|
||||
define copy_bundled_data
|
||||
@set -e; \
|
||||
echo "==> 複製 server/data → $(1)（.nef 白名單：$(words $(BUNDLED_NEFS)) 個）"; \
|
||||
if [ ! -d server/data ]; then echo "!! ERROR: server/data 不存在 !!"; exit 1; fi; \
|
||||
mkdir -p "$(1)"; \
|
||||
dest="$$(cd "$(1)" && pwd)"; \
|
||||
( cd server/data && \
|
||||
find . -path ./nef -prune -o -type d -print | while read -r d; do mkdir -p "$$dest/$$d"; done && \
|
||||
find . -path ./nef -prune -o -type f -print | while read -r f; do cp "$$f" "$$dest/$$f"; done ); \
|
||||
if [ ! -f "$$dest/models.json" ]; then \
|
||||
echo "!! ERROR: models.json 沒有被複製到 $$dest !!"; exit 1; \
|
||||
fi; \
|
||||
for nef in $(BUNDLED_NEFS); do \
|
||||
if [ ! -f "server/data/$$nef" ]; then \
|
||||
echo "!! ERROR: BUNDLED_NEFS 列出的 server/data/$$nef 不存在 !!"; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
mkdir -p "$$dest/$$(dirname $$nef)"; \
|
||||
cp "server/data/$$nef" "$$dest/$$nef"; \
|
||||
echo " + $$nef"; \
|
||||
done; \
|
||||
copied=$$(find "$$dest" -name '*.nef' | wc -l | tr -d ' '); \
|
||||
if [ "$$copied" != "$(words $(BUNDLED_NEFS))" ]; then \
|
||||
echo "!! ERROR: 預期 $(words $(BUNDLED_NEFS)) 個 .nef，實際 $$copied 個 !!"; exit 1; \
|
||||
fi; \
|
||||
echo " models.json + $$copied 個 .nef 已就位"
|
||||
endef
|
||||
|
||||
.PHONY: help \
|
||||
vendor-sync vendor-python vendor-wheels vendor-ffmpeg vendor-ffmpeg-macos-build \
|
||||
vendor-python-windows vendor-wheels-windows vendor-ffmpeg-windows \
|
||||
@ -245,7 +300,7 @@ payload-macos: build-server vendor-python vendor-wheels vendor-ffmpeg ## 準備
|
||||
cp vendor/ffmpeg/macos/ffprobe payload/darwin/bin/
|
||||
cp vendor/ffmpeg/macos/COPYING.LGPLv3 payload/darwin/bin/ffmpeg-COPYING.LGPLv3
|
||||
chmod +x payload/darwin/bin/ffmpeg payload/darwin/bin/ffprobe
|
||||
cp -R server/data/* payload/darwin/data/
|
||||
$(call copy_bundled_data,payload/darwin/data)
|
||||
cp -R server/scripts/* payload/darwin/scripts/
|
||||
cp vendor/python/darwin/python.tar.gz payload/darwin/python/
|
||||
@cp vendor/wheels/darwin/*.whl payload/darwin/wheels/ 2>/dev/null || true
|
||||
@ -360,7 +415,7 @@ payload-windows: build-server-windows vendor-python-windows vendor-wheels-window
|
||||
@# LGPL 授權條款（BtbN build 自帶 LICENSE.txt；COPYING.LGPLv3 不一定在壓縮檔內，失敗不致命）
|
||||
@cp vendor/ffmpeg/windows/LICENSE.txt payload/windows/bin/ffmpeg-LICENSE.txt 2>/dev/null || true
|
||||
@cp vendor/ffmpeg/windows/COPYING.LGPLv3 payload/windows/bin/ffmpeg-COPYING.LGPLv3 2>/dev/null || true
|
||||
cp -R server/data/. payload/windows/data/
|
||||
$(call copy_bundled_data,payload/windows/data)
|
||||
cp -R server/scripts/. payload/windows/scripts/
|
||||
cp vendor/python/windows/python.tar.gz payload/windows/python/
|
||||
@cp vendor/wheels/windows/*.whl payload/windows/wheels/ 2>/dev/null || true
|
||||
@ -444,7 +499,7 @@ payload-linux: build-server-linux vendor-python-linux vendor-wheels-linux vendor
|
||||
@cp vendor/ffmpeg/linux/ffmpeg payload/linux/bin/ 2>/dev/null && chmod +x payload/linux/bin/ffmpeg || echo "!! WARN: ffmpeg 缺失"
|
||||
@cp vendor/ffmpeg/linux/ffprobe payload/linux/bin/ 2>/dev/null && chmod +x payload/linux/bin/ffprobe || echo "!! WARN: ffprobe 缺失"
|
||||
@cp vendor/ffmpeg/linux/LICENSE.txt payload/linux/bin/ffmpeg-LICENSE.txt 2>/dev/null || true
|
||||
@if [ -d server/data ]; then cp -R server/data/. payload/linux/data/; fi
|
||||
$(call copy_bundled_data,payload/linux/data)
|
||||
@if [ -d server/scripts ]; then cp -R server/scripts/. payload/linux/scripts/; fi
|
||||
@if [ ! -f vendor/python/linux/python.tar.gz ]; then \
|
||||
echo "!! ERROR: vendor/python/linux/python.tar.gz 不存在，vendor-python-linux 應該已先跑過 !!"; \
|
||||
|
||||
@ -40,8 +40,8 @@
|
||||
},
|
||||
{
|
||||
"id": "kl520-fcos-detection",
|
||||
"name": "FCOS Detection (KL520)",
|
||||
"description": "FCOS (Fully Convolutional One-Stage) object detection with DarkNet53s backbone, compiled for KL520. Anchor-free detection at 512x512.",
|
||||
"name": "物件辨識",
|
||||
"description": "通用物件偵測模型，可辨識人、車輛、動物等常見物件，適合一般場景的多物件偵測。",
|
||||
"thumbnail": "/images/models/fcos-det.png",
|
||||
"taskType": "object_detection",
|
||||
"categories": [
|
||||
@ -109,8 +109,8 @@
|
||||
},
|
||||
{
|
||||
"id": "kl520-tiny-yolov3",
|
||||
"name": "Tiny YOLOv3 (KL520)",
|
||||
"description": "Tiny YOLOv3 object detection model compiled for KL520. Compact and fast model for general-purpose multi-object detection on edge devices.",
|
||||
"name": "人型監測",
|
||||
"description": "輕量快速的人員偵測模型，適合即時監控場景，可在邊緣裝置上高速偵測畫面中的人員。",
|
||||
"thumbnail": "/images/models/tiny-yolov3.png",
|
||||
"taskType": "object_detection",
|
||||
"categories": [
|
||||
|
||||
@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
@ -13,6 +14,15 @@ type Repository struct {
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// NewRepository 載入 models.json 的內建模型目錄。
|
||||
//
|
||||
// models.json 會列出所有「產品支援」的模型定義，但安裝包不一定會帶上每個
|
||||
// 對應的 .nef（見 Makefile 的 BUNDLED_NEFS 白名單）。因此載入後會過濾掉
|
||||
// .nef 檔案實際不存在的 model —— 否則使用者會在 UI 看到選不了的模型，選下去
|
||||
// 才在 flash 階段拿到 "model file not found" 這種沒頭沒尾的錯誤。
|
||||
//
|
||||
// 過濾只作用在 models.json 的內建模型。使用者上傳的自訂模型走 Add()，
|
||||
// 路徑是絕對路徑且必定存在，不受影響。
|
||||
func NewRepository(dataPath string) *Repository {
|
||||
r := &Repository{}
|
||||
data, err := os.ReadFile(dataPath)
|
||||
@ -20,12 +30,57 @@ func NewRepository(dataPath string) *Repository {
|
||||
fmt.Printf("Warning: could not load models from %s: %v\n", dataPath, err)
|
||||
return r
|
||||
}
|
||||
if err := json.Unmarshal(data, &r.models); err != nil {
|
||||
var declared []Model
|
||||
if err := json.Unmarshal(data, &declared); err != nil {
|
||||
fmt.Printf("Warning: could not parse models JSON: %v\n", err)
|
||||
return r
|
||||
}
|
||||
r.models = filterAvailableModels(declared, filepath.Dir(dataPath))
|
||||
return r
|
||||
}
|
||||
|
||||
// filterAvailableModels 只保留 .nef 檔案實際存在的 model。
|
||||
//
|
||||
// dataDir 是 models.json 所在的目錄（即 bundle 內的 data/），models.json 的
|
||||
// filePath 以它為基準解析。
|
||||
func filterAvailableModels(models []Model, dataDir string) []Model {
|
||||
available := make([]Model, 0, len(models))
|
||||
for _, m := range models {
|
||||
path := resolveBuiltInModelPath(m.FilePath, dataDir)
|
||||
// 沒宣告 filePath 的 model 不做檔案檢查（沒有東西可以檢查），保留原行為。
|
||||
if path == "" {
|
||||
available = append(available, m)
|
||||
continue
|
||||
}
|
||||
if info, err := os.Stat(path); err != nil || info.IsDir() {
|
||||
fmt.Printf("[INFO] Skipping model %q (%s): .nef not bundled at %s\n", m.ID, m.Name, path)
|
||||
continue
|
||||
}
|
||||
available = append(available, m)
|
||||
}
|
||||
return available
|
||||
}
|
||||
|
||||
// resolveBuiltInModelPath 把 models.json 的 filePath 解析成實際的檔案路徑。
|
||||
//
|
||||
// 規則與 flash.Service.StartFlash 一致：
|
||||
// - 絕對路徑 → 原樣使用
|
||||
// - "data/nef/..." → 去掉 "data/" 前綴後接在 dataDir 之下
|
||||
// （因為 dataDir 本身就是那個 data/ 目錄，不去掉會變成 data/data/nef/...）
|
||||
// - 其他相對路徑 → 直接接在 dataDir 之下
|
||||
func resolveBuiltInModelPath(filePath, dataDir string) string {
|
||||
if filePath == "" {
|
||||
return ""
|
||||
}
|
||||
if filepath.IsAbs(filePath) {
|
||||
return filePath
|
||||
}
|
||||
if strings.HasPrefix(filePath, "data/") || strings.HasPrefix(filePath, "data\\") {
|
||||
return filepath.Join(dataDir, filePath[len("data/"):])
|
||||
}
|
||||
return filepath.Join(dataDir, filePath)
|
||||
}
|
||||
|
||||
func (r *Repository) List(filter ModelFilter) ([]ModelSummary, int) {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
|
||||
@ -1,6 +1,9 @@
|
||||
package model
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@ -98,6 +101,193 @@ func TestRepository_Add(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// writeModelsJSON 在 dir 底下建立 models.json，回傳它的路徑。
|
||||
func writeModelsJSON(t *testing.T, dir string, models []Model) string {
|
||||
t.Helper()
|
||||
data, err := json.MarshalIndent(models, "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("marshal models: %v", err)
|
||||
}
|
||||
path := filepath.Join(dir, "models.json")
|
||||
if err := os.WriteFile(path, data, 0o644); err != nil {
|
||||
t.Fatalf("write models.json: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
// touchNef 在 dir 底下建立一個假的 .nef（內容不重要，只檢查存在性）。
|
||||
func touchNef(t *testing.T, dir, relPath string) {
|
||||
t.Helper()
|
||||
full := filepath.Join(dir, relPath)
|
||||
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
|
||||
t.Fatalf("mkdir for %s: %v", relPath, err)
|
||||
}
|
||||
if err := os.WriteFile(full, []byte("fake nef"), 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", relPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveBuiltInModelPath(t *testing.T) {
|
||||
dataDir := filepath.Join("/bundle", "data")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
filePath string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "strips data/ prefix so it does not become data/data/",
|
||||
filePath: "data/nef/kl520/a.nef",
|
||||
want: filepath.Join("/bundle", "data", "nef", "kl520", "a.nef"),
|
||||
},
|
||||
{
|
||||
name: "relative path without data/ prefix joins directly",
|
||||
filePath: "nef/kl520/a.nef",
|
||||
want: filepath.Join("/bundle", "data", "nef", "kl520", "a.nef"),
|
||||
},
|
||||
{
|
||||
name: "absolute path is used as-is",
|
||||
filePath: filepath.Join("/custom", "models", "x", "model.nef"),
|
||||
want: filepath.Join("/custom", "models", "x", "model.nef"),
|
||||
},
|
||||
{
|
||||
name: "empty file path stays empty",
|
||||
filePath: "",
|
||||
want: "",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := resolveBuiltInModelPath(tt.filePath, dataDir); got != tt.want {
|
||||
t.Errorf("resolveBuiltInModelPath(%q) = %q, want %q", tt.filePath, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterAvailableModels(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
touchNef(t, dataDir, "nef/kl520/bundled.nef")
|
||||
|
||||
absent := filepath.Join(dataDir, "nef", "kl520", "abs-missing.nef")
|
||||
present := filepath.Join(dataDir, "nef", "kl520", "abs.nef")
|
||||
touchNef(t, dataDir, "nef/kl520/abs.nef")
|
||||
|
||||
// 目錄而非檔案：不該被當成可用的 model
|
||||
if err := os.MkdirAll(filepath.Join(dataDir, "nef/kl520/dir.nef"), 0o755); err != nil {
|
||||
t.Fatalf("mkdir dir.nef: %v", err)
|
||||
}
|
||||
|
||||
models := []Model{
|
||||
{ID: "bundled", FilePath: "data/nef/kl520/bundled.nef"},
|
||||
{ID: "not-bundled", FilePath: "data/nef/kl520/nope.nef"},
|
||||
{ID: "abs-present", FilePath: present},
|
||||
{ID: "abs-absent", FilePath: absent},
|
||||
{ID: "no-file-path"},
|
||||
{ID: "dir-not-file", FilePath: "data/nef/kl520/dir.nef"},
|
||||
}
|
||||
|
||||
got := filterAvailableModels(models, dataDir)
|
||||
|
||||
var gotIDs []string
|
||||
for _, m := range got {
|
||||
gotIDs = append(gotIDs, m.ID)
|
||||
}
|
||||
want := []string{"bundled", "abs-present", "no-file-path"}
|
||||
|
||||
if len(gotIDs) != len(want) {
|
||||
t.Fatalf("filterAvailableModels() = %v, want %v", gotIDs, want)
|
||||
}
|
||||
for i := range want {
|
||||
if gotIDs[i] != want[i] {
|
||||
t.Errorf("filterAvailableModels()[%d] = %q, want %q", i, gotIDs[i], want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRepository_FiltersUnbundledModels(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
touchNef(t, dataDir, "nef/kl520/kl520_20004_fcos-drk53s_w512h512.nef")
|
||||
touchNef(t, dataDir, "nef/kl520/kl520_tiny_yolo_v3.nef")
|
||||
|
||||
// 模擬正式情境：models.json 宣告 4 個 model，但只打包了其中 2 個 .nef
|
||||
path := writeModelsJSON(t, dataDir, []Model{
|
||||
{ID: "kl520-fcos-detection", Name: "物件辨識", TaskType: "object_detection",
|
||||
FilePath: "data/nef/kl520/kl520_20004_fcos-drk53s_w512h512.nef"},
|
||||
{ID: "kl520-tiny-yolov3", Name: "人型監測", TaskType: "object_detection",
|
||||
FilePath: "data/nef/kl520/kl520_tiny_yolo_v3.nef"},
|
||||
{ID: "kl520-yolov5-detection", Name: "YOLOv5", TaskType: "object_detection",
|
||||
FilePath: "data/nef/kl520/kl520_20005_yolov5-noupsample_w640h640.nef"},
|
||||
{ID: "kl720-resnet18-classification", Name: "ResNet18", TaskType: "classification",
|
||||
FilePath: "data/nef/kl720/kl720_20001_resnet18_w224h224.nef"},
|
||||
})
|
||||
|
||||
repo := NewRepository(path)
|
||||
|
||||
if repo.Count() != 2 {
|
||||
t.Fatalf("Count() = %d, want 2 (only bundled .nef should load)", repo.Count())
|
||||
}
|
||||
for _, id := range []string{"kl520-fcos-detection", "kl520-tiny-yolov3"} {
|
||||
if _, err := repo.GetByID(id); err != nil {
|
||||
t.Errorf("GetByID(%q) failed, expected it to be available: %v", id, err)
|
||||
}
|
||||
}
|
||||
for _, id := range []string{"kl520-yolov5-detection", "kl720-resnet18-classification"} {
|
||||
if _, err := repo.GetByID(id); err == nil {
|
||||
t.Errorf("GetByID(%q) succeeded, expected it to be filtered out", id)
|
||||
}
|
||||
}
|
||||
|
||||
// 過濾後的清單也不該出現在 List()
|
||||
results, count := repo.List(ModelFilter{})
|
||||
if count != 2 || len(results) != 2 {
|
||||
t.Errorf("List() = %d results (count %d), want 2", len(results), count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRepository_CustomModelsUnaffectedByFilter(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
path := writeModelsJSON(t, dataDir, []Model{
|
||||
{ID: "built-in-missing", FilePath: "data/nef/kl520/missing.nef"},
|
||||
})
|
||||
|
||||
repo := NewRepository(path)
|
||||
if repo.Count() != 0 {
|
||||
t.Fatalf("Count() = %d, want 0 after filtering", repo.Count())
|
||||
}
|
||||
|
||||
// 自訂模型走 Add()，不經過過濾
|
||||
repo.Add(Model{ID: "custom-1", IsCustom: true, FilePath: "/anywhere/model.nef"})
|
||||
if repo.Count() != 1 {
|
||||
t.Errorf("Count() = %d after Add(), want 1", repo.Count())
|
||||
}
|
||||
if _, err := repo.GetByID("custom-1"); err != nil {
|
||||
t.Errorf("GetByID(custom-1) failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRepository_MissingOrInvalidFile(t *testing.T) {
|
||||
t.Run("missing models.json yields empty repo", func(t *testing.T) {
|
||||
repo := NewRepository(filepath.Join(t.TempDir(), "nope.json"))
|
||||
if repo.Count() != 0 {
|
||||
t.Errorf("Count() = %d, want 0", repo.Count())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("invalid JSON yields empty repo", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "models.json")
|
||||
if err := os.WriteFile(path, []byte("{not json"), 0o644); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
repo := NewRepository(path)
|
||||
if repo.Count() != 0 {
|
||||
t.Errorf("Count() = %d, want 0", repo.Count())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRepository_Remove(t *testing.T) {
|
||||
repo := newTestRepo()
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user