feat: 啟用 staging 真實轉檔 + 修 ref_images/promote OAuth bug #1
34
.dockerignore
Normal file
34
.dockerignore
Normal file
@ -0,0 +1,34 @@
|
||||
# Build context 瘦身 — 套用於「以專案根目錄為 build context」的 docker build
|
||||
# （Dockerfile.real / Dockerfile.stub）。
|
||||
#
|
||||
# Dockerfile.real 用精準 COPY，不會把以下內容放進 image；此檔只是避免把它們
|
||||
# 上傳給 daemon（省 ~4.3GB context 傳輸）。
|
||||
|
||||
# 大型版本控制目錄
|
||||
.git
|
||||
.gitmodules
|
||||
|
||||
# 被 USE_PREBUILD 取代的舊系統路徑 binaries（image 不需要，省 3.2GB）
|
||||
libs/dynasty
|
||||
libs/compiler
|
||||
|
||||
# c_sim_* 模擬器（NEF 轉檔不需要）
|
||||
libs/c_sim_520
|
||||
libs/c_sim_530
|
||||
libs/c_sim_630
|
||||
libs/c_sim_720
|
||||
|
||||
# Python / OS caches
|
||||
**/__pycache__
|
||||
**/*.pyc
|
||||
**/*.pyo
|
||||
.pytest_cache
|
||||
.tmp
|
||||
.mypy_cache
|
||||
|
||||
# 個人層 / 文件
|
||||
.autoflow
|
||||
docs
|
||||
|
||||
# macOS metadata
|
||||
**/.DS_Store
|
||||
@ -720,11 +720,14 @@ describe('POST /api/v1/jobs — 201 happy path', () => {
|
||||
// MinIO：model + 2 個 ref_images
|
||||
expect(ctx.minio.uploadToMinIO).toHaveBeenCalledTimes(3);
|
||||
const keys = ctx.minio._uploaded.map((u) => u.key);
|
||||
// ref_images 必須在 `input/ref_images/` 下，與 bie worker 讀取端
|
||||
// (consumer.py `jobs/{id}/input/ref_images`) + local backend + legacy.js
|
||||
// 約定一致。2026-06-22 修復：原本少 `input/` 一層導致 bie e2e blocker。
|
||||
expect(keys).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.stringMatching(/^jobs\/[^/]+\/input\/model\.onnx$/),
|
||||
expect.stringMatching(/^jobs\/[^/]+\/ref_images\/0_a\.jpg$/),
|
||||
expect.stringMatching(/^jobs\/[^/]+\/ref_images\/1_b\.png$/),
|
||||
expect.stringMatching(/^jobs\/[^/]+\/input\/ref_images\/0_a\.jpg$/),
|
||||
expect.stringMatching(/^jobs\/[^/]+\/input\/ref_images\/1_b\.png$/),
|
||||
])
|
||||
);
|
||||
|
||||
|
||||
@ -105,9 +105,13 @@ describe('jobService.writeInputToMinIO', () => {
|
||||
);
|
||||
|
||||
expect(result.inputObjectKey).toBe('jobs/job-123/input/model.onnx');
|
||||
// ref_images 必須在 `input/ref_images/` 下，與 bie worker
|
||||
// (consumer.py download `jobs/{id}/input/ref_images`) + local backend
|
||||
// (storage/local.js `<id>/input/ref_images/`) + legacy.js 三方約定一致。
|
||||
// 2026-06-22 修復：原本少了 `input/` 一層導致 bie worker 抓空 prefix → e2e fail。
|
||||
expect(result.refImageObjectKeys).toEqual([
|
||||
'jobs/job-123/ref_images/0_a.jpg',
|
||||
'jobs/job-123/ref_images/1_b.png',
|
||||
'jobs/job-123/input/ref_images/0_a.jpg',
|
||||
'jobs/job-123/input/ref_images/1_b.png',
|
||||
]);
|
||||
expect(result.uploadedKeys).toHaveLength(3);
|
||||
expect(minio.uploadToMinIO).toHaveBeenCalledTimes(3);
|
||||
@ -409,10 +413,27 @@ describe('jobService._internals (object key naming)', () => {
|
||||
minio: makeFakeMinio(),
|
||||
});
|
||||
expect(svc._internals.buildRefImageObjectKey('j-1', 0, 'a.jpg')).toBe(
|
||||
'jobs/j-1/ref_images/0_a.jpg'
|
||||
'jobs/j-1/input/ref_images/0_a.jpg'
|
||||
);
|
||||
expect(svc._internals.buildRefImageObjectKey('j-1', 1, 'a.jpg')).toBe(
|
||||
'jobs/j-1/ref_images/1_a.jpg'
|
||||
'jobs/j-1/input/ref_images/1_a.jpg'
|
||||
);
|
||||
});
|
||||
|
||||
// Read/write contract guard：寫入端 (jobService) 產出的 ref_images key prefix
|
||||
// 必須等於 bie worker (consumer.py) download_prefix 的 `jobs/{id}/input/ref_images`。
|
||||
// 這條鎖死 scheduler↔worker 的 S3 key 約定，避免兩端路徑再次漂移
|
||||
// （2026-06-22 real-mode e2e blocker 的根因）。
|
||||
it('buildRefImageObjectKey key prefix matches bie worker download_prefix contract', () => {
|
||||
const svc = createJobService({
|
||||
redis: makeFakeRedis(),
|
||||
sseService: makeFakeSseService(),
|
||||
minio: makeFakeMinio(),
|
||||
});
|
||||
const jobId = 'contract-job';
|
||||
// consumer.py:93-94 — bie worker download_prefix(f"{jobs/{job_id}}/input/ref_images")
|
||||
const workerDownloadPrefix = `jobs/${jobId}/input/ref_images`;
|
||||
const writeKey = svc._internals.buildRefImageObjectKey(jobId, 0, 'cal.jpg');
|
||||
expect(writeKey.startsWith(`${workerDownloadPrefix}/`)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@ -335,12 +335,25 @@ function createJobService(deps) {
|
||||
* Ref image object key（對齊 TDD §6.1）。
|
||||
* 加入 index 前綴避免同名衝突。
|
||||
*
|
||||
* 路徑約定：`jobs/{jobId}/input/ref_images/...`。
|
||||
* ref_images 屬於 job 的「input」資產，與 model 檔（buildInputObjectKey 的
|
||||
* `jobs/{jobId}/input/{filename}`）同處 `input/` 命名空間下，並與以下三處
|
||||
* 既有約定一致：
|
||||
* - local backend：storage/local.js 寫到 `<jobId>/input/ref_images/`
|
||||
* - legacy MinIO path：routes/legacy.js 寫到 `jobs/{jobId}/input/ref_images/`
|
||||
* - bie worker 讀取端：services/workers/consumer.py 從
|
||||
* `jobs/{jobId}/input/ref_images/` download（且 _build_input_paths 對
|
||||
* local/minio 兩種 backend 共用 `input/ref_images` 落地路徑）
|
||||
* 2026-06-22 修復：原本此處少了 `input/` 一層（v1 path 的孤例），導致
|
||||
* bie worker 抓空 prefix → dataset 目錄不存在 → bie job fail（real-mode
|
||||
* e2e blocker）。對齊到 `input/ref_images` 後與上述三處一致。
|
||||
*
|
||||
* @param {string} jobId
|
||||
* @param {number} index
|
||||
* @param {string} safeFilename
|
||||
*/
|
||||
function buildRefImageObjectKey(jobId, index, safeFilename) {
|
||||
return `jobs/${jobId}/ref_images/${index}_${safeFilename}`;
|
||||
return `jobs/${jobId}/input/ref_images/${index}_${safeFilename}`;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
42
docker-compose.real.yml
Normal file
42
docker-compose.real.yml
Normal file
@ -0,0 +1,42 @@
|
||||
##
|
||||
# Kneron Model Converter — REAL worker override（真實轉檔版）
|
||||
#
|
||||
# 用途：把 onnx/bie/nef 三個 worker 從 stub 切成真實轉檔 image
|
||||
# （jim800121/kneron-model-converter-worker:real-20260622）
|
||||
#
|
||||
# 部署（staging）：
|
||||
# export DOCKER_HOST=tcp://192.168.0.130:2375
|
||||
# docker compose -f docker-compose.yml -f docker-compose.real.yml pull onnx-worker bie-worker nef-worker
|
||||
# docker compose -f docker-compose.yml -f docker-compose.real.yml up -d onnx-worker bie-worker nef-worker
|
||||
#
|
||||
# Rollback 回 stub（秒退、不靠 Docker Hub）：
|
||||
# export DOCKER_HOST=tcp://192.168.0.130:2375
|
||||
# # 直接用 base compose（不帶 -f docker-compose.real.yml）重啟，會用回本地 stub image：
|
||||
# docker compose -f docker-compose.yml up -d --build onnx-worker bie-worker nef-worker
|
||||
# # （stub image 還在 staging 本地；--build 確保用回 Dockerfile.stub。
|
||||
# # 或更快：base compose 的 stub image 還在，省略 --build 直接 up -d 即用回原 stub container）
|
||||
#
|
||||
# 設計：override 只覆寫「image / WORKER_MODE」，其餘（STAGE / REDIS / MinIO env / volumes /
|
||||
# depends_on / restart）沿用 base compose。base 的 build: 區段保留不刪 → rollback 乾淨。
|
||||
# STORAGE_BACKEND=minio 由 .env 提供（已驗證），三個 worker 沿用 base 的
|
||||
# STORAGE_BACKEND=${STORAGE_BACKEND:-local} → 最終生效 minio。
|
||||
##
|
||||
|
||||
services:
|
||||
onnx-worker:
|
||||
image: jim800121/kneron-model-converter-worker:real-20260622
|
||||
pull_policy: always
|
||||
environment:
|
||||
- WORKER_MODE=real
|
||||
|
||||
bie-worker:
|
||||
image: jim800121/kneron-model-converter-worker:real-20260622
|
||||
pull_policy: always
|
||||
environment:
|
||||
- WORKER_MODE=real
|
||||
|
||||
nef-worker:
|
||||
image: jim800121/kneron-model-converter-worker:real-20260622
|
||||
pull_policy: always
|
||||
environment:
|
||||
- WORKER_MODE=real
|
||||
@ -211,7 +211,7 @@ visionA-backend Member Center Converter
|
||||
```
|
||||
inputObjectKey = `jobs/${jobId}/input/${safeFilename}`
|
||||
^uuidv4 ^server-controlled prefix ^sanitized
|
||||
refImageKey = `jobs/${jobId}/ref_images/${index}_${safeFilename}`
|
||||
refImageKey = `jobs/${jobId}/input/ref_images/${index}_${safeFilename}`
|
||||
```
|
||||
|
||||
attacker 無法控制：
|
||||
|
||||
112
services/workers/Dockerfile.real
Normal file
112
services/workers/Dockerfile.real
Normal file
@ -0,0 +1,112 @@
|
||||
# Real Worker Dockerfile — 自帶 prebuild toolchain，可實際把 BIE→NEF 真實轉檔
|
||||
#
|
||||
# 與 Dockerfile.stub 的差異：本 image 內含 ktc/ 真實轉檔素材 + toolchain/prebuild
|
||||
# 的 x86-64 編譯 binaries（batch_compile / kneron_nef_utils / model_converter），
|
||||
# 並設好 USE_PREBUILD/LD_LIBRARY_PATH/PYTHONPATH，讓 import ktc + ktc.compile 走
|
||||
# repo 自帶 prebuild 路徑（不抓系統 libs/compiler、libs/dynasty）。
|
||||
#
|
||||
# Build（從專案根目錄、本機 Docker daemon）：
|
||||
# docker build -f services/workers/Dockerfile.real -t kneron-worker-real:verify .
|
||||
#
|
||||
# 注意：build context 含 toolchain/prebuild (~2GB)。搭配「專案根 .dockerignore」
|
||||
# （docker build 以專案根為 context，只讀 context 根目錄的 .dockerignore；不存在
|
||||
# services/workers/.dockerignore）排除 libs/dynasty + libs/compiler（被 USE_PREBUILD
|
||||
# 取代，省 ~3.2GB）。
|
||||
|
||||
# python:3.9 對齊既有 worker；bullseye(Debian 11, glibc 2.31) 對舊 toolchain
|
||||
# （prebuild ELF 最低需求 GNU/Linux 3.2.0）相容性最佳。
|
||||
FROM python:3.9-slim-bullseye
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# --- System libraries ---------------------------------------------------------
|
||||
# opencv-python runtime: libgl1 / libglib2.0-0
|
||||
# OpenMP / OpenBLAS（compiler binary 會用）: libgomp1
|
||||
# gfortran runtime（prebuild/lib 雖自帶 libgfortran.so.5，仍裝保險）: libgfortran5
|
||||
# 其餘缺的 .so 靠 image 內 `ldd toolchain/prebuild/batch_compile` 的 "not found" 補。
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
libgl1 \
|
||||
libglib2.0-0 \
|
||||
libgomp1 \
|
||||
libgfortran5 \
|
||||
ca-certificates \
|
||||
procps \
|
||||
file \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# --- Python dependencies ------------------------------------------------------
|
||||
# 1) queue / storage（consumer.py 需要）
|
||||
# 2) worker 既有 requirements（fastapi/onnx/opencv…）
|
||||
# 3) ktc → sys_flow / sys_flow_v2 + extract_bie_info + kneronnxopt module-load 額外
|
||||
# 需要（全部由本機實跑 `import ktc` 逐個盤出）：
|
||||
# pandas blinker docopt snoop pyzipper dict_recursive_update beautifulsoup4
|
||||
# jinja2 matplotlib tabulate commentjson IPython adjustText onnx_tool onnxsim
|
||||
#
|
||||
# 特別注意的版本鎖：
|
||||
# - onnx==1.14.1：與 root requirements 一致；ktc/toolchain.py 對 onnx==1.7.0 會
|
||||
# 關閉 730 支援，必須避開 1.7.0。
|
||||
# - onnx_tool==0.7.0：未鎖時裝到的新版在 import 期會踩
|
||||
# "unsupported operand type(s) for |: MessageMeta and type"（protobuf 衝突）；
|
||||
# 0.7.0 import 乾淨。
|
||||
# - onnxsim==0.4.36（--only-binary）：py3.9/bullseye 上新版 onnxsim 無 wheel、需
|
||||
# cmake 從源碼編；0.4.36 有 prebuilt wheel，避免裝 build toolchain。
|
||||
# 4) numpy<2 最後裝：prebuild 鏈的 C-extension 以 numpy 1.x 編譯，numpy 2.x 會出
|
||||
# "numpy.core.multiarray failed to import"。最後安裝確保不被其他套件升級回 2.x。
|
||||
COPY services/workers/nef/requirements.txt /tmp/worker-requirements.txt
|
||||
RUN pip install --no-cache-dir \
|
||||
redis>=5.0 boto3>=1.28 \
|
||||
&& pip install --no-cache-dir -r /tmp/worker-requirements.txt \
|
||||
&& pip install --no-cache-dir \
|
||||
"onnx==1.14.1" \
|
||||
scipy \
|
||||
pandas blinker docopt snoop \
|
||||
pyzipper dict_recursive_update beautifulsoup4 jinja2 matplotlib tabulate \
|
||||
commentjson IPython adjustText "onnx_tool==0.7.0" \
|
||||
&& pip install --no-cache-dir --only-binary :all: "onnxsim==0.4.36" \
|
||||
&& pip install --no-cache-dir "numpy<2"
|
||||
|
||||
# --- Application + toolchain source ------------------------------------------
|
||||
# 精準 COPY，避免把 libs/dynasty(2.0G) + libs/compiler(1.2G) 帶進來。
|
||||
COPY ktc/ /app/ktc/
|
||||
COPY vendor/ /app/vendor/
|
||||
COPY services/ /app/services/
|
||||
COPY toolchain/prebuild/ /app/toolchain/prebuild/
|
||||
COPY libs/kneronnxopt/ /app/libs/kneronnxopt/
|
||||
COPY libs/ONNX_Convertor/ /app/libs/ONNX_Convertor/
|
||||
COPY libs/fpAnalyser/ /app/libs/fpAnalyser/
|
||||
COPY E2E_Simulator/python_flow/ /app/E2E_Simulator/python_flow/
|
||||
# 驗證用 fixtures + conftest（production 部署可拿掉，這裡為了在 image 內端到端跑）
|
||||
COPY tests/ /app/tests/
|
||||
|
||||
RUN mkdir -p /data/jobs
|
||||
|
||||
# --- Toolchain / runtime environment（即 tests/conftest.py 的 production 化版本）---
|
||||
ENV USE_PREBUILD=/app/toolchain/prebuild
|
||||
ENV LD_LIBRARY_PATH=/app/toolchain/prebuild/lib
|
||||
ENV PYTHONPATH=/app:/app/vendor:/app/libs:/app/libs/kneronnxopt:/app/E2E_Simulator/python_flow
|
||||
ENV KTC_DISABLE_MP=1
|
||||
|
||||
# --- Worker runtime config ----------------------------------------------------
|
||||
ENV WORKER_MODE=real
|
||||
ENV REDIS_URL=redis://redis:6379
|
||||
ENV JOB_DATA_DIR=/data/jobs
|
||||
# STORAGE_BACKEND 刻意「不」在此 image 設預設值。
|
||||
#
|
||||
# 為什麼：consumer.py 第 52 行 `os.environ.get("STORAGE_BACKEND", "local")` 本身
|
||||
# 已會 fall back 到 local。若這支真實版 image 又寫死 ENV STORAGE_BACKEND=local，會給
|
||||
# 部署者「image 預設就能上雲」的錯覺；一旦 staging compose 漏設 STORAGE_BACKEND=minio，
|
||||
# worker 會 *安靜* 走 local（產出寫 container 本機磁碟、不上 MinIO）→ 轉檔成功但
|
||||
# visionA poll 拿不到結果（即 commit b8457dd / cbd1b9d 修過的同類隱患）。
|
||||
#
|
||||
# 處理姿態：
|
||||
# - 本 image 不設預設 → 由 compose / 部署環境「顯式」提供。
|
||||
# - staging（用 MinIO）必設 `STORAGE_BACKEND=minio`（見部署 checklist 硬性項）。
|
||||
# - 本機 / dev 不設時，consumer.py 仍 fall back local，stub/local 開發流程不受影響。
|
||||
# - 註：要根除 silent fallback 的最穩做法是在 consumer.py 對 WORKER_MODE=real 做
|
||||
# fail-fast（漏設 minio 即報錯），但那屬 application code、需 backend 處理，
|
||||
# 不在本 Dockerfile 範圍。
|
||||
# STAGE: onnx / bie / nef（三個 worker 共用此 image，差在 entrypoint）
|
||||
ENV STAGE=nef
|
||||
|
||||
# 用 shell form 以便 ${STAGE} 變數展開（三個 worker 共用 image、差在此變數）。
|
||||
CMD ["sh", "-c", "python -m services.workers.${STAGE}.worker"]
|
||||
Loading…
x
Reference in New Issue
Block a user