Refactor server.js (647 → 99 lines) into 30+ modules under src/: - auth/: JWKS validation, JWT middleware, OAuth client_credentials - routes/v1/: jobs (POST/GET/:id) + promote with input validation - routes/legacy.js: existing /jobs multipart path (backward compatible) - services/: jobService, healthService, sseService, statusMapper, doneListener - middleware/: requestId, errorHandler, perClientRateLimit, uploadConcurrency, upload (multer + storage) - redis/: Lua scripts for atomic claim/release_active_job - storage/: local + minio adapters; fileAccessAgent/: PUT promote client - config.js: env var validation with fail-fast Phase 1 features (T1–T11): - T1 Auth middleware + JWKS (Member Center OAuth2 resource server) - T2 OAuth client (Member Center client_credentials, Basic auth) - T3 /api/v1/* router skeleton - T4 server.js refactor (legacy endpoints fully preserved, real-Redis regression verified — existing worker consumer group untouched) - T5 POST /api/v1/jobs (multipart, OWASP-audited, 2 Critical / 6 Major fixed; Risk-A/B documented as accepted) - T6 GET /api/v1/jobs + GET /:id (cursor pagination, ETag, IDOR-safe) - T7 POST /jobs/:id/promote (FAA PUT with own service token, 300s timeout, fail-fast on missing FAA URL) - T8 /health upgrade (healthy/degraded/unhealthy + 30s background cache) - T9 stage_timings (release_active_job in terminal states) - T10 env + Docker integration (MULTIPART_* + concurrency limiter) - T11 README (498 lines) + OpenAPI 3.0 spec (1588 lines) Tests: 630 pass across 29 suites. Updated Dockerfile + .dockerignore + docker-compose.yml env passthrough (no hardcoded secrets, fail-fast on missing required vars). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
186 lines
6.1 KiB
YAML
186 lines
6.1 KiB
YAML
##
|
||
# Kneron Model Converter — Development docker-compose
|
||
#
|
||
# Usage:
|
||
# docker-compose up # local mode (shared volume)
|
||
# STORAGE_BACKEND=s3 docker-compose up # S3/MinIO mode
|
||
# docker-compose up --scale bie-worker=3 # scale BIE workers
|
||
##
|
||
|
||
volumes:
|
||
job-data:
|
||
|
||
services:
|
||
# ---------- Infrastructure ----------
|
||
|
||
redis:
|
||
image: redis:7-alpine
|
||
expose:
|
||
- "6379"
|
||
command: redis-server --save ""
|
||
healthcheck:
|
||
test: ["CMD", "redis-cli", "ping"]
|
||
interval: 5s
|
||
timeout: 3s
|
||
retries: 3
|
||
|
||
# ---------- Web UI ----------
|
||
|
||
web:
|
||
build: ./apps/web
|
||
ports:
|
||
- "9500:3000"
|
||
depends_on:
|
||
scheduler:
|
||
condition: service_healthy
|
||
restart: unless-stopped
|
||
|
||
# ---------- Scheduler ----------
|
||
#
|
||
# T10:Phase 1 env 透傳清單。所有值都用 ${VAR} 從 .env / shell 讀取,
|
||
# 不在 docker-compose.yml hardcode(避免 secret 被 commit)。
|
||
# 必填變數缺漏 → scheduler container 會啟動失敗(fail-fast)。
|
||
|
||
scheduler:
|
||
build: ./apps/task-scheduler
|
||
ports:
|
||
- "9501:4000"
|
||
depends_on:
|
||
redis:
|
||
condition: service_healthy
|
||
volumes:
|
||
- job-data:/data/jobs
|
||
environment:
|
||
# === 應用基本 ===
|
||
- PORT=4000
|
||
- NODE_ENV=${NODE_ENV:-development}
|
||
- LOG_LEVEL=${LOG_LEVEL:-info}
|
||
|
||
# === Redis ===
|
||
- REDIS_URL=redis://redis:6379
|
||
|
||
# === Job 資料目錄 / CORS ===
|
||
- JOB_DATA_DIR=/data/jobs
|
||
- FRONTEND_URL=${FRONTEND_URL:-http://localhost:9500}
|
||
|
||
# === Storage backend ===
|
||
- STORAGE_BACKEND=${STORAGE_BACKEND:-local}
|
||
- MINIO_ENDPOINT_URL=${MINIO_ENDPOINT_URL:-http://192.168.0.130:9000}
|
||
- MINIO_BUCKET=${MINIO_BUCKET:-convertet-working-space}
|
||
- MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY:-convuser}
|
||
- MINIO_SECRET_KEY=${MINIO_SECRET_KEY}
|
||
- MINIO_REGION=${MINIO_REGION:-us-east-1}
|
||
- MINIO_LIFECYCLE_DAYS=${MINIO_LIFECYCLE_DAYS:-7}
|
||
|
||
# === OAuth / Member Center(必填,缺漏 fail-fast)===
|
||
- MEMBER_CENTER_ISSUER=${MEMBER_CENTER_ISSUER}
|
||
- MEMBER_CENTER_JWKS_URL=${MEMBER_CENTER_JWKS_URL}
|
||
- MEMBER_CENTER_TOKEN_URL=${MEMBER_CENTER_TOKEN_URL}
|
||
|
||
# === Converter 身份(必填)===
|
||
- KNERON_CONVERTER_AUDIENCE=${KNERON_CONVERTER_AUDIENCE}
|
||
- KNERON_CONVERTER_CLIENT_ID=${KNERON_CONVERTER_CLIENT_ID}
|
||
- KNERON_CONVERTER_CLIENT_SECRET=${KNERON_CONVERTER_CLIENT_SECRET}
|
||
- CONVERTER_TENANT_ID=${CONVERTER_TENANT_ID:-}
|
||
|
||
# === File Access Agent(必填)===
|
||
- FILE_ACCESS_AGENT_BASE_URL=${FILE_ACCESS_AGENT_BASE_URL}
|
||
- FILE_ACCESS_AGENT_AUDIENCE=${FILE_ACCESS_AGENT_AUDIENCE}
|
||
|
||
# === Scope(可選,預設 TDD §8)===
|
||
- CONVERTER_SCOPE_WRITE=${CONVERTER_SCOPE_WRITE:-converter:job.write}
|
||
- CONVERTER_SCOPE_READ=${CONVERTER_SCOPE_READ:-converter:job.read}
|
||
|
||
# === JWKS / JWT cache 行為(可選)===
|
||
- JWKS_CACHE_MAX_AGE_MS=${JWKS_CACHE_MAX_AGE_MS:-600000}
|
||
- JWKS_COOLDOWN_MS=${JWKS_COOLDOWN_MS:-30000}
|
||
- JWT_CLOCK_TOLERANCE_SEC=${JWT_CLOCK_TOLERANCE_SEC:-60}
|
||
|
||
# === OAuth Client cache(可選)===
|
||
- OAUTH_TOKEN_REFRESH_SKEW_MS=${OAUTH_TOKEN_REFRESH_SKEW_MS:-60000}
|
||
- OAUTH_TOKEN_TIMEOUT_MS=${OAUTH_TOKEN_TIMEOUT_MS:-10000}
|
||
|
||
# === Promote 行為(可選)===
|
||
- PROMOTE_TIMEOUT_MS=${PROMOTE_TIMEOUT_MS:-300000}
|
||
|
||
# === Multipart 上限(T10 修 D5)===
|
||
- MULTIPART_MODEL_MAX_BYTES=${MULTIPART_MODEL_MAX_BYTES:-524288000}
|
||
- MULTIPART_REF_IMAGE_MAX_BYTES=${MULTIPART_REF_IMAGE_MAX_BYTES:-10485760}
|
||
- MULTIPART_REF_IMAGES_MAX_COUNT=${MULTIPART_REF_IMAGES_MAX_COUNT:-100}
|
||
|
||
# === Upload concurrency(T10 修 D5)===
|
||
- MAX_CONCURRENT_UPLOADS=${MAX_CONCURRENT_UPLOADS:-5}
|
||
- UPLOAD_RETRY_AFTER_SECONDS=${UPLOAD_RETRY_AFTER_SECONDS:-30}
|
||
restart: unless-stopped
|
||
|
||
# ---------- Workers (stub mode) ----------
|
||
|
||
onnx-worker:
|
||
build:
|
||
context: .
|
||
dockerfile: services/workers/Dockerfile.stub
|
||
depends_on:
|
||
redis:
|
||
condition: service_healthy
|
||
volumes:
|
||
- job-data:/data/jobs
|
||
environment:
|
||
- STAGE=onnx
|
||
- REDIS_URL=redis://redis:6379
|
||
- JOB_DATA_DIR=/data/jobs
|
||
- WORKER_MODE=${WORKER_MODE:-stub}
|
||
- STORAGE_BACKEND=${STORAGE_BACKEND:-local}
|
||
- MINIO_ENDPOINT_URL=${MINIO_ENDPOINT_URL:-http://192.168.0.130:9000}
|
||
- MINIO_BUCKET=${MINIO_BUCKET:-convertet-working-space}
|
||
- MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY:-convuser}
|
||
- MINIO_SECRET_KEY=${MINIO_SECRET_KEY}
|
||
- MINIO_REGION=${MINIO_REGION:-us-east-1}
|
||
- MINIO_LIFECYCLE_DAYS=${MINIO_LIFECYCLE_DAYS:-7}
|
||
restart: unless-stopped
|
||
|
||
bie-worker:
|
||
build:
|
||
context: .
|
||
dockerfile: services/workers/Dockerfile.stub
|
||
depends_on:
|
||
redis:
|
||
condition: service_healthy
|
||
volumes:
|
||
- job-data:/data/jobs
|
||
environment:
|
||
- STAGE=bie
|
||
- REDIS_URL=redis://redis:6379
|
||
- JOB_DATA_DIR=/data/jobs
|
||
- WORKER_MODE=${WORKER_MODE:-stub}
|
||
- STORAGE_BACKEND=${STORAGE_BACKEND:-local}
|
||
- MINIO_ENDPOINT_URL=${MINIO_ENDPOINT_URL:-http://192.168.0.130:9000}
|
||
- MINIO_BUCKET=${MINIO_BUCKET:-convertet-working-space}
|
||
- MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY:-convuser}
|
||
- MINIO_SECRET_KEY=${MINIO_SECRET_KEY}
|
||
- MINIO_REGION=${MINIO_REGION:-us-east-1}
|
||
- MINIO_LIFECYCLE_DAYS=${MINIO_LIFECYCLE_DAYS:-7}
|
||
restart: unless-stopped
|
||
|
||
nef-worker:
|
||
build:
|
||
context: .
|
||
dockerfile: services/workers/Dockerfile.stub
|
||
depends_on:
|
||
redis:
|
||
condition: service_healthy
|
||
volumes:
|
||
- job-data:/data/jobs
|
||
environment:
|
||
- STAGE=nef
|
||
- REDIS_URL=redis://redis:6379
|
||
- JOB_DATA_DIR=/data/jobs
|
||
- WORKER_MODE=${WORKER_MODE:-stub}
|
||
- STORAGE_BACKEND=${STORAGE_BACKEND:-local}
|
||
- MINIO_ENDPOINT_URL=${MINIO_ENDPOINT_URL:-http://192.168.0.130:9000}
|
||
- MINIO_BUCKET=${MINIO_BUCKET:-convertet-working-space}
|
||
- MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY:-convuser}
|
||
- MINIO_SECRET_KEY=${MINIO_SECRET_KEY}
|
||
- MINIO_REGION=${MINIO_REGION:-us-east-1}
|
||
- MINIO_LIFECYCLE_DAYS=${MINIO_LIFECYCLE_DAYS:-7}
|
||
restart: unless-stopped
|