jim800121chen
1c53253e6e
問題：同一顆含 Softmax 的 tflite，正式站能編出 KL520 真 .nef， 本專案 worker 在 nef 階段撞 UnimplementedFeature [Softmax] exit 6。 根因：runtime ktc 的 eliminate_tail 是 no-op（閹割版、只印警告）， 尾端 Softmax 未被移除；正式站產物實測為 logits 輸出（Softmax 已砍）。 修法（staging 實測驗證、與正式站產物權重段逐 byte 相同）： - onnx/core.py：新增 remove_tail_softmax()，onnx2onnx_flow 後對所有 platform 移除 terminal Softmax（cut_nodes、不用 cut_types 避免誤砍 中間層下游）、多 output rewire 逐項驗證 fail-loud、onnx.checker、 removed_tail_softmax metadata - precheck.py：R3 誤擋收窄——刪 hw_not_support_col 欄位名誤命中、 op capture 改必須、抽不到具體 op 名改放行+warning（真 fail-open）、 刪自由文字猜 op fallback、加噪音 token 過濾 - tests：+16（29 passed；terminal/no-op/多 output/fail-open 全覆蓋） - docs：TDD §12、design-doc ADR-012、PRD 輸出語意變更（機率→logits） 行為變更：所有 platform NEF 輸出改 logits（與正式站一致）、 分類後處理需呼叫端自行補 softmax。 Review：兩輪（0C/3M→0C/0M）；W3 docker integration 6/6 PASS （520+Softmax e2e 產真 .nef 799,844 bytes、720 回歸、precheck 放行）。 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
265 lines
9.1 KiB
Python
265 lines
9.1 KiB
Python
"""Unit tests for platform/operator compatibility pre-check.
|
||
|
||
這些測試刻意不 import onnx / ktc（本機環境沒有 toolchain），只透過 mock 的
|
||
evaluator backend 驗證 pre-check 的判斷邏輯。
|
||
"""
|
||
|
||
from pathlib import Path
|
||
import sys
|
||
|
||
import pytest
|
||
|
||
ROOT = Path(__file__).resolve().parents[2]
|
||
sys.path.insert(0, str(ROOT))
|
||
|
||
from services.backends.precheck import (
|
||
UnsupportedOperatorError,
|
||
build_error_message,
|
||
find_unsupported_operators,
|
||
run_precheck,
|
||
)
|
||
|
||
|
||
class _FakeEvaluator:
|
||
"""可注入的假 evaluator：回傳固定字串，或在呼叫時 raise。"""
|
||
|
||
def __init__(self, *, report=None, raises=None):
|
||
self._report = report
|
||
self._raises = raises
|
||
self.calls = []
|
||
|
||
def evaluate(self, onnx_path, **kwargs):
|
||
self.calls.append((onnx_path, kwargs))
|
||
if self._raises is not None:
|
||
raise self._raises
|
||
return self._report
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# find_unsupported_operators
|
||
# --------------------------------------------------------------------------- #
|
||
|
||
def test_find_unsupported_empty_report_returns_empty():
|
||
assert find_unsupported_operators("") == []
|
||
assert find_unsupported_operators(None) == []
|
||
|
||
|
||
def test_find_unsupported_clean_report_returns_empty():
|
||
report = "kdp520/FPS: 1200, cpu_node: None, docker_version: 1.0"
|
||
assert find_unsupported_operators(report) == []
|
||
|
||
|
||
def test_find_unsupported_detects_empty_node_op():
|
||
report = "compiler: creating an EmptyNode instance for op_type: Softmax"
|
||
assert find_unsupported_operators(report) == ["Softmax"]
|
||
|
||
|
||
def test_find_unsupported_detects_unimplemented_feature_with_bracket_op():
|
||
report = "UnimplementedFeature: undefined CPU op [Softmax]"
|
||
assert find_unsupported_operators(report) == ["Softmax"]
|
||
|
||
|
||
def test_find_unsupported_detects_hw_not_support_with_op():
|
||
report = "HW_NOT_SUPPORT: Softmax"
|
||
assert find_unsupported_operators(report) == ["Softmax"]
|
||
|
||
|
||
def test_find_unsupported_hw_not_supported_suffix_variants():
|
||
# marker 的 "supported" 字尾要被 marker 本身吃掉：
|
||
# 後面跟真 op → 照擋；後面跟表格值 → 放行（"ed" 不可被誤捕成 op 名）。
|
||
assert find_unsupported_operators("HW_NOT_SUPPORTED: Softmax") == ["Softmax"]
|
||
assert find_unsupported_operators("Hardware not supported: None") == []
|
||
assert find_unsupported_operators("hardware not supported") == []
|
||
|
||
|
||
def test_find_unsupported_free_text_fragment_passes_fail_open():
|
||
# unimplemented 的自由文字片段不做「首字母大寫字」猜測：
|
||
# "Not" 是真 ONNX op、"CPU" 不是對的 op —— 猜錯（誤擋 / 錯訊息）比放行更糟。
|
||
assert (
|
||
find_unsupported_operators("UnimplementedFeature: Not supported in this mode")
|
||
== []
|
||
)
|
||
# 無中括號的自由文字（抓不到明確 op）→ 放行，而不是抓錯成 "CPU"。
|
||
assert (
|
||
find_unsupported_operators("UnimplementedFeature: undefined CPU op Softmax")
|
||
== []
|
||
)
|
||
|
||
|
||
def test_find_unsupported_marker_without_op_passes_fail_open():
|
||
# marker 字樣但抽不到具體 op 名稱 → 放行（真 fail-open）。
|
||
# "HW not support" 很可能只是 model_fx_report 的欄位分類名（表頭），
|
||
# 光是出現不代表模型有不支援 op（R3 誤擋根因）。
|
||
assert find_unsupported_operators("kdp520/ERROR, HW not support") == []
|
||
# 欄位名後面接表格值（非 op）也不能擋。
|
||
assert find_unsupported_operators("HW not support: 0") == []
|
||
assert find_unsupported_operators("Hardware not support: None") == []
|
||
assert find_unsupported_operators("kdp520/ip_eval/HW not support: N/A") == []
|
||
# 光禿禿的 "UnimplementedFeature" 欄位名（無後續內容）也放行。
|
||
assert find_unsupported_operators("UnimplementedFeature") == []
|
||
|
||
|
||
def test_find_unsupported_does_not_false_positive_on_supported_wording():
|
||
# "int16 is not supported in 520" 這種 bitwidth 說明不應被誤判為 op 不支援。
|
||
report = "note: int16 is not supported in 520; datapath int8 ok"
|
||
assert find_unsupported_operators(report) == []
|
||
|
||
|
||
def test_find_unsupported_detects_multiple_ops_across_markers():
|
||
# 模型含多個不支援 op，且分散在不同 marker pattern（EmptyNode + HW_NOT_SUPPORT）。
|
||
# 應回傳兩者、順序穩定（依報告出現順序）。
|
||
report = (
|
||
"compiler: creating an EmptyNode instance for op_type: Softmax\n"
|
||
"HW_NOT_SUPPORT: LSTM"
|
||
)
|
||
assert find_unsupported_operators(report) == ["Softmax", "LSTM"]
|
||
|
||
|
||
def test_find_unsupported_dedups_repeated_op():
|
||
# 同一個不支援 op 在報告中出現多次（多個節點）→ 只回一次（precheck.py 的 dedup）。
|
||
report = (
|
||
"creating an EmptyNode instance for op_type: Softmax\n"
|
||
"creating an EmptyNode instance for op_type: Softmax\n"
|
||
"HW_NOT_SUPPORT: Softmax"
|
||
)
|
||
assert find_unsupported_operators(report) == ["Softmax"]
|
||
|
||
|
||
def test_run_precheck_raises_lists_multiple_ops_in_message():
|
||
# 多個不支援 op → 擋掉時錯誤訊息應同時列出（去重後）。
|
||
evaluator = _FakeEvaluator(
|
||
report=(
|
||
"creating an EmptyNode instance for op_type: Softmax\n"
|
||
"HW_NOT_SUPPORT: LSTM\n"
|
||
"creating an EmptyNode instance for op_type: Softmax"
|
||
)
|
||
)
|
||
with pytest.raises(UnsupportedOperatorError) as exc:
|
||
run_precheck(
|
||
"/tmp/out.onnx",
|
||
model_id=6,
|
||
version="v1",
|
||
platform="520",
|
||
evaluator=evaluator,
|
||
)
|
||
msg = str(exc.value)
|
||
assert "Softmax" in msg
|
||
assert "LSTM" in msg
|
||
assert "520" in msg
|
||
# 去重：Softmax 只出現一次
|
||
assert msg.count("Softmax") == 1
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# build_error_message
|
||
# --------------------------------------------------------------------------- #
|
||
|
||
def test_build_error_message_with_op_names():
|
||
msg = build_error_message("520", ["Softmax"])
|
||
assert "520" in msg
|
||
assert "Softmax" in msg
|
||
# 帶有引導使用者的補救建議
|
||
assert "720" in msg or "730" in msg
|
||
|
||
|
||
def test_build_error_message_without_op_names_falls_back_to_generic():
|
||
msg = build_error_message("520", [])
|
||
assert "520" in msg
|
||
assert "operator" in msg
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# run_precheck
|
||
# --------------------------------------------------------------------------- #
|
||
|
||
def test_run_precheck_raises_on_unsupported_report():
|
||
evaluator = _FakeEvaluator(
|
||
report="compiler: creating an EmptyNode instance for op_type: Softmax"
|
||
)
|
||
with pytest.raises(UnsupportedOperatorError) as exc:
|
||
run_precheck(
|
||
"/tmp/out.onnx",
|
||
model_id=1,
|
||
version="v1",
|
||
platform="520",
|
||
evaluator=evaluator,
|
||
)
|
||
assert "Softmax" in str(exc.value)
|
||
assert "520" in str(exc.value)
|
||
assert len(evaluator.calls) == 1
|
||
|
||
|
||
def test_run_precheck_passes_on_clean_report():
|
||
evaluator = _FakeEvaluator(report="kdp720/FPS: 900, cpu_node: None")
|
||
# 不應 raise
|
||
run_precheck(
|
||
"/tmp/out.onnx",
|
||
model_id=2,
|
||
version="v1",
|
||
platform="720",
|
||
evaluator=evaluator,
|
||
)
|
||
assert len(evaluator.calls) == 1
|
||
|
||
|
||
def test_run_precheck_raises_when_evaluate_raises_with_signal():
|
||
# evaluate() 直接 raise，但例外訊息含明確不支援訊號 → 應轉成清楚錯誤。
|
||
evaluator = _FakeEvaluator(
|
||
raises=AssertionError("UnimplementedFeature: undefined CPU op [Softmax]")
|
||
)
|
||
with pytest.raises(UnsupportedOperatorError) as exc:
|
||
run_precheck(
|
||
"/tmp/out.onnx",
|
||
model_id=3,
|
||
version="v1",
|
||
platform="520",
|
||
evaluator=evaluator,
|
||
)
|
||
assert "Softmax" in str(exc.value)
|
||
|
||
|
||
def test_run_precheck_passes_on_marker_only_report():
|
||
# 報告帶「HW not support」欄位名但無具體 op → 放行不 raise（真 fail-open）。
|
||
evaluator = _FakeEvaluator(
|
||
report="kdp520/ip_eval/HW not support: N/A, kdp520/FPS: 1200"
|
||
)
|
||
# 不應 raise
|
||
run_precheck(
|
||
"/tmp/out.onnx",
|
||
model_id=6,
|
||
version="v1",
|
||
platform="520",
|
||
evaluator=evaluator,
|
||
)
|
||
assert len(evaluator.calls) == 1
|
||
|
||
|
||
def test_run_precheck_passes_when_evaluate_raises_without_signal():
|
||
# evaluate() raise，但訊息不含不支援訊號（transient / 環境問題）→ 放行不誤擋。
|
||
evaluator = _FakeEvaluator(
|
||
raises=RuntimeError("Quantization model generation failed. See above message.")
|
||
)
|
||
# 不應 raise
|
||
run_precheck(
|
||
"/tmp/out.onnx",
|
||
model_id=4,
|
||
version="v1",
|
||
platform="530",
|
||
evaluator=evaluator,
|
||
)
|
||
|
||
|
||
def test_run_precheck_passes_target_platform_to_evaluator():
|
||
evaluator = _FakeEvaluator(report="")
|
||
run_precheck(
|
||
"/tmp/out.onnx",
|
||
model_id=5,
|
||
version="abc",
|
||
platform="730",
|
||
evaluator=evaluator,
|
||
)
|
||
_path, kwargs = evaluator.calls[0]
|
||
assert kwargs["platform"] == "730"
|
||
assert kwargs["model_id"] == 5
|
||
assert kwargs["version"] == "abc"
|