#! /usr/bin/env python3 """ CLI interface for flow_constants.py Usage: flow_constants.py print_binaries flow_constants.py (-h | --help) flow_constants.py --version Options: -h --help Show this screen. --version Show version. """ from docopt import docopt import itertools import pathlib import os import pandas as pd # from IPython import embed DEBUG = True if os.environ.get("REGRESSION_DEBUG", False) else False P_FLOW = pathlib.Path(__file__).resolve().parent P_BIN_CACHE = pathlib.Path("/opt/data/piano_bin_cache") MODE_HARDWARE = [520, 720, 530, 730, 630, 540] HW_CODE = {520: "mozart", 530: "bach", 720: "beethoven", 730: "wagner", 630: "schubert ii", 540: "dvorak"} MODE_HW_LIMIT = {} MODE_HW_LIMIT["inc_in_toolchain"] = [520, 530, 540, 630, 720, 730] MODE_HW_LIMIT["weight_compress"] = [720, 730] MODE_HW_LIMIT["weight_4bit"] = [530, 630, 730, 540] MODE_HW_LIMIT["16bit"] = [720, 530, 730, 630, 540] # no 520 MODE_HW_LIMIT["fm_cut"] = [720, 530, 730, 630, 540] # no 520 MODE_HW_LIMIT["nef_v0"] = [520] # pure bin. 720 may have old format MODE_HW_LIMIT["nef_v1"] = [530, 630, 720] # flatbuffer MODE_HW_LIMIT["nef_v2"] = [540, 730] # this is for mapping from DYNASTY to PIANO NUM2MODE = {} NUM2MODE[0] = "Float" NUM2MODE[1] = "DataPathQuan" # dq NUM2MODE[2] = "Scaled" NUM2MODE[3] = "DataPathClamp" # dc for hw_mode in MODE_HARDWARE: NUM2MODE[hw_mode] = f"kl{hw_mode}" MODE_DYNASTY = NUM2MODE.keys() # use this to display on report NUM2PLATFORM = {} NUM2PLATFORM[0] = "general/dyn_fl" NUM2PLATFORM[1] = "general/dyn_dq" # dq NUM2PLATFORM[2] = "general/dyn_scl" NUM2PLATFORM[3] = "general/dyn_dc" # dc for hw_mode in MODE_HARDWARE: NUM2PLATFORM[hw_mode] = f"kdp{hw_mode}/dynasty" KNEREX_UPDATER_TYPE = { 520: 7, 720: 6, 530: 8, 730: 9, 630: 10, 540: 11 } CSIM_REPO_TAGS = { 520: ("csim_520", "npu_sim"), # bin name different for 520 720: ("csim_720", "npu_csim"), 530: ("csim_530", "npu_csim"), 730: ("csim_730", "npu_csim"), 630: ("csim_630", "npu_csim"), 540: ("csim_540", "npu_csim") } CSIM_REPO_DEBUG_TAGS = { 520: ("csim_520_debug", "npu_sim"), # bin name different for 520 720: ("csim_720_debug", "npu_csim"), 530: ("csim_530_debug", "npu_csim"), 730: ("csim_730_debug", "npu_csim"), 630: ("csim_630_debug", "npu_csim"), 540: ("csim_540_debug", "npu_csim") } SNR_REPORT_COLS = ["SNR", "SNR_With_Mean"] REPORT_COLS_SUCCESS = ["HW not support", "unimplemented feature"] def is_success(e): """Tell a flow is success or not by RegressionError. Inputs: - e: RegressionError """ col_name = e.module_name.split("/")[1] good_cols = ["Success"] + REPORT_COLS_SUCCESS return col_name in good_cols ######################################################################################## # binary name convention BIN_KEY_SKIP = ["bin_dir", "pld.py", "lib_cuda.so", "lib_cuda.bin"] BIN_KEY_OPTIONAL = ["get_run_val_py", "libonnxruntime"] def binary_in_toolchain(): # NOTE: confirm with Jiyuan for path in docker d_binary = {} d1 = "/workspace/libs" d_binary["data_converter"] = {} d2 = "compiler" # data converter v2, independent binary d_binary["data_converter"]["v2"] = f"{d1}/{d2}/dataConverter" d_binary["knerex"] = {} d_binary["knerex"]["normal"] = f"{d1}/fpAnalyser/updater/run_updater" d2 = "dynasty" d_binary["dynasty"] = {} d_binary["dynasty"]["binary"] = f"{d1}/{d2}/run_fix_inference" # TODO: need to update # libdynasty.so is for e2e d_binary["dynasty"]["lib.so"] = f"{d1}/{d2}/E2ESimulator/libdynasty.so" d_binary["dynasty"]["lib.bin"] = f"{d1}/{d2}/libdynasty.bin" # libdynasty with cuda need to compile from docker with GPU. d_binary["dynasty"]["lib_cuda.so"] = f"{d1}/{d2}/E2ESimulator/libdynastycuda.so" d_binary["dynasty"]["lib_cuda.bin"] = f"{d1}/{d2}/libdynastycuda.bin" # actually from compiler d_binary["dynasty"]["libonnxruntime"] = f"{d1}/{d2}/libonnxruntime.so" d_binary["compiler"] = {} d2 = "compiler" d_binary["compiler"]["bin_dir"] = f"{d1}/{d2}" d_binary["compiler"]["opt_bin_dir"] = f"{d1}/{d2}/opt_compile" d_binary["compiler"]["lib_dir"] = f"{d1}/{d2}/lib" d_binary["compiler"]["compiler"] = f"{d1}/{d2}/compile" d_binary["compiler"]["gen_py"] = f"{d1}/{d2}/gen_config.py" d_binary["compiler"]["gen_test_conf_py"] = f"{d1}/{d2}/gen_test_conf.py" d_binary["compiler"]["batch_compiler"] = f"{d1}/{d2}/batch_compile" d_binary["compiler"]["get_run_val_py"] = f"{d1}/{d2}/get_run_val.py" # for gen nef v2. d_binary["compiler"]["kneron_kne_utils"] = f"{d1}/{d2}/kneron_kne_utils" d_binary["compiler"]["kneron_nef_utils"] = f"{d1}/{d2}/kneron_nef_utils" d_binary["compiler"]["kneron_setup_parser"] = f"{d1}/{d2}/kneron_setup_parser" # TODO: model_converter d_binary["compiler"]["model_converter"] = f"{d1}/{d2}/model_converter" d_binary["ip_eval"] = {} for platform in MODE_HARDWARE: d_binary["ip_eval"][platform] = f"/workspace/scripts/res/ip_config_{platform}.json" d_binary["csim"] = {} for platform in MODE_HW_LIMIT["inc_in_toolchain"]: d_binary["csim"][platform] = f"{d1}/c_sim_{platform}/{CSIM_REPO_TAGS[platform][1]}" return d_binary def binary_in_prebuild(p_prebuild): # NOTE: treat this as piano release interface. d_binary = {} d_binary["regression"] = {} # NOTE: the whl name may change. d_binary["regression"]["sys_flow.whl"] = f"{p_prebuild}/SAMENAME" d_binary["data_converter"] = {} # data converter v2, independent binary d_binary["data_converter"]["v2"] = f"{p_prebuild}/dataConverter" d_binary["knerex"] = {} d_binary["knerex"]["normal"] = f"{p_prebuild}/run_updater" d_binary["dynasty"] = {} d_binary["dynasty"]["binary"] = f"{p_prebuild}/run_fix_inference" # libdynasty.so is for e2e d_binary["dynasty"]["lib.so"] = f"{p_prebuild}/libdynasty.so" d_binary["dynasty"]["lib.bin"] = f"{p_prebuild}/libdynasty.bin" # libdynasty with cuda need to compile from docker with GPU. d_binary["dynasty"]["lib_cuda.so"] = f"{p_prebuild}/libdynastycuda.so" d_binary["dynasty"]["lib_cuda.bin"] = f"{p_prebuild}/libdynastycuda.bin" # actually from compiler d_binary["dynasty"]["libonnxruntime"] = f"{p_prebuild}/libonnxruntime.so" d_binary["compiler"] = {} d_binary["compiler"]["bin_dir"] = f"{p_prebuild}" d_binary["compiler"]["opt_bin_dir"] = f"{p_prebuild}/opt_compile" d_binary["compiler"]["lib_dir"] = f"{p_prebuild}/lib" d_binary["compiler"]["compiler"] = f"{p_prebuild}/compile" d_binary["compiler"]["gen_py"] = f"{p_prebuild}/gen_config.py" # temp csim for data converter d_binary["compiler"]["batch_compiler"] = f"{p_prebuild}/batch_compile" d_binary["compiler"]["get_run_val_py"] = f"{p_prebuild}/get_run_val.py" d_binary["compiler"]["model_converter"] = f"{p_prebuild}/model_converter" d_binary["compiler"]["kneron_kne_utils"] = f"{p_prebuild}/kne_utils/kneron_kne_utils" d_binary["compiler"]["kneron_nef_utils"] = f"{p_prebuild}/nef_utils/kneron_nef_utils" d_binary["compiler"]["kneron_setup_parser"] = f"{p_prebuild}/nef_utils/kneron_setup_parser" d_binary["ip_eval"] = {} for platform in MODE_HARDWARE: d_binary["ip_eval"][platform] = f"{p_prebuild}/ip_eval_{platform}.json" d_binary["csim"] = {} for hw_mode in MODE_HARDWARE: d_binary["csim"][hw_mode] = f"{p_prebuild}/npu_csim_{hw_mode}" d_binary["csim_debug"] = {} for hw_mode in MODE_HARDWARE: d_binary["csim_debug"][hw_mode] = f"{p_prebuild}/npu_csim_{hw_mode}_debug" # internal use. not released to toolchain d_binary["pld"] = {} d_binary["pld"]["pld.dir"] = pathlib.Path(f"{p_prebuild}/pld") d_binary["pld"]["pld.py"] = d_binary["pld"]["pld.dir"] / "pld.py" return d_binary def binary_in_piano(p_piano=None): """Get binaries out of piano repo. if given None as input, will dry-run with fake path """ def create_default_repo_config(p_piano): repo_config = {} repo_config["piano_dynasty"] = { "path": p_piano, "dir_build": "build/dynasty" } repo_config["piano_knerex"] = { "path": p_piano, "dir_build": "knerex/updater/release/bin" } repo_config["compiler_piano"] = { "path": p_piano, "dir_build": "install" } repo_config["csim_520"] = { "path": p_piano / "csim/kdp520_hw_csim", "dir_build": "build" } repo_config["csim_520_debug"] = { "path": p_piano / "csim/kdp520_hw_csim", "dir_build_debug": "build_debug" } for platform in [720, 530, 540, 630, 730]: repo_config[f"csim_{platform}"] = { "path": p_piano / f"csim/kdp{platform}_hw_csim", "dir_build": "build/src" } for platform in [720, 530, 540, 630, 730]: repo_config[f"csim_{platform}_debug"] = { "path": p_piano / f"csim/kdp{platform}_hw_csim", "dir_build_debug": "build_debug/src" } return repo_config if p_piano: p_piano = pathlib.Path(p_piano) assert p_piano.exists(), f"piano directory ({p_piano}) does not exist!" else: # dry-run p_piano = pathlib.Path("PIANO_REPO") repo_config = create_default_repo_config(p_piano) d_binary = {} d_binary["regression"] = {} # sys_flow.whl file. the name cannot be changed p_whl_s = list(p_piano.glob("regression/dist/kneron*.whl")) assert len(p_whl_s) <= 1, f"Multiple sys_flow.whl found: {p_whl_s}" # local developer may not prebuild sys_flow.whl so this file may be missing if len(p_whl_s) > 0: d_binary["regression"]["sys_flow.whl"] = p_whl_s[0] else: print("WARNING: no sys_flow.whl ready!!!") d_binary["data_converter"] = {} # data converter v2, independent binary d_binary["data_converter"]["v2"] = p_piano / "data_converter/build/src/dataConverter" # knerex d_binary["knerex"] = {} d1 = repo_config["piano_knerex"]["path"] d2 = repo_config["piano_knerex"]["dir_build"] d3 = f"{d1}/{d2}" d_binary["knerex"]["normal"] = f"{d3}/run_updater" # dynasty d1 = repo_config["piano_dynasty"]["path"] d2 = repo_config["piano_dynasty"]["dir_build"] d_binary["dynasty"] = {} d_binary["dynasty"]["binary"] = f"{d1}/{d2}/run_fix_inference" # so files d_binary["dynasty"]["lib.so"] = f"{d1}/{d2}/E2ESimulator/libdynasty.so" d_binary["dynasty"]["lib.bin"] = f"{d1}/{d2}/E2ESimulator/dynasty_executable" # NOTE: libdynasty with cuda need to compile from docker with GPU. # lib.so and lib_cuda.so are same file but compiled in different docker containter # d_binary["dynasty"]["lib_cuda.so"] = f"{d1}/{d2}/E2ESimulator/libdynasty.so" # d_binary["dynasty"]["lib_cuda.bin"] = f"{d1}/{d2}/E2ESimulator/dynasty_executable" d_binary["compiler"] = {} # TODO: update path from `install` p_piano = repo_config["compiler_piano"]["path"] p_install = repo_config["compiler_piano"]["dir_build"] d_binary["compiler"]["bin_dir"] = f"{p_piano}/{p_install}" d_binary["compiler"]["lib_dir"] = f"{p_piano}/{p_install}/lib" d_binary["compiler"]["compiler"] = f"{p_piano}/{p_install}/compile" d_binary["compiler"]["model_converter"] = f"{p_piano}/{p_install}/model_converter" d_binary["compiler"]["batch_compiler"] = f"{p_piano}/{p_install}/batch_compile" # for gen nef v2 d_binary["compiler"]["kneron_kne_utils"] = f"{p_piano}/{p_install}/kneron_kne_utils" d_binary["compiler"]["kneron_nef_utils"] = f"{p_piano}/{p_install}/kneron_nef_utils" d_binary["compiler"]["kneron_setup_parser"] = f"{p_piano}/{p_install}/kneron_setup_parser" d3 = "compiler/test/gen_rtl_test/utils" # opt_bin_dir is for scripts included in compiler repo, not compiled bin d_binary["compiler"]["opt_bin_dir"] = f"{p_piano}/compiler/bin/opt_compile" d_binary["compiler"]["gen_py"] = f"{p_piano}/{d3}/gen_config.py" d_binary["compiler"]["get_run_val_py"] = f"{p_piano}/{d3}/get_run_val.py" # for release rtl. internal # d_binary["compiler"]["link_bin"] = f"{p_piano}/{d3}/bin/link_bin.sh" # TODELETE on 0.24.0 d_binary["compiler"]["compile_and_gen_conv_all"] = f"{p_piano}/compiler/test/gen_rtl_test/conv/compile_and_gen_conv_all.sh" # so for dynasty float, actually from compiler # TODO: wait for compiler to merge branch and make .so without append in name, such as libonnxruntime.so.1.15.0 d_binary["dynasty"]["libonnxruntime"] = f"{p_piano}/piano/common_header_lib/external/lib/ubuntu_x64/msft/libonnxruntime.so" d_binary["ip_eval"] = {} for platform in MODE_HARDWARE: d_binary["ip_eval"][platform] = f"{p_piano}/compiler/resource/ip_eval/ip_eval_{platform}.json" d_binary["csim"] = {} for hw_code, (repo_name, bin_name) in CSIM_REPO_TAGS.items(): try: d1 = repo_config[repo_name]["path"] d2 = repo_config[repo_name]["dir_build"] d_binary["csim"][hw_code] = f"{d1}/{d2}/{bin_name}" except KeyError: print(f"repo config does not include {repo_name}") d_binary["csim_debug"] = {} for hw_code, (repo_name, bin_name) in CSIM_REPO_DEBUG_TAGS.items(): try: d1 = repo_config[repo_name]["path"] d2 = repo_config[repo_name]["dir_build_debug"] d_binary["csim_debug"][hw_code] = f"{d1}/{d2}/{bin_name}" except KeyError: print(f"repo config does not include {repo_name}") # internal use. compare command.bin inst.hex.opt . should be same on all platforms # TODELETE? d1 = repo_config["csim_530"]["path"] # randomly pick 530 d_binary["csim"]["rtl_cmd_cmp"] = f"{d1}/scripts/rtlCmdCmpBinTxt.py" # internal use. not released to toolchain d_binary["pld"] = {} d_binary["pld"]["pld.dir"] = P_FLOW.parent / "library" / "pld_ng" d_binary["pld"]["pld.py"] = d_binary["pld"]["pld.dir"] / "pld.py" return d_binary def get_binary(use_prebuild=None, regression_script=None): """Get binary set from prebuild / local piano / toolchain. Input: - use_prebuild: may be set from parameter. - regression_script: regression may be set seperately, not using piano submoudle. 1st check: prebuild could: - passed in via function parameter - passed in via command line - passed in via environment "USE_PREBUILD" """ use_prebuild = get_prebuild_path(use_prebuild) use_piano = None use_toolchain = False if use_prebuild is None: # 2nd check: use_piano # use environment "USE_PIANO" to spefify which piano to use use_piano = get_piano_path() if use_piano is None: # 3rd check: use_toolchain, fallback option use_toolchain = True if use_toolchain: # NOTE: use docker/toolchain only # for customers p_fn = pathlib.Path("/workspace/version.txt") assert p_fn.exists(), "Specified to use toolchain, but not running in docker" bin_set = binary_in_toolchain() msg = """use toolchain binaries\n""" elif use_prebuild: # NOTE: the binaries are in given folder bin_set = binary_in_prebuild(use_prebuild) msg = f"""use prebuild piano: {use_prebuild}\n""" elif use_piano: # NOTE: get the binaries from the local piano repo bin_set = binary_in_piano(use_piano) msg = f"""use piano repo at: {use_piano}\n""" else: raise NotImplemented if regression_script: bin_set.update(regression_script) return bin_set, msg, use_prebuild, use_piano, use_toolchain def get_prebuild_path(commit=None): """Get prebuild pathfrom parameter/environment. The parameter could be * pathlib path * full path in string * short commit in string (assume in P_BIN_CACHE) * bash enviroent $USE_PREBUILD """ if type(commit) is pathlib.PosixPath: if commit.exists(): return commit else: print(f"ERROR: given prebuild commit {commit} does NOT EXISTS!") return None if type(commit) is str: if commit.startswith("/"): # full path string given return get_prebuild_path(pathlib.Path(commit)) else: return get_prebuild_path(P_BIN_CACHE / commit) if commit is None: # get from environ cmt = os.environ.get("USE_PREBUILD", None) if cmt: return get_prebuild_path(cmt) return None def get_piano_path(p_piano=None): """Get piano pathfrom parameter/environment. The parameter could be * pathlib path * full path in string * bash enviroent $USE_PIANO """ if p_piano is None: # get from environ as str p_piano = os.environ.get("USE_PIANO", None) if type(p_piano) is str: p_piano = pathlib.Path(p_piano) if type(p_piano) is pathlib.PosixPath: if p_piano.exists(): return p_piano return None def binary_notes(): """Add some notes for some binaries.""" d_binary = {} d_binary["data_converter"] = {} # d_binary["data_converter"]["csim530"] = "to remove" # d_binary["data_converter"][720] = "to remove" # for platform in MODE_HW_LIMIT["data_converter_wrap"]: # d_binary["data_converter"][platform] = "to remove" d_binary["data_converter"]["v2"] = f"new from 0.21.0" d_binary["compiler"] = {} d_binary["compiler"]["kneron_kne_utils"] = "new from 0.21.0" d_binary["compiler"]["kneron_nef_utils"] = "new from 0.21.0" d_binary["compiler"]["kneron_setup_parser"] = "new from 0.21.0" d_binary["compiler"]["batch_compiler_v2"] = "delete from 0.21.1" d_binary["compiler"]["opt_bin_dir"] = "script dir included in compiler" d_binary["regression"]["sys_flow.whl"] = "new from 0.24.0" return d_binary def gen_table_binaries(fn_save=None): """Generate a document table for toolchain release.""" def reform(d): # from https://stackoverflow.com/questions/24988131/nested-dictionary-to-multiindex-dataframe-where-dictionary-keys-are-column-label return {(outerKey, innerKey): values for outerKey, innerDict in d.items() for innerKey, values in innerDict.items()} d = {"piano": reform(binary_in_piano()), "copy to prebuild": reform(binary_in_prebuild("/opt/data/piano_bin_cache/COMMIT")), "copy to toolchain": reform(binary_in_toolchain()), "note": reform(binary_notes())} df = pd.DataFrame.from_dict(d).fillna("") df.sort_index(key=lambda x: x.str.lower(), inplace=True) if fn_save is None: print(df) else: with open(fn_save, "w") as f: f.write(df.to_markdown()) return df # NOTE: the binary set has been choosen with environment vairiables. BIN_SET, bin_msg, is_use_prebuild, is_use_piano, is_use_toolchain = get_binary() print(bin_msg) ######################################################################################## # define onnx/bie file names MODEL_OPTIMIZE = ["scaled", "graphopt", "decomp", "quan", "wqbi", "hwbi", "hwbi-mse"] MODEL_BIAS_ADJUST = ["wqbi", "hwbi", "hwbi-mse"] # TODO: the is a two-step mapping. too difficult. need refactor later. # ONLY release below formats to compiler/nef # e.g., use scaled.onnx to generate model_520_piano MODEL_RELEASE = { # onnx : appendix in dynasty mode "scaled": "", "wqbi": "-wqbi", "hwbi": "-hwbi", "hwbi-mse": "-hwbi-mse", } MODEL_FORMAT = ["bie", "onnx"] MODEL_KEY = [f"kdp{hw_mode}_{m_opt}_piano_{fmt}" for hw_mode in MODE_HARDWARE for m_opt in MODEL_OPTIMIZE for fmt in MODEL_FORMAT] MODEL_KEY.append("origin") # MODEL_KEY should be same as map_onnx.keys() ######################################################################################## # appendix for piano/dynasty dump DYNASTY_OUT_DIR_APPENDIX = {} DYNASTY_OUT_DIR_APPENDIX["renaissance"] = "" DYNASTY_OUT_DIR_APPENDIX["piano"] = "_piano" def get_dynasty_mode_setting(): # here defines each mode to use which onnx, which mode number dynasty_mode_settings = {} # NOTE: define configs for dynasty inference: which onnx, which mode? for hw_fmt in ["piano_onnx", "piano_bie"]: # float mode t = [["float", "origin", 0]] for hw_mode in MODE_HARDWARE: fn_onnx_scaled = "kdp{}_scaled_{}".format(hw_mode, hw_fmt) # name_mode, fn_onnx, n_mode t.append([f"{hw_mode}scl", fn_onnx_scaled, 2]) t.append([f"{hw_mode}dq", fn_onnx_scaled, 1]) t.append([f"{hw_mode}dc", fn_onnx_scaled, 3]) t.append([f"{hw_mode}", fn_onnx_scaled, hw_mode]) # NOTE: this is same as float mode. # but using hardware related decomposed graph (knerex generated) fn_onnx_decomp = f"kdp{hw_mode}_decomp_{hw_fmt}" t.append([f"{hw_mode}decomp", fn_onnx_decomp, 0]) # NOTE: this is same as decomp mode. # but using compiler dumped graph optimized fn_onnx_graphopt = f"kdp{hw_mode}_opt_{hw_fmt}" t.append([f"{hw_mode}graphopt", fn_onnx_graphopt, 0]) # wq related to use quantized model fn_onnx_quan = f"kdp{hw_mode}_quan_{hw_fmt}" t.append([f"{hw_mode}wq", fn_onnx_quan, 2]) t.append([f"{hw_mode}wqdq", fn_onnx_quan, 1]) t.append([f"{hw_mode}wqdc", fn_onnx_quan, 3]) # use bias adjusted onnx. they are quantized already # no need for dq / dc only modes. for bi_onnx in MODEL_BIAS_ADJUST: fn_onnx_bi = f"kdp{hw_mode}_{bi_onnx}_{hw_fmt}" t.append([f"{hw_mode}wq-{bi_onnx}", fn_onnx_bi, 2]) t.append([f"{hw_mode}wqdq-{bi_onnx}", fn_onnx_bi, 1]) t.append([f"{hw_mode}wqdc-{bi_onnx}", fn_onnx_bi, 3]) t.append([f"{hw_mode}-{bi_onnx}", fn_onnx_bi, hw_mode]) dynasty_mode_settings[hw_fmt] = t return dynasty_mode_settings DYNASTY_MODE_SETTINGS = get_dynasty_mode_setting() ######################################################################################## def get_snr_pairs(): """Define pairs of dynasty various mode dump to calculate SNR. [mode_degraded, mode_reference] You should put the actual folder name name. Rule from mode_name to folder name is as below: * We have the mode_name, e.g., "520scl" * a prefix of "mode_" is added before the mode_name, e.g., "mode_520scl" * a appendix is added after mode_name, depends which dynasty are used: * if "renaissance", the output has an appendix of "" (aka, no appendix. to be compatatiable with old results) * e.g., the output dir is "mode_520scl" * if "piano", the output dir name has an appendix of "_piano" * e.g., the output dir is "mode_520scl_piano" * Try to define snr pairs you are interested as MANY as you can. * SNR pairs defined here does not guaranty the calculation of SNR. * SNR calculate process will loop through the defined SNR_pairs * If both modes in this pair have dynasty dump exist, the snr between them will be __calculated__ * If any mode not on (or without any corresponding dynasty dump), SNR for this pair will be __skipped__. """ # TODO: move dynasty onnx definitions here? # NOTE: some SNR pairs are NOT related to HW mode snr_pairs = [ # pairs for tflite ("mode_tflite_float_noise", "mode_720_piano"), ("mode_tflite_float_noise", "mode_720decomp_piano"), ("mode_tflite_float_noise", "mode_float_piano"), ("mode_tflite_fix_noise", "mode_720_piano"), ("mode_tflite_fix_noise", "mode_720decomp_piano"), ("mode_tflite_fix_noise", "mode_float_piano"), ("mode_onnxruntime_noise", "mode_float_piano") ] # noise snr_pairs.append(("mode_float_noise3_piano", "mode_float_piano")) snr_pairs.append(("mode_float_noise6_piano", "mode_float_piano")) for hw_mode in MODE_HARDWARE: # NOTE: use decomposed graph float mode as reference mode_decomp = f"mode_{hw_mode}decomp_piano" snr_pairs.append((mode_decomp, "mode_float_piano")) mode_graphopt = f"mode_{hw_mode}graphopt_piano" snr_pairs.append((mode_graphopt, "mode_float_piano")) # there maybe 3 possible reference for mode_float in [mode_graphopt, mode_decomp, "mode_float_piano"]: # first are normal dynasty modes t = [ (f"mode_{hw_mode}scl_piano", mode_float), (f"mode_{hw_mode}wq_piano", mode_float), (f"mode_{hw_mode}dq_piano", mode_float), (f"mode_{hw_mode}dc_piano", mode_float), (f"mode_{hw_mode}wqdq_piano", f"mode_{hw_mode}wq_piano"), (f"mode_{hw_mode}wqdq_piano", f"mode_{hw_mode}dq_piano"), (f"mode_{hw_mode}wqdq_piano", mode_float), (f"mode_{hw_mode}wqdc_piano", f"mode_{hw_mode}wq_piano"), (f"mode_{hw_mode}wqdc_piano", f"mode_{hw_mode}dc_piano"), (f"mode_{hw_mode}wqdc_piano", mode_float), (f"mode_{hw_mode}_piano", f"mode_{hw_mode}wqdq_piano"), (f"mode_{hw_mode}_piano", f"mode_{hw_mode}wqdc_piano"), (f"mode_{hw_mode}_piano", mode_float), ] # seconds are bias adjust related. for bi_mode in MODEL_BIAS_ADJUST: t2 = [ (f"mode_{hw_mode}wq-{bi_mode}_piano", mode_float), (f"mode_{hw_mode}wqdq-{bi_mode}_piano", mode_float), (f"mode_{hw_mode}wqdc-{bi_mode}_piano", mode_float), (f"mode_{hw_mode}-{bi_mode}_piano", mode_float), (f"mode_{hw_mode}-{bi_mode}_piano", f"mode_{hw_mode}wqdq-{bi_mode}_piano"), (f"mode_{hw_mode}-{bi_mode}_piano", f"mode_{hw_mode}wqdc-{bi_mode}_piano"), ] t.extend(t2) snr_pairs.extend(t) # make it unique snr_pairs = sorted(list(set(snr_pairs))) # print(snr_pairs) # for debug return snr_pairs def get_snr_pair_bi(): """Get snr pairs interested for bias_adjust snr report. Each report is determined by its three dependant modes e.g., if mode "float", "520wq", "520wqbi" are turned on, regression will create a report for "520wqbi_improve" if all modes are turned on, all report will be generated. """ snr_pair_bi = {} for hw_mode in MODE_HARDWARE: refs = ["float", f"{hw_mode}decomp", f"{hw_mode}graphopt"] for ref in refs: # this is for wqbi only k1 = f"{ref}-{hw_mode}wq-wqbi_improve" snr_pair_bi[k1] = (ref, f"{hw_mode}wq", f"{hw_mode}wq-wqbi") # these are for hw mode improvements for bi_mode in MODEL_BIAS_ADJUST: k1 = f"{ref}-{hw_mode}-{bi_mode}_improve" snr_pair_bi[k1] = (ref, f"{hw_mode}", f"{hw_mode}-{bi_mode}") return snr_pair_bi SNR_PAIRS = get_snr_pairs() SNR_BI_IMPROVE = get_snr_pair_bi() def contain_valid_snr_pairs(lst_mode_run): """Determine whether need to run SNR. For example, if only run dynasty float, there will be no necessary to run SNR calculation. """ if len(lst_mode_run) < 2: return False for (d1, d2) in itertools.combinations(lst_mode_run, 2): if [d1, d2] in SNR_PAIRS or [d2, d1] in SNR_PAIRS: return True return False ############################################################################### if __name__ == "__main__": arguments = docopt(__doc__, version="flow_constants v0.2") # print(arguments) if arguments["print_binaries"]: gen_table_binaries(P_FLOW.parent / "doc" / "bins_piano_prebuild_toolchain.md")