#! /usr/bin/env python3 import ctypes import os import re import pathlib from collections import defaultdict import numpy as np import sys_flow_v2.flow_constants as fconsts import sys_flow_v2.flow_utils as futils from sys_flow_v2.exceptions import RegressionError import snoop DEBUG = True if os.environ.get("REGRESSION_DEBUG", False) else False snoop.install(enabled=DEBUG) def gen_dynasty_mode_settings(name_mode, fn_onnx=None, onnx_map=None, which_onnx="piano_onnx", model_id="tc/model_1"): """ Generate dynasty settings for `one` mode. Args: name_mode (str): could be "720", "720wq". fn_onnx (pathlib / str): location of onnx file. May skip this if `onnx_map` given. onnx_map (dict): a dict of multiple onnx files. Will use `name_mode` to find the proper onnx. which_onnx (str): "piano_onnx" / "piano_bie", depends on the picked model. model_id (str): default "tc/model_1". it should have the format of "CATEGORY/MODEL_NAME" Returns: mode settings for this input. """ # predefined settings select_dynasty_setting = [a for a in fconsts.DYNASTY_MODE_SETTINGS[which_onnx] if a[0] == name_mode] n_settings = len(select_dynasty_setting) assert n_settings == 1, f"found {n_settings} dynasty settings, expect 1" name_mode, k_onnx, n_mode = select_dynasty_setting[0] # k_onnx can be used to select specific onnx in kneron regression for debug i_mode = {} i_mode["name_mode"] = name_mode i_mode["n_mode"] = n_mode i_mode["dir_out"] = f"mode_{name_mode}" i_mode["platform"] = "piano_dynasty" i_mode["model_id"] = model_id # will use this in error report if fn_onnx and pathlib.Path(fn_onnx).exists(): i_mode["fn_onnx"] = fn_onnx else: assert type(onnx_map) is dict, "must specify at least one parameter 'fn_onnx' or 'onnx_map'" i_mode["fn_onnx"] = onnx_map[k_onnx] return i_mode def gen_dynasty_list(mode_list, input_list, info_in, p_output, dump_level=0, do_print=False, use_cuda=False, shape_in="onnx_shape", round_mode=1): """Create list of input `x` mode for this model to run. base_dump: Now we support more flexible calling of dynasty which may dump to different folder. Args: mode_list: list of dynasty mode to run, e.g., `["520", "520wq"]` input_list: list of input groups, if you want to run batch inference. info_in: list of input node names. To get correct order input from numpy input. p_output: regression will specify "model_name/output/results" by default. The results for input `xxx.txt` inference in `dynasty mode` are dumped in "`model_name/output/results`/xxx.txt/dynasty_mode_name" folder. Change this to "yyy" will cause dump to "`yyy`/xxx.txt/dynasty_mode_name". shape_in: choose `onnx_shape` (current default) / `channel_last` (obsolete). round_mode: Only effective for platform 540/730. - 0: round to inf. 520/530/630/720 will only support this. 540/730 may set to 0 in case of debug. - 1: (default for 540/730). round to even. Returns: tuple. - dynasty_list: list of dynasty run parameters, combination of all inputs and all mode configs. - dir_out_list: path to dynasty dump for each of `dynasty_list` items. """ # create a list with all input images. dynasty_list = [] dir_out_list = [] # this dir_out corresponding to xxx.txt, not including mode in it for i_mode, mode_config in enumerate(mode_list): for fn_input in input_list: fn_in_base = pathlib.Path(fn_input[0]).name this_out = f"""{p_output}/{fn_in_base}""" # HACK: save test_input.txt / test_input.npy to test_input this_out = str(pathlib.Path(this_out).with_suffix('')) if i_mode == 0: # only save for 0th mode dir_out_list.append(this_out) pathlib.Path(this_out).mkdir(mode=0o770, parents=True, exist_ok=True) i_para = { "model_id": mode_config["model_id"], "fn_onnx": mode_config["fn_onnx"], "n_mode": mode_config["n_mode"], "name_mode": mode_config["name_mode"], "dir_out": "{}/{}".format(this_out, mode_config["dir_out"]), "fn_input": fn_input, "node_input": info_in, "dump_level": dump_level, "print_command": do_print, "platform": mode_config["platform"], "cuda": use_cuda, # only False for now "round_mode": round_mode, "input_shape": shape_in, } dynasty_list.append(i_para) return dynasty_list, dir_out_list def build_dynasty_cmd_1(dynasty_para, bin_dynasty): """Create command for running dynasty.""" fn_input = dynasty_para["fn_input"] assert type(fn_input) is list node_input = dynasty_para["node_input"] dir_out = dynasty_para["dir_out"] + fconsts.DYNASTY_OUT_DIR_APPENDIX["piano"] pathlib.Path(dir_out).mkdir(parents=True, exist_ok=True) dump_level = dynasty_para["dump_level"] round_mode = dynasty_para["round_mode"] fn_onnx = dynasty_para["fn_onnx"] n_mode = dynasty_para["n_mode"] # name_mode = dynasty_para["name_mode"] # model_name = dynasty_para["model_name"] # cat_name = dynasty_para["cat_name"] model_id = dynasty_para["model_id"] base_name_input = pathlib.PurePath(fn_input[0]).name shape_convert = {"onnx_shape": 1, "channel_last": 0} shapeOrder = shape_convert.get(dynasty_para["input_shape"], 1) # -m path_to_model # -r path_to_radix_json # -d path_to_input_data # -n The name of the input node # -p platform: kl520, kl720, generic # -u dump level: 0: only dump final output, 2: dump all intermediate results # -s shapeOrder: 0: hwc, 1: chw # -o path to output data dump folder # -e if bie, not onnx. inputs = [] for fn_in, node_in in zip(fn_input, node_input): inputs.append("--data") inputs.append(f"\"{fn_in}\"") inputs.append("--name") inputs.append(f"\"{node_in}\"") # TODO: export LD_LIBRARY_PATH="{lib_dir}:${{LD_LIBRARY_PATH}}" for libonnxruntime parent path command = [ str(bin_dynasty), "--model", f"\"{fn_onnx}\"", "--radix", f"\"{fn_onnx}.json\"", "--platform", fconsts.NUM2MODE[n_mode], "--dump", str(dump_level), "--shapeOrder", str(shapeOrder), "--output", f"\"{dir_out}\"", "--roundMode", str(round_mode), ] + inputs if pathlib.Path(fn_onnx).name.endswith(".bie"): command.append("-e") if dynasty_para["cuda"]: command.append("--cuda") # TODO: remove model_id thing. return only command to make things easier return command, fconsts.NUM2PLATFORM[n_mode], model_id, base_name_input def build_dynasty_cmd(dynasty_list, dynasty_bin, fn_dynasty=None): """Generate commands of ALL dynasty inference.""" cmds = [" ".join(build_dynasty_cmd_1(d_para, dynasty_bin)[0]) for d_para in dynasty_list] if fn_dynasty: with open(fn_dynasty, 'w') as f: f.write("\n".join(cmds)) return cmds def guess_platform(d_cmd): """Guess platform from dynasty command line used onnx or bie.""" s = re.compile("\.kdp(\d\d\d)\..*?\.(?:onnx|bie)") try: platform = "kdp{}".format(s.findall(d_cmd)[0]) except: platform = "general" return platform def run_dynasty_so(lib, model, num_inputs, input_files, input_names, output_folder, ort=False): """Call Dynasty C inferencer. Arguments: lib: String path to the Dynasty shared library. model: String path to the ONNX model. num_inputs: Integer number of input nodes. input_files: List of pathlib paths to each of the input image text files. input_names: List of string names of the input nodes. output_folder: String path to directory where outputs will be stored. ort: Flag to use onnxruntime as inferencer for Dynasty float. """ dynasty_lib = ctypes.CDLL(lib) c_function = dynasty_lib.inference_wrapper c_function.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p * num_inputs, ctypes.c_char_p * num_inputs, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p] c_function.restype = None cuda = "True" # non cuda lib will not use this debug = "False" dump = 0 encrypt = "False" platform = "Float" radix_file = "" shape_order = "0" files_arr = (ctypes.c_char_p * num_inputs)() files_arr[:] = [str(input_file).encode() for input_file in input_files] names_arr = (ctypes.c_char_p * num_inputs)() names_arr[:] = [input_name.encode() for input_name in input_names] c_function(platform.encode(), encrypt.encode(), shape_order.encode(), dump, model.encode(), radix_file.encode(), num_inputs, files_arr, names_arr, output_folder.encode(), debug.encode(), cuda.encode(), str(ort).encode()) def run_dynasty_command_parallel(model_id, fn_cmds, n_thread=None, fn_err=None, timeout=60*60*2): """Use gnu parallel to run dynasty in parallel.""" if n_thread is None: n_str = "" else: n_str = f"--jobs {n_thread}" if fn_err is None: joblog = "" else: joblog = f"--joblog {fn_err}" command = f"parallel {n_str} --halt now,fail=1 {joblog} < {fn_cmds}" # TODO: if fn_cmds contains only 1 command, skip parallel cp = futils.run_bash_script(command, timeout=timeout) # some known errors if cp.returncode != 0: platform = guess_platform(cp.stderr) if cp.returncode == 20: raise RegressionError(f"{platform}/knerex", model_id, msg="dynasty: found wrong bw") else: try: with open(fn_err, "a") as f: f.write("\n\n") f.write("\n\n".join([cp.stderr, cp.stdout])) except Exception as e: pass if DEBUG: print(cp.stderr) raise RegressionError(f"{platform}/dynasty", model_id, msg=f"Exit code={cp.returncode}") elif fn_err is not None: # check fn_err. sometime parallel cannot grab the failed dynasty # e.g., if dynasty killed by segmental fault msg, e1, e2 = futils.check_parallel_log(fn_err) platform = guess_platform(cp.stderr) if len(msg) > 3: if DEBUG: print(f"dynasty parallel failed: {msg}") raise RegressionError(f"{platform}/dynasty", model_id, msg=msg) def np2txt(input_np, input_nodes, p_working, dump_prefix="knerex_input", ch_last=False): """Convert np input to text for knerex / dynasty when do inference from toolchain. NOTE: - the numpy array in input_np must be "onnx_shape". - move this function to flow_utils. """ # TODELETE raise FutureWarning("This function is obsolete. call flow_utils.npy2txt") assert set(input_nodes) == set(input_np.keys()), \ f"ERROR: input name does not match: onnx input ({input_nodes}) vs given np ({input_np.keys()})" # TODO: what if input_np is str or path is_dumped = [] p_dumped = [] pair_names = [] for i_dp, dp_in in enumerate(input_nodes): p_dump = pathlib.Path(p_working) / f"{dump_prefix}_{i_dp}" p_dumped.append(p_dump) is_dumped.append(p_dump.exists()) if all(is_dumped): # all text exists already. skip for now pp(f"{input_nodes} texts are dumped already. continue.") else: # save numpy to txt files # TODO: call common function to verify inputs # check input_np size, which is dict of list of np (onnx input shape) # check pairs of inputs. n_pairs = [len(v) for k, v in input_np.items()] assert len(set(n_pairs)) == 1, f"input group numbers not match: got len of {n_pairs}" # check input shape for i_dp, dp_in in enumerate(input_nodes): p_dump = p_dumped[i_dp] # pre-defined above p_dump.mkdir(parents=True, exist_ok=True) for i_group, np_1 in enumerate(input_np[dp_in]): fn_dump = p_dump / f"in_{i_group:06d}.txt" if i_dp == 0: pair_names.append(fn_dump.name) # NOTE: will not support channel last text dump # this is only used for dynasty float shared library if ch_last: input_shape = np_1.shape if len(input_shape) > 2: axes = range(len(input_shape)) axes = [axes[0], *axes[2:], axes[1]] np_1 = np.transpose(np_1, axes) np.savetxt(fn_dump, np_1.flat, fmt="%.15f") # do search to collect all text files input_list = list_multiple_input(p_dumped, pair_names, verify_exist=False) return p_dumped, input_list, pair_names def txt2np_so(out_nodes_shape, output_folder): """Convert the dynasty dumped results from the shared library to np. Args: out_nodes_shape: output_folder: basically `list(working_dir)` Returns: - np_out_fl: dict of list of numpy array, each array is onnx shape, in float format. """ collect_txt_fl = defaultdict(list) for dp_out, shape_out in out_nodes_shape.items(): fl_output = "layer_output_{}_fl.txt".format(futils.clean_name(dp_out)) p_fl = output_folder / fl_output assert p_fl.exists() collect_txt_fl[dp_out].append(futils.txt2np_fl(p_fl, shape_out)) return collect_txt_fl def txt2np(out_nodes_shape, output_list, dmode, load_fl=True, load_fx=False): """Convert the dynasty dumped results to np. Args: out_nodes_shape: output_list: basically `list(p_dump.glob(f"*.txt/mode_{dmode}_piano"))` dmode: which dynasty mode. Returns: tuple - np_out_fx: dict of list of numpy array, each array is onnx shape, in fix-point format. - np_out_fl: dict of list of numpy array, each array is onnx shape, in float format. """ collect_txt_fl = defaultdict(list) collect_txt_fx = defaultdict(list) for dp_out, shape_out in out_nodes_shape.items(): doname = futils.clean_name(dp_out) fl_output = f"layer_output_{doname}_fl.txt" fx_output = f"layer_output_{doname}_fx.txt" for p_dump in output_list: if load_fl: p_fl = pathlib.Path(p_dump) / f"mode_{dmode}_piano" / fl_output assert p_fl.exists() collect_txt_fl[dp_out].append(futils.txt2np_fl(p_fl, shape_out)) if load_fx: p_fx = pathlib.Path(p_dump) / f"mode_{dmode}_piano" / fx_output assert p_fx.exists() collect_txt_fx[dp_out].append(futils.txt2np_fx(p_fx, shape_out)) return collect_txt_fl, collect_txt_fx def np_fx2fl(np_in: dict, d_ch_dim: dict, d_scale: dict, d_radix: dict): """Convert fx data in numpy format to float.""" np_fl = {} for dp_name, dp_arr in np_in.items(): ch_dim = d_ch_dim[dp_name] scale_expand_dim = list(range(len(dp_arr[0].shape))) scale_expand_dim.remove(ch_dim) scale1 = np.array(d_scale[dp_name]) * 2.0**np.array(d_radix[dp_name]) scale = np.expand_dims(scale1, scale_expand_dim) these_fl = [] for i_dp, dp in enumerate(dp_arr): these_fl.append(dp / scale) np_fl[dp_name] = these_fl return np_fl