""" Utility functions to help perform CSIM inference. """ import ctypes import glob import re import subprocess import struct import sys from typing import List, Mapping, Optional, Tuple import numpy as np import numpy.typing as npt from sys_flow.inference import inference_csim_v2 as inference_csim_v2_v1 from sys_flow_v2.inference import inference_csim_v2 as inference_csim_v2_v2 from python_flow.common import constants from python_flow.common import directory_manager as dm from python_flow.common import exceptions from python_flow.utils import utils # CSIM def csim_inference(nef: str, inputs: Mapping[str, List[npt.ArrayLike]], reordering: Optional[List[str]], use_onnx_shape: bool, model_maps: Mapping[int, Tuple], platform: int = 0, model_id: Optional[int] = None, data_type: str = "float", out_dir: Optional[str] = None, oup_trans: Optional[List[int]] = [0, 2, 3, 1]) -> List[npt.ArrayLike]: """Performs inference on the specified NEF file. Arguments: nef: String path to the NEF model. inputs: Mapping of string node names to lists of input NumPy arrays. reordering: List of string node names specifying the output order. If not provided or empty, use the output keys as order. use_onnx_shape: Flag indicating if outputs should be returned in ONNX shape order. model_maps: Mapping from int model number to a tuple of data. This should come from the unpack_nefs API call. platform: Integer platform for NEF model. model_id: Integer ID of model to perform inference on. Only needed for combined NEF. data_type: String format of the resulting output, "fixed" or "float". out_dir: String path to where the output dumps will be saved. oup_trans: List of integers indicating the axes order to transpose the outputs. Returns: A list of NumPy arrays of either floats or integers, depending on 'data_type'. If data_type is "float", results will be float NumPy arrays. If data_type is "fixed", results will be fixed NumPy arrays. """ if platform in constants.PLATFORMS_MO3: inference_csim_v2 = inference_csim_v2_v1 else: inference_csim_v2 = inference_csim_v2_v2 if model_id is None: model_id = list(model_maps.keys())[0] # single model id p_nef, ioinfo = model_maps[model_id] if data_type == "float": out_dict = inference_csim_v2( p_nef, ioinfo, inputs, platform, out_fmt="fl", p_working=out_dir) else: out_dict = inference_csim_v2( p_nef, ioinfo, inputs, platform, out_fmt="fx", p_working=out_dir) # 520 output names will just be integers if platform == 520: reordering = [str(index) for index in range(len(out_dict))] elif reordering is None or not len(reordering): # if no reordering list, just use provided output dictionary keys reordering = out_dict.keys() output_data = [] for output_name in reordering: output_data.append(out_dict[output_name][0]) if oup_trans is not None: # use axes to transpose if provided output_data = [data.transpose(oup_trans) if np.ndim(data) == len(oup_trans) else data for data in output_data] elif not use_onnx_shape: output_data = [utils.convert_first_to_last(data) for data in output_data] return output_data ## Rest of these functions are unused after 0.21.0 API change. # Preparation def get_node_type(setup: str, data_size: int = 4) -> Optional[int]: """Get the node type at the current position. Only used for 720 setup binaries. Arguments: setup: String path to the model setup binary. data_size: Integer indicating the number of bytes of a single data point. Returns: An integer read from the next data_size bytes in the setup binary. Will return None if the data is empty. """ data = setup.read(data_size) if b'' == data: return None return int.from_bytes(data, byteorder="little", signed=False) def get_input_dims(setup_file: str, index: int, platform: int, platform_version: int) -> Tuple[int]: """Parses the input dimensions from the setup binary. The returned tuple is in format (row, column, channel). Argument: setup_file: String path to the model setup binary. index: Integer of the input node to get the radix for. platform: Integer CSIM version the setup binary. platform_version: Integer version of the specific CSIM platform. Returns: A tuple of 3 integers, (row, column, channel), indicating the dimensions of the specified input node. Default value will be empty tuple. """ try: with open(setup_file, "rb") as setup: if platform == 520: # 520 input shape is in header setup.read(7 * 4) row = int.from_bytes(setup.read(4), byteorder="little", signed=False) col = int.from_bytes(setup.read(4), byteorder="little", signed=False) ch = int.from_bytes(setup.read(4), byteorder="little", signed=False) return (row, col, ch) elif platform in constants.FLATBUFFERS or (platform == 720 and platform_version == 1): # 720 version 1 uses flatbuffers data = setup.read() inf_data = INFContent.INFContent.GetRootAsINFContent(bytearray(data), 0) inode = inf_data.Inputs(index) return (inode.Shape(2), inode.Shape(3), inode.Shape(1)) # row, col, ch elif platform == 720 and platform_version == 0: count = 0 setup.read(15 * 4) # cnn header is 15 values for 720 node_type = get_node_type(setup) while node_type is not None: if node_type == 5: setup.read(8) # index, format if count == index: row = int.from_bytes(setup.read(4), byteorder="little", signed=False) col = int.from_bytes(setup.read(4), byteorder="little", signed=False) ch = int.from_bytes(setup.read(4), byteorder="little", signed=False) return (row, col, ch) setup.read(6 * 4) # rest of network info count += 1 else: setup.read(constants.NODE_MAP720[node_type] * 4) node_type = get_node_type(setup) except FileNotFoundError: print(f"Could not find setup file, '{setup_file}'. Using default value of ().") return () # default value def get_input_format(setup_file: str, index: int, platform: int, platform_version: int) -> str: """Parses the input format from the setup binary. Only needed for non 520 platforms at the moment. The return string will be used as a parameter to the data converter to generate the input RGBA. Argument: setup_file: String path to the model setup binary. index: Integer of the input node to get the radix for. platform: Integer CSIM version the setup binary. platform_version: Integer version of the specific CSIM platform. Returns: A string indicating how the specific input node data will be formatted in the RGBA. Default format will be set to 4W4C8B. """ try: with open(setup_file, "rb") as setup: if platform in constants.FLATBUFFERS or (platform == 720 and platform_version == 1): data = setup.read() inf_data = INFContent.INFContent.GetRootAsINFContent(bytearray(data), 0) inode = inf_data.Inputs(index) format_num = inode.Format() if platform == 720: return constants.FORMAT_MAP720[format_num] else: return constants.FORMAT_MAP530[format_num] elif platform == 720 and platform_version == 0: count = 0 setup.read(15 * 4) # cnn header is 15 values for 720 node_type = get_node_type(setup) while node_type is not None: if node_type == 5: setup.read(4) # index if count == index: format_num = int.from_bytes( setup.read(4), byteorder="little", signed=False) return constants.FORMAT_MAP720[format_num] setup.read(7 * 4) # rest of network info count += 1 else: setup.read(constants.NODE_MAP720[node_type] * 4) node_type = get_node_type(setup) except FileNotFoundError: print(f"Could not find setup file, '{setup_file}'. Using default value of '4W4C8B'.") return "4W4C8B" # default value def get_radix(setup_file: str, index: int, platform: int, platform_version: int) -> int: """Parses the input radix from the setup binary. Argument: setup_file: String path to the model setup binary. index: Integer of the input node to get the radix for. platform: Integer CSIM version the setup binary. platform_version: Integer version of the specific CSIM platform. Returns: An integer indicating the radix of the specific input node used to convert between float and fixed data. Default radix will be set to 8. """ try: with open(setup_file, "rb") as setup: if platform == 520: # 520 input radix is in header setup.read(16 * 4) return int.from_bytes(setup.read(4), byteorder="little", signed=True) elif platform in constants.FLATBUFFERS or (platform == 720 and platform_version == 1): # 720 version 1 uses flatbuffers data = setup.read() inf_data = INFContent.INFContent.GetRootAsINFContent(bytearray(data), 0) inode = inf_data.Inputs(index) return inode.Quantization().FxpInfo(0).Radix() elif platform == 720 and platform_version == 0: count = 0 setup.read(15 * 4) # cnn header is 15 values for 720 node_type = get_node_type(setup) while node_type is not None: if node_type == 5: setup.read(7 * 4) # last value if count == index: return int.from_bytes(setup.read(4), byteorder="little", signed=True) setup.read(4) # rest of network info count += 1 else: setup.read(constants.NODE_MAP720[node_type] * 4) node_type = get_node_type(setup) except FileNotFoundError: print(f"Could not find setup file, '{setup_file}'. Using default value of 8.") return 8 def prep_rgba(pre_results: List[npt.ArrayLike], inputs: List[str], setup: str, platform: int, platform_version: int, use_dongle: bool = False, radix: int = 8, input_format: int = 0) -> List[str]: """Generates the input RGBAs used for CSIM inference. Inputs generated may be a litte different from the input paths provided in the case the same model inference is run multiple times for the same image. If using dongle, dimension checking and radix extraction will be skipped, and the provided radix will be used. Arguments: pre_results: List of preprocessed NumPy arrays in channel last format. inputs: List of string paths to each of the input image RGBA binaries. setup: String path to the model setup binary. platform: Integer CSIM version to use. platform_version: Integer version of the specific CSIM platform that was used. use_dongle: Flag indicating if RGBAs will be for Dongle use. radix: Integer radix used for all of the nodes. input_format: Integer indicating shape format of preprocessed data. 0: channel_last 1: ONNX shape Returns: A list of string paths to the newly generated RGBA input binaries. """ new_inputs = [] converted_data = [] # convert to channel last if input_format == 1: # ONNX shape for pre_data in pre_results: new_data = utils.convert_first_to_last(pre_data) converted_data.append(new_data) else: converted_data = pre_results for index, result in enumerate(converted_data): if not use_dongle: # verify input shapes are the same input_shape = result.shape if len(input_shape) == 4: # cut off batch input_shape = input_shape[1:] model_dims = get_input_dims(setup, index, platform, platform_version) # "golden" dims if input_shape != model_dims: if input_format == 1: dims_last = (model_dims[-1], *model_dims[0:-1]) sys.exit("Input dimensions do not match. Please make sure your input data " f"is of shape {dims_last}") else: sys.exit("Input dimensions do not match. Please make sure your input data " f"is of shape {model_dims}") radix = get_radix(setup, index, platform, platform_version) file_name = utils.get_new_dump_name(inputs[index]) utils.convert_pre_numpy_to_rgba(result, file_name, radix, platform, setup, index, platform_version) new_inputs.append(file_name) return new_inputs # Inference. def create_ini(base_ini: str, out_ini: str, command: str, weight: str, setup: str, inputs: List[str], register: str) -> None: """Creates INI file used as CSIM input. Assumes all files except 'out_ini' already exist. Arguments: base_ini: String path to the template INI file. out_ini: String path to the newly generated INI file. command: String path to the model command binary. weight: String path to the model weight binary. setup: String path to the model setup binary. inputs: List of string paths to each of the input image RGBA binaries. register: String path to the CSIM file register. Should be related to the CSIM version used. In most cases, the default register will be valid. """ file_inputs = ",".join(inputs) updated_lines = { "file_command": "".join(["file_command = ", command, "\n"]), "file_weight": "".join(["file_weight = ", weight, "\n"]), "file_setup": "".join(["file_setup = ", setup, "\n"]), "file_input": "".join(["file_input = ", file_inputs, "\n"]), "file_register": "".join(["file_register = ", register, "\n"]) } with open(base_ini, "r") as in_file, open(out_ini, "w") as out_file: for line in in_file.readlines(): ini_input = line.split(" ", 1)[0] if ini_input in updated_lines: out_file.write(updated_lines[ini_input]) else: out_file.write(line) def run_csim(output_folder: str, command: str, weight: str, setup: str, platform: int, inputs: List[str], ini: str = "", dump: int = 0, toolchain: str = "", platform_version: int = 0, from_toolchain: bool = False) -> None: """Performs CSIM inference. For specific toolchain versions and platform versions, the CSIM executable and register should be modified prior to calling this function. The ini argument will only be necessary for non-520 CSIM versions. The dump value will only be used for 520 CSIM versions. The toolchain parameter will only be used for error messages. Arguments: output_folder: String path to directory where outputs will be stored. command: String path to the model command binary. weight: String path to the model weight binary. setup: String path to the model setup binary. platform: Integer CSIM version to use. inputs: List of string paths to each of the input image RGBA binaries. ini: String path to the newly generated INI file. dump: Integer indicating if all intermediate node values should be dumped. toolchain: String of the toolchain version that was specified. platform_version: Integer version of the specific CSIM platform that was used. Only used if from_toolchain is set to True. from_toolchain: Flag indicating if function call comes from kneron_inference function. """ bin_map = { 0: { 530: constants.CSIM530, 540: constants.CSIM540, 630: constants.CSIM630, 720: constants.CSIM720, 730: constants.CSIM730, }, 1: { 720: constants.CSIM720_1 } } apb_map = { 0: { 530: constants.APB530, 540: constants.APB540, 630: constants.APB630, 720: constants.APB720, 730: constants.APB730, }, 1: { 720: constants.APB720_1, } } with dm.DirectoryManager(output_folder): try: if platform == 520: subprocess.run([constants.CSIM520, "-d", str(dump), command, weight, *inputs, setup, "--thread", "1"], check=True) elif from_toolchain: # used for kneron_inference, dont really need to worry about platform version toolchain_map = { "apb": { 530: constants.APB530, 540: constants.APB540, 630: constants.APB630, 720: constants.APB720_1, 730: constants.APB730, }, "bin": { 530: constants.CSIM530, 540: constants.CSIM540, 630: constants.CSIM630, 720: constants.CSIM720_1, 730: constants.CSIM730, } } create_ini(constants.TEMPLATE_INI, ini, command, weight, setup, inputs, toolchain_map["apb"][platform]) subprocess.run([toolchain_map["bin"][platform], ini], check=True) else: create_ini(constants.TEMPLATE_INI, ini, command, weight, setup, inputs, apb_map[platform_version][platform]) subprocess.run([bin_map[platform_version][platform], ini], check=True) except subprocess.CalledProcessError as error: sys.exit(f"Hardware CSIM {platform} failed. Please verify that the model is " f"compatible with the {toolchain} toolchain.\n\n{error}") # Results. def convert_results(results: List[npt.ArrayLike], radices: List[int], scales: List[float], conversion: str) -> List[npt.ArrayLike]: """Converts the results using the provided radices, scales, and conversion. "fx2fl" indicates a conversion from floating point data to fixed point data. "fl2fx" indicated a conversion from fixed point data to floating point data. Arguments: results: List of NumPy arrays to convert. radices: List of integer radices. scales: List of float multipliers. conversion: String indicating conversion type, "fx2fl" or "fl2fx". Returns: A list of NumPy arrays of either floats or integers, depending on 'conversion'. If conversion is 'fx2fl', results will be float arrays. If conversion is 'fl2fx', results will be fixed arrays. """ converted = [] for result, radix, scale in zip(results, radices, scales): if radix < 0: value = 1.0 / (1 << -radix) else: value = 1 << radix if conversion == "fx2fl": data = result.flatten() / float(scale) / value for i, val in enumerate(data.flatten()): # could not find np function to set precision data[i] = format(ctypes.c_float(val).value, ".8g") # to match C version of Dynasty data = data.reshape(result.shape) else: data = result * scale * value data = np.round(data).astype(np.int8) converted.append(data) return converted def convert_csim_data_type(results: List[npt.ArrayLike], output_folder: str, platform: int, conversion: str, platform_version: int) -> List[npt.ArrayLike]: """Converts the data type of the CSIM results. Assumes the input results are in the same order as in the output folder. The platform_version parameter currently is only necessary with the new 720 flatbuffers version. Arguments: results: List of NumPy arrays to convert. output_folder: String path to directory where outputs were stored. platform: Integer CSIM version that was used. conversion: String indicating conversion type, "fx2fl" or "fl2fx". platform_version: Integer version of the specific CSIM platform that was used. Returns: A list of NumPy arrays of either floats or integers, depending on 'conversion'. If conversion is 'fx2fl', results will be float arrays. If conversion is 'fl2fx', results will be fixed arrays. """ # get radix/scale information from output folder if platform == 520: with open("".join([output_folder, "/radix_scale_info.txt"])) as params: values = [line.strip().split(" ") for line in params.readlines()] radices = [struct.unpack("i", struct.pack("I", int(radix)))[0] for _, radix, _ in values] # convert "int" scale value into float value using its bytes scales = [struct.unpack("f", struct.pack("I", int(scale)))[0] for _, _, scale in values] else: radices = [] scales = [] info = glob.glob(glob.escape(output_folder) + "/dma2seq*.info") # sort by the integer in the file name, so 10 comes after 9, split on '_' and '.' info_files = sorted(info, key=lambda x: int(re.split("_|\.", x)[-2])) for info_file in info_files: with open(info_file) as info_file_p: # channel/row/column/radix/scale info = [int(value) for value in info_file_p.read().splitlines()] if platform == 720 and platform_version == 1: radices.append(info[3]) else: if len(info) == 6: # newer version, added batch to element 0 if info[4] < 0: # given as negative value radices.append(info[4]) else: radices.append(struct.unpack("i", struct.pack("I", info[4]))[0]) else: if info[3] < 0: # given as negative value radices.append(info[3]) else: radices.append(struct.unpack("i", struct.pack("I", info[3]))[0]) if len(info) == 6: scales.append(struct.unpack("f", struct.pack("I", info[5]))[0]) else: scales.append(struct.unpack("f", struct.pack("I", info[4]))[0]) return convert_results(results, radices, scales, conversion) def csim_520_to_np(output_folder: str, data_type: str, reordering: List[str], ioinfo: str, first: bool) -> List[npt.ArrayLike]: """Converts 520 CSIM output into a list of NumPy arrays. The resulting NumPy arrays are values in channel first format if first is true (bchw). Otherwise, it will be in channel last format (bhwc). The type of the resulting arrays depends on the given data_type. The ordering will depend on the reordering parameter. Normally, this will be specified based off of the order needed for postprocessing. Arguments: output_folder: String path to directory where outputs were stored. data_type: String specifying type of data to load, "float" or "fixed". reordering: List of strings correponding to the return node order. ioinfo: String path to file that maps output node name to node number. first: Flag indicating if results will be returned in channel first format. Returns: A list of NumPy arrays of either floats or integers, depending on 'data_type'. If data_type is "float", results will be float arrays. If data_type is "fixed", results will be fixed arrays. """ np_output = [] matrices = sorted(glob.glob(glob.escape(output_folder) + "/node*matrix.txt")) data_files = sorted(glob.glob(glob.escape(output_folder) + "/node*output.txt")) for matrix, data_file in zip(matrices, data_files): with open(matrix) as dim_file: for line in dim_file: if line.startswith("="): channel = int("".join(filter(str.isdigit, line))) elif line.startswith("Rectangle"): # 520 CSIM will always have two values for this height, width = [int("".join(filter(str.isdigit, x))) for x in line.split("x")] data = np.loadtxt(data_file, dtype=np.int8) data = np.reshape(data, (1, channel, height, width)) if not first: data = np.transpose(data, (0, 2, 3, 1)) np_output.append(data) if data_type == "float": np_output = convert_csim_data_type(np_output, output_folder, 520, "fx2fl", 0) return utils.reorder_outputs(np_output, reordering, ioinfo) def csim_to_np(output_folder: str, platform: int, data_type: str, reordering: List[str], ioinfo: str, first: bool, platform_version: int) -> List[npt.ArrayLike]: """Converts non-520 CSIM output into a list of NumPy arrays. The resulting NumPy arrays are values in channel first format if first is true (bchw). Otherwise, it will be in channel last format (bhwc). The type of the resulting arrays depends on the given data_type. The ordering will depend on the reordering parameter. Normally, this will be specified based off of the order needed for postprocessing. The platform_version parameter currently is only necessary with the new 720 flatbuffers version. Arguments: output_folder: String path to directory where outputs were stored. platform: Integer CSIM version that was used. data_type: String specifying type of data to load, "float" or "fixed". reordering: List of strings correponding to the return node order. ioinfo: String path to file that maps output node name to node number. first: Flag indicating if results will be returned in channel first format. platform_version: Integer version of the specific CSIM platform that was used. Returns: A list of NumPy arrays of either floats or integers, depending on 'data_type'. If data_type is "float", results will be float arrays. If data_type is "fixed", results will be fixed arrays. """ np_output = [] info = glob.glob(glob.escape(output_folder) + "/dma2seq*.info") seq = glob.glob(glob.escape(output_folder) + "/dma2seq*.seq") # sort by the integer in the file name, so 10 comes after 9, split on '_' and '.' info_files = sorted(info, key=lambda x: int(re.split("_|\.", x)[-2])) data_files = sorted(seq, key=lambda x: int(re.split("_|\.", x)[-2])) for data_file, info_file in zip(data_files, info_files): with open(info_file) as info_file_p: # channel/row/column/radix/scale info = [int(value) for value in info_file_p.read().splitlines()] if len(info) == 6: # newer version, added batch to element 0 channel, height, width = info[1:4] else: channel, height, width = info[:3] data = np.loadtxt(data_file, dtype=np.int8) data = np.reshape(data, (1, channel, height, width)) if not first: data = np.transpose(data, (0, 2, 3, 1)) np_output.append(data) if data_type == "float": np_output = convert_csim_data_type(np_output, output_folder, platform, "fx2fl", platform_version) return utils.reorder_outputs(np_output, reordering, ioinfo) def get_csim_outputs(platform: int, output_folder: str, data_type: str, reordering: List[str], ioinfo: str, first: bool, platform_version: int = 0) -> List[npt.ArrayLike]: """Gets the CSIM inference results as a list of NumPy arrays. The platform_version parameter currently is only necessary with the new 720 flatbuffers version. Arguments: platform: Integer CSIM version that was used. output_folder: String path to directory where outputs were stored. data_type: String specifying type of data to load, "float" or "fixed". reordering: List of strings correponding to the return node order. ioinfo: String path to file that maps output node name to node number. first: Flag indicating if results will be returned in channel first format. platform_version: Integer version of the specific CSIM platform that was used. Returns: A list of NumPy arrays with length n, where n is the number of outputs in the model. Each array will follow channel first format (bchw) unless the first flag is disabled; each array will then follow channel last format (bhwc). """ try: if platform == 520: return csim_520_to_np(output_folder, data_type, reordering, ioinfo, first) return csim_to_np(output_folder, platform, data_type, reordering, ioinfo, first, platform_version) except exceptions.ConfigError as error: sys.exit(error)