382 lines
17 KiB
Python
382 lines
17 KiB
Python
"""
|
|
Utility functions to help perform Dynasty inference.
|
|
"""
|
|
import csv
|
|
import ctypes
|
|
import glob
|
|
import json
|
|
import re
|
|
from typing import Callable, List, Mapping, Optional, Union
|
|
|
|
import numpy as np
|
|
import numpy.typing as npt
|
|
from packaging import version
|
|
from sys_flow.inference import inference_dynasty_fx
|
|
from sys_flow_v2.inference import inference_dynasty_fx as inference_dynasty_fx_v2
|
|
from sys_flow_v2.inference import inference_dynasty_fl_so
|
|
|
|
from python_flow.common import constants
|
|
from python_flow.common import exceptions
|
|
from python_flow.utils import utils
|
|
|
|
RADIX_DUMP = "radix.json"
|
|
|
|
# Dynasty
|
|
def dynasty_inference(model_file: str, inputs: Mapping[str, List[npt.ArrayLike]],
|
|
reordering: Optional[List[str]], use_onnx_shape: bool,
|
|
input_nodes: List[str], out_node_shape: Mapping[str, List[int]],
|
|
is_fixed: bool = False, platform: int = 0,
|
|
data_type: str = "float", out_dir: Optional[str] = None,
|
|
use_cuda: bool = False, use_ort: bool = False,
|
|
oup_trans: Optional[List[int]] = [0, 2, 3, 1],
|
|
d_ioinfo: Optional[Mapping] = None) -> List[npt.ArrayLike]:
|
|
"""Performs inference on the specified ONNX/BIE file.
|
|
|
|
Arguments:
|
|
model_file: String path to the ONNX or BIE model.
|
|
inputs: Mapping of string node names to lists of input NumPy arrays.
|
|
reordering: List of string node names specifying the output order. If not provided or
|
|
empty, use the output keys as order.
|
|
use_onnx_shape: Flag indicating if outputs should be returned in ONNX shape order.
|
|
input_nodes: List of string node names for the inputs.
|
|
out_node_shape: Mapping of string output node names to a list of integers indicating
|
|
the node shapes.
|
|
is_fixed: Flag indicating Dynasty fixed inference.
|
|
platform: Integer platform for Dynasty fixed inference. Unused for float inference.
|
|
data_type: String format of the resulting output, "fixed" or "float".
|
|
out_dir: String path to where the output dumps will be saved.
|
|
use_cuda: Flag indicating if CUDA Dynasty library should be used.
|
|
use_ort: Flag indicating if MSFT inferencer should be used.
|
|
oup_trans: List of integers indicating the axes order to transpose the outputs.
|
|
d_ioinfo: Mapping needed to bit-match fixed inference results with CSIM inference results.
|
|
|
|
Returns:
|
|
A list of NumPy arrays of either floats or integers, depending on 'data_type'. If
|
|
data_type is "float", results will be float NumPy arrays. If data_type is "fixed",
|
|
results will be fixed NumPy arrays.
|
|
"""
|
|
if is_fixed:
|
|
if platform in constants.PLATFORMS_MO3:
|
|
float_dict, fixed_dict = inference_dynasty_fx(
|
|
model_file, platform, inputs, p_working=out_dir,
|
|
input_nodes=input_nodes, out_node_shape=out_node_shape,
|
|
d_ioinfo=d_ioinfo)
|
|
else:
|
|
float_dict, fixed_dict = inference_dynasty_fx_v2(
|
|
model_file, platform, inputs, p_working=out_dir,
|
|
input_nodes=input_nodes, out_node_shape=out_node_shape,
|
|
d_ioinfo=d_ioinfo)
|
|
|
|
if data_type == "float":
|
|
out_dict = float_dict
|
|
else:
|
|
out_dict = fixed_dict
|
|
else:
|
|
if use_ort:
|
|
mode = "ort"
|
|
else:
|
|
mode = "dynasty"
|
|
if use_cuda:
|
|
device = f"{mode}_gpu"
|
|
else:
|
|
device = f"{mode}_cpu"
|
|
out_dict = inference_dynasty_fl_so(
|
|
model_file, inputs, p_working=out_dir, device=device,
|
|
input_nodes=input_nodes, out_node_shape=out_node_shape,
|
|
shape_in="channel_last")
|
|
|
|
# if no reordering list, just use provided output dictionary keys
|
|
if reordering is None or not len(reordering):
|
|
reordering = out_dict.keys()
|
|
|
|
output_data = []
|
|
for output_name in reordering:
|
|
output_data.append(out_dict[output_name][0])
|
|
if oup_trans is not None: # use axes to transpose if provided
|
|
output_data = [data.transpose(oup_trans) if np.ndim(data) == len(oup_trans) else data for data in output_data]
|
|
elif not use_onnx_shape:
|
|
output_data = [utils.convert_first_to_last(data) for data in output_data]
|
|
return output_data
|
|
|
|
## Rest of these functions are unused after 0.21.0 API change.
|
|
|
|
# Preparation.
|
|
def check_input_dims(platform: int, model: str, dims: List[List[int]], is_bie: bool,
|
|
input_names: List[str]) -> None:
|
|
"""Check that the inputs are in channel last format to match the model dimensions.
|
|
|
|
Original dimensions should be in channel last format (hwc), but the dimensions passed in
|
|
should be channel first format (chw).
|
|
|
|
Arguments:
|
|
platform: Integer platform version to use.
|
|
model: String path to the ONNX or BIE model.
|
|
dims: List of list of integers of the expected dimensions of each input node (chw).
|
|
is_bie: Flag indicating if model is a BIE model.
|
|
input_names: List of strings of the input node names, used for debug message.
|
|
"""
|
|
dynasty_lib = ctypes.CDLL(constants.DYNASTY_LIB)
|
|
c_function = dynasty_lib.check_input_dims
|
|
c_function.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p,
|
|
ctypes.POINTER(ctypes.c_int)]
|
|
c_function.restype = ctypes.c_int
|
|
|
|
# dims should be passed in as chw format to match the C ordering
|
|
c_dims = (ctypes.c_int * len(dims))(*dims)
|
|
|
|
# C function will return 1 over actual index, 0 for success
|
|
if is_bie:
|
|
index = c_function(str(platform).encode(), "True".encode(), model.encode(),
|
|
"".encode(), c_dims) - 1
|
|
else:
|
|
index = c_function("Float".encode(), "False".encode(), model.encode(),
|
|
"".encode(), c_dims) - 1
|
|
|
|
if index != -1:
|
|
raise exceptions.InvalidInputError(
|
|
f"Input dimensions for '{input_names[index]}' (index: {index}) do not match. "
|
|
"Please make sure your data is the correct shape.")
|
|
|
|
def dump_radix_json(platform: int, model: str, out_json_folder: str) -> None:
|
|
"""Dump radix JSON file for a BIE file.
|
|
|
|
Arguments:
|
|
platform: Integer platform version to use.
|
|
model: String path to the BIE model.
|
|
out_json_folder: String path to the folder to dump the JSON file.
|
|
"""
|
|
dynasty_lib = ctypes.CDLL(constants.DYNASTY_LIB)
|
|
c_function = dynasty_lib.dump_radix_json
|
|
c_function.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_char_p]
|
|
c_function.restype = ctypes.c_int
|
|
c_function(platform, model.encode(), out_json_folder.encode())
|
|
|
|
def get_bie_radices(platform: int, model: str, out_json_folder: str,
|
|
input_names: List[str], input_shape: int = 0) -> List[npt.ArrayLike]:
|
|
"""Gets the radices for each input node from the BIE model.
|
|
|
|
Arguments:
|
|
platform: Integer platform version to use.
|
|
model: String path to the BIE model.
|
|
out_json_folder: String path to the folder to dump the JSON file.
|
|
input_names: List of input node names to model.
|
|
input_shape: Integer indicating shape format of preprocessed data.
|
|
0: channel_last
|
|
1: ONNX shape
|
|
|
|
Returns:
|
|
A list of NumPy arrays, each of size 3, that holds the radices for each dimension of
|
|
each input node. Each NumPy array of radices will also be in channel last format to
|
|
match the input to the model.
|
|
"""
|
|
dump_radix_json(platform, model, out_json_folder)
|
|
radix_json = "/".join([out_json_folder, RADIX_DUMP])
|
|
with open(radix_json) as radix_file:
|
|
radix_data = json.load(radix_file)
|
|
|
|
radices = []
|
|
for name in input_names:
|
|
radix = radix_data.get(name, {}).get("output_datapath_radix", [8, 8, 8])
|
|
if input_shape == 1:
|
|
radices.append(np.array(radix))
|
|
else:
|
|
radices.append(np.array([*radix[1:], radix[0]])) # set to channel last to match input
|
|
|
|
return radices
|
|
|
|
def prep_txt(pre_results: List[npt.ArrayLike], inputs: List[str],
|
|
radix: Union[int, List[int]] = 8, dump_rgba: bool = False,
|
|
platform: int = 520, input_shape: int = 0) -> List[str]:
|
|
"""Generates the input text files used for Dynasty inference.
|
|
|
|
Radix and platform will only be needed if dump_rgba is set to true.
|
|
|
|
Arguments:
|
|
pre_results: List of preprocessed NumPy arrays in channel last format.
|
|
inputs: List of string paths to each of the input image text files.
|
|
radix: Integer radix of the input node, or a list of integer radices for each input node.
|
|
dump_rgba: Flag indicating to dump RGBA file as well.
|
|
platform: Integer platform version to use.
|
|
input_shape: Integer indicating shape format of preprocessed data.
|
|
0: channel_last
|
|
1: ONNX shape
|
|
|
|
Returns:
|
|
A list of string paths to the newly generated input text files.
|
|
"""
|
|
new_inputs = []
|
|
converted_data = []
|
|
# convert to channel last
|
|
if input_shape == 1: # ONNX shape
|
|
for pre_data in pre_results:
|
|
new_data = utils.convert_first_to_last(pre_data)
|
|
converted_data.append(new_data)
|
|
else:
|
|
converted_data = pre_results
|
|
|
|
for index, result in enumerate(converted_data):
|
|
file_name = utils.get_new_dump_name(inputs[index])
|
|
utils.convert_pre_numpy_to_txt(result, file_name)
|
|
new_inputs.append(file_name)
|
|
|
|
if dump_rgba:
|
|
rgba = file_name + ".bin"
|
|
if isinstance(radix, int):
|
|
utils.convert_pre_numpy_to_rgba(result, rgba, radix, platform)
|
|
else:
|
|
utils.convert_pre_numpy_to_rgba(result, rgba, radix[index], platform)
|
|
return new_inputs
|
|
|
|
# Inference.
|
|
def get_dynasty_function(cuda: str, tc_version: str) -> Callable:
|
|
"""Get the Dynasty inference function from the appropriate library.
|
|
|
|
Arguments:
|
|
cuda: String to use the CUDA version, "True" or "False". Will only work if CUDA is
|
|
installed on the device.
|
|
tc_version: String to specify which Dynasty library to use.
|
|
|
|
Returns:
|
|
A callable function for Dynasty inference from the appropriate library.
|
|
"""
|
|
if cuda == "True":
|
|
if tc_version != "current":
|
|
dynasty_lib = ctypes.CDLL(
|
|
constants.DYNASTY_CUDA_LIB_SPECIFIC.replace("{version}", tc_version))
|
|
else:
|
|
dynasty_lib = ctypes.CDLL(constants.DYNASTY_CUDA_LIB)
|
|
else:
|
|
if tc_version != "current":
|
|
dynasty_lib = ctypes.CDLL(
|
|
constants.DYNASTY_LIB_SPECIFIC.replace("{version}", tc_version))
|
|
else:
|
|
dynasty_lib = ctypes.CDLL(constants.DYNASTY_LIB)
|
|
# account for function name change in 15.5
|
|
if (tc_version != "current" and tc_version != "solution_regression" and
|
|
version.parse(tc_version) <= version.parse("0.15.5")):
|
|
return dynasty_lib.inference
|
|
else:
|
|
return dynasty_lib.inference_wrapper
|
|
|
|
def run_dynasty(platform: str, encrypt: str, dump: int, model: str,
|
|
radix_file: str, num_inputs: int, input_files: List[str],
|
|
input_names: List[str], output_folder: str, debug: str,
|
|
cuda: str, tc_version: str, ort: bool = False) -> None:
|
|
"""Wrapper to call Dynasty C inferencer.
|
|
|
|
Arguments:
|
|
platform: String version to use, "Float" for float or "number" for specific fixed
|
|
version, e.g. "520".
|
|
encrypt: String to use BIE input or not, "True" for BIE models and "False" otherwise.
|
|
dump: Integer to dump the intermediate nodes, 2 to dump or 0 to only dump final nodes.
|
|
model: String path to the ONNX or BIE model.
|
|
radix_file: String path to the JSON file when running fixed mode with an ONNX file. Only
|
|
used for debugging.
|
|
num_inputs: Integer number of input nodes.
|
|
input_files: List of string paths to each of the input image text files.
|
|
input_names: List of string names of the input nodes.
|
|
output_folder: String path to directory where outputs will be stored.
|
|
debug: String to enable some debug messages, "True" or "False"
|
|
cuda: String to use the CUDA version, "True" or "False". Will only work if CUDA is
|
|
installed on the device.
|
|
tc_version: String to specify which Dynasty library to use.
|
|
ort: Flag to use onnxruntime as inferencer for Dynasty float.
|
|
"""
|
|
c_function = get_dynasty_function(cuda, tc_version)
|
|
c_function.restype = None
|
|
|
|
shape_order = "0"
|
|
# change from channel last to ONNX shape after 0.21.0
|
|
if (tc_version == "current" or version.parse(tc_version) > version.parse("0.21.0")):
|
|
shape_order = "1"
|
|
|
|
files_arr = (ctypes.c_char_p * num_inputs)()
|
|
files_arr[:] = [input_file.encode() for input_file in input_files]
|
|
names_arr = (ctypes.c_char_p * num_inputs)()
|
|
names_arr[:] = [input_name.encode() for input_name in input_names]
|
|
|
|
# use onnxruntime flag added after 0.20.0
|
|
if (tc_version == "current" or version.parse(tc_version) > version.parse("0.20.0")):
|
|
c_function.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_int,
|
|
ctypes.c_char_p, ctypes.c_char_p, ctypes.c_int,
|
|
ctypes.c_char_p * num_inputs, ctypes.c_char_p * num_inputs,
|
|
ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p]
|
|
c_function(
|
|
platform.encode(), encrypt.encode(), shape_order.encode(), dump, model.encode(),
|
|
radix_file.encode(), num_inputs, files_arr, names_arr, output_folder.encode(),
|
|
debug.encode(), cuda.encode(), str(ort).encode())
|
|
else:
|
|
c_function.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_int,
|
|
ctypes.c_char_p, ctypes.c_char_p, ctypes.c_int,
|
|
ctypes.c_char_p * num_inputs, ctypes.c_char_p * num_inputs,
|
|
ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p]
|
|
c_function(
|
|
platform.encode(), encrypt.encode(), shape_order.encode(), dump, model.encode(),
|
|
radix_file.encode(), num_inputs, files_arr, names_arr, output_folder.encode(),
|
|
debug.encode(), cuda.encode())
|
|
|
|
# Results.
|
|
def dynasty_to_dict(output_folder: str, data_type: str,
|
|
first: bool) -> Mapping[str, npt.ArrayLike]:
|
|
"""Converts Dynasty output into a mapping of output node names to NumPy arrays.
|
|
|
|
The resulting NumPy arrays are values in channel first format if first is true (bchw).
|
|
Otherwise, it will be in channel last format (bhwc).
|
|
|
|
Arguments:
|
|
output_folder: String path to directory where outputs were stored.
|
|
data_type: String specifying type of data to load, "float" or "fixed".
|
|
first: Flag indicating if results will be returned in channel first format.
|
|
|
|
Returns:
|
|
A mapping of strings to NumPy arrays of either floats or integers, depending on
|
|
'data_type'. The keys are the names of the output nodes, and the values are the arrays.
|
|
If data_type is "float", results will be float arrays. If data_type is "fixed", results
|
|
will be fixed arrays.
|
|
"""
|
|
if data_type == "float":
|
|
suffix = "_fl.txt"
|
|
else:
|
|
suffix = "_fx.txt"
|
|
np_output = {}
|
|
dim_files = sorted(glob.glob(glob.escape(output_folder) + "/layer_output*.csv"))
|
|
data_files = sorted(glob.glob(glob.escape(output_folder) + "/layer_output_*" + suffix))
|
|
outputs = [re.sub(f"(.*layer_output_|{suffix})", "", data_file) for data_file in data_files]
|
|
|
|
dims = []
|
|
for dim_file in dim_files:
|
|
with open(dim_file, newline="") as in_file:
|
|
reader = csv.reader(in_file)
|
|
data = [int(value[0]) for value in reader]
|
|
dims.append(data)
|
|
|
|
for name, out_file, dim in zip(outputs, data_files, dims):
|
|
data = np.loadtxt(out_file)
|
|
data = np.reshape(data, dim)
|
|
if not first:
|
|
data = utils.convert_first_to_last(data)
|
|
np_output[name] = np.ascontiguousarray(data)
|
|
|
|
return np_output
|
|
|
|
def dynasty_to_np(output_folder: str, data_type: str, reordering: List[str],
|
|
first: bool) -> List[npt.ArrayLike]:
|
|
"""Converts Dynasty output into a list of NumPy arrays.
|
|
|
|
The resulting NumPy arrays are values in channel first format if first is true (bchw).
|
|
Otherwise, it will be in channel last format (bhwc).
|
|
|
|
Arguments:
|
|
output_folder: String path to directory where outputs were stored.
|
|
data_type: String specifying type of data to load, "float" or "fixed".
|
|
reordering: List of strings correponding to the return node order.
|
|
first: Flag indicating if results will be returned in channel first format.
|
|
|
|
Returns:
|
|
A list of NumPy arrays of either floats or integers, depending on 'data_type'. If
|
|
data_type is "float", results will be float arrays. If data_type is "fixed", results
|
|
will be fixed arrays.
|
|
"""
|
|
np_dict = dynasty_to_dict(output_folder, data_type, first)
|
|
return utils.reorder_outputs(np_dict, reordering)
|