464 lines
16 KiB
Python
464 lines
16 KiB
Python
from . import io
|
|
from . import utils
|
|
from . import general_funcs as funcs
|
|
from ctypes import c_float
|
|
import numpy as np
|
|
import math
|
|
from PIL import Image
|
|
|
|
def inproc_520_flow(image,return_float=False,raw_fmt=None,raw_size=None,npu_size=None,crop_box=None,pad_mode=None,resize_size=None,pad_length=None,norm='kneron', gray=False, rotate=0, radix=8, bit_width=8, round_w_to_16=True, NUM_BANK_LINE=32,BANK_ENTRY_CNT=512,MAX_IMG_PREPROC_ROW_NUM=511,MAX_IMG_PREPROC_COL_NUM=256):
|
|
"""
|
|
inproc_520 python simulator
|
|
|
|
Parameters
|
|
----------
|
|
image : np.ndarray / string
|
|
Image input, can be image file, bin or hex file
|
|
|
|
return_float : bool
|
|
If False, return RGBA fix format
|
|
If True, return RGB float format
|
|
default = False
|
|
|
|
raw_fmt : string
|
|
If image is raw file, set the raw fmt
|
|
|
|
raw_size : tuple
|
|
(src_w, src_h)
|
|
|
|
npu_size : tuple
|
|
(target_w, target_h)
|
|
|
|
crop_box : tuple
|
|
(x1, y1, x2, y2), will skip crop if set None
|
|
|
|
pad_mode : int
|
|
0: pad 2 sides, 1: pad 1 side, 2: no pad. default = 0
|
|
|
|
pad_val : int
|
|
|
|
norm : str
|
|
Norm mode, default = 'kneron'
|
|
|
|
rotate: int
|
|
0 / 1(90 degree) / 2(180 degree), default = 0
|
|
|
|
radix : int
|
|
Default = 8
|
|
|
|
bit_width : int
|
|
Default = 8
|
|
|
|
round_w_to_16 : bool
|
|
Default = True
|
|
|
|
gray : bool
|
|
Default = False
|
|
|
|
pad_limit_on : bool
|
|
turn on the hw pad limit(up to 255 pre side) or not, default = True(on)
|
|
|
|
Returns
|
|
----------
|
|
out : np.ndarray
|
|
|
|
Examples
|
|
----------
|
|
>>> image_data = kp.inproc_520(image_data,npu_size=(56,56),crop_box=(272,145,460,341),pad_mode=1)
|
|
"""
|
|
# ============================================================================================
|
|
# assert
|
|
# ============================================================================================
|
|
assert(npu_size is not None)
|
|
assert(isinstance(npu_size,tuple))
|
|
assert(len(npu_size) == 2)
|
|
|
|
# ============================================================================================
|
|
# load image
|
|
# ============================================================================================
|
|
## if input is jpg/bin
|
|
if not isinstance(image, np.ndarray):
|
|
## assume input is jpg
|
|
if raw_size == None:
|
|
image = Image.open(image).convert("RGB")
|
|
image = np.array(image).astype('uint8')
|
|
raw_fmt = 'rgb888'
|
|
## assume input is bin
|
|
else:
|
|
assert raw_fmt is not None
|
|
image = io.load_bin(input_file=image, src_w=raw_size[0], src_h=raw_size[1], image_fmt=raw_fmt)
|
|
else:
|
|
assert(image.ndim >= 2)
|
|
if raw_fmt is None:
|
|
if image.ndim == 2:
|
|
raw_fmt = 'nir'
|
|
elif image.shape[2] == 4:
|
|
raw_fmt = 'rgba'
|
|
else:
|
|
raw_fmt = 'rgb'
|
|
|
|
# ============================================================================================
|
|
# init para
|
|
# ============================================================================================
|
|
originH = image.shape[0]
|
|
originW = image.shape[1]
|
|
if npu_size is not None:
|
|
npu_img_w = npu_size[0]
|
|
npu_img_h = npu_size[1]
|
|
|
|
## Crop
|
|
if crop_box != None:
|
|
assert isinstance(crop_box, list) | isinstance(crop_box, tuple)
|
|
assert len(crop_box) == 4
|
|
startW = crop_box[0]
|
|
startH = crop_box[1]
|
|
cropW = crop_box[2] - crop_box[0]
|
|
cropH = crop_box[3] - crop_box[1]
|
|
else:
|
|
startW = 0
|
|
startH = 0
|
|
cropW = originW
|
|
cropH = originH
|
|
|
|
## Resize & Pad
|
|
# if pad_length is not None:
|
|
# assert isinstance(pad_length, list) | isinstance(pad_length, tuple)
|
|
# if pad_length[1] != 0:
|
|
# assert ( (pad_length[2]==0) & (pad_length[3]==0) )
|
|
# if pad_length[0] != 0:
|
|
# pad_mode = 0
|
|
# else:
|
|
# pad_mode = 1
|
|
# elif pad_length[3] != 0:
|
|
# assert ( (pad_length[0]==0) & (pad_length[1]==0) )
|
|
# if pad_length[2] != 0:
|
|
# pad_mode = 0
|
|
# else:
|
|
# pad_mode = 1
|
|
# else:
|
|
# pad_mode = 2
|
|
# else:
|
|
pad_mode = utils.str2int(pad_mode)
|
|
|
|
if pad_mode == 0:
|
|
keep_ratio = True
|
|
elif pad_mode == 1:
|
|
keep_ratio = True
|
|
else:
|
|
keep_ratio = False
|
|
|
|
if norm.lower() in ['tf']:
|
|
val = np.uint8(-128 >> (7 - radix))
|
|
elif norm.lower() in ['yolo']:
|
|
val = np.uint8(0 >> (8 - radix))
|
|
elif norm.lower() in [ 'kneron']:
|
|
val = np.uint8(-128 >> (8 - radix))
|
|
else:
|
|
val = np.uint8(-128 >> (8 - radix))
|
|
pad_val = [val,val,val,val]
|
|
|
|
# ============================================================================================
|
|
# main flow
|
|
# ============================================================================================
|
|
crop_num = [0] * 4
|
|
crop_num[0] = startW #left
|
|
crop_num[1] = startH #top
|
|
crop_num[2] = originW - (startW + cropW) #right
|
|
crop_num[3] = originH - (startH + cropH) #bottom
|
|
|
|
# calculate scaleW scaleH
|
|
if keep_ratio:
|
|
scaleW, scaleH = funcs.calculate_keep_ratio_size(tar_size=(npu_img_w,npu_img_h), ori_size=(cropW,cropH), platform_type='520')
|
|
else:
|
|
scaleW = npu_img_w
|
|
scaleH = npu_img_h
|
|
|
|
if resize_size is not None:
|
|
assert ( (scaleW == resize_size[0]) & (scaleH == resize_size[1])) , f'pad_mode:{pad_mode},scaleW:{scaleW},scaleH:{scaleH},resize_size[0]:{resize_size[0]},resize_size[1]:{resize_size[1]},npu_img_w:{npu_img_w},npu_img_h:{npu_img_h},cropW:{cropW},cropH:{cropH}'
|
|
|
|
|
|
# calculate pad_top pad_bottom pad_left pad_right
|
|
if (pad_mode == 0):
|
|
pad_left, pad_right, pad_top, pad_bottom = funcs.calculate_pad_length_center(tar_size=(npu_img_w,npu_img_h),ori_size=(scaleW,scaleH))
|
|
elif (pad_mode == 1):
|
|
# only pad right and bottom
|
|
pad_left, pad_right, pad_top, pad_bottom = funcs.calculate_pad_length_corner(tar_size=(npu_img_w,npu_img_h),ori_size=(scaleW,scaleH))
|
|
else:
|
|
pad_left, pad_right, pad_top, pad_bottom = 0, 0, 0, 0
|
|
|
|
if pad_length is not None:
|
|
assert ( (pad_length[0]==pad_left) & (pad_length[1]==pad_right) & (pad_length[2]==pad_top) & (pad_length[3]==pad_bottom))
|
|
|
|
# calculate padW padH
|
|
padW = pad_left + pad_right
|
|
padH = pad_top + pad_bottom
|
|
|
|
# calculate cut_total
|
|
valid_in_row = cropH
|
|
valid_in_col = cropW
|
|
out_row = scaleH + padH
|
|
out_col = scaleW + padW
|
|
|
|
max_row = int(math.floor(BANK_ENTRY_CNT * NUM_BANK_LINE / (out_col / 4)))
|
|
max_row = min(max_row, MAX_IMG_PREPROC_ROW_NUM)
|
|
|
|
##
|
|
if ( (pad_right > 127 or pad_bottom > 127) ) or ( max(pad_top,pad_bottom) > (max_row-4)):
|
|
## generic
|
|
image = generic_520(image,crop_box,raw_fmt,gray,scaleW,scaleH,radix,norm,pad_left,pad_top,pad_right,pad_bottom,pad_val,round_w_to_16)
|
|
else:
|
|
orig_pad_num = [0] * 4
|
|
orig_pad_num[0] = pad_left
|
|
orig_pad_num[1] = pad_top
|
|
orig_pad_num[2] = pad_right
|
|
orig_pad_num[3] = pad_bottom
|
|
|
|
if (pad_mode == 0):
|
|
big_pad_row = (out_row % max_row) < (pad_bottom + 4)
|
|
if (big_pad_row):
|
|
last_row = int(pad_bottom + 4)
|
|
cut_total = int(math.ceil( float(out_row - last_row) / max_row) + 1)
|
|
else:
|
|
cut_total = int(math.ceil( float(out_row) / max_row))
|
|
elif (pad_mode == 1):
|
|
big_pad_row = (out_row % max_row) < (pad_bottom + 4)
|
|
last_row = max_row
|
|
if (big_pad_row):
|
|
cut_total = int(math.ceil( float(out_row - last_row) / max_row) + 1)
|
|
else:
|
|
cut_total = int(math.ceil( float(out_row) / max_row))
|
|
else:
|
|
big_pad_row = False
|
|
cut_total = int(math.ceil( float(out_row) / max_row))
|
|
|
|
# calculate seg_cnt
|
|
max_col = MAX_IMG_PREPROC_COL_NUM
|
|
last_col = 0
|
|
if (out_col % max_col):
|
|
if (pad_mode == 0):
|
|
big_pad_col = (out_col % max_col) < (pad_right + 4)
|
|
if (big_pad_col):
|
|
last_col = round_up_n(pad_right + 4, 4)
|
|
seg_cnt = math.ceil( float(out_col - last_col) / max_col) + 1
|
|
else:
|
|
seg_cnt = math.ceil( float(out_col) / max_col)
|
|
elif (pad_mode == 1):
|
|
big_pad_col = (out_col % max_col) < (pad_right + 4)
|
|
last_col = max_col
|
|
if (big_pad_col):
|
|
seg_cnt = math.ceil( float(out_col - last_col) / max_col) + 1
|
|
else:
|
|
seg_cnt = math.ceil( float(out_col) / max_col)
|
|
else:
|
|
big_pad_col = False
|
|
seg_cnt = math.ceil( float(out_col) / max_col)
|
|
else:
|
|
big_pad_col = False
|
|
seg_cnt = math.ceil( float(out_col) / max_col)
|
|
|
|
# start loop
|
|
if (big_pad_row):
|
|
remain_row = out_row - last_row
|
|
else:
|
|
remain_row = out_row
|
|
start_row = 0
|
|
row_num = 0
|
|
for r in range(0, cut_total):
|
|
start_row += row_num
|
|
block_start_row = cal_img_row_offset(crop_num, orig_pad_num, start_row, out_row, originH)
|
|
if (big_pad_row) and (r == (cut_total - 1)):
|
|
row_num = last_row
|
|
else:
|
|
row_num = min(max_row, remain_row)
|
|
|
|
# due to HW only support max col = 256, we may need to process data in segments */
|
|
if(big_pad_col):
|
|
remain_col = (out_col - last_col)
|
|
else:
|
|
remain_col = out_col
|
|
start_col = 0
|
|
col_num = 0
|
|
block_start_col = crop_num[0]
|
|
block_col = 0
|
|
for c in range(0,seg_cnt):
|
|
start_col += col_num
|
|
block_start_col += block_col
|
|
if (big_pad_col) and (c == (seg_cnt - 1)):
|
|
col_num = last_col
|
|
else:
|
|
col_num = min(remain_col, MAX_IMG_PREPROC_COL_NUM)
|
|
|
|
pad_num = get_pad_num(orig_pad_num, (c == 0), (r == 0), (c == seg_cnt - 1), (r == cut_total - 1))
|
|
block_row = int(valid_in_row * (row_num - pad_num[1] - pad_num[3]) / (out_row - orig_pad_num[1] - orig_pad_num[3]))
|
|
block_col = int(valid_in_col * (col_num - pad_num[0] - pad_num[2]) / (out_col - orig_pad_num[0] - orig_pad_num[2]))
|
|
#/* (src_w * byte_per_pixel) should align to multiple of 4-byte and 2 cols */
|
|
byte_per_pixel = get_byte_per_pixel(raw_fmt)
|
|
new_block_col = round_up_n(round_up_n(block_col, (4 / byte_per_pixel)), 2)
|
|
|
|
if (new_block_col > block_col):
|
|
if byte_per_pixel == 1:
|
|
block_col = new_block_col - 4
|
|
elif byte_per_pixel == 4:
|
|
block_col = new_block_col - 2
|
|
else:
|
|
block_col = new_block_col - 2
|
|
|
|
# crop
|
|
image_temp = funcs.crop(image=image, box=(block_start_col, block_start_row, block_start_col+block_col, block_start_row+block_row))
|
|
|
|
# color
|
|
image_temp = funcs.color_convert(image=image_temp, input_fmt=raw_fmt, output_fmt='RGB')
|
|
if gray:
|
|
image_temp = funcs.color_convert(image=image_temp, input_fmt='RGB', output_fmt='NIR')
|
|
|
|
# resize
|
|
image_temp = funcs.resize_fixed(image=image_temp, size=(col_num - pad_num[0] - pad_num[2], row_num - pad_num[1] - pad_num[3]), platform_type='520')
|
|
|
|
# normalize
|
|
image_temp = funcs.channelize(image=image_temp, radix=radix, mode='kneron')
|
|
|
|
# padding
|
|
image_temp = funcs.pad(image=image_temp, pad_l=pad_num[0],pad_t=pad_num[1],pad_r=pad_num[2],pad_b=pad_num[3], pad_val=pad_val)
|
|
|
|
##
|
|
remain_col -= col_num
|
|
if c == 0:
|
|
image_temp_H = image_temp
|
|
else:
|
|
image_temp_H = np.concatenate((image_temp_H, image_temp), axis=1)
|
|
|
|
##
|
|
remain_row -= row_num
|
|
if r == 0:
|
|
image_temp_V = image_temp_H
|
|
else:
|
|
image_temp_V = np.concatenate((image_temp_V, image_temp_H), axis=0)
|
|
|
|
|
|
image = image_temp_V
|
|
## round_w_to_16
|
|
if round_w_to_16:
|
|
out_w_16 = round_up_n(out_col,16)
|
|
image_16 = np.ones((out_row,out_w_16 - out_col,4)) *128
|
|
image = np.concatenate((image, image_16), axis=1)
|
|
|
|
# rotate
|
|
if not (rotate == 0):
|
|
image = rotate(image=image,rotate_direction=rotate)
|
|
|
|
image = image.astype('uint8')
|
|
|
|
if return_float:
|
|
image = image[0:npu_size[1],0:npu_size[0],:3]
|
|
if norm.lower() in ['tf']:
|
|
shift = 7 - radix
|
|
sub = 128
|
|
elif norm.lower() in ['yolo']:
|
|
shift = 8 - radix
|
|
sub = 0
|
|
elif norm.lower() in ['kneron']:
|
|
shift = 8 - radix
|
|
sub = 128
|
|
else:
|
|
shift = 8 - radix
|
|
sub = 128
|
|
|
|
image = (np.left_shift(image, shift))
|
|
image = (image + sub).astype('uint8').astype('float')
|
|
image = (image - sub) / (2**radix)
|
|
|
|
return image
|
|
|
|
def generic_520(image,crop_box,raw_fmt,gray,scaleW,scaleH,radix,norm,pad_left,pad_top,pad_right,pad_bottom,pad_val,round_w_to_16):
|
|
print('take generic_520')
|
|
# crop
|
|
if crop_box != None:
|
|
image = funcs.crop(image=image, box=crop_box)
|
|
|
|
# color
|
|
image = funcs.color_convert(image=image, input_fmt=raw_fmt, output_fmt='RGB')
|
|
if gray:
|
|
image = funcs.color_convert(image=image, input_fmt='RGB', output_fmt='NIR')
|
|
|
|
# resize
|
|
image = funcs.resize_fixed(image=image, size=(scaleW,scaleH), platform_type='520')
|
|
|
|
# normalize
|
|
image = funcs.channelize(image=image, radix=radix, mode=norm)
|
|
|
|
# padding
|
|
image = funcs.pad(image=image, pad_l=pad_left,pad_t=pad_top,pad_r=pad_right,pad_b=pad_bottom, pad_val=pad_val)
|
|
|
|
## round_w_to_16
|
|
if round_w_to_16:
|
|
out_w_16 = round_up_n(image.shape[1],16)
|
|
image_16 = np.ones((image.shape[0],out_w_16 - image.shape[1],4)) *128
|
|
image = np.concatenate((image, image_16), axis=1)
|
|
|
|
return image
|
|
|
|
def round_up_16(num):
|
|
return ((num + (16 - 1)) & ~(16 - 1))
|
|
|
|
def round_up_n(num, n):
|
|
if (num > 0):
|
|
temp = float(num) / n
|
|
return math.ceil(temp) * n
|
|
else:
|
|
return -math.ceil(float(-num) / n) * n
|
|
|
|
def cal_img_row_offset(crop_num, pad_num, start_row, out_row, orig_row):
|
|
|
|
scaled_img_row = int(out_row - (pad_num[1] + pad_num[3]))
|
|
if ((start_row - pad_num[1]) > 0):
|
|
img_str_row = int((start_row - pad_num[1]))
|
|
else:
|
|
img_str_row = 0
|
|
valid_row = int(orig_row - (crop_num[1] + crop_num[3]))
|
|
img_str_row = int(valid_row * img_str_row / scaled_img_row)
|
|
return int(img_str_row + crop_num[1])
|
|
|
|
def get_pad_num(pad_num_orig, left, up, right, bottom):
|
|
pad_num = [0]*4
|
|
for i in range(0,4):
|
|
pad_num[i] = pad_num_orig[i]
|
|
|
|
if not (left):
|
|
pad_num[0] = 0
|
|
if not (up):
|
|
pad_num[1] = 0
|
|
if not (right):
|
|
pad_num[2] = 0
|
|
if not (bottom):
|
|
pad_num[3] = 0
|
|
|
|
return pad_num
|
|
|
|
def get_byte_per_pixel(raw_fmt):
|
|
if raw_fmt.lower() in ['rgb','rgb888','rgba','rgba8888']:
|
|
return 4
|
|
elif raw_fmt.lower() in ['YUV', 'yuv', 'YUV422', 'yuv422']:
|
|
return 2
|
|
elif raw_fmt.lower() in ['RGB565', 'rgb565']:
|
|
return 2
|
|
elif raw_fmt.lower() in ['NIR888', 'nir888', 'NIR', 'nir']:
|
|
return 1
|
|
else:
|
|
return -1
|
|
|
|
def rotate(image, rotate_direction):
|
|
if rotate_direction == 1 or rotate_direction == 2:
|
|
col, row, unit = image.shape
|
|
pInBuf = image.reshape((-1,1))
|
|
pOutBufTemp = np.zeros((col* row* unit))
|
|
for r in range(row):
|
|
for c in range(col):
|
|
for u in range(unit):
|
|
if rotate_direction == 1:
|
|
pOutBufTemp[unit * (c * row + (row - r - 1))+u] = pInBuf[unit * (r * col + c)+u]
|
|
elif rotate_direction == 2:
|
|
pOutBufTemp[unit * (row * (col - c - 1) + r)+u] = pInBuf[unit * (r * col + c)+u]
|
|
|
|
img = pOutBufTemp.reshape((col,row,unit))
|
|
|
|
return img |