from . import io from . import utils from . import general_funcs as funcs from ctypes import c_float import numpy as np import math from PIL import Image def inproc_520_flow(image,return_float=False,raw_fmt=None,raw_size=None,npu_size=None,crop_box=None,pad_mode=None,resize_size=None,pad_length=None,norm='kneron', gray=False, rotate=0, radix=8, bit_width=8, round_w_to_16=True, NUM_BANK_LINE=32,BANK_ENTRY_CNT=512,MAX_IMG_PREPROC_ROW_NUM=511,MAX_IMG_PREPROC_COL_NUM=256): """ inproc_520 python simulator Parameters ---------- image : np.ndarray / string Image input, can be image file, bin or hex file return_float : bool If False, return RGBA fix format If True, return RGB float format default = False raw_fmt : string If image is raw file, set the raw fmt raw_size : tuple (src_w, src_h) npu_size : tuple (target_w, target_h) crop_box : tuple (x1, y1, x2, y2), will skip crop if set None pad_mode : int 0: pad 2 sides, 1: pad 1 side, 2: no pad. default = 0 pad_val : int norm : str Norm mode, default = 'kneron' rotate: int 0 / 1(90 degree) / 2(180 degree), default = 0 radix : int Default = 8 bit_width : int Default = 8 round_w_to_16 : bool Default = True gray : bool Default = False pad_limit_on : bool turn on the hw pad limit(up to 255 pre side) or not, default = True(on) Returns ---------- out : np.ndarray Examples ---------- >>> image_data = kp.inproc_520(image_data,npu_size=(56,56),crop_box=(272,145,460,341),pad_mode=1) """ # ============================================================================================ # assert # ============================================================================================ assert(npu_size is not None) assert(isinstance(npu_size,tuple)) assert(len(npu_size) == 2) # ============================================================================================ # load image # ============================================================================================ ## if input is jpg/bin if not isinstance(image, np.ndarray): ## assume input is jpg if raw_size == None: image = Image.open(image).convert("RGB") image = np.array(image).astype('uint8') raw_fmt = 'rgb888' ## assume input is bin else: assert raw_fmt is not None image = io.load_bin(input_file=image, src_w=raw_size[0], src_h=raw_size[1], image_fmt=raw_fmt) else: assert(image.ndim >= 2) if raw_fmt is None: if image.ndim == 2: raw_fmt = 'nir' elif image.shape[2] == 4: raw_fmt = 'rgba' else: raw_fmt = 'rgb' # ============================================================================================ # init para # ============================================================================================ originH = image.shape[0] originW = image.shape[1] if npu_size is not None: npu_img_w = npu_size[0] npu_img_h = npu_size[1] ## Crop if crop_box != None: assert isinstance(crop_box, list) | isinstance(crop_box, tuple) assert len(crop_box) == 4 startW = crop_box[0] startH = crop_box[1] cropW = crop_box[2] - crop_box[0] cropH = crop_box[3] - crop_box[1] else: startW = 0 startH = 0 cropW = originW cropH = originH ## Resize & Pad # if pad_length is not None: # assert isinstance(pad_length, list) | isinstance(pad_length, tuple) # if pad_length[1] != 0: # assert ( (pad_length[2]==0) & (pad_length[3]==0) ) # if pad_length[0] != 0: # pad_mode = 0 # else: # pad_mode = 1 # elif pad_length[3] != 0: # assert ( (pad_length[0]==0) & (pad_length[1]==0) ) # if pad_length[2] != 0: # pad_mode = 0 # else: # pad_mode = 1 # else: # pad_mode = 2 # else: pad_mode = utils.str2int(pad_mode) if pad_mode == 0: keep_ratio = True elif pad_mode == 1: keep_ratio = True else: keep_ratio = False if norm.lower() in ['tf']: val = np.uint8(-128 >> (7 - radix)) elif norm.lower() in ['yolo']: val = np.uint8(0 >> (8 - radix)) elif norm.lower() in [ 'kneron']: val = np.uint8(-128 >> (8 - radix)) else: val = np.uint8(-128 >> (8 - radix)) pad_val = [val,val,val,val] # ============================================================================================ # main flow # ============================================================================================ crop_num = [0] * 4 crop_num[0] = startW #left crop_num[1] = startH #top crop_num[2] = originW - (startW + cropW) #right crop_num[3] = originH - (startH + cropH) #bottom # calculate scaleW scaleH if keep_ratio: scaleW, scaleH = funcs.calculate_keep_ratio_size(tar_size=(npu_img_w,npu_img_h), ori_size=(cropW,cropH), platform_type='520') else: scaleW = npu_img_w scaleH = npu_img_h if resize_size is not None: assert ( (scaleW == resize_size[0]) & (scaleH == resize_size[1])) , f'pad_mode:{pad_mode},scaleW:{scaleW},scaleH:{scaleH},resize_size[0]:{resize_size[0]},resize_size[1]:{resize_size[1]},npu_img_w:{npu_img_w},npu_img_h:{npu_img_h},cropW:{cropW},cropH:{cropH}' # calculate pad_top pad_bottom pad_left pad_right if (pad_mode == 0): pad_left, pad_right, pad_top, pad_bottom = funcs.calculate_pad_length_center(tar_size=(npu_img_w,npu_img_h),ori_size=(scaleW,scaleH)) elif (pad_mode == 1): # only pad right and bottom pad_left, pad_right, pad_top, pad_bottom = funcs.calculate_pad_length_corner(tar_size=(npu_img_w,npu_img_h),ori_size=(scaleW,scaleH)) else: pad_left, pad_right, pad_top, pad_bottom = 0, 0, 0, 0 if pad_length is not None: assert ( (pad_length[0]==pad_left) & (pad_length[1]==pad_right) & (pad_length[2]==pad_top) & (pad_length[3]==pad_bottom)) # calculate padW padH padW = pad_left + pad_right padH = pad_top + pad_bottom # calculate cut_total valid_in_row = cropH valid_in_col = cropW out_row = scaleH + padH out_col = scaleW + padW max_row = int(math.floor(BANK_ENTRY_CNT * NUM_BANK_LINE / (out_col / 4))) max_row = min(max_row, MAX_IMG_PREPROC_ROW_NUM) ## if ( (pad_right > 127 or pad_bottom > 127) ) or ( max(pad_top,pad_bottom) > (max_row-4)): ## generic image = generic_520(image,crop_box,raw_fmt,gray,scaleW,scaleH,radix,norm,pad_left,pad_top,pad_right,pad_bottom,pad_val,round_w_to_16) else: orig_pad_num = [0] * 4 orig_pad_num[0] = pad_left orig_pad_num[1] = pad_top orig_pad_num[2] = pad_right orig_pad_num[3] = pad_bottom if (pad_mode == 0): big_pad_row = (out_row % max_row) < (pad_bottom + 4) if (big_pad_row): last_row = int(pad_bottom + 4) cut_total = int(math.ceil( float(out_row - last_row) / max_row) + 1) else: cut_total = int(math.ceil( float(out_row) / max_row)) elif (pad_mode == 1): big_pad_row = (out_row % max_row) < (pad_bottom + 4) last_row = max_row if (big_pad_row): cut_total = int(math.ceil( float(out_row - last_row) / max_row) + 1) else: cut_total = int(math.ceil( float(out_row) / max_row)) else: big_pad_row = False cut_total = int(math.ceil( float(out_row) / max_row)) # calculate seg_cnt max_col = MAX_IMG_PREPROC_COL_NUM last_col = 0 if (out_col % max_col): if (pad_mode == 0): big_pad_col = (out_col % max_col) < (pad_right + 4) if (big_pad_col): last_col = round_up_n(pad_right + 4, 4) seg_cnt = math.ceil( float(out_col - last_col) / max_col) + 1 else: seg_cnt = math.ceil( float(out_col) / max_col) elif (pad_mode == 1): big_pad_col = (out_col % max_col) < (pad_right + 4) last_col = max_col if (big_pad_col): seg_cnt = math.ceil( float(out_col - last_col) / max_col) + 1 else: seg_cnt = math.ceil( float(out_col) / max_col) else: big_pad_col = False seg_cnt = math.ceil( float(out_col) / max_col) else: big_pad_col = False seg_cnt = math.ceil( float(out_col) / max_col) # start loop if (big_pad_row): remain_row = out_row - last_row else: remain_row = out_row start_row = 0 row_num = 0 for r in range(0, cut_total): start_row += row_num block_start_row = cal_img_row_offset(crop_num, orig_pad_num, start_row, out_row, originH) if (big_pad_row) and (r == (cut_total - 1)): row_num = last_row else: row_num = min(max_row, remain_row) # due to HW only support max col = 256, we may need to process data in segments */ if(big_pad_col): remain_col = (out_col - last_col) else: remain_col = out_col start_col = 0 col_num = 0 block_start_col = crop_num[0] block_col = 0 for c in range(0,seg_cnt): start_col += col_num block_start_col += block_col if (big_pad_col) and (c == (seg_cnt - 1)): col_num = last_col else: col_num = min(remain_col, MAX_IMG_PREPROC_COL_NUM) pad_num = get_pad_num(orig_pad_num, (c == 0), (r == 0), (c == seg_cnt - 1), (r == cut_total - 1)) block_row = int(valid_in_row * (row_num - pad_num[1] - pad_num[3]) / (out_row - orig_pad_num[1] - orig_pad_num[3])) block_col = int(valid_in_col * (col_num - pad_num[0] - pad_num[2]) / (out_col - orig_pad_num[0] - orig_pad_num[2])) #/* (src_w * byte_per_pixel) should align to multiple of 4-byte and 2 cols */ byte_per_pixel = get_byte_per_pixel(raw_fmt) new_block_col = round_up_n(round_up_n(block_col, (4 / byte_per_pixel)), 2) if (new_block_col > block_col): if byte_per_pixel == 1: block_col = new_block_col - 4 elif byte_per_pixel == 4: block_col = new_block_col - 2 else: block_col = new_block_col - 2 # crop image_temp = funcs.crop(image=image, box=(block_start_col, block_start_row, block_start_col+block_col, block_start_row+block_row)) # color image_temp = funcs.color_convert(image=image_temp, input_fmt=raw_fmt, output_fmt='RGB') if gray: image_temp = funcs.color_convert(image=image_temp, input_fmt='RGB', output_fmt='NIR') # resize image_temp = funcs.resize_fixed(image=image_temp, size=(col_num - pad_num[0] - pad_num[2], row_num - pad_num[1] - pad_num[3]), platform_type='520') # normalize image_temp = funcs.channelize(image=image_temp, radix=radix, mode='kneron') # padding image_temp = funcs.pad(image=image_temp, pad_l=pad_num[0],pad_t=pad_num[1],pad_r=pad_num[2],pad_b=pad_num[3], pad_val=pad_val) ## remain_col -= col_num if c == 0: image_temp_H = image_temp else: image_temp_H = np.concatenate((image_temp_H, image_temp), axis=1) ## remain_row -= row_num if r == 0: image_temp_V = image_temp_H else: image_temp_V = np.concatenate((image_temp_V, image_temp_H), axis=0) image = image_temp_V ## round_w_to_16 if round_w_to_16: out_w_16 = round_up_n(out_col,16) image_16 = np.ones((out_row,out_w_16 - out_col,4)) *128 image = np.concatenate((image, image_16), axis=1) # rotate if not (rotate == 0): image = rotate(image=image,rotate_direction=rotate) image = image.astype('uint8') if return_float: image = image[0:npu_size[1],0:npu_size[0],:3] if norm.lower() in ['tf']: shift = 7 - radix sub = 128 elif norm.lower() in ['yolo']: shift = 8 - radix sub = 0 elif norm.lower() in ['kneron']: shift = 8 - radix sub = 128 else: shift = 8 - radix sub = 128 image = (np.left_shift(image, shift)) image = (image + sub).astype('uint8').astype('float') image = (image - sub) / (2**radix) return image def generic_520(image,crop_box,raw_fmt,gray,scaleW,scaleH,radix,norm,pad_left,pad_top,pad_right,pad_bottom,pad_val,round_w_to_16): print('take generic_520') # crop if crop_box != None: image = funcs.crop(image=image, box=crop_box) # color image = funcs.color_convert(image=image, input_fmt=raw_fmt, output_fmt='RGB') if gray: image = funcs.color_convert(image=image, input_fmt='RGB', output_fmt='NIR') # resize image = funcs.resize_fixed(image=image, size=(scaleW,scaleH), platform_type='520') # normalize image = funcs.channelize(image=image, radix=radix, mode=norm) # padding image = funcs.pad(image=image, pad_l=pad_left,pad_t=pad_top,pad_r=pad_right,pad_b=pad_bottom, pad_val=pad_val) ## round_w_to_16 if round_w_to_16: out_w_16 = round_up_n(image.shape[1],16) image_16 = np.ones((image.shape[0],out_w_16 - image.shape[1],4)) *128 image = np.concatenate((image, image_16), axis=1) return image def round_up_16(num): return ((num + (16 - 1)) & ~(16 - 1)) def round_up_n(num, n): if (num > 0): temp = float(num) / n return math.ceil(temp) * n else: return -math.ceil(float(-num) / n) * n def cal_img_row_offset(crop_num, pad_num, start_row, out_row, orig_row): scaled_img_row = int(out_row - (pad_num[1] + pad_num[3])) if ((start_row - pad_num[1]) > 0): img_str_row = int((start_row - pad_num[1])) else: img_str_row = 0 valid_row = int(orig_row - (crop_num[1] + crop_num[3])) img_str_row = int(valid_row * img_str_row / scaled_img_row) return int(img_str_row + crop_num[1]) def get_pad_num(pad_num_orig, left, up, right, bottom): pad_num = [0]*4 for i in range(0,4): pad_num[i] = pad_num_orig[i] if not (left): pad_num[0] = 0 if not (up): pad_num[1] = 0 if not (right): pad_num[2] = 0 if not (bottom): pad_num[3] = 0 return pad_num def get_byte_per_pixel(raw_fmt): if raw_fmt.lower() in ['rgb','rgb888','rgba','rgba8888']: return 4 elif raw_fmt.lower() in ['YUV', 'yuv', 'YUV422', 'yuv422']: return 2 elif raw_fmt.lower() in ['RGB565', 'rgb565']: return 2 elif raw_fmt.lower() in ['NIR888', 'nir888', 'NIR', 'nir']: return 1 else: return -1 def rotate(image, rotate_direction): if rotate_direction == 1 or rotate_direction == 2: col, row, unit = image.shape pInBuf = image.reshape((-1,1)) pOutBufTemp = np.zeros((col* row* unit)) for r in range(row): for c in range(col): for u in range(unit): if rotate_direction == 1: pOutBufTemp[unit * (c * row + (row - r - 1))+u] = pInBuf[unit * (r * col + c)+u] elif rotate_direction == 2: pOutBufTemp[unit * (row * (col - c - 1) + r)+u] = pInBuf[unit * (r * col + c)+u] img = pOutBufTemp.reshape((col,row,unit)) return img