cluster4npu/cluster4npu_ui/example.py

from typing import Union, Tuple
import os
import sys
import argparse
import time
import threading
import queue
import numpy as np
import kp
import cv2
import time
from abc import ABC, abstractmethod
from typing import Callable, Optional, Any, Dict


# class PreProcessor(DataProcessor): # type: ignore
#     def __init__(self, resize_fn: Optional[Callable] = None, 
#                  format_convert_fn: Optional[Callable] = None):
#         self.resize_fn = resize_fn or self._default_resize
#         self.format_convert_fn = format_convert_fn or self._default_format_convert
    
#     def process(self, frame: np.ndarray, target_size: tuple, target_format: str) -> np.ndarray:
#         """Main processing pipeline"""
#         resized = self.resize_fn(frame, target_size)
#         return self.format_convert_fn(resized, target_format)
    
#     def _default_resize(self, frame: np.ndarray, target_size: tuple) -> np.ndarray:
#         """Default resize implementation"""
#         return cv2.resize(frame, target_size)
    
#     def _default_format_convert(self, frame: np.ndarray, target_format: str) -> np.ndarray:
#         """Default format conversion"""
#         if target_format == 'BGR565':
#             return cv2.cvtColor(frame, cv2.COLOR_BGR2BGR565)
#         elif target_format == 'RGB8888':
#             return cv2.cvtColor(frame, cv2.COLOR_BGR2RGBA)
#         return frame

class MultiDongle:
    # Curently, only BGR565, RGB8888, YUYV, and RAW8 formats are supported
    _FORMAT_MAPPING = { 
        'BGR565': kp.ImageFormat.KP_IMAGE_FORMAT_RGB565,
        'RGB8888': kp.ImageFormat.KP_IMAGE_FORMAT_RGBA8888,
        'YUYV': kp.ImageFormat.KP_IMAGE_FORMAT_YUYV,
        'RAW8': kp.ImageFormat.KP_IMAGE_FORMAT_RAW8,
        # 'YCBCR422_CRY1CBY0': kp.ImageFormat.KP_IMAGE_FORMAT_YCBCR422_CRY1CBY0,
        # 'YCBCR422_CBY1CRY0': kp.ImageFormat.KP_IMAGE_FORMAT_CBY1CRY0,
        # 'YCBCR422_Y1CRY0CB': kp.ImageFormat.KP_IMAGE_FORMAT_Y1CRY0CB,
        # 'YCBCR422_Y1CBY0CR': kp.ImageFormat.KP_IMAGE_FORMAT_Y1CBY0CR,
        # 'YCBCR422_CRY0CBY1': kp.ImageFormat.KP_IMAGE_FORMAT_CRY0CBY1,
        # 'YCBCR422_CBY0CRY1': kp.ImageFormat.KP_IMAGE_FORMAT_CBY0CRY1,
        # 'YCBCR422_Y0CRY1CB': kp.ImageFormat.KP_IMAGE_FORMAT_Y0CRY1CB,
        # 'YCBCR422_Y0CBY1CR': kp.ImageFormat.KP_IMAGE_FORMAT_Y0CBY1CR,
    }

    def __init__(self, port_id: list, scpu_fw_path: str, ncpu_fw_path: str, model_path: str, upload_fw: bool = False):
        """
        Initialize the MultiDongle class.
        :param port_id: List of USB port IDs for the same layer's devices.
        :param scpu_fw_path: Path to the SCPU firmware file.
        :param ncpu_fw_path: Path to the NCPU firmware file.
        :param model_path: Path to the model file.
        :param upload_fw: Flag to indicate whether to upload firmware.
        """
        self.port_id = port_id
        self.upload_fw = upload_fw
        
        # Check if the firmware is needed
        if self.upload_fw:
            self.scpu_fw_path = scpu_fw_path
            self.ncpu_fw_path = ncpu_fw_path
            
        self.model_path = model_path
        self.device_group = None
        
        # generic_inference_input_descriptor will be prepared in initialize
        self.model_nef_descriptor = None
        self.generic_inference_input_descriptor = None
        # Queues for data
        # Input queue for images to be sent
        self._input_queue = queue.Queue()
        # Output queue for received results
        self._output_queue = queue.Queue()

        # Threading attributes
        self._send_thread = None
        self._receive_thread = None
        self._stop_event = threading.Event() # Event to signal threads to stop
        
        self._inference_counter = 0

    def initialize(self):
        """
        Connect devices, upload firmware (if upload_fw is True), and upload model.
        Must be called before start().
        """
        # Connect device and assign to self.device_group
        try:
            print('[Connect Device]')
            self.device_group = kp.core.connect_devices(usb_port_ids=self.port_id)
            print(' - Success')
        except kp.ApiKPException as exception:
            print('Error: connect device fail, port ID = \'{}\', error msg: [{}]'.format(self.port_id, str(exception)))
            sys.exit(1)

        # setting timeout of the usb communication with the device
        # print('[Set Device Timeout]')
        # kp.core.set_timeout(device_group=self.device_group, milliseconds=5000)
        # print(' - Success')

        if self.upload_fw:
            try:
                print('[Upload Firmware]')
                kp.core.load_firmware_from_file(device_group=self.device_group,
                                                scpu_fw_path=self.scpu_fw_path,
                                                ncpu_fw_path=self.ncpu_fw_path)
                print(' - Success')
            except kp.ApiKPException as exception:
                print('Error: upload firmware failed, error = \'{}\''.format(str(exception)))
                sys.exit(1)

        # upload model to device
        try:
            print('[Upload Model]')
            self.model_nef_descriptor = kp.core.load_model_from_file(device_group=self.device_group,
                                                                file_path=self.model_path)
            print(' - Success')
        except kp.ApiKPException as exception:
            print('Error: upload model failed, error = \'{}\''.format(str(exception)))
            sys.exit(1)
            
        # Extract model input dimensions automatically from model metadata
        if self.model_nef_descriptor and self.model_nef_descriptor.models:
            model = self.model_nef_descriptor.models[0]
            if hasattr(model, 'input_nodes') and model.input_nodes:
                input_node = model.input_nodes[0]
                # From your JSON: "shape_npu": [1, 3, 128, 128] -> (width, height)
                shape = input_node.tensor_shape_info.data.shape_npu
                self.model_input_shape = (shape[3], shape[2])  # (width, height)
                self.model_input_channels = shape[1]  # 3 for RGB
                print(f"Model input shape detected: {self.model_input_shape}, channels: {self.model_input_channels}")
            else:
                self.model_input_shape = (128, 128)  # fallback
                self.model_input_channels = 3
                print("Using default input shape (128, 128)")
        else:
            self.model_input_shape = (128, 128)
            self.model_input_channels = 3
            print("Model info not available, using default shape")

        # Prepare generic inference input descriptor after model is loaded
        if self.model_nef_descriptor:
            self.generic_inference_input_descriptor = kp.GenericImageInferenceDescriptor(
                model_id=self.model_nef_descriptor.models[0].id,
            )
        else:
            print("Warning: Could not get generic inference input descriptor from model.")
            self.generic_inference_input_descriptor = None
            
    def preprocess_frame(self, frame: np.ndarray, target_format: str = 'BGR565') -> np.ndarray:
        """
        Preprocess frame for inference
        """
        resized_frame = cv2.resize(frame, self.model_input_shape)
        
        if target_format == 'BGR565':
            return cv2.cvtColor(resized_frame, cv2.COLOR_BGR2BGR565)
        elif target_format == 'RGB8888':
            return cv2.cvtColor(resized_frame, cv2.COLOR_BGR2RGBA)
        elif target_format == 'YUYV':
            return cv2.cvtColor(resized_frame, cv2.COLOR_BGR2YUV_YUYV)
        else:
            return resized_frame  # RAW8 or other formats

    def get_latest_inference_result(self, timeout: float = 0.01) -> Tuple[float, str]:
        """
        Get the latest inference result
        Returns: (probability, result_string) or (None, None) if no result
        """
        output_descriptor = self.get_output(timeout=timeout)
        if not output_descriptor:
            return None, None
            
        # Process the output descriptor
        if hasattr(output_descriptor, 'header') and \
        hasattr(output_descriptor.header, 'num_output_node') and \
        hasattr(output_descriptor.header, 'inference_number'):
            
            inf_node_output_list = []
            retrieval_successful = True
            
            for node_idx in range(output_descriptor.header.num_output_node):
                try:
                    inference_float_node_output = kp.inference.generic_inference_retrieve_float_node(
                        node_idx=node_idx,
                        generic_raw_result=output_descriptor,
                        channels_ordering=kp.ChannelOrdering.KP_CHANNEL_ORDERING_CHW
                    )
                    inf_node_output_list.append(inference_float_node_output.ndarray.copy())
                except kp.ApiKPException as e:
                    retrieval_successful = False
                    break 
                except Exception as e:
                    retrieval_successful = False
                    break

            if retrieval_successful and len(inf_node_output_list) > 0:
                # Process output nodes
                if output_descriptor.header.num_output_node == 1:
                    raw_output_array = inf_node_output_list[0].flatten()
                else:
                    concatenated_outputs = [arr.flatten() for arr in inf_node_output_list]
                    raw_output_array = np.concatenate(concatenated_outputs) if concatenated_outputs else np.array([])
                
                if raw_output_array.size > 0:
                    probability = postprocess(raw_output_array)
                    result_str = "Fire" if probability > 0.5 else "No Fire"
                    return probability, result_str
            
        return None, None

    # Modified _send_thread_func to get data from input queue
    def _send_thread_func(self):
        """Internal function run by the send thread, gets images from input queue."""
        print("Send thread started.")
        while not self._stop_event.is_set():
            if self.generic_inference_input_descriptor is None:
                # Wait for descriptor to be ready or stop
                self._stop_event.wait(0.1) # Avoid busy waiting
                continue

            try:
                # Get image and format from the input queue
                # Blocks until an item is available or stop event is set/timeout occurs
                try:
                    # Use get with timeout or check stop event in a loop
                    # This pattern allows thread to check stop event while waiting on queue
                    item = self._input_queue.get(block=True, timeout=0.1)
                    # Check if this is our sentinel value
                    if item is None:
                        continue
                        
                    # Now safely unpack the tuple
                    image_data, image_format_enum = item
                except queue.Empty:
                    # If queue is empty after timeout, check stop event and continue loop
                    continue

                # Configure and send the image
                self._inference_counter += 1  # Increment counter for each image
                self.generic_inference_input_descriptor.inference_number = self._inference_counter
                self.generic_inference_input_descriptor.input_node_image_list = [kp.GenericInputNodeImage(
                    image=image_data,
                    image_format=image_format_enum,  # Use the format from the queue
                    resize_mode=kp.ResizeMode.KP_RESIZE_ENABLE,
                    padding_mode=kp.PaddingMode.KP_PADDING_CORNER,
                    normalize_mode=kp.NormalizeMode.KP_NORMALIZE_KNERON
                )]

                kp.inference.generic_image_inference_send(device_group=self.device_group,
                                                          generic_inference_input_descriptor=self.generic_inference_input_descriptor)
                # print("Image sent.") # Optional: add log
                # No need for sleep here usually, as queue.get is blocking
            except kp.ApiKPException as exception:
                print(f' - Error in send thread: inference send failed, error = {exception}')
                self._stop_event.set() # Signal other thread to stop
            except Exception as e:
                 print(f' - Unexpected error in send thread: {e}')
                 self._stop_event.set()

        print("Send thread stopped.")

    # _receive_thread_func remains the same
    def _receive_thread_func(self):
        """Internal function run by the receive thread, puts results into output queue."""
        print("Receive thread started.")
        while not self._stop_event.is_set():
            try:
                generic_inference_output_descriptor = kp.inference.generic_image_inference_receive(device_group=self.device_group)
                self._output_queue.put(generic_inference_output_descriptor)
            except kp.ApiKPException as exception:
                if not self._stop_event.is_set(): # Avoid printing error if we are already stopping
                     print(f' - Error in receive thread: inference receive failed, error = {exception}')
                     self._stop_event.set()
            except Exception as e:
                 print(f' - Unexpected error in receive thread: {e}')
                 self._stop_event.set()

        print("Receive thread stopped.")

    def start(self):
        """
        Start the send and receive threads.
        Must be called after initialize().
        """
        if self.device_group is None:
            raise RuntimeError("MultiDongle not initialized. Call initialize() first.")

        if self._send_thread is None or not self._send_thread.is_alive():
            self._stop_event.clear() # Clear stop event for a new start
            self._send_thread = threading.Thread(target=self._send_thread_func, daemon=True)
            self._send_thread.start()
            print("Send thread started.")

        if self._receive_thread is None or not self._receive_thread.is_alive():
            self._receive_thread = threading.Thread(target=self._receive_thread_func, daemon=True)
            self._receive_thread.start()
            print("Receive thread started.")
    
    def stop(self):
        """Improved stop method with better cleanup"""
        if self._stop_event.is_set():
            return  # Already stopping
            
        print("Stopping threads...")
        self._stop_event.set()
        
        # Clear queues to unblock threads
        while not self._input_queue.empty():
            try:
                self._input_queue.get_nowait()
            except queue.Empty:
                break
        
        # Signal send thread to wake up
        self._input_queue.put(None)
        
        # Join threads with timeout
        for thread, name in [(self._send_thread, "Send"), (self._receive_thread, "Receive")]:
            if thread and thread.is_alive():
                thread.join(timeout=2.0)
                if thread.is_alive():
                    print(f"Warning: {name} thread didn't stop cleanly")
        
    def put_input(self, image: Union[str, np.ndarray], format: str, target_size: Tuple[int, int] = None):
        """
        Put an image into the input queue with flexible preprocessing
        """
        if isinstance(image, str):
            image_data = cv2.imread(image)
            if image_data is None:
                raise FileNotFoundError(f"Image file not found at {image}")
            if target_size:
                image_data = cv2.resize(image_data, target_size)
        elif isinstance(image, np.ndarray):
            # Don't modify original array, make copy if needed
            image_data = image.copy() if target_size is None else cv2.resize(image, target_size)
        else:
            raise ValueError("Image must be a file path (str) or a numpy array (ndarray).")

        if format in self._FORMAT_MAPPING:
            image_format_enum = self._FORMAT_MAPPING[format]
        else:
            raise ValueError(f"Unsupported format: {format}")

        self._input_queue.put((image_data, image_format_enum))

    def get_output(self, timeout: float = None):
        """
        Get the next received data from the output queue.
        This method is non-blocking by default unless a timeout is specified.
        :param timeout: Time in seconds to wait for data. If None, it's non-blocking.
        :return: Received data (e.g., kp.GenericInferenceOutputDescriptor) or None if no data available within timeout.
        """
        try:
            return self._output_queue.get(block=timeout is not None, timeout=timeout)
        except queue.Empty:
            return None

    def __del__(self):
         """Ensure resources are released when the object is garbage collected."""
         self.stop()
         if self.device_group:
             try:
                 kp.core.disconnect_devices(device_group=self.device_group)
                 print("Device group disconnected in destructor.")
             except Exception as e:
                  print(f"Error disconnecting device group in destructor: {e}")

def postprocess(raw_model_output: list) -> float:
    """
    Post-processes the raw model output.
    Assumes the model output is a list/array where the first element is the desired probability.
    """
    if raw_model_output is not None and len(raw_model_output) > 0:
        probability = raw_model_output[0]
        return float(probability)
    return 0.0 # Default or error value

class WebcamInferenceRunner:
    def __init__(self, multidongle: MultiDongle, image_format: str = 'BGR565'):
        self.multidongle = multidongle
        self.image_format = image_format
        self.latest_probability = 0.0
        self.result_str = "No Fire"
        
        # Statistics tracking
        self.processed_inference_count = 0
        self.inference_fps_start_time = None
        self.display_fps_start_time = None
        self.display_frame_counter = 0
        
    def run(self, camera_id: int = 0):
        cap = cv2.VideoCapture(camera_id)
        if not cap.isOpened():
            raise RuntimeError("Cannot open webcam")
            
        try:
            while True:
                ret, frame = cap.read()
                if not ret:
                    break
                
                # Track display FPS
                if self.display_fps_start_time is None:
                    self.display_fps_start_time = time.time()
                self.display_frame_counter += 1
                
                # Preprocess and send frame
                processed_frame = self.multidongle.preprocess_frame(frame, self.image_format)
                self.multidongle.put_input(processed_frame, self.image_format)
                
                # Get inference result
                prob, result = self.multidongle.get_latest_inference_result()
                if prob is not None:
                    # Track inference FPS
                    if self.inference_fps_start_time is None:
                        self.inference_fps_start_time = time.time()
                    self.processed_inference_count += 1
                    
                    self.latest_probability = prob
                    self.result_str = result
                
                # Display frame with results
                self._display_results(frame)
                
                if cv2.waitKey(1) & 0xFF == ord('q'):
                    break
                    
        finally:
            # self._print_statistics()
            cap.release()
            cv2.destroyAllWindows()
    
    def _display_results(self, frame):
        display_frame = frame.copy()
        text_color = (0, 255, 0) if "Fire" in self.result_str else (0, 0, 255)
        
        # Display inference result
        cv2.putText(display_frame, f"{self.result_str} (Prob: {self.latest_probability:.2f})", 
                   (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.7, text_color, 2)
        
        # Calculate and display inference FPS
        if self.inference_fps_start_time and self.processed_inference_count > 0:
            elapsed_time = time.time() - self.inference_fps_start_time
            if elapsed_time > 0:
                inference_fps = self.processed_inference_count / elapsed_time
                cv2.putText(display_frame, f"Inference FPS: {inference_fps:.2f}",
                           (10, 60), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 255), 2)
        
        cv2.imshow('Fire Detection', display_frame)
    
    # def _print_statistics(self):
    #     """Print final statistics"""
    #     print(f"\n--- Summary ---")
    #     print(f"Total inferences processed: {self.processed_inference_count}")
        
    #     if self.inference_fps_start_time and self.processed_inference_count > 0:
    #         elapsed = time.time() - self.inference_fps_start_time
    #         if elapsed > 0:
    #             avg_inference_fps = self.processed_inference_count / elapsed
    #             print(f"Average Inference FPS: {avg_inference_fps:.2f}")
        
    #     if self.display_fps_start_time and self.display_frame_counter > 0:
    #         elapsed = time.time() - self.display_fps_start_time
    #         if elapsed > 0:
    #             avg_display_fps = self.display_frame_counter / elapsed
    #             print(f"Average Display FPS: {avg_display_fps:.2f}")

if __name__ == "__main__":
    PORT_IDS = [32]
    SCPU_FW = r'C:/Users/mason/Downloads/kneron_plus_v3.1.2/kneron_plus/res/firmware/KL520/fw_scpu.bin'
    NCPU_FW = r'C:/Users/mason/Downloads/kneron_plus_v3.1.2/kneron_plus/res/firmware/KL520/fw_ncpu.bin'
    MODEL_PATH = r'C:/Users/mason/AppData/Local/Kneron_Academy/utils/yolov5s/yolov5s/kl520_20005_yolov5-noupsample_w640h640.nef'
    
    try:
        # Initialize inference engine
        print("Initializing MultiDongle...")
        multidongle = MultiDongle(PORT_IDS, SCPU_FW, NCPU_FW, MODEL_PATH, upload_fw=True)
        multidongle.initialize()
        multidongle.start()
        
        # Run using the new runner class
        print("Starting webcam inference...")
        runner = WebcamInferenceRunner(multidongle, 'BGR565')
        runner.run()
        
    except Exception as e:
        print(f"Error: {e}")
        import traceback
        traceback.print_exc()
    finally:
        if 'multidongle' in locals():
            multidongle.stop()