From d92c05c159fa5e9e5d19a01715d35d8078a2d27f Mon Sep 17 00:00:00 2001 From: Atmadeep Arya Date: Tue, 9 Mar 2021 14:02:30 +0530 Subject: [PATCH] perf(multiprocessing and multithreading): Skeleton now works completely. 1. skeleton now works completely from taking video to putting together a file. 2. outputframe uses mp.Queue for taking frames from. 3. Still cannot display frames as they are being received in outputframeThread owing to a bug related to PyQt5. --- config/config.py | 1 - skeleton.py | 103 +++++++++++++++++++---------- src/objectDetection.py | 7 +- src/ocrNumberPlate.py | 2 +- src/trafficApp.py | 146 +++++++++++++++-------------------------- 5 files changed, 127 insertions(+), 132 deletions(-) diff --git a/config/config.py b/config/config.py index efe1d5d..4e6dbcb 100644 --- a/config/config.py +++ b/config/config.py @@ -11,7 +11,6 @@ class PARAMS(object): _YOLOV3_OD_INPUT_IMAGE_SIZE = (416,416) #(Width, Height) #_YOLOV3_OD_NUMBER_PLATE_OBJECT_LIST = ['car','bus','truck','motorbike'] _YOLOV3_OD_NUMBER_PLATE_OBJECT_LIST = ['car'] - _OBJECT_DETECTION_PROCESS_CORES = 2 # Number Plate Detection _YOLOV3_NP_WEIGHT = os.path.join(_CMD_PMT_,'resources/trainedModels/yolov3_number_plate_detection/lapi.weights') diff --git a/skeleton.py b/skeleton.py index 1501c96..3670012 100644 --- a/skeleton.py +++ b/skeleton.py @@ -1,91 +1,116 @@ + import argparse import multiprocessing as mp import os -import random import threading as th import time +from datetime import datetime from queue import Queue +import numpy as np + +os.environ['DISPLAY'] = ':0' import cv2 inputQueue = mp.Queue() vehicleDetectionQueue = mp.Queue() outputQueue = mp.Queue() +IMAGE_HEIGHT = mp.Value('i',0) +IMAGE_WIDTH = mp.Value('i',0) + class ReadFrame(th.Thread): + global inputQueue + global IMAGE_HEIGHT,IMAGE_WIDTH def __init__(self,source,name='Input thread',custom_id=1) -> None: super().__init__() self.frameId = 1 self.stopped = False self.grabbed = True self.name = f'{name} {custom_id}' - - self.videoCaptureObject = cv2.VideoCapture(source) + if(self.videoCaptureObject.isOpened()): + IMAGE_HEIGHT.value = int(self.videoCaptureObject.get(cv2.CAP_PROP_FRAME_HEIGHT)) + IMAGE_WIDTH.value = int(self.videoCaptureObject.get(cv2.CAP_PROP_FRAME_WIDTH)) print(f'Reading from source = {source}') - - global inputQueue def run(self): while (self.grabbed): (self.grabbed, self.frame) = self.videoCaptureObject.read() + IMAGE_HEIGHT.value = int(self.videoCaptureObject.get(cv2.CAP_PROP_FRAME_HEIGHT)) + IMAGE_WIDTH.value = int(self.videoCaptureObject.get(cv2.CAP_PROP_FRAME_WIDTH)) inputQueue.put((self.frame,self.frameId)) - print(f"{self.name}frame added with id {self.frameId}\n") + print(f"{self.name} frame added with id {self.frameId}\n") self.frameId+=1 print('--Done reading frames--\n') + self.videoCaptureObject.release() return - class VehicleDetection(mp.Process): + global inputQueue + global vehicleDetectionQueue def __init__(self,name='Vehicle Detection Process',custom_id=1): super(VehicleDetection,self).__init__() self.name = f'{name} {custom_id}' - - global inputQueue + def run(self): while (True): if(inputQueue.qsize() == 0): + vehicleDetectionQueue.put(None) + print(f'{self.name} exiting !! \n') return (frame,frameId) = inputQueue.get() - #inputQueue.task_done() - print(f"{self.name}Got frame with ID {frameId} qsize = {inputQueue.qsize()}\n") + print(f"{self.name} Got frame with ID {frameId} qsize = {inputQueue.qsize()}\n") #do some processing here. time.sleep(.5) - vehicleDetectionQueue.put_nowait((frame,frameId)) + vehicleDetectionQueue.put((frame,frameId)) class NumberPlateOcr(mp.Process): + global vehicleDetectionQueue + global outputQueue def __init__(self,name='Number plate OCR Process',custom_id=1): super(NumberPlateOcr,self).__init__() self.name=f'{name} {custom_id}' - global inputQueue - global vehicleDetectionQueue - global outputQueue - def run(self): while True: - (frame,frameId) = vehicleDetectionQueue.get() - #inputQueue.task_done() + value = vehicleDetectionQueue.get() + if(value == None): + print(f'{self.name} exiting !! \n') + outputQueue.put(None) + return + (frame,frameId) = value print(f"{self.name} Got frame with ID {frameId}\n") #do some processing here. time.sleep(.25) - outputQueue.put_nowait((frame,frameId)) - if((inputQueue.empty()) and (vehicleDetectionQueue.empty())): - return - - -class outputframe(th.Thread): - def __init__(self,name='output thread',custom_id=1): + outputQueue.put((frame,frameId)) + +class OutputFrame(th.Thread): + global IMAGE_HEIGHT,IMAGE_WIDTH + global outputQueue + def __init__(self,name='output thread',custom_id=1,outputfilename="output.avi"): super().__init__() self.name = f'{name} {custom_id}' - + self.outputfilename = outputfilename + print(f'frame size {IMAGE_HEIGHT.value} {IMAGE_WIDTH.value}') + self.videoWriterObject = cv2.VideoWriter(outputfilename,cv2.VideoWriter_fourcc(*'MJPG'),30,(IMAGE_WIDTH.value,IMAGE_HEIGHT.value)) + self.winName = "DISPLAY" + def run(self): + cv2.namedWindow(self.winName,cv2.WINDOW_NORMAL) while True: - (frame,frameId) = outputQueue.get() - print(f'{self.name} got frame {frameId}\n') - - - - + try: + value = outputQueue.get() + if(value == None): + return + (frame,frameId) = value + print(f'{self.name} got frame with ID {frameId} shape = {frame.shape}') + cv2.imshow(self.winName,frame) + self.videoWriterObject.write(frame) + except(AttributeError): + continue + + + if __name__ == '__main__': import cProfile @@ -97,16 +122,25 @@ if __name__ == '__main__': parser.add_argument('--realtime',help='Camera Connected Input') args = parser.parse_args() + #Name of the video file. + outputvideo = f'output {os.path.basename(args.video)[:-4]} {datetime.now()}.avi' + print(f'-----> Writing to file {outputvideo} <-------\n') #enable profiler here. app_profiler.enable() readFramesThread = ReadFrame(args.video) vehicleDetectionProcess = VehicleDetection() numberPlateOcrProcess = NumberPlateOcr() + outputframeThread = OutputFrame(outputfilename = outputvideo) readFramesThread.start() time.sleep(.25) vehicleDetectionProcess.start() numberPlateOcrProcess.start() + outputframeThread.start() + + print(f'{vehicleDetectionProcess.name} {vehicleDetectionProcess.pid} \n') + print(f'{numberPlateOcrProcess.name} {numberPlateOcrProcess.pid} \n') + readFramesThread.join() print(f'readframesthread {readFramesThread.is_alive()}\n') @@ -114,11 +148,14 @@ if __name__ == '__main__': print(f'vehicleDetectionProcess {vehicleDetectionProcess.is_alive()}\n') numberPlateOcrProcess.join() print(f'numberPlateOcrProcess {numberPlateOcrProcess.is_alive()}\n') + + outputframeThread.join() + print(f'{outputframeThread.name} {outputframeThread.is_alive()}') #disable profiler here. app_profiler.disable() - profile_name = str('temp.prof'.format(os.path.basename(args.video)[0:-4])) + profile_name = str('{}.prof'.format(os.path.basename(args.video)[0:-4])) print("------------------------\nEnd of execution, dumping profile stats\n-------------------------") app_profiler.dump_stats(profile_name) diff --git a/src/objectDetection.py b/src/objectDetection.py index 74ba71a..04456fc 100644 --- a/src/objectDetection.py +++ b/src/objectDetection.py @@ -115,7 +115,7 @@ class ObjectDetection(object): return boxes, confidences, classids - def run_object_detection(self,img,frameId,imageH,imageW,vehicleDetectionQueue,doPlotBoxNLabel = True): + def run_object_detection(self,img,imageH,imageW,doPlotBoxNLabel = True): status = True # Image preprocess - make RGB,Resize,Scale by 1/255 blob = cv.dnn.blobFromImage(img, 1 / 255.0, PARAMS._YOLOV3_OD_INPUT_IMAGE_SIZE, @@ -149,6 +149,5 @@ class ObjectDetection(object): if doPlotBoxNLabel: # Draw labels and boxes on the image img = self.draw_labels_and_boxes(img, boxes, confidences, classids, idxs, self.colors, self.labels) - - return img, frameId, boxes, confidences, classids, idxs, status - + + return img, boxes, confidences, classids, idxs,status diff --git a/src/ocrNumberPlate.py b/src/ocrNumberPlate.py index 61f963a..25e857c 100644 --- a/src/ocrNumberPlate.py +++ b/src/ocrNumberPlate.py @@ -160,7 +160,7 @@ def extract_all_number_plates_text(img, region_of_interests): # ============================================================================ -def get_number_plate_ocr_from_rois(img, np_rois_info,save_number_plate): +def get_number_plate_ocr_from_rois(img, np_rois_info, save_number_plate): np_roi_text_dict= {} for idx,roiinfo in enumerate(np_rois_info): conf, classID, roi = roiinfo diff --git a/src/trafficApp.py b/src/trafficApp.py index fec2026..b021ca7 100644 --- a/src/trafficApp.py +++ b/src/trafficApp.py @@ -1,16 +1,11 @@ import argparse import cv2 as cv - import numpy as np - +from tqdm import tqdm import os -import signal -import multiprocessing as mp os.environ['DISPLAY'] = ':0' - - from config.config import PARAMS from src.numberPlateRoiDetection import NumberPlateROIDetection from src.objectDetection import ObjectDetection @@ -18,9 +13,6 @@ from src.ocrNumberPlate import get_number_plate_ocr_from_rois from src.parkingDetection import ParkingDetection from src.trackingManager import TrackerManager -# TODO 2 processes for objectdetection using queues (queue1) -# TODO implement queue for taking frames, frameid from queue1 - class TrafficApp(object): def __init__(self,args): self.args = args @@ -36,16 +28,9 @@ class TrafficApp(object): if self.args.video is not None: self.vid_writer = None self.runVideoFlow() - + def runVideoFlow(self): frame_count = 0 - frame1_id=0 - frame2_id= 1 - - vehicleDetectionQueue = mp.Queue() - framesList=[] - outputfile = "output-{}.mp4".format(os.path.basename(args.video)[:-4]) - if args.video is not None: try: videoObj = cv.VideoCapture(args.video) @@ -82,85 +67,62 @@ class TrafficApp(object): print('Frames-{},Height-{}, Width-{}'.format(totalFrames,imgH,imgW)) if self.args.saveoutput and (imgH > 0 and imgW > 0): - self.vid_writer = cv.VideoWriter(outputfile, + self.vid_writer = cv.VideoWriter(self.outputfile, cv.VideoWriter_fourcc(*"MJPG"), 30, (round(imgW),round(imgH))) - + progress_bar=tqdm(total = totalFrames) # start reading frame - while True : - grabbed1, frame1 = videoObj.read() - grabbed2, frame2 = videoObj.read() - frame1_id+=1 - frame2_id+=2 - - + while True: + grabbed, frame = videoObj.read() #frame[:,450:,:] = 0 # end of frame - if not grabbed1: + if not grabbed: break frame_count +=1 + #print('Frame_count-',frame_count) #Use jump argument to skip frames. - # get object detection on this frame - objectDetectionProcess1=mp.Process(name='Object Detection Process 1',target=self.objectDetection.run_object_detection, args=(frame1.copy(),frame1_id,imgH,imgW,vehicleDetectionQueue)) - objectDetectionProcess2=mp.Process(name='Object Detection Process 2',target=self.objectDetection.run_object_detection, args=(frame2.copy(),frame2_id,imgH,imgW,vehicleDetectionQueue)) - - objectDetectionProcess1.start() - objectDetectionProcess2.start() - - print(f'{objectDetectionProcess1.name},{objectDetectionProcess1.pid},\n' ) - print(f'{objectDetectionProcess2.name},{objectDetectionProcess2.pid} \n' ) - - #print(f'Vehicle detection Queue size = {vehicleDetectionQueue.qsize()}') - #img_objectMarking, boxes, confidences, classids, idxs,status = self.objectDetection.run_object_detection(frame.copy(),imageH=imgH,imageW=imgW) - img, frameId, boxes, confidences, classids, idxs,status = vehicleDetectionQueue.get() - #append the frames and frameid. - framesList.append((img,frameId)) - print(f'frames stored = {len(framesList)} \n') - '''Assign Trackers''' - object_detect_info = [boxes, confidences, classids, idxs, status] - bbox_labels_tracking = self.parseObjDetectInfo(object_detect_info) - TrackerManager.FrameCount = frame_count - TrackerManager.manageTracker(bbox_labels_tracking) - - ''' Get Parking Status''' - if PARAMS._ALGO_MODE_PARKING: - self.parkingDetection.getParkingStatus(TrackerManager.TrackerList) - - '''Filter ROIs for Number Plate Detection''' - tentative_numberplate_rios = self.objectDetection.filterRoiforNumberPlate(boxes, classids, idxs) - - - ''' Get Number Plate ROI''' - detected_np_info = self.numberPlateDetection.run_number_plate_detection_rois(image=img,rois=tentative_numberplate_rios) - - - ''' Get Number plate OCR ''' - number_plate_ocr_dict = get_number_plate_ocr_from_rois(img,detected_np_info, False) - - objectDetectionProcess1.join() - objectDetectionProcess2.join() - - #Display frame - displayFrame = self.displayFrame(img,detected_np_info,number_plate_ocr_dict,object_detect_info) - # ISSUE how to kil the processes? New processes spawn on every iteration. - print(f'objectDetectionProcess1 is alive = {objectDetectionProcess1.is_alive()}\n') - print(f'objectDetectionProcess2 is alive = {objectDetectionProcess2.is_alive()}\n') - print("+++++++++++++++++++end of cycle++++++++++++++++++") - - #winName = 'YOLOV3 Object Detection' - #cv.namedWindow(winName, cv.WINDOW_NORMAL) - #cv.imshow(winName, displayFrame) - #cv.resizeWindow('objectDetection',680,420) - if self.vid_writer: - self.vid_writer.write(displayFrame.astype(np.uint8)) - c = cv.waitKey(1) - if c & 0xFF == ord('q'): - self.vid_writer.release() - videoObj.release() - break - + if (frame_count % self.args.jump == 0): + + # get object detection on this frame + img_objectMarking, boxes, confidences, classids, idxs,status = self.objectDetection.run_object_detection(frame.copy(),imageH=imgH,imageW=imgW) + '''Assign Trcakers''' + object_detect_info = [boxes, confidences, classids, idxs, status] + bbox_labels_tracking = self.parseObjDetectInfo(object_detect_info) + TrackerManager.FrameCount = frame_count + TrackerManager.manageTracker(bbox_labels_tracking) + + ''' Get Parking Status''' + if PARAMS._ALGO_MODE_PARKING: + self.parkingDetection.getParkingStatus(TrackerManager.TrackerList) + + '''Filter ROIs for Number Plate Detection''' + tentative_numberplate_rios = self.objectDetection.filterRoiforNumberPlate(boxes, classids, idxs) + + + ''' Get Number Plate ROI''' + detected_np_info = self.numberPlateDetection.run_number_plate_detection_rois(image=frame.copy(),rois=tentative_numberplate_rios) + + + ''' Get Number plate OCR ''' + number_plate_ocr_dict = get_number_plate_ocr_from_rois(frame.copy(),detected_np_info, False) + + #Display frame + displayFrame = self.displayFrame(frame.copy(),detected_np_info,number_plate_ocr_dict,object_detect_info) + + winName = 'YOLOV3 Object Detection' + cv.namedWindow(winName, cv.WINDOW_NORMAL) + cv.imshow(winName, displayFrame) + cv.resizeWindow('objectDetection',680,420) + if self.vid_writer: + self.vid_writer.write(displayFrame.astype(np.uint8)) + c = cv.waitKey(1) + if c & 0xFF == ord('q'): + self.vid_writer.release() + videoObj.release() + break + progress_bar.close() def parseObjDetectInfo(self,object_roi_info): boxes, confidences, classids, idxs, status = object_roi_info #[[list of bbox ][list of conf and labels]] @@ -285,7 +247,7 @@ class TrafficApp(object): if __name__ == '__main__': - import cProfile + import cProfile, pstats app_profiler = cProfile.Profile() parser = argparse.ArgumentParser(description='BitSilica Traffic Analysis Solution') @@ -294,14 +256,12 @@ if __name__ == '__main__': parser.add_argument('--realtime', help='Camera Connected Input') parser.add_argument('--target', type=str,default = 'CPU',help='Target for CNN to run') parser.add_argument('--saveoutput',type=bool,default=True, help='save video or not') - parser.add_argument('--debug',type=bool,default=False, help='print time taken by function') + parser.add_argument('--outputfile',type=str,default='./result.avi', help='save video path') + parser.add_argument('--debug',type=bool,default=False, help='print time taken by function') + parser.add_argument('--jump',type=int,default=1,help='integer value for jumping frames') args = parser.parse_args() - #enable profiler here. app_profiler.enable() - app = TrafficApp(args = args) - - #disable profiler here. app_profiler.disable() - profile_name = str('{}.prof'.format(os.path.basename(args.video)[0:-4])) + profile_name = str('profile_info-{}.prof'.format(args.jump)) app_profiler.dump_stats(profile_name) \ No newline at end of file