Extract and analyze frames while recording using PiCamera, OpenCV

Question

I'm streaming the video from my raspberryPi using piCamera to a web socket, so that I can view it within my local network.

I want to make my own motion detection script from scratch, therefore I want to get the first image from the video stream (which is going to be the plain background) then compare with a function next frames to the first one to check whether something has changed (I have written those functions separately), I am not really worrying about efficiency here.

MAIN ISSUE: I want to get the data from those frames in a BytesIO object, then convert them to a 2D numpy array in B&W so I can perform operations. All this while keeping the stream going (I have in fact reduced the frame rate to 4 per second to make it run faster on my computer).

PROBLEM ENCOUNTERED WITH THE FOLLOWING CODE: One part of the problem that I have identified is that the numbers are way off. In my settings my camera to have a resolution of around 640*480 (= 307,200 length numpy array pixels data in B&W) whereas my computations in len() return less that 100k pixels.

    def main():
    print('Initializing camera')
    base_image = io.BytesIO()
    image_captured = io.BytesIO()
    with picamera.PiCamera() as camera:
        camera.resolution = (WIDTH, HEIGHT)
        camera.framerate = FRAMERATE
        camera.vflip = VFLIP # flips image rightside up, as needed
        camera.hflip = HFLIP # flips image left-right, as needed
        sleep(1) # camera warm-up time
        print('Initializing websockets server on port %d' % WS_PORT)
        WebSocketWSGIHandler.http_version = '1.1'
        websocket_server = make_server(
            '', WS_PORT,
            server_class=WSGIServer,
            handler_class=WebSocketWSGIRequestHandler,
            app=WebSocketWSGIApplication(handler_cls=StreamingWebSocket))
        websocket_server.initialize_websockets_manager()
        websocket_thread = Thread(target=websocket_server.serve_forever)
        print('Initializing HTTP server on port %d' % HTTP_PORT)
        http_server = StreamingHttpServer()
        http_thread = Thread(target=http_server.serve_forever)
        print('Initializing broadcast thread')
        output = BroadcastOutput(camera)
        broadcast_thread = BroadcastThread(output.converter, websocket_server)
        print('Starting recording')
        camera.start_recording(output, 'yuv')

        try:
            print('Starting websockets thread')
            websocket_thread.start()
            print('Starting HTTP server thread')
            http_thread.start()
            print('Starting broadcast thread')
            broadcast_thread.start()
            time.sleep(0.5)

            camera.capture(base_image, use_video_port=True, format='jpeg')

            base_data = np.asarray(bytearray(base_image.read()), dtype=np.uint64)

            base_img_matrix = cv2.imdecode(base_data, cv2.IMREAD_GRAYSCALE)

            while True:
                camera.wait_recording(1)

                #insert here the code for frame analysis

                camera.capture(image_captured, use_video_port=True, format='jpeg')
                data_next = np.asarray(bytearray(image_captured.read()), dtype=np.uint64)
                next_img_matrix = cv2.imdecode(data_next, cv2.IMREAD_GRAYSCALE)

                monitor_changes(base_img_matrix, next_img_matrix)

        except KeyboardInterrupt:
            pass
        finally:
            print('Stopping recording')
            camera.stop_recording()
            print('Waiting for broadcast thread to finish')
            broadcast_thread.join()
            print('Shutting down HTTP server')
            http_server.shutdown()
            print('Shutting down websockets server')
            websocket_server.shutdown()
            print('Waiting for HTTP server thread to finish')
            http_thread.join()
            print('Waiting for websockets thread to finish')
            websocket_thread.join()


if __name__ == '__main__':
    main()

score 1 · Answer 1 · answered Mar 20 '20 at 11:21

Solved, basically the problem was all in the way I was handling data and BytesIO files. First of all I needed to use unsigned int8 as type of the file to decode id. Then I have switched to np.frombuffer to read the files in its entirety, because the base image is not going to change, hence it will read always the same thing, and the next one will be inizialized and eliminated in every while loop. Also I can replace cv2.IMREAD_GRAYSCALE with 0 in the function.

camera.start_recording(output, 'yuv')
base_image = io.BytesIO()

    try:
        print('Starting websockets thread')
        websocket_thread.start()
        print('Starting HTTP server thread')
        http_thread.start()
        print('Starting broadcast thread')
        broadcast_thread.start()
        time.sleep(0.5)

        camera.capture(base_image, use_video_port=True, format='jpeg')

        base_data = np.frombuffer(base_image.getvalue(), dtype=np.uint8)

        base_img_matrix = cv2.imdecode(base_data, 0)

        while True:
            camera.wait_recording(0.25)
            image_captured = io.BytesIO()

            #insert here the code for frame analysis

            camera.capture(image_captured, use_video_port=True, format='jpeg')
            data_next = np.frombuffer(image_captured.getvalue(), dtype=np.uint8)
            next_img_matrix = cv2.imdecode(data_next, cv2.IMREAD_GRAYSCALE)

            monitor_changes(base_img_matrix, next_img_matrix)
            image_captured.close()

Extract and analyze frames while recording using PiCamera, OpenCV

1 Answers1