object detect on external rtsp feed pic max

2024-01-25 14:10

Hi All
I need a bit of help . I want to infer an external rtsp stream using yolo5 on my pico max. What i have at the moment is use the RKNN Model Inference Test and ffmpeg to grab images and put on the sd card. My problem is that it is very slow and only infers ever second and i would like to get to may 10 frames.

Anyone know of a better way to do this. I tried the opencv-mobile but it does not support RTSP. Python3 in buildroot does not have a opencv module either.
I think i am stuck with trying to do it with RKNN Model Inference but i cant figure out how to pull a rtsp stream without ffmepg libarys or Opencv librarys.

Any help would be great

Thanks

Robby

2024-01-31 9:08

Hi Guys

Hope this helps other people. I have managed to get an rtsp stream on my pico MAX using the following. The next step is to add the RKNN infering but I dont have enough C skills so if anyone can build on this or add, please go ahead.

I could be wrong but this is the closest i have come so far.

I suggest first doing the RKNN tutorial and making a basic cc program first to understand the build script.

First you need to buildroot and add :
ffmpeg
jpeglib

Then build a new image for you pico. I have only done this on the Pico Max.

Then i used the RKNN Model Inference tutorial and build my own C program.

Here is a copy of the CmakeLists.txt

Code: Select all

cmake_minimum_required(VERSION 3.4.1)

project(apex)



set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wl,--allow-shlib-undefined")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wl,--allow-shlib-undefined")

# install target and libraries
set(CMAKE_INSTALL_PREFIX ${CMAKE_SOURCE_DIR}/install/apex_${CMAKE_SYSTEM_NAME})

set(CMAKE_SKIP_INSTALL_RPATH FALSE)
set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")

# rknn api
set(RKNN_API_PATH ${CMAKE_SOURCE_DIR}/../../../runtime/RV1106/${CMAKE_SYSTEM_NAME}/librknn_api)
set(RKNN_RT_LIB ${RKNN_API_PATH}/armhf/librknnmrt.so)

include_directories(${RKNN_API_PATH}/include)
include_directories(${CMAKE_SOURCE_DIR}/../../3rdparty)
include_directories(/home/Projects/luckfox/luckfox-pico/sysdrv/source/buildroot/buildroot-2023.02.6/output/host/arm-buildroot-linux-uclibcgnueabihf/sysroot/usr/include)

link_directories(/home/Projects/luckfox/luckfox-pico/sysdrv/source/buildroot/buildroot-2023.02.6/output/host/arm-buildroot-linux-uclibcgnueabihf/sysroot/usr/lib)

# Apex
include_directories(${CMAKE_SOURCE_DIR}/include)


add_executable(apex
            src/main.cc
            src/postprocess.cc
)

target_link_libraries(apex avformat avcodec avutil jpeg
${RKNN_RT_LIB}
)

# install target and libraries
set(CMAKE_INSTALL_PREFIX ${CMAKE_SOURCE_DIR}/install/apex_${CMAKE_SYSTEM_NAME})
install(TARGETS apex DESTINATION ./)

install(PROGRAMS ${RKNN_RT_LIB} DESTINATION lib)
install(DIRECTORY model DESTINATION ./)

And here is a copy of my .cc file.

Code: Select all

#include "rknn_api.h"

#include <float.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/time.h>
#include <vector>
#include <iostream>
#include <filesystem>
#include <string>
#include <stdio.h>
#include <dirent.h>
#include <string.h>
#include <unistd.h>
#include <jpeglib.h>

extern "C"{
    #include <libavformat/avformat.h>
    #include <libavcodec/avcodec.h>
}




// #define STB_IMAGE_IMPLEMENTATION
// #include "stb/stb_image.h"
// #define STB_IMAGE_RESIZE_IMPLEMENTATION
// #include <stb/stb_image_resize.h>

// #include "postprocess.h"

// #define PERF_WITH_POST 1

//Using Hikvision Camera
#define RTSP_URL "rtsp://xxx:xxx@10.0.0.86:554/Streaming/Channels/102"


/*-------------------------------------------
                  Main Functions
-------------------------------------------*/

int main() {
    av_register_all();

    // Initialize network components (needed for RTSP)
    avformat_network_init();

    AVFormatContext *formatContext = avformat_alloc_context();

    // Open the RTSP stream
    if (avformat_open_input(&formatContext, RTSP_URL, NULL, NULL) != 0) {
        fprintf(stderr, "Error opening RTSP stream\n");
        return 1;
    }

    // Find stream information
    if (avformat_find_stream_info(formatContext, NULL) < 0) {
        fprintf(stderr, "Error finding stream information\n");
        avformat_close_input(&formatContext);
        avformat_network_deinit();
        return 1;
    }

    // Find the first video stream
    int videoStreamIndex = -1;
    for (int i = 0; i < formatContext->nb_streams; i++) {
        if (formatContext->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
            videoStreamIndex = i;
            break;
        }
    }

    if (videoStreamIndex == -1) {
        fprintf(stderr, "Error finding video stream\n");
        avformat_close_input(&formatContext);
        avformat_network_deinit();
        return 1;
    }

    AVCodecContext *codecContext = avcodec_alloc_context3(NULL);
    avcodec_parameters_to_context(codecContext, formatContext->streams[videoStreamIndex]->codecpar);

    AVCodec *codec = avcodec_find_decoder(codecContext->codec_id);
    if (!codec) {
        fprintf(stderr, "Error finding codec\n");
        avcodec_free_context(&codecContext);
        avformat_close_input(&formatContext);
        avformat_network_deinit();
        return 1;
    }

    if (avcodec_open2(codecContext, codec, NULL) < 0) {
        fprintf(stderr, "Error opening codec\n");
        avcodec_free_context(&codecContext);
        avformat_close_input(&formatContext);
        avformat_network_deinit();
        return 1;
    }

    AVPacket packet;
    av_init_packet(&packet);

    AVFrame *frame = av_frame_alloc();

    printf("Ready to receive frames...\n");

    while (av_read_frame(formatContext, &packet) >= 0) {
        if (packet.stream_index == videoStreamIndex) {
            // Decode video frame
            if (avcodec_send_packet(codecContext, &packet) == 0) {
                while (avcodec_receive_frame(codecContext, frame) == 0) {
                    // Print frame information
                    printf("Received frame: width=%d, height=%d\n", frame->width, frame->height);

                    // Save the frame to an image file (you can implement this function)
                    //save_frame_to_image(frame->data[0], frame->linesize[0] * frame->height, frame->width, frame->height);

                    // Sleep for a short duration (adjust as needed)
                    usleep(10000);  // 10ms
                }
            }
        }

        av_packet_unref(&packet);
    }

    printf("Finished receiving frames...\n");

    av_frame_free(&frame);
    avcodec_free_context(&codecContext);
    avformat_close_input(&formatContext);

    // Clean up
    avformat_network_deinit();

    return 0;
}

Hope this helps someone

Thanks
Robby

2024-01-31 9:22

Congratulations on successfully obtaining an RTSP stream on your pico MAX! I am sure that the method you've shared will be of great help to many others in the community.