When running the sc3336 camera, the first frame is always dropped

2025-02-10 8:45

The camera is turned on for 1 second and then turned off. The program uses VIChn to obtain 25 frames of images.

However, by capturing the camera output through an oscilloscope, it can be seen that the camera outputs 26 frames of images.

Please ask what may be the cause of this.

2025-02-10 11:32

Hello, can you provide a test program? Theoretically, frames will be lost when the instantaneous bitrate exceeds the ISP threshold, but if you are not sure whether you are getting all the frame data accurately for 1 second, you can determine the duration of 1 frame

2025-02-11 2:17

I can reproduce this problem using the official "simple_vi_venc" example.

Will the image data obtained from "RK_MPI_VI_GetChnFrame" pass through ISP?

Through the oscilloscope, the duration of one frame is measured to be 40ms.

I can't upload the file, so I have to paste the code here.

Code: Select all


#include <errno.h>
#include <pthread.h>
#include <signal.h>
#include <stdio.h>
#include <sys/poll.h>
#include <time.h>
#include <unistd.h>

#include "sample_comm.h"

static FILE *file;

static RK_S32 g_s32FrameCnt = -1;
static bool quit = false;

static void sigterm_handler(int sig) {
	fprintf(stderr, "signal %d\n", sig);
	quit = true;
}

RK_U64 TEST_COMM_GetNowUs() {
	struct timespec time = {0, 0};
	clock_gettime(CLOCK_MONOTONIC, &time);
	return (RK_U64)time.tv_sec * 1000000 + (RK_U64)time.tv_nsec / 1000; /* microseconds */
}

static void *GetMediaBuffer0(void *arg) {
	(void)arg;
	printf("========%s========\n", __func__);
	void *pData = RK_NULL;
	int loopCount = 1;
	int s32Ret;
	char jpeg_path[128];
	VENC_STREAM_S stFrame;
	stFrame.pstPack = malloc(sizeof(VENC_PACK_S));

	while (!quit) {
		memset(jpeg_path, 0, sizeof(jpeg_path));
		sprintf(jpeg_path, "image_%d.jpeg", loopCount);
		file = fopen(jpeg_path, "w");

		s32Ret = RK_MPI_VENC_GetStream(0, &stFrame, 1000);
		printf("s32Ret %d loopCount %d\n", s32Ret, loopCount);
		if (s32Ret == RK_SUCCESS) {
			if (file) {
				pData = RK_MPI_MB_Handle2VirAddr(stFrame.pstPack->pMbBlk);
				fwrite(pData, 1, stFrame.pstPack->u32Len, file);
				fflush(file);
			}
			RK_U64 nowUs = TEST_COMM_GetNowUs();

			RK_LOGD("chn:0, loopCount:%d enc->seq:%d wd:%d pts=%lld delay=%lldus\n",
			        loopCount, stFrame.u32Seq, stFrame.pstPack->u32Len,
			        stFrame.pstPack->u64PTS, nowUs - stFrame.pstPack->u64PTS);

			s32Ret = RK_MPI_VENC_ReleaseStream(0, &stFrame);
			if (s32Ret != RK_SUCCESS) {
				RK_LOGE("RK_MPI_VENC_ReleaseStream fail %x", s32Ret);
			}
		} else {
			RK_LOGE("RK_MPI_VI_GetChnFrame fail %x", s32Ret);
		}
		loopCount++;

		if ((g_s32FrameCnt >= 0) && (loopCount > g_s32FrameCnt)) {
			system("i2ctransfer -f -y 4 w3@0x30 0x01 0x00 0x00");
		}

		if ((g_s32FrameCnt >= 0) && (loopCount > g_s32FrameCnt + 10)) {
			quit = true;
			break;
		}

		usleep(10 * 1000);
	}

	if (file)
		fclose(file);

	free(stFrame.pstPack);
	return NULL;
}

static RK_S32 test_venc_init(int chnId, int width, int height, RK_CODEC_ID_E enType) {
	printf("========%s========\n", __func__);
	VENC_RECV_PIC_PARAM_S stRecvParam;
	VENC_RC_PARAM_S stRcParam;
	VENC_CHN_ATTR_S stAttr;
	memset(&stAttr, 0, sizeof(VENC_CHN_ATTR_S));

	stAttr.stVencAttr.enType = enType;
	stAttr.stVencAttr.enPixelFormat = RK_FMT_YUV420SP;
	stAttr.stVencAttr.u32PicWidth = width;
	stAttr.stVencAttr.u32PicHeight = height;
	stAttr.stVencAttr.u32VirWidth = width;
	stAttr.stVencAttr.u32VirHeight = height;
	stAttr.stVencAttr.u32StreamBufCnt = 2;
	stAttr.stVencAttr.u32BufSize = width * height ;
	stAttr.stVencAttr.enMirror = MIRROR_NONE;
	stAttr.stRcAttr.enRcMode = VENC_RC_MODE_MJPEGFIXQP;
	stAttr.stRcAttr.stMjpegFixQp.u32Qfactor = 95;
	RK_MPI_VENC_CreateChn(chnId, &stAttr);
	stRcParam.stParamMjpeg.u32Qfactor = 95;
	RK_MPI_VENC_SetRcParam(chnId, &stRcParam);


	memset(&stRecvParam, 0, sizeof(VENC_RECV_PIC_PARAM_S));
	stRecvParam.s32RecvPicNum = -1;
	RK_MPI_VENC_StartRecvFrame(chnId, &stRecvParam);

	return 0;
}

// demo板dev默认都是0，根据不同的channel 来选择不同的vi节点
int vi_dev_init() {
	printf("%s\n", __func__);
	int ret = 0;
	int devId = 0;
	int pipeId = devId;

	VI_DEV_ATTR_S stDevAttr;
	VI_DEV_BIND_PIPE_S stBindPipe;
	memset(&stDevAttr, 0, sizeof(stDevAttr));
	memset(&stBindPipe, 0, sizeof(stBindPipe));
	// 0. get dev config status
	ret = RK_MPI_VI_GetDevAttr(devId, &stDevAttr);
	if (ret == RK_ERR_VI_NOT_CONFIG) {
		// 0-1.config dev
		ret = RK_MPI_VI_SetDevAttr(devId, &stDevAttr);
		if (ret != RK_SUCCESS) {
			printf("RK_MPI_VI_SetDevAttr %x\n", ret);
			return -1;
		}
	} else {
		printf("RK_MPI_VI_SetDevAttr already\n");
	}
	// 1.get dev enable status
	ret = RK_MPI_VI_GetDevIsEnable(devId);
	if (ret != RK_SUCCESS) {
		// 1-2.enable dev
		ret = RK_MPI_VI_EnableDev(devId);
		if (ret != RK_SUCCESS) {
			printf("RK_MPI_VI_EnableDev %x\n", ret);
			return -1;
		}
		// 1-3.bind dev/pipe
		stBindPipe.u32Num = pipeId;
		stBindPipe.PipeId[0] = pipeId;
		ret = RK_MPI_VI_SetDevBindPipe(devId, &stBindPipe);
		if (ret != RK_SUCCESS) {
			printf("RK_MPI_VI_SetDevBindPipe %x\n", ret);
			return -1;
		}
	} else {
		printf("RK_MPI_VI_EnableDev already\n");
	}

	return 0;
}

int vi_chn_init(int channelId, int width, int height) {
	int ret;
	int buf_cnt = 2;
	// VI init
	VI_CHN_ATTR_S vi_chn_attr;
	memset(&vi_chn_attr, 0, sizeof(vi_chn_attr));
	vi_chn_attr.stIspOpt.u32BufCount = buf_cnt;
	vi_chn_attr.stIspOpt.enMemoryType =
	    VI_V4L2_MEMORY_TYPE_DMABUF; // VI_V4L2_MEMORY_TYPE_MMAP;
	vi_chn_attr.stSize.u32Width = width;
	vi_chn_attr.stSize.u32Height = height;
	vi_chn_attr.enPixelFormat = RK_FMT_YUV420SP;
	vi_chn_attr.enCompressMode = COMPRESS_MODE_NONE; // COMPRESS_AFBC_16x16;
	vi_chn_attr.u32Depth = 2;
	ret = RK_MPI_VI_SetChnAttr(0, channelId, &vi_chn_attr);
	ret |= RK_MPI_VI_EnableChn(0, channelId);
	if (ret) {
		printf("ERROR: create VI error! ret=%d\n", ret);
		return ret;
	}

	return ret;
}

static RK_CHAR optstr[] = "?::w:h:c:I:e:o:";
static void print_usage(const RK_CHAR *name) {
	printf("usage example:\n");
	printf("\t%s -I 0 -w 1920 -h 1080 -o /tmp/venc.h264\n", name);
	printf("\t-w | --width: VI width, Default:1920\n");
	printf("\t-h | --heght: VI height, Default:1080\n");
	printf("\t-c | --frame_cnt: frame number of output, Default:150\n");
	printf("\t-I | --camid: camera ctx id, Default 0. "
	       "0:rkisp_mainpath,1:rkisp_selfpath,2:rkisp_bypasspath\n");
	printf("\t-e | --encode: encode type, Default:h264, Value:h264, h265, mjpeg\n");
	printf("\t-o: output path, Default:NULL\n");
}

int main(int argc, char *argv[]) {
	RK_S32 s32Ret = RK_FAILURE;
	RK_U32 u32Width = 1920;
	RK_U32 u32Height = 1080;
	RK_CHAR *pOutPath = NULL;
	RK_CODEC_ID_E enCodecType = RK_VIDEO_ID_AVC;
	RK_CHAR *pCodecName = "H264";
	RK_S32 s32chnlId = 0;
	int c;

	while ((c = getopt(argc, argv, optstr)) != -1) {
		switch (c) {
		case 'w':
			u32Width = atoi(optarg);
			break;
		case 'h':
			u32Height = atoi(optarg);
			break;
		case 'I':
			s32chnlId = atoi(optarg);
			break;
		case 'c':
			g_s32FrameCnt = atoi(optarg);
			break;
		case 'e':
			if (!strcmp(optarg, "h264")) {
				enCodecType = RK_VIDEO_ID_AVC;
				pCodecName = "H264";
			} else if (!strcmp(optarg, "h265")) {
				enCodecType = RK_VIDEO_ID_HEVC;
				pCodecName = "H265";
			} else if (!strcmp(optarg, "mjpeg")) {
				enCodecType = RK_VIDEO_ID_MJPEG;
				pCodecName = "MJPEG";
			} else {
				printf("ERROR: Invalid encoder type.\n");
				return 0;
			}
			break;
		case 'o':
			pOutPath = optarg;
			break;
		case '?':
		default:
			print_usage(argv[0]);
			return 0;
		}
	}

	printf("#CodecName:%s\n", pCodecName);
	printf("#Resolution: %dx%d\n", u32Width, u32Height);
	printf("#Output Path: %s\n", pOutPath);
	printf("#CameraIdx: %d\n\n", s32chnlId);
	printf("#Frame Count to save: %d\n", g_s32FrameCnt);

	// if (pOutPath) {
	// 	venc0_file = fopen(pOutPath, "w");
	// 	if (!venc0_file) {
	// 		printf("ERROR: open file: %s fail, exit\n", pOutPath);
	// 		return 0;
	// 	}
	// }
	signal(SIGINT, sigterm_handler);

	if (RK_MPI_SYS_Init() != RK_SUCCESS) {
		RK_LOGE("rk mpi sys init fail!");
		goto __FAILED;
	}

	vi_dev_init();
	vi_chn_init(s32chnlId, u32Width, u32Height);

	// venc  init
	test_venc_init(0, u32Width, u32Height,
	               enCodecType); // RK_VIDEO_ID_AVC RK_VIDEO_ID_HEVC

	MPP_CHN_S stSrcChn, stDestChn;
	// bind vi to venc
	stSrcChn.enModId = RK_ID_VI;
	stSrcChn.s32DevId = 0;
	stSrcChn.s32ChnId = s32chnlId;

	stDestChn.enModId = RK_ID_VENC;
	stDestChn.s32DevId = 0;
	stDestChn.s32ChnId = 0;
	printf("====RK_MPI_SYS_Bind vi0 to venc0====\n");
	s32Ret = RK_MPI_SYS_Bind(&stSrcChn, &stDestChn);
	if (s32Ret != RK_SUCCESS) {
		RK_LOGE("bind 0 ch venc failed");
		goto __FAILED;
	}

	pthread_t main_thread;
	pthread_create(&main_thread, NULL, GetMediaBuffer0, NULL);

	while (!quit) {
		usleep(50000);
	}
	pthread_join(&main_thread, NULL);

	s32Ret = RK_MPI_SYS_UnBind(&stSrcChn, &stDestChn);
	if (s32Ret != RK_SUCCESS) {
		RK_LOGE("RK_MPI_SYS_UnBind fail %x", s32Ret);
	}

	s32Ret = RK_MPI_VI_DisableChn(0, s32chnlId);
	RK_LOGE("RK_MPI_VI_DisableChn %x", s32Ret);

	s32Ret = RK_MPI_VENC_StopRecvFrame(0);
	if (s32Ret != RK_SUCCESS) {
		return s32Ret;
	}

	s32Ret = RK_MPI_VENC_DestroyChn(0);
	if (s32Ret != RK_SUCCESS) {
		RK_LOGE("RK_MPI_VDEC_DestroyChn fail %x", s32Ret);
	}

	s32Ret = RK_MPI_VI_DisableDev(0);
	RK_LOGE("RK_MPI_VI_DisableDev %x", s32Ret);

__FAILED:
	RK_LOGE("test running exit:%d", s32Ret);
	RK_MPI_SYS_Exit();

	return 0;
}

2025-02-11 3:36

The test program you provided seems to be just an example of transmitting VI to VENC and encoding it as JPEG, with modifications to the program's exit conditions. I still cannot confirm whether your frame data is acquired within the standard 1 second.

From the oscilloscope, the time to acquire one frame is 40ms, meaning at most 25 frames can be acquired in 1 second, which aligns with the driver settings. The 26 frames captured by the oscilloscope might be due to inaccurate timing, causing frame data to be output but not counted by the program. Since I don’t have your source code, I cannot determine the exact reason.

The images obtained by RK_MPI_VI_GetChnFrame are all processed by the ISP. If RKAIQ is not started, the ISP will not work, and the acquired image will be close to RAW format.

2025-02-11 5:17

My question is not how many frames the camera outputs per second, and from the first oscilloscope photo it can be confirmed that the camera outputs 25 fps per second.

My question is why the camera outputs 26 frames, but I can only get 25 frames of data from VI.

2025-02-11 8:07

I'm sorry I misunderstood your question, but please provide your test program or command and test method, and only if I can reproduce the phenomenon in the same software environment can solve the problem.
The program you provided above does not correspond to the phenomenon in the first post, the initial value of loopCount in the second post is 1, and the print result is also as expected, I don't understand what the problem is?

2025-02-11 8:30

The code in the second post is modified by me according to the official example, and the problem exposed is essentially the same as the first post.

After correctly acquiring one frame of image data in the program, the "i2ctransfer" command is used to force the camera to close. However, the program does not stop, and the VI can receive data normally.

Through the oscilloscope, it can be seen that the camera outputs two frames of data.

However, only one frame of image is successfully acquired from the VI, and the rest are failures.

This indicates that the VI acquires the second frame of image, and the first frame of image is lost.

2025-02-11 14:06

Based on my tests, whether using RKMPI's VI to capture images or using V4L2, there is always a situation where the waveform frame count is greater than the software-configured frame count. This is because when V4L2 sends the streamon command, it triggers an I2C command to the sensor, instructing it to continuously send data. To stop the data flow, V4L2 needs to control the I2C to send a streamoff command.

During the process where the program calls V4L2, V4L2 controls I2C, and I2C sends the streamoff command, the system takes time. Compared to the speed of CSI, this delay is quite noticeable. As a result, even after capturing the last frame and sending streamoff, the oscilloscope can still detect CSI data.

You can add a delay before calling:
s32Ret = RK_MPI_VI_DisableChn(0, s32chnlId);
This will make the phenomenon even more noticeable.

2025-02-12 2:19

可能我还是没有表达清楚我测试的情况。

我是在从VI中获取一帧图像数据后，立即使用"i2ctransfer"命令直接将相机关闭，相机不再输出图像数据，没有调用任何API关闭相关通道。

Code: Select all

system("i2ctransfer -f -y 4 w3@0x30 0x01 0x00 0x00");

此时程序是正常运行的，VI也是正常运行的。

按照我的理解: 相机一共发出了两帧图像数据，如果VI获取到了数据是相机发出的第一帧图像，那么即使相机关闭了，VI也应该能正常获取第二帧图像。

现在的现象是VI只获取到了一帧图像数据，我认为这帧数据是相机发出的第二帧数据。相机发出的第一帧数据不知道为什么VI没有收到。有可能是第一帧数据是错误的VI拒收了。

2025-02-12 6:45

您的测试程序中在VENC计数到指定数值后就会发送 i2c 强制摄像头停止然后再获取10帧后才推出程序，这里示波器已经获取到了第三帧的图像在中间被断开，根据您的推测此时理论上VI获取到了两帧数据存储在RKMPI的缓存区中等待发送到VENC中，实际现象只有一帧数据被成功编码，这是目前我对您问题的理解。

我注意到您是根据VENC帧数据是否成功被获取来计数的，我不确定当VI不能正常获取数据时是否会影响下一级VENC的编码，所以我通过VI帧数据是否被成功接收来判断。

: simple_vi_get_frame.7z (2.67 KiB)
Downloaded 17 times

执行命令

Code: Select all

./simple_vi_get_frame -I 0 -w 1920 -h 1080 -c 1 -o 1

结果时在第一帧结束时发送 i2ctransfer -f -y 4 w3@0x30 0x01 0x00 0x00 后程序继续进行，VI会成功获取下一个帧数据，示波器显示第三帧仅捕获到一半所以VI一共只能捕获两帧在第三帧时停止。

我一开始觉得第一帧可能因为要统计3A所以在rkmpi框架中被丢弃，但是获取RAW图数据时前面曝光不正确的帧也都被捕获了没有丢帧，所以我更倾向于第一帧有被获取，多出的帧数是软件框架已经退出捕获的猜测。

很遗憾的是 rockit 是闭源框架，所以在能够正常运行的情况下，即使首帧确实被框架使用导致没有被VI捕获，都无法通过修改库的方式来获取首帧，如果您有这方面的需求，建议直接向 Rockchip 反馈，我们并不参与rockit 库的开发维护，仅能对您的测试或问题提供建议。