FFmpeg+SDL实时解码和渲染H264视频流

技术分享 3年前 (2023-07-11) 0 999+

前言

之前实现了Android手机摄像头数据的TCP实时传输，今天接着聊聊，如何在PC端把接收到的H264视频流实时解码并渲染出来。这次使用的语言是C++，框架有FFmpeg和SDL2。

解码

解码部分使用FFmpeg，首先，需要初始化H264解码器：

int H264Decoder::init() {     codec = avcodec_find_decoder(AV_CODEC_ID_H264);     if (codec == nullptr) {         printf("No H264 decoder foundn");         return -1;     }     codecCtx = avcodec_alloc_context3(codec);     codecCtx->flags |= AV_CODEC_FLAG_LOW_DELAY;     if (avcodec_open2(codecCtx, codec, nullptr) < 0) {         printf("Failed to open codecn");         return -2;     }     packet = av_packet_alloc();     m_Frame = av_frame_alloc();     parser = av_parser_init(AV_CODEC_ID_H264);     return 0; }

然后，使用创建TCP连接到我们的Android端，读取数据包：

bool read_data(SOCKET socket, void* data, unsigned int len) {     while (len > 0) {         int ret = recv(socket, (char*)data, len, 0);         if (ret <= 0) {             return false;         }         len -= ret;         data = (char*)data + ret;     }     return true; }  bool read_int(SOCKET socket, ULONG* value) {     bool ret = read_data(socket, value, 4);     if (ret) {         *value = ntohl(*value);     }     return ret; }  int PacketReceiver::readPacket(unsigned char** data, unsigned long* size) {     ULONG pkgSize = 0;     bool ret = read_int(m_Socket, &pkgSize);     if (!ret) {         printf("Failed to read packet sizen");         return -1;     }     if (m_DataLen < pkgSize) {         if (m_Data != nullptr) {             delete[] m_Data;         }         m_Data = new unsigned char[pkgSize];         m_DataLen = pkgSize;     }     if (!read_data(m_Socket, m_Data, pkgSize)) {         printf("Failed to read packet datan");         return -2;     }     *data = m_Data;     *size = pkgSize;     return 0; }

再把每个数据包传送给H264解码器解码

int H264Decoder::decode(unsigned char* data, int size, AVFrame** frame) {     int new_pkg_ret = av_new_packet(packet, size);     if (new_pkg_ret != 0) {         printf("Failed to create new packetn");         return -1;     }     memcpy(packet->data, data, size);     int ret = avcodec_send_packet(codecCtx, packet);     if (ret < 0 && ret != AVERROR(EAGAIN)) {         printf("Failed to parse packetn");         return -1;     }     ret = avcodec_receive_frame(codecCtx, m_Frame);     if (ret == AVERROR(EAGAIN)) {         *frame = nullptr;         return 0;     }     if (ret != 0) {         printf("Failed to read framen");         return -1;     }     *frame = m_Frame;     av_packet_unref(packet);     return 0; }

解码器解码后，最终得到的是AVFrame对象，代表一帧画面，数据格式一般为YUV格式（跟编码端选择的像素格式有关）。

渲染

通过使用SDL2，我们可以直接渲染YUV数据，无需手动转成RGB。

首先，我们先初始化SDL2并创建渲染窗口：

int YuvRender::init(int video_width, int video_height) {     SDL_Init(SDL_INIT_VIDEO);     SDL_Rect bounds;     SDL_GetDisplayUsableBounds(0, &bounds);     int winWidth = video_width;     int winHeight = video_height;     if (winWidth > bounds.w || winHeight > bounds.h) {         float widthRatio = 1.0 * winWidth / bounds.w;         float heightRatio = 1.0 * winHeight / bounds.h;         float maxRatio = widthRatio > heightRatio ? widthRatio : heightRatio;         winWidth = int(winWidth / maxRatio);         winHeight = int(winHeight / maxRatio);     }     SDL_Window* window = SDL_CreateWindow(         "NetCameraViewer",         SDL_WINDOWPOS_UNDEFINED,         SDL_WINDOWPOS_UNDEFINED,         winWidth,         winHeight,         SDL_WINDOW_OPENGL     );     m_Renderer = SDL_CreateRenderer(window, -1, 0);     m_Texture = SDL_CreateTexture(         m_Renderer,         SDL_PIXELFORMAT_IYUV,         SDL_TEXTUREACCESS_STREAMING,         video_width,         video_height     );     m_VideoWidth = video_width;     m_VideoHeight = video_height;     m_Rect.x = 0;     m_Rect.y = 0;     m_Rect.w = winWidth;     m_Rect.h = winHeight;     return 0; }

每次解码出一帧画面的时候，再调用render函数渲染：

int YuvRender::render(unsigned char* data[], int pitch[]) {     int uvHeight = m_VideoHeight / 2;     int ySize = pitch[0] * m_VideoHeight;     int uSize = pitch[1] * uvHeight;     int vSize = pitch[2] * uvHeight;     int buffSize =  ySize + uSize + vSize;     if (m_FrameBufferSize < buffSize) {         if (m_FrameBuffer != nullptr) {             delete[] m_FrameBuffer;         }         m_FrameBuffer = new unsigned char[buffSize];         m_FrameBufferSize = buffSize;     }     SDL_memcpy(m_FrameBuffer, data[0], ySize);     SDL_memcpy(m_FrameBuffer + ySize, data[1], uSize);     SDL_memcpy(m_FrameBuffer + ySize + uSize, data[2], vSize);     SDL_UpdateTexture(m_Texture, NULL, m_FrameBuffer, pitch[0]);     SDL_RenderClear(m_Renderer);     SDL_RenderCopy(m_Renderer, m_Texture, NULL, &m_Rect);     SDL_RenderPresent(m_Renderer);     SDL_PollEvent(&m_Event);     if (m_Event.type == SDL_QUIT) {         exit(0);     }     return 0; }