FFMPEG SDK 开发介绍
1.简介:
ffmpeg是一套可以用来记录、转换数字音频、视频,并能将其转化为流的开源计算机程序。
使用ffmpeg能够完成如下功能:parse,demux,decode,filter(preprocessing),encode,mux,stream和player等.
2.下载和编译:
下载地址: http://ffmpeg.org/download.html
编译:
1)windows平台static library/shared library, 编译工具:mingw-gcc或者在linux平台下交叉编译(推荐)
2)linux平台static library/shared library, 编译工具:gcc
模块:
libavcodec - 编码解码器
libavdevice - 输入输出设备的支持
libavfilter - 视音频滤镜支持
libavformat - 视音频等格式的解析
libavutil - 工具库
libpostproc - 后期效果处理
libswscale - 图像颜色、尺寸转换
3.SDK介绍和开发(基于ffmpeg 0.8 sdk)
ffmpeg每部分功能都采用plugin的方式,使用统一的接口调用,这样就能够非常方便的使用和扩展。
plugin分为几种:muxer,demuxer,protocol,hwaccel,encoder,decoder,parser,bitstream,filter,...
因此在使用SDK的时候第一步就是注册plugin
avcodec_register_all() : 注册 hwaccel,encoder,decoder,parser,bitstream
av_register_all() : 注册 muxer,demuxer,protocol
avfilter_register_all() : 注册 滤镜filter
下面根据不同的应用场景,给出主要的代码示例(仅是代码片断,不一定能编译通过):
1)如何获取媒体文件的信息(Parser):
// 参考V3代码: interface IFileDecoder, media/impl/filedecoderimpl.cpp
{ av_register_all(); AVFormatContext * pFormatCtx = NULL; int err = 0; const char *fileName = "c:\\test.mp4"; err = av_open_input_file(&pFormatCtx, fileName, NULL, 0, NULL); if(err != 0) { // break ; } err = av_find_stream_info(pFormatCtx); if(err < 0) { // break ; } for(uint32_t i = 0; i < pFormatCtx->nb_streams; i ++) { // stream 结构数据 AVStream *pStream = pFormatCtx->streams[i]; // 帧率信息 AVRational frameRate = pStream->r_frame_rate; // 时间单位比率 AVRational timeBase = pStream->time_base; // stream duration int64_t duration = pStream->duration; // 获取Codec数据结构 AVCodecContext *pCodecCtx = pStream->codec; AVMediaType codecType = pCodecCtx->codec_type; /* 判断codec类型:Video/Audio/Subtitle... AVMEDIA_TYPE_VIDEO, AVMEDIA_TYPE_AUDIO, AVMEDIA_TYPE_DATA, AVMEDIA_TYPE_SUBTITLE, AVMEDIA_TYPE_ATTACHMENT, */ CodecID codecId = pCodecCtx->codec_id; /* // video codec CODEC_ID_H261, CODEC_ID_H263, CODEC_ID_RV10, CODEC_ID_RV20, // ... // audio codec CODEC_ID_MP2= 0x15000, CODEC_ID_MP3, CODEC_ID_AAC, CODEC_ID_AC3, CODEC_ID_DTS, // ... // subtitle codec CODEC_ID_DVD_SUBTITLE= 0x17000, CODEC_ID_DVB_SUBTITLE, CODEC_ID_TEXT, ///< raw UTF-8 text CODEC_ID_XSUB, CODEC_ID_SSA, CODEC_ID_MOV_TEXT, // ... */ if(codecType == AVMEDIA_TYPE_VIDEO) { // 获取Video基本信息 int width = pCodecCtx->width; int height = pCodecCtx->height; PixelFormat pixelFormat = pCodecCtx->pix_fmt; } else if(codecType == AVMEDIA_TYPE_AUDIO) { // 获取Audio基本信息 int channels = pCodecCtx->channels; int sample_rate = pCodecCtx->sample_rate; AVSampleFormat sampleFmt = pCodecCtx->sample_fmt; } } // 释放 if(pFormatCtx != NULL) { av_close_input_file(pFormatCtx); pFormatCtx = NULL; } }
2)读取sample数据(Read raw sample不解码)
// 参考V3代码: interface IFileDecoder, media/impl/filedecoderimpl.cpp
{ // 参考Parser代码 // av_register_all(); // AVFormatContext * pFormatCtx = NULL; // err = av_open_input_file(&pFormatCtx, fileName, NULL, 0, NULL); AVPacket packet; av_init_packet(&packet); int ret = av_read_frame(pFormatCtx, &packet); if(ret >= 0) { int streamIndex = packet.stream_index; AVStream *pStream = pFormatCtx->streams[streamIndex]; AVCodecContext *pCodecCtx = pStream->codec; // 计算timestamp // 转换时间到1/1000000秒 AVRational time_base; time_base.num = 1; time_base.den = 1000000; // 25.0 1/25, 29.97 1001/30000 // 获取 dts/pts const int64_t dts = av_rescale_q(packet.dts, pStream->time_base, time_base); const int64_t pts = av_rescale_q(packet.pts, pStream->time_base, time_base); uint8_t *data = packet.data; int size = packet.size; bool isKey = ((packet.flags & AV_PKT_FLAG_KEY) == AV_PKT_FLAG_KEY); } av_free_packet(&packet); }
3)解码sample(Video ES=>YUV/RGB, Audio ES=>PCM)
// 参考V3代码: interface IVideoDecoder/IAudioDecoder, media/impl/videodecoderimpl.cpp/audiodecoderimpl.cpp
{ // 参考Parser,Read raw sample代码 // AVMediaType codecType = pCodecCtx->codec_type; AVMediaType codecType = AVMEDIA_TYPE_VIDEO; // CodecId codecId = pCodecCtx->codec_id; CodecId codecId = CODEC_ID_H264; // 通过Codec ID查找解码器 AVCodec *pCodec = avcodec_find_decoder(codecId); // 分配codec关联结构 AVCodecContext *pCodecCtx = avcodec_alloc_context(); // 设置一些必要的信息 pCodecCtx->codec_type = AVMEDIA_TYPE_VIDEO / AVMEDIA_TYPE_AUDIO; pCodecCtx->codec_id = codecId; if(pCodec->capabilities & CODEC_CAP_TRUNCATED) pCodecCtx->flags |= CODEC_FLAG_TRUNCATED; // 在open codec时要加锁,否则多个codec同时打开时时会出现错误 gMutexFFmpeg.lock(); // 打开Codec avcodec_open(pCodecCtx, pCodec); gMutexFFmpeg.unlock(); if(codecType == AVMEDIA_TYPE_VIDEO) { AVFrame *pSrcFrame = avcodec_alloc_frame(); AVFrame *pDstFrame = avcodec_alloc_frame(); // 因为内存的原因,所以需要多分配一些数据, FF_INPUT_BUFFER_PADDING_SIZE uint8_t *data = ...; int size = ...; while(size > 0)) { AVPacket pkt; av_init_packet(&pkt); pkt.data = data; pkt.size = size; int frameFinished = 0; int bytesDecoded = avcodec_decode_video2(pCodecCtx, pSrcFrame, &frameFinished, &pkt); if(bytesDecoded > 0) { data += bytesDecoded; size -= bytesDecoded; } if(frameFinished) { int numBytes = avpicture_get_size(pCodecCtx->pix_fmt, pCodecCtx->width, pCodecCtx->height); uint8_t *pBuffer = new uint8_t[numBytes]; avpicture_fill((AVPicture *)pDstFrame, pBuffer, pCodecCtx->pix_fmt, pCodecCtx->width, pCodecCtx->height); av_picture_copy((AVPicture *)pDstFrame, (AVPicture *)pSrcFrame, pCodecCtx->pix_fmt, pCodecCtx->width, pCodecCtx->height); // pBuffer/numBytes/pCodecCtx->pix_fmt : YUV/RGB数据 delete []pBuffer; } if(bytesDecoded < 0) break ; } av_free(pSrcFrame); av_free(pDstFrame); } else if(codecType == AVMEDIA_TYPE_AUDIO) { // 分配解码内存空间 uint8_t *pBuffer = new uint8_t[AVCODEC_MAX_AUDIO_FRAME_SIZE]; // 因为内存的原因,所以需要多分配一些数据, FF_INPUT_BUFFER_PADDING_SIZE uint8_t *data = ...; int size = ...; while(size > 0) { AVPacket pkt; av_init_packet(&pkt); pkt.data = data; pkt.size = size; int outSize = AVCODEC_MAX_AUDIO_FRAME_SIZE; int bytesDecoded = avcodec_decode_audio3(pCodecCtx, (int16_t *)pBuffer, &outSize, &pkt); if(bytesDecoded > 0) { data += bytesDecoded; size -= bytesDecoded; } if((bytesDecoded >= 0) && (outSize > 0)) { // pBuffer/outSize : PCM数据 // 格式 // pCodecCtx->channels; // pCodecCtx->sample_fmt; // pCodecCtx->sample_rate; } } } gMutexFFmpeg.lock(); // 关闭和释放 avcodec_close(pCodecCtx); gMutexFFmpeg.unlock(); av_free(pCodecCtx); }
4)视音频编码(YUV/RGB=>Video ES, PCM=>Audio ES)
// 参考V3代码: media/videoencoder.cpp/audioencoder.cpp
{ // video encode avcodec_register_all(); // 查找编码器 AVCodec *avCodec = avcodec_find_encoder((CodecID)mConfig.codec); AVCodecContext *codecCtx = avcodec_alloc_context(); codecCtx->codec_type = AVMEDIA_TYPE_VIDEO; codecCtx->codec_id = (CodecID)mConfig.codec; codecCtx->width = mOutFormat.width; codecCtx->height = mOutFormat.height; codecCtx->pix_fmt = (PixelFormat)mOutFormat.pixelFormat; uint32 num = 0; uint32 den = 0; SampleUtil::FPS2Timescale(mOutFormat.frameRate, num, den); codecCtx->time_base.num = num; codecCtx->time_base.den = den; codecCtx->bit_rate = mConfig.bitRate*1000; codecCtx->max_b_frames = 0; codecCtx->gop_size = 100; if(codecCtx->codec_id == CODEC_ID_MPEG1VIDEO) { codecCtx->mb_decision = FF_MB_DECISION_RD; } else { codecCtx->mb_decision = FF_MB_DECISION_RD; } avcodec_open(codecCtx, avCodec); // 分配编码后的内存,分配为1MB mOutputBuffer.resize(1*1024*1024); AVFrame *pSrcFrame = avcodec_alloc_frame(); avcodec_get_frame_defaults(pSrcFrame); int ret = avpicture_fill((AVPicture *)pSrcFrame, (uint8_t *)inData.data, (PixelFormat)mOutFormat.pixelFormat, mOutFormat.width, mOutFormat.height); AVRational time_base; time_base.num = 1; time_base.den = 1000000; pSrcFrame->pts = av_rescale_q(inData.dts, time_base, codecCtx->time_base); int bytesWritten = avcodec_encode_video(codecCtx, (uint8 *)mOutputBuffer.data(), mOutputBuffer.size(), isEmpty ? NULL : pSrcFrame); outData.data = (char *)mOutputBuffer.data(); outData.size = bytesWritten; outData.isKey = (mCodecCtx->coded_frame->key_frame != 0); av_free(pSrcFrame); avcodec_close(codecCtx); av_free(codecCtx); // audio encode请看audioencoder.cpp 文件 }
5)图像格式转换(YUV/RGB <=> YUV/RGB & Resize)
// 参考代码: media/imageconverter.cpp
{ SwsContext *pSwsCtx = NULL; /* srcWidth - 源图像 width srcHeight - 源图像 height srcFmt - 源图像像素格式 enum PixelFormat dstWidth - 目标图像 width dstHeight - 目标图像 height dstFmt - 目标图像像素格式 enum PixelFormat */ // resize 算法 int swsFlags = SWS_LANCZOS; // SWS_FAST_BILINEAR; // 初始化 pSwsCtx = sws_getCachedContext(NULL, srcWidth, srcHeight, srcFmt, dstWidth, dstHeight, dstFmt, swsFlags, NULL, NULL, NULL); // 设置数据到结构 AVPicture AVPicture avSrcPic; AVPicture avDstPic; memset(&avSrcPic, 0, sizeof(avSrcPic)); memset(&avDstPic, 0, sizeof(avDstPic)); int dstRet = avpicture_fill(&avDstPic, (uint8_t *)pDstBuffer, dstFmt, dstWidth, dstHeight); { // pSrcBuffer - 源数据 // pDstBuffer - 目标数据 int srcRet = avpicture_fill(&avSrcPic, (uint8_t *)pSrcBuffer, srcFmt, srcWidth, srcHeight); // 执行转换 sws_scale(pSwsCtx, avSrcPic.data, avSrcPic.linesize, 0, abs(srcHeight), avDstPic.data, avDstPic.linesize); } // 释放 sws_freeContext(pSwsCtx); }
6)封装格式(Muxer, .mp4/.avi/.mkv...)
// 参考代码: interface IFileWriter, media/impl/filewriterimpl.cpp
{ av_register_all(); AVFormatContext * pFormatCtx; avformat_alloc_output_context2(&pFormatCtx, NULL, "mp4", "c:\\out.mp4"); { // new video stream AVStream * avStream = av_new_stream(pFormatCtx, pFormatCtx->nb_streams; avcodec_get_context_defaults3(avStream->codec, NULL); AVCodecContext *codecCtx = avStream->codec; codecCtx->codec_id = (CodecID)format->codecId; codecCtx->codec_type = AVMEDIA_TYPE_VIDEO; codecCtx->width = format->width; codecCtx->height = format->height; codecCtx->bit_rate = 800000; uint32 num = 0; uint32 den = 0; SampleUtil::FPS2Timescale(format->frameRate, num, den); codecCtx->time_base.num = num; codecCtx->time_base.den = den; av_set_pts_info(streamInfo->avStream, 64, num, den); if(pFormatCtx->oformat->flags & AVFMT_GLOBALHEADER) { codecCtx->flags |= CODEC_FLAG_GLOBAL_HEADER; } switch(codecCtx->codec_id) { case CODEC_ID_H264: { AVBitStreamFilterContext * avFilter = av_bitstream_filter_init("h264_mp4toannexb"); } break ; case CODEC_ID_AAC: { codecCtx->frame_size = 1024; AVBitStreamFilterContext * avFilter = av_bitstream_filter_init("aac_adtstoasc"); } break ; } // 设置解码相关数据, 比如H264要设置:SPS & PPS codecCtx->extradata_size = ;// size; codecCtx->extradata = ;// (uint8_t *)av_malloc(size + FF_INPUT_BUFFER_PADDING_SIZE); } { // new stream AVStream * avStream = av_new_stream(pFormatCtx, pFormatCtx->nb_streams; avcodec_get_context_defaults3(avStream->codec, NULL); } err = av_set_parameters(pFormatCtx, NULL); // 以写的方式打开文件 err = avio_open(&pFormatCtx->pb, "c:\\out.mp4", AVIO_FLAG_WRITE); // 写文件头信息 err = av_write_header(pFormatCtx); { const AVRational in_time_base = { 1, 1000000 }; AVRational out_time_base = avStream->time_base; AVPacket pkt = { 0 }; av_init_packet(&pkt); pkt.stream_index = streamId; // 流的id pkt.data = ;//(uint8_t *)mediaSample->data(); pkt.size = ;//mediaSample->size(); // 转换dts/pts时间单位 1/1000000=>avStream->time_base pkt.dts = av_rescale_q(mediaSample->dts(), in_time_base, out_time_base); pkt.pts = av_rescale_q(mediaSample->pts(), in_time_base, out_time_base); pkt.flags = mediaSample->isKey() ? AV_PKT_FLAG_KEY : 0; // 写入一帧数据 int err = av_interleaved_write_frame(pFormatCtx, &pkt); av_free_packet(&pkt); } // 写文件尾信息 av_write_trailer(pFormatCtx); // 释放 // av_bitstream_filter_close(avFilter); avio_close(pFormatCtx->pb); avformat_free_context(pFormatCtx); }
7)滤镜filter的使用(crop, resize, deinterlace, drawtext, overlay, vflip, ...)
通过搭建若干个filter可以对视音频进行一系列的处理.
a).Simple filtergraphs:
reencode filter graph:
_________ __________ ______________
| | | | | |
| decoded | simple filtergraph | filtered | encoder | encoded data |
| frames | -------------------> | frames | ---------> | packets |
|_________| |__________| |______________|
filter graph:
_______ _____________ _______ _____ ________
| | | | | | | | | |
| input | ---> | deinterlace | ---> | scale | ---> | fps | ---> | output |
|_______| |_____________| |_______| |_____| |________|
int ret = av_vsink_buffer_get_video_buffer_ref(mBufferDstCtx, &picRef, 0);
request_frame
start_frame
draw_slice
end_frame
b).Complex filtergraphs:
_________
| |
| input 0 |\ __________
|_________| \ | |
\ _________ /| output 0 |
\ | | / |__________|
_________ \| complex | /
| | | |/
| input 1 |---->| filter |\
|_________| | | \ __________
/| graph | \ | |
/ | | \| output 1 |
_________ / |_________| |__________|
| | /
| input 2 |/
|_________|
V3组件中实现的 interface IVideoPreprocess代码示例:
代码文件:media/impl/videopreprocessimpl.cpp
我们搭建的filter graph:
/1-->pad----\
input-->deinterlace-->fps-->logo remove-->color-->image overlaps-->crop-->resize<-0----------->output
\2-->crop---/
{ avcodec_register_all(); avfilter_register_all(); AVFilterGraph * pFilterGraph = NULL; AVFilterContext * pBufferSrcCtx = NULL; AVFilterContext * pBufferDstCtx = NULL; AVFrame * pSrcFrame = avcodec_alloc_frame(); AVFrame * pSinkFrame = avcodec_alloc_frame(); AVFrame * pDstFrame = avcodec_alloc_frame(); // 设定输出格式列表,我们仅支持PIX_FMT_YUV420P PixelFormat pix_fmts[] = { PIX_FMT_YUV420P, PIX_FMT_NONE }; char args[512]; AVFilterContext *lastFilterCtx = NULL; // 我们使用到的filter,其中"nl_"开头的是我们自己写的filter // 输入buffer filter AVFilter *bufferFilter = avfilter_get_by_name("buffer"); // deinterlace filter, 目前使用yadif filter AVFilter *yadifFilter = avfilter_get_by_name("yadif"); // 我们自己实现的fps转换filter AVFilter *fpsFilter = avfilter_get_by_name("nl_fps"); // 我们自己实现的遮logo的filter,支持多个,动态设置,能够设定区间范围 AVFilter *delogosFilter = avfilter_get_by_name("nl_delogos"); // 我们自己实现的调节对比度和亮度的filter AVFilter *colorFilter = avfilter_get_by_name("nl_color"); // 我们自己实现的叠加图片的filter,支持多个,动态设置,能够设定区间范围 AVFilter *overlaysFilter = avfilter_get_by_name("nl_overlays"); // crop filter AVFilter *cropFilter = avfilter_get_by_name("crop"); // resize filter AVFilter *resizeFilter = avfilter_get_by_name("scale"); // 图像扩展filter,可以在图像边界填充特定的颜色 AVFilter *padFilter = avfilter_get_by_name("pad"); // 输出buffer filter AVFilter *buffersinkFilter = avfilter_get_by_name("buffersink"); // 创建graph pFilterGraph = avfilter_graph_alloc(); // 开始创建filter AVRational tb = { 1, 1000000 }; AVRational sar = { 0, 1 }; // 计算图像宽度比 av_reduce(&sar.num, &sar.den, mConfig.width, mConfig.height, 1000*1000); // 设定 buffer filter的参数 // w:h:pixfmt:time_base.num:time_base.den:sample_aspect_ratio.num:sample_aspect_ratio.den:sws_param sprintf(args, "%d:%d:%d:%d:%d:%d:%d", mConfig.width, mConfig.height, mConfig.pixelFormat, tb.num, tb.den, sar.num, sar.den); // input filter err = avfilter_graph_create_filter(&pBufferSrcCtx, bufferFilter, "in", args, NULL, pFilterGraph); // 记录前一个filter context lastFilterCtx = pBufferSrcCtx; // 如果需要 deinterlace,则创建 yadif filter,同时和前一个filter进行连接 // deinterlace : yadif if(mConfig.deinterlace > 0) { if(yadifFilter == NULL) break ; // yadif filter的参数 // mode:parity sprintf(args, "%d:%d", 0, -1); // 创建filter,同时加入到graph AVFilterContext *deinterlaceCtx = NULL; err = avfilter_graph_create_filter(&deinterlaceCtx, yadifFilter, "yadif", args, NULL, pFilterGraph); if(err < 0) break ; // 和前一个filter进行连接 err = avfilter_link(lastFilterCtx, 0, deinterlaceCtx, 0); if(err < 0) break ; lastFilterCtx = deinterlaceCtx; } // ... 中间略过 // 创建output filter err = avfilter_graph_create_filter(&pBufferDstCtx, buffersinkFilter, "out", NULL, pix_fmts, pFilterGraph); if(err < 0) break ; // 和前一个filter进行连接 err = avfilter_link(lastFilterCtx, 0, pBufferDstCtx, 0); if(err < 0) break ; // 配置 graph err = avfilter_graph_config(pFilterGraph, NULL); // 把输入frame填充到结构AVFrame avpicture_fill((AVPicture *)pSrcFrame, (uint8_t *)inMediaSample->data(), (PixelFormat)mConfig.pixelFormat, mConfig.width, mConfig.height); pSrcFrame->width = mConfig.width; pSrcFrame->height = mConfig.height; pSrcFrame->format = mConfig.pixelFormat; pSrcFrame->pts = inMediaSample->dts(); // 开始写input写入frame ret = av_vsrc_buffer_add_frame(pBufferSrcCtx, pSrcFrame, AV_VSRC_BUF_FLAG_OVERWRITE); // 从输出filter查看输入是否可以获取数据,返回可获取的数目 int count = avfilter_poll_frame(pBufferDstCtx->inputs[0]); if(count > 0) { AVFilterBufferRef *picRef = NULL; // 从输出filter中获取结果 int ret = av_vsink_buffer_get_video_buffer_ref(pBufferDstCtx, &picRef, 0); if(picRef != NULL) { // 转换AVFilterBufferRef到AVFrame avfilter_fill_frame_from_video_buffer_ref(pSinkFrame, picRef); pSinkFrame->format = picRef->format; pSinkFrame->width = picRef->video->w; pSinkFrame->height = picRef->video->h; const int numBytes = avpicture_get_size((PixelFormat)pSinkFrame->format, pSinkFrame->width, pSinkFrame->height); // 转换时间单位 AVRational tb = { 1, 1000000 }; const int64 dts = av_rescale_q(picRef->pts, mBufferDstCtx->inputs[0]->time_base, tb); // 获取图像数据 avpicture_fill((AVPicture *)pDstFrame, (uint8_t *)mediaSample->data(), (PixelFormat)pSinkFrame->format, pSinkFrame->width, pSinkFrame->height); av_picture_copy((AVPicture *)pDstFrame, (AVPicture *)pSinkFrame, (PixelFormat)pSinkFrame->format, pSinkFrame->width, pSinkFrame->height); // 释放buffer计数器 avfilter_unref_buffer(picRef); } } }
微信
支付宝