From 76863482d019eb1bf136af2a3f69cbbeaa16d357 Mon Sep 17 00:00:00 2001 From: pingkai Date: Fri, 5 Jun 2020 21:21:20 +0800 Subject: [PATCH] feat(avafpacket): support AVAFPacket to PBAFFrame Signed-off-by: pingkai --- framework/base/media/AVAFPacket.cpp | 15 +- framework/base/media/AVAFPacket.h | 7 + framework/base/media/PBAFFrame.cpp | 37 ++- framework/base/media/PBAFFrame.h | 43 +-- framework/base/media/avFrame2pixelBuffer.c | 276 ++++++++++++++++++++ framework/base/media/avFrame2pixelBuffer.h | 19 ++ framework/codec/Apple/AppleVideoToolBox.cpp | 4 + 7 files changed, 360 insertions(+), 41 deletions(-) create mode 100644 framework/base/media/avFrame2pixelBuffer.c create mode 100644 framework/base/media/avFrame2pixelBuffer.h diff --git a/framework/base/media/AVAFPacket.cpp b/framework/base/media/AVAFPacket.cpp index 7a3c4b46e..68f70f736 100644 --- a/framework/base/media/AVAFPacket.cpp +++ b/framework/base/media/AVAFPacket.cpp @@ -6,6 +6,10 @@ #include "base/media/IAFPacket.h" #include "AVAFPacket.h" #include "utils/ffmpeg_utils.h" +#ifdef __APPLE__ +#include "PBAFFrame.h" +#include "avFrame2pixelBuffer.h" +#endif using namespace std; @@ -197,4 +201,13 @@ void AVAFFrame::updateInfo() { copyInfo(); } - +#ifdef __APPLE__ +AVAFFrame::operator PBAFFrame *() +{ + CVPixelBufferRef pixelBuffer = avFrame2pixelBuffer(mAvFrame); + if (pixelBuffer) { + return new PBAFFrame(pixelBuffer, mInfo.pts, mInfo.duration); + } + return nullptr; +} +#endif diff --git a/framework/base/media/AVAFPacket.h b/framework/base/media/AVAFPacket.h index 21a888319..635daf918 100644 --- a/framework/base/media/AVAFPacket.h +++ b/framework/base/media/AVAFPacket.h @@ -11,6 +11,9 @@ extern "C" { #include }; +#ifdef __APPLE__ +class PBAFFrame; +#endif class AVAFPacket : public IAFPacket { public: @@ -67,6 +70,10 @@ class AVAFFrame : public IAFFrame { explicit operator AVFrame *() const; +#ifdef __APPLE__ + explicit operator PBAFFrame *(); +#endif + void updateInfo(); diff --git a/framework/base/media/PBAFFrame.cpp b/framework/base/media/PBAFFrame.cpp index 1a0435e9a..7a9219cab 100644 --- a/framework/base/media/PBAFFrame.cpp +++ b/framework/base/media/PBAFFrame.cpp @@ -2,12 +2,47 @@ // Created by moqi on 2019-08-28. // #include "PBAFFrame.h" +#include "AVAFPacket.h" extern "C" { #include #include } +PBAFFrame::PBAFFrame(CVPixelBufferRef pixelBuffer, int64_t pts, int64_t duration) : mPBuffer(CVPixelBufferRetain(pixelBuffer)) +{ + + mInfo.pts = pts; + mInfo.duration = duration; + mInfo.video.format = AF_PIX_FMT_APPLE_PIXEL_BUFFER; + mInfo.video.width = (int) CVPixelBufferGetWidth(mPBuffer); + mInfo.video.height = (int) CVPixelBufferGetHeight(mPBuffer); + + OSType pixel_format = CVPixelBufferGetPixelFormatType(pixelBuffer); + if (pixel_format == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) { + mInfo.video.colorRange = COLOR_RANGE_FULL; + } else if (pixel_format == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) { + mInfo.video.colorRange = COLOR_RANGE_LIMITIED; + } else { + mInfo.video.colorRange = COLOR_RANGE_UNSPECIFIED; + } + + CFTypeRef colorAttachments = CVBufferGetAttachment((CVPixelBufferRef) pixelBuffer, kCVImageBufferYCbCrMatrixKey, NULL); + if (colorAttachments != nullptr) { + if (CFStringCompare((CFStringRef) colorAttachments, kCVImageBufferYCbCrMatrix_ITU_R_601_4, 0) == kCFCompareEqualTo) { + mInfo.video.colorSpace = COLOR_SPACE_BT601; + } else if (CFStringCompare((CFStringRef) colorAttachments, kCVImageBufferYCbCrMatrix_ITU_R_709_2, 0) == kCFCompareEqualTo) { + mInfo.video.colorSpace = COLOR_SPACE_BT709; + } else if (CFStringCompare((CFStringRef) colorAttachments, kCVImageBufferYCbCrMatrix_ITU_R_2020, 0) == kCFCompareEqualTo) { + mInfo.video.colorSpace = COLOR_SPACE_BT2020; + } else { + mInfo.video.colorSpace = COLOR_SPACE_UNSPECIFIED; + } + } else { + mInfo.video.colorSpace = COLOR_SPACE_UNSPECIFIED; + } +} + PBAFFrame::operator AVAFFrame *() { CVReturn err; @@ -57,4 +92,4 @@ PBAFFrame::operator AVAFFrame *() AVAFFrame *pAvFrame = new AVAFFrame(pFrame, FrameTypeVideo); av_frame_free(&pFrame); return pAvFrame; -} +} \ No newline at end of file diff --git a/framework/base/media/PBAFFrame.h b/framework/base/media/PBAFFrame.h index 6bb16bcc1..ab556ad8e 100644 --- a/framework/base/media/PBAFFrame.h +++ b/framework/base/media/PBAFFrame.h @@ -8,47 +8,12 @@ #include "base/media/IAFPacket.h" #include #include -#include + +class AVAFFrame; class PBAFFrame : public IAFFrame { public: - PBAFFrame(CVPixelBufferRef pixelBuffer, int64_t pts, int64_t duration) : - mPBuffer(CVPixelBufferRetain(pixelBuffer)) - { - - mInfo.pts = pts; - mInfo.duration = duration; - mInfo.video.format = AF_PIX_FMT_APPLE_PIXEL_BUFFER; - mInfo.video.width = (int) CVPixelBufferGetWidth(mPBuffer); - mInfo.video.height = (int) CVPixelBufferGetHeight(mPBuffer); - - OSType pixel_format = CVPixelBufferGetPixelFormatType(pixelBuffer); - if(pixel_format == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange){ - mInfo.video.colorRange = COLOR_RANGE_FULL; - } else if(pixel_format == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) { - mInfo.video.colorRange = COLOR_RANGE_LIMITIED; - } else { - mInfo.video.colorRange = COLOR_RANGE_UNSPECIFIED; - } - - CFTypeRef colorAttachments = CVBufferGetAttachment((CVPixelBufferRef)pixelBuffer, kCVImageBufferYCbCrMatrixKey, NULL); - if (colorAttachments != NULL) { - if(CFStringCompare((CFStringRef)colorAttachments, kCVImageBufferYCbCrMatrix_ITU_R_601_4, 0) == kCFCompareEqualTo) { - mInfo.video.colorSpace = COLOR_SPACE_BT601; - } - else if(CFStringCompare((CFStringRef)colorAttachments, kCVImageBufferYCbCrMatrix_ITU_R_709_2, 0) == kCFCompareEqualTo) { - mInfo.video.colorSpace = COLOR_SPACE_BT709; - } - else if(CFStringCompare((CFStringRef)colorAttachments, kCVImageBufferYCbCrMatrix_ITU_R_2020, 0) == kCFCompareEqualTo) { - mInfo.video.colorSpace = COLOR_SPACE_BT2020; - } else { - mInfo.video.colorSpace = COLOR_SPACE_UNSPECIFIED; - } - } else { - mInfo.video.colorSpace = COLOR_SPACE_UNSPECIFIED; - } - - } + PBAFFrame(CVPixelBufferRef pixelBuffer, int64_t pts, int64_t duration); ~PBAFFrame() override { @@ -80,7 +45,7 @@ class PBAFFrame : public IAFFrame { return nullptr; } - operator AVAFFrame *(); + explicit operator AVAFFrame *(); private: CVPixelBufferRef mPBuffer; diff --git a/framework/base/media/avFrame2pixelBuffer.c b/framework/base/media/avFrame2pixelBuffer.c new file mode 100644 index 000000000..ed00ab655 --- /dev/null +++ b/framework/base/media/avFrame2pixelBuffer.c @@ -0,0 +1,276 @@ +// +// Created by moqi on 2020/6/5. +// + +#include "avFrame2pixelBuffer.h" +static int copy_avframe_to_pixel_buffer(const AVFrame *frame, CVPixelBufferRef cv_img, const size_t *plane_strides, + const size_t *plane_rows) +{ + int i, j; + size_t plane_count; + int status; + int rows; + int src_stride; + int dst_stride; + uint8_t *src_addr; + uint8_t *dst_addr; + size_t copy_bytes; + + status = CVPixelBufferLockBaseAddress(cv_img, 0); + if (status) { + av_log(NULL, AV_LOG_ERROR, "Error: Could not lock base address of CVPixelBuffer: %d.\n", status); + } + + if (CVPixelBufferIsPlanar(cv_img)) { + plane_count = CVPixelBufferGetPlaneCount(cv_img); + for (i = 0; frame->data[i]; i++) { + if (i == plane_count) { + CVPixelBufferUnlockBaseAddress(cv_img, 0); + av_log(NULL, AV_LOG_ERROR, "Error: different number of planes in AVFrame and CVPixelBuffer.\n"); + + return AVERROR_EXTERNAL; + } + + dst_addr = (uint8_t *) CVPixelBufferGetBaseAddressOfPlane(cv_img, i); + src_addr = (uint8_t *) frame->data[i]; + dst_stride = CVPixelBufferGetBytesPerRowOfPlane(cv_img, i); + src_stride = plane_strides[i]; + rows = plane_rows[i]; + + if (dst_stride == src_stride) { + memcpy(dst_addr, src_addr, src_stride * rows); + } else { + copy_bytes = dst_stride < src_stride ? dst_stride : src_stride; + + for (j = 0; j < rows; j++) { + memcpy(dst_addr + j * dst_stride, src_addr + j * src_stride, copy_bytes); + } + } + } + } else { + if (frame->data[1]) { + CVPixelBufferUnlockBaseAddress(cv_img, 0); + av_log(NULL, AV_LOG_ERROR, "Error: different number of planes in AVFrame and non-planar CVPixelBuffer.\n"); + + return AVERROR_EXTERNAL; + } + + dst_addr = (uint8_t *) CVPixelBufferGetBaseAddress(cv_img); + src_addr = (uint8_t *) frame->data[0]; + dst_stride = CVPixelBufferGetBytesPerRow(cv_img); + src_stride = plane_strides[0]; + rows = plane_rows[0]; + + if (dst_stride == src_stride) { + memcpy(dst_addr, src_addr, src_stride * rows); + } else { + copy_bytes = dst_stride < src_stride ? dst_stride : src_stride; + + for (j = 0; j < rows; j++) { + memcpy(dst_addr + j * dst_stride, src_addr + j * src_stride, copy_bytes); + } + } + } + + status = CVPixelBufferUnlockBaseAddress(cv_img, 0); + if (status) { + av_log(NULL, AV_LOG_ERROR, "Error: Could not unlock CVPixelBuffer base address: %d.\n", status); + return AVERROR_EXTERNAL; + } + + return 0; +} + +static int get_cv_pixel_format(enum AVPixelFormat fmt, enum AVColorRange range, int *av_pixel_format, int *range_guessed) +{ + if (range_guessed) *range_guessed = range != AVCOL_RANGE_MPEG && range != AVCOL_RANGE_JPEG; + + //MPEG range is used when no range is set + if (fmt == AV_PIX_FMT_NV12) { + *av_pixel_format = range == AVCOL_RANGE_JPEG ? kCVPixelFormatType_420YpCbCr8BiPlanarFullRange + : kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange; + } else if (fmt == AV_PIX_FMT_YUV420P) { + *av_pixel_format = range == AVCOL_RANGE_JPEG ? kCVPixelFormatType_420YpCbCr8PlanarFullRange : kCVPixelFormatType_420YpCbCr8Planar; + } else if (fmt == AV_PIX_FMT_P010LE) { + *av_pixel_format = range == AVCOL_RANGE_JPEG ? kCVPixelFormatType_420YpCbCr10BiPlanarFullRange + : kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange; + *av_pixel_format = kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange; + } else { + return AVERROR(EINVAL); + } + + return 0; +} + +static int get_cv_pixel_info(const AVFrame *frame, int *color, int *plane_count, size_t *widths, size_t *heights, size_t *strides, + size_t *contiguous_buf_size) +{ + int av_format = frame->format; + int av_color_range = frame->color_range; + int i; + int range_guessed; + int status; + + status = get_cv_pixel_format(av_format, av_color_range, color, &range_guessed); + if (status) { + // av_log(avctx, + // AV_LOG_ERROR, + // "Could not get pixel format for color format '%s' range '%s'.\n", + // av_get_pix_fmt_name(av_format), + // av_color_range > AVCOL_RANGE_UNSPECIFIED && + // av_color_range < AVCOL_RANGE_NB ? + // av_color_range_name(av_color_range) : + // "Unknown"); + + return AVERROR(EINVAL); + } + + switch (av_format) { + case AV_PIX_FMT_NV12: + *plane_count = 2; + + widths[0] = frame->width; + heights[0] = frame->height; + strides[0] = frame ? frame->linesize[0] : frame->width; + + widths[1] = (frame->width + 1) / 2; + heights[1] = (frame->height + 1) / 2; + strides[1] = frame ? frame->linesize[1] : (frame->width + 1) & -2; + break; + + case AV_PIX_FMT_YUV420P: + *plane_count = 3; + + widths[0] = frame->width; + heights[0] = frame->height; + strides[0] = frame ? frame->linesize[0] : frame->width; + + widths[1] = (frame->width + 1) / 2; + heights[1] = (frame->height + 1) / 2; + strides[1] = frame ? frame->linesize[1] : (frame->width + 1) / 2; + + widths[2] = (frame->width + 1) / 2; + heights[2] = (frame->height + 1) / 2; + strides[2] = frame ? frame->linesize[2] : (frame->width + 1) / 2; + break; + + case AV_PIX_FMT_P010LE: + *plane_count = 2; + widths[0] = frame->width; + heights[0] = frame->height; + strides[0] = frame ? frame->linesize[0] : (frame->width * 2 + 63) & -64; + + widths[1] = (frame->width + 1) / 2; + heights[1] = (frame->height + 1) / 2; + strides[1] = frame ? frame->linesize[1] : ((frame->width + 1) / 2 + 63) & -64; + break; + + default: + av_log(NULL, AV_LOG_ERROR, "Could not get frame format info for color %d range %d.\n", av_format, av_color_range); + + return AVERROR(EINVAL); + } + + *contiguous_buf_size = 0; + for (i = 0; i < *plane_count; i++) { + if (i < *plane_count - 1 && frame->data[i] + strides[i] * heights[i] != frame->data[i + 1]) { + *contiguous_buf_size = 0; + break; + } + + *contiguous_buf_size += strides[i] * heights[i]; + } + + return 0; +} + +static AVFrame *yuv420p2nv12(AVFrame *frame) +{ + int x, y; + AVFrame *outFrame = av_frame_alloc(); + outFrame->format = AV_PIX_FMT_NV12; + outFrame->width = frame->width; + outFrame->height = frame->height; + + int ret = av_frame_get_buffer(outFrame, 32); + if (ret < 0) { + av_frame_free(&outFrame); + return NULL; + } + ret = av_frame_make_writable(outFrame); + if (ret < 0) { + av_frame_free(&outFrame); + return NULL; + } + if (frame->linesize[0] == frame->width) { + memcpy(outFrame->data[0], frame->data[0], outFrame->linesize[0]); + } else { + for (y = 0; y < outFrame->height; ++y) { + for (x = 0; x < outFrame->width; ++x) { + outFrame->data[0][y * outFrame->linesize[0] + x] = frame->data[0][y * frame->linesize[0] + x]; + } + } + } + + for (y = 0; y < outFrame->height / 2; ++y) { + for (x = 0; x < outFrame->width / 2; ++x) { + outFrame->data[1][y * outFrame->linesize[1] + 2 * x] = frame->data[1][y * frame->linesize[1] + x]; + outFrame->data[1][y * outFrame->linesize[1] + 2 * x + 1] = frame->data[2][y * frame->linesize[2] + x]; + } + } + + return outFrame; +} + +CVPixelBufferRef avFrame2pixelBuffer(AVFrame *frame) +{ + int plane_count; + int color; + size_t widths[AV_NUM_DATA_POINTERS]; + size_t heights[AV_NUM_DATA_POINTERS]; + size_t strides[AV_NUM_DATA_POINTERS]; + int status; + size_t contiguous_buf_size; + CVPixelBufferPoolRef pix_buf_pool; + memset(widths, 0, sizeof(widths)); + memset(heights, 0, sizeof(heights)); + memset(strides, 0, sizeof(strides)); + status = get_cv_pixel_info(frame, &color, &plane_count, widths, heights, strides, &contiguous_buf_size); + if (status) { + // av_log( + // avctx, + // AV_LOG_ERROR, + // "Error: Cannot convert format %d color_range %d: %d\n", + // frame->format, + // frame->color_range, + // status + // ); + + return NULL; + } + + CVPixelBufferRef pixelBuffer; + OSType pixelFormat; + if (frame->color_range == AVCOL_RANGE_MPEG) { + pixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange; + } else + pixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange; + pixelFormat = kCVPixelFormatType_420YpCbCr8Planar; + status = CVPixelBufferCreate(kCFAllocatorDefault, frame->width, frame->height, pixelFormat, NULL, &pixelBuffer); + + + if (status) { + av_log(NULL, AV_LOG_ERROR, "Could not create pixel buffer from pool: %d.\n", status); + return NULL; + } + + // AVFrame * nv12Frame = yuv420p2nv12(frame); + + status = copy_avframe_to_pixel_buffer(frame, pixelBuffer, strides, heights); + // av_frame_free(&nv12Frame); + if (status) { + CFRelease(pixelBuffer); + pixelBuffer = NULL; + } + return pixelBuffer; +} \ No newline at end of file diff --git a/framework/base/media/avFrame2pixelBuffer.h b/framework/base/media/avFrame2pixelBuffer.h new file mode 100644 index 000000000..6a68fa6e4 --- /dev/null +++ b/framework/base/media/avFrame2pixelBuffer.h @@ -0,0 +1,19 @@ +// +// Created by moqi on 2020/6/5. +// + +#ifndef CICADAMEDIA_AVFRAME2PIXELBUFFER_H +#define CICADAMEDIA_AVFRAME2PIXELBUFFER_H + +#include +#ifdef __cplusplus +extern "C" { +#endif +#include +CVPixelBufferRef avFrame2pixelBuffer(AVFrame *frame); + +#ifdef __cplusplus +}; +#endif + +#endif//CICADAMEDIA_AVFRAME2PIXELBUFFER_H diff --git a/framework/codec/Apple/AppleVideoToolBox.cpp b/framework/codec/Apple/AppleVideoToolBox.cpp index 386f9661f..c1eb0e659 100644 --- a/framework/codec/Apple/AppleVideoToolBox.cpp +++ b/framework/codec/Apple/AppleVideoToolBox.cpp @@ -6,6 +6,10 @@ #include "utils/errors/framework_error.h" #include "utils/timer.h" #include +extern "C" { +#include +}; +#include #include "video_tool_box_utils.h"