diff --git a/packages/multimedia/ffmpeg/patches/v4l2-drmprime/ffmpeg-001-v4l2-drmprime.patch b/packages/multimedia/ffmpeg/patches/v4l2-drmprime/ffmpeg-001-v4l2-drmprime.patch new file mode 100644 index 0000000000..9313375a37 --- /dev/null +++ b/packages/multimedia/ffmpeg/patches/v4l2-drmprime/ffmpeg-001-v4l2-drmprime.patch @@ -0,0 +1,970 @@ +From b29b5b9f529bbb10cd9880adebf0fb287dcf233b Mon Sep 17 00:00:00 2001 +From: Andriy Gelman +Date: Tue, 28 Apr 2020 22:54:21 -0400 +Subject: [PATCH 01/11] avcodec/v4l2_m2m: Adapt to call close() on init fail + +This fixes several mem leaks when init of encoder/decoder failed. + +Fixes ticket #8285 + +Signed-off-by: Andriy Gelman +--- + libavcodec/v4l2_m2m.c | 8 ++++++++ + libavcodec/v4l2_m2m_dec.c | 10 ++-------- + libavcodec/v4l2_m2m_enc.c | 1 + + 3 files changed, 11 insertions(+), 8 deletions(-) + +diff --git a/libavcodec/v4l2_m2m.c b/libavcodec/v4l2_m2m.c +index e48b3a8ccf..bfea70ff0c 100644 +--- a/libavcodec/v4l2_m2m.c ++++ b/libavcodec/v4l2_m2m.c +@@ -338,6 +338,13 @@ int ff_v4l2_m2m_codec_end(V4L2m2mPriv *priv) + V4L2m2mContext *s = priv->context; + int ret; + ++ if (!s) ++ return 0; ++ ++ if (av_codec_is_decoder(s->avctx->codec)) ++ av_packet_unref(&s->buf_pkt); ++ ++ if (s->fd >= 0) { + ret = ff_v4l2_context_set_status(&s->output, VIDIOC_STREAMOFF); + if (ret) + av_log(s->avctx, AV_LOG_ERROR, "VIDIOC_STREAMOFF %s\n", s->output.name); +@@ -345,6 +352,7 @@ int ff_v4l2_m2m_codec_end(V4L2m2mPriv *priv) + ret = ff_v4l2_context_set_status(&s->capture, VIDIOC_STREAMOFF); + if (ret) + av_log(s->avctx, AV_LOG_ERROR, "VIDIOC_STREAMOFF %s\n", s->capture.name); ++ } + + ff_v4l2_context_release(&s->output); + +diff --git a/libavcodec/v4l2_m2m_dec.c b/libavcodec/v4l2_m2m_dec.c +index 3e17e0fcac..a2ea0ff73a 100644 +--- a/libavcodec/v4l2_m2m_dec.c ++++ b/libavcodec/v4l2_m2m_dec.c +@@ -212,9 +212,6 @@ static av_cold int v4l2_decode_init(AVCodecContext *avctx) + ret = ff_v4l2_m2m_codec_init(priv); + if (ret) { + av_log(avctx, AV_LOG_ERROR, "can't configure decoder\n"); +- s->self_ref = NULL; +- av_buffer_unref(&priv->context_ref); +- + return ret; + } + +@@ -223,10 +220,7 @@ static av_cold int v4l2_decode_init(AVCodecContext *avctx) + + static av_cold int v4l2_decode_close(AVCodecContext *avctx) + { +- V4L2m2mPriv *priv = avctx->priv_data; +- V4L2m2mContext *s = priv->context; +- av_packet_unref(&s->buf_pkt); +- return ff_v4l2_m2m_codec_end(priv); ++ return ff_v4l2_m2m_codec_end(avctx->priv_data); + } + + #define OFFSET(x) offsetof(V4L2m2mPriv, x) +@@ -261,7 +255,7 @@ static const AVOption options[] = { + .close = v4l2_decode_close, \ + .bsfs = bsf_name, \ + .capabilities = AV_CODEC_CAP_HARDWARE | AV_CODEC_CAP_DELAY | AV_CODEC_CAP_AVOID_PROBING, \ +- .caps_internal = FF_CODEC_CAP_SETS_PKT_DTS, \ ++ .caps_internal = FF_CODEC_CAP_SETS_PKT_DTS | FF_CODEC_CAP_INIT_CLEANUP, \ + .wrapper_name = "v4l2m2m", \ + } + +diff --git a/libavcodec/v4l2_m2m_enc.c b/libavcodec/v4l2_m2m_enc.c +index 32321f392f..9f1b2c2ffc 100644 +--- a/libavcodec/v4l2_m2m_enc.c ++++ b/libavcodec/v4l2_m2m_enc.c +@@ -416,6 +416,7 @@ static const AVCodecDefault v4l2_m2m_defaults[] = { + .close = v4l2_encode_close, \ + .defaults = v4l2_m2m_defaults, \ + .capabilities = AV_CODEC_CAP_HARDWARE | AV_CODEC_CAP_DELAY, \ ++ .caps_internal = FF_CODEC_CAP_INIT_CLEANUP, \ + .wrapper_name = "v4l2m2m", \ + } + + +From 7aaac68934c0e03a78c9f477ec64e522729a64b7 Mon Sep 17 00:00:00 2001 +From: Andriy Gelman +Date: Tue, 5 May 2020 01:54:54 -0400 +Subject: [PATCH 02/11] avcodec/v4l2_m2m_dec: Use av_packet_move_ref() + +Signed-off-by: Andriy Gelman +--- + libavcodec/v4l2_m2m_dec.c | 3 +-- + 1 file changed, 1 insertion(+), 2 deletions(-) + +diff --git a/libavcodec/v4l2_m2m_dec.c b/libavcodec/v4l2_m2m_dec.c +index a2ea0ff73a..45e9a8e9fe 100644 +--- a/libavcodec/v4l2_m2m_dec.c ++++ b/libavcodec/v4l2_m2m_dec.c +@@ -142,8 +142,7 @@ static int v4l2_receive_frame(AVCodecContext *avctx, AVFrame *frame) + int ret; + + if (s->buf_pkt.size) { +- avpkt = s->buf_pkt; +- memset(&s->buf_pkt, 0, sizeof(AVPacket)); ++ av_packet_move_ref(&avpkt, &s->buf_pkt); + } else { + ret = ff_decode_get_packet(avctx, &avpkt); + if (ret < 0 && ret != AVERROR_EOF) + +From c6b85ed30f06ea99513b13cc768a922ebe4d68c2 Mon Sep 17 00:00:00 2001 +From: Lukas Rusak +Date: Tue, 24 Apr 2018 23:00:23 -0700 +Subject: [PATCH 03/11] libavcodec: v4l2m2m: output AVDRMFrameDescriptor + +This allows for a zero-copy output by exporting the v4l2 buffer then wrapping that buffer +in the AVDRMFrameDescriptor like it is done in rkmpp. + +This has been in use for quite some time with great success on many platforms including: + - Amlogic S905 + - Raspberry Pi + - i.MX6 + - Dragonboard 410c + +This was developed in conjunction with Kodi to allow handling the zero-copy buffer rendering. +A simply utility for testing is also available here: https://github.com/BayLibre/ffmpeg-drm + +todo: + - allow selecting pixel format output from decoder + - allow configuring amount of output and capture buffers + +V2: + - allow selecting AV_PIX_FMT_DRM_PRIME + +V3: + - use get_format to select AV_PIX_FMT_DRM_PRIME + - use hw_configs + - add handling of AV_PIX_FMT_YUV420P format (for raspberry pi) + - add handling of AV_PIX_FMT_YUYV422 format (for i.MX6 coda decoder) + +V4: + - rebased on 4.2.2 + +V5: + - rebased on 4.3 +--- + libavcodec/v4l2_buffers.c | 155 ++++++++++++++++++++++++++++++++++++-- + libavcodec/v4l2_buffers.h | 4 + + libavcodec/v4l2_context.c | 40 +++++++++- + libavcodec/v4l2_m2m.h | 3 + + libavcodec/v4l2_m2m_dec.c | 23 ++++++ + 5 files changed, 213 insertions(+), 12 deletions(-) + +diff --git a/libavcodec/v4l2_buffers.c b/libavcodec/v4l2_buffers.c +index 02f23d954b..4bb2bf6f87 100644 +--- a/libavcodec/v4l2_buffers.c ++++ b/libavcodec/v4l2_buffers.c +@@ -21,6 +21,7 @@ + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + ++#include + #include + #include + #include +@@ -30,6 +31,7 @@ + #include "libavcodec/avcodec.h" + #include "libavcodec/internal.h" + #include "libavutil/pixdesc.h" ++#include "libavutil/hwcontext.h" + #include "v4l2_context.h" + #include "v4l2_buffers.h" + #include "v4l2_m2m.h" +@@ -210,7 +212,79 @@ static enum AVColorTransferCharacteristic v4l2_get_color_trc(V4L2Buffer *buf) + return AVCOL_TRC_UNSPECIFIED; + } + +-static void v4l2_free_buffer(void *opaque, uint8_t *unused) ++static uint8_t * v4l2_get_drm_frame(V4L2Buffer *avbuf) ++{ ++ AVDRMFrameDescriptor *drm_desc = &avbuf->drm_frame; ++ AVDRMLayerDescriptor *layer; ++ ++ /* fill the DRM frame descriptor */ ++ drm_desc->nb_objects = avbuf->num_planes; ++ drm_desc->nb_layers = 1; ++ ++ layer = &drm_desc->layers[0]; ++ layer->nb_planes = avbuf->num_planes; ++ ++ for (int i = 0; i < avbuf->num_planes; i++) { ++ layer->planes[i].object_index = i; ++ layer->planes[i].offset = 0; ++ layer->planes[i].pitch = avbuf->plane_info[i].bytesperline; ++ } ++ ++ switch (avbuf->context->av_pix_fmt) { ++ case AV_PIX_FMT_YUYV422: ++ ++ layer->format = DRM_FORMAT_YUYV; ++ layer->nb_planes = 1; ++ ++ break; ++ ++ case AV_PIX_FMT_NV12: ++ case AV_PIX_FMT_NV21: ++ ++ layer->format = avbuf->context->av_pix_fmt == AV_PIX_FMT_NV12 ? ++ DRM_FORMAT_NV12 : DRM_FORMAT_NV21; ++ ++ if (avbuf->num_planes > 1) ++ break; ++ ++ layer->nb_planes = 2; ++ ++ layer->planes[1].object_index = 0; ++ layer->planes[1].offset = avbuf->plane_info[0].bytesperline * ++ avbuf->context->format.fmt.pix.height; ++ layer->planes[1].pitch = avbuf->plane_info[0].bytesperline; ++ break; ++ ++ case AV_PIX_FMT_YUV420P: ++ ++ layer->format = DRM_FORMAT_YUV420; ++ ++ if (avbuf->num_planes > 1) ++ break; ++ ++ layer->nb_planes = 3; ++ ++ layer->planes[1].object_index = 0; ++ layer->planes[1].offset = avbuf->plane_info[0].bytesperline * ++ avbuf->context->format.fmt.pix.height; ++ layer->planes[1].pitch = avbuf->plane_info[0].bytesperline >> 1; ++ ++ layer->planes[2].object_index = 0; ++ layer->planes[2].offset = layer->planes[1].offset + ++ ((avbuf->plane_info[0].bytesperline * ++ avbuf->context->format.fmt.pix.height) >> 2); ++ layer->planes[2].pitch = avbuf->plane_info[0].bytesperline >> 1; ++ break; ++ ++ default: ++ drm_desc->nb_layers = 0; ++ break; ++ } ++ ++ return (uint8_t *) drm_desc; ++} ++ ++static void v4l2_free_buffer(void *opaque, uint8_t *data) + { + V4L2Buffer* avbuf = opaque; + V4L2m2mContext *s = buf_to_m2mctx(avbuf); +@@ -234,6 +308,36 @@ static void v4l2_free_buffer(void *opaque, uint8_t *unused) + } + } + ++static int v4l2_buffer_export_drm(V4L2Buffer* avbuf) ++{ ++ struct v4l2_exportbuffer expbuf; ++ int i, ret; ++ ++ for (i = 0; i < avbuf->num_planes; i++) { ++ memset(&expbuf, 0, sizeof(expbuf)); ++ ++ expbuf.index = avbuf->buf.index; ++ expbuf.type = avbuf->buf.type; ++ expbuf.plane = i; ++ ++ ret = ioctl(buf_to_m2mctx(avbuf)->fd, VIDIOC_EXPBUF, &expbuf); ++ if (ret < 0) ++ return AVERROR(errno); ++ ++ if (V4L2_TYPE_IS_MULTIPLANAR(avbuf->buf.type)) { ++ /* drm frame */ ++ avbuf->drm_frame.objects[i].size = avbuf->buf.m.planes[i].length; ++ avbuf->drm_frame.objects[i].fd = expbuf.fd; ++ } else { ++ /* drm frame */ ++ avbuf->drm_frame.objects[0].size = avbuf->buf.length; ++ avbuf->drm_frame.objects[0].fd = expbuf.fd; ++ } ++ } ++ ++ return 0; ++} ++ + static int v4l2_buf_increase_ref(V4L2Buffer *in) + { + V4L2m2mContext *s = buf_to_m2mctx(in); +@@ -254,6 +358,24 @@ static int v4l2_buf_increase_ref(V4L2Buffer *in) + return 0; + } + ++static int v4l2_buf_to_bufref_drm(V4L2Buffer *in, AVBufferRef **buf) ++{ ++ int ret; ++ ++ *buf = av_buffer_create((uint8_t *) &in->drm_frame, ++ sizeof(in->drm_frame), ++ v4l2_free_buffer, ++ in, AV_BUFFER_FLAG_READONLY); ++ if (!*buf) ++ return AVERROR(ENOMEM); ++ ++ ret = v4l2_buf_increase_ref(in); ++ if (ret) ++ av_buffer_unref(buf); ++ ++ return ret; ++} ++ + static int v4l2_buf_to_bufref(V4L2Buffer *in, int plane, AVBufferRef **buf) + { + int ret; +@@ -303,13 +425,24 @@ static int v4l2_buffer_buf_to_swframe(AVFrame *frame, V4L2Buffer *avbuf) + + frame->format = avbuf->context->av_pix_fmt; + +- for (i = 0; i < avbuf->num_planes; i++) { +- ret = v4l2_buf_to_bufref(avbuf, i, &frame->buf[i]); ++ if (buf_to_m2mctx(avbuf)->output_drm) { ++ /* 1. get references to the actual data */ ++ ret = v4l2_buf_to_bufref_drm(avbuf, &frame->buf[0]); + if (ret) + return ret; + +- frame->linesize[i] = avbuf->plane_info[i].bytesperline; +- frame->data[i] = frame->buf[i]->data; ++ frame->data[0] = (uint8_t *) v4l2_get_drm_frame(avbuf); ++ frame->format = AV_PIX_FMT_DRM_PRIME; ++ } else { ++ /* 1. get references to the actual data */ ++ for (i = 0; i < avbuf->num_planes; i++) { ++ ret = v4l2_buf_to_bufref(avbuf, i, &frame->buf[i]); ++ if (ret) ++ return ret; ++ ++ frame->linesize[i] = avbuf->plane_info[i].bytesperline; ++ frame->data[i] = frame->buf[i]->data; ++ } + } + + /* fixup special cases */ +@@ -543,9 +676,6 @@ int ff_v4l2_buffer_initialize(V4L2Buffer* avbuf, int index) + + avbuf->status = V4L2BUF_AVAILABLE; + +- if (V4L2_TYPE_IS_OUTPUT(ctx->type)) +- return 0; +- + if (V4L2_TYPE_IS_MULTIPLANAR(ctx->type)) { + avbuf->buf.m.planes = avbuf->planes; + avbuf->buf.length = avbuf->num_planes; +@@ -555,6 +685,15 @@ int ff_v4l2_buffer_initialize(V4L2Buffer* avbuf, int index) + avbuf->buf.length = avbuf->planes[0].length; + } + ++ if (V4L2_TYPE_IS_OUTPUT(ctx->type)) ++ return 0; ++ ++ if (buf_to_m2mctx(avbuf)->output_drm) { ++ ret = v4l2_buffer_export_drm(avbuf); ++ if (ret) ++ return ret; ++ } ++ + return ff_v4l2_buffer_enqueue(avbuf); + } + +diff --git a/libavcodec/v4l2_buffers.h b/libavcodec/v4l2_buffers.h +index 8dbc7fc104..037e667997 100644 +--- a/libavcodec/v4l2_buffers.h ++++ b/libavcodec/v4l2_buffers.h +@@ -27,6 +27,7 @@ + #include + #include + ++#include "libavutil/hwcontext_drm.h" + #include "avcodec.h" + + enum V4L2Buffer_status { +@@ -42,6 +43,9 @@ typedef struct V4L2Buffer { + /* each buffer needs to have a reference to its context */ + struct V4L2Context *context; + ++ /* DRM descriptor */ ++ AVDRMFrameDescriptor drm_frame; ++ + /* This object is refcounted per-plane, so we need to keep track + * of how many context-refs we are holding. */ + AVBufferRef *context_ref; +diff --git a/libavcodec/v4l2_context.c b/libavcodec/v4l2_context.c +index 29b144ed73..7a92df2c3e 100644 +--- a/libavcodec/v4l2_context.c ++++ b/libavcodec/v4l2_context.c +@@ -455,22 +455,54 @@ static int v4l2_release_buffers(V4L2Context* ctx) + struct v4l2_requestbuffers req = { + .memory = V4L2_MEMORY_MMAP, + .type = ctx->type, +- .count = 0, /* 0 -> unmaps buffers from the driver */ ++ .count = 0, /* 0 -> unmap all buffers from the driver */ + }; +- int i, j; ++ int ret, i, j; + + for (i = 0; i < ctx->num_buffers; i++) { + V4L2Buffer *buffer = &ctx->buffers[i]; + + for (j = 0; j < buffer->num_planes; j++) { + struct V4L2Plane_info *p = &buffer->plane_info[j]; ++ ++ if (V4L2_TYPE_IS_OUTPUT(ctx->type)) { ++ /* output buffers are not EXPORTED */ ++ goto unmap; ++ } ++ ++ if (ctx_to_m2mctx(ctx)->output_drm) { ++ /* use the DRM frame to close */ ++ if (buffer->drm_frame.objects[j].fd >= 0) { ++ if (close(buffer->drm_frame.objects[j].fd) < 0) { ++ av_log(logger(ctx), AV_LOG_ERROR, "%s close drm fd " ++ "[buffer=%2d, plane=%d, fd=%2d] - %s \n", ++ ctx->name, i, j, buffer->drm_frame.objects[j].fd, ++ av_err2str(AVERROR(errno))); ++ } ++ } ++ } ++unmap: + if (p->mm_addr && p->length) + if (munmap(p->mm_addr, p->length) < 0) +- av_log(logger(ctx), AV_LOG_ERROR, "%s unmap plane (%s))\n", ctx->name, av_err2str(AVERROR(errno))); ++ av_log(logger(ctx), AV_LOG_ERROR, "%s unmap plane (%s))\n", ++ ctx->name, av_err2str(AVERROR(errno))); + } + } + +- return ioctl(ctx_to_m2mctx(ctx)->fd, VIDIOC_REQBUFS, &req); ++ ret = ioctl(ctx_to_m2mctx(ctx)->fd, VIDIOC_REQBUFS, &req); ++ if (ret < 0) { ++ av_log(logger(ctx), AV_LOG_ERROR, "release all %s buffers (%s)\n", ++ ctx->name, av_err2str(AVERROR(errno))); ++ ++ if (ctx_to_m2mctx(ctx)->output_drm) ++ av_log(logger(ctx), AV_LOG_ERROR, ++ "Make sure the DRM client releases all FB/GEM objects before closing the codec (ie):\n" ++ "for all buffers: \n" ++ " 1. drmModeRmFB(..)\n" ++ " 2. drmIoctl(.., DRM_IOCTL_GEM_CLOSE,... )\n"); ++ } ++ ++ return ret; + } + + static inline int v4l2_try_raw_format(V4L2Context* ctx, enum AVPixelFormat pixfmt) +diff --git a/libavcodec/v4l2_m2m.h b/libavcodec/v4l2_m2m.h +index 456281f48c..4ee0be653b 100644 +--- a/libavcodec/v4l2_m2m.h ++++ b/libavcodec/v4l2_m2m.h +@@ -63,6 +63,9 @@ typedef struct V4L2m2mContext { + + /* reference back to V4L2m2mPriv */ + void *priv; ++ ++ /* generate DRM frames */ ++ int output_drm; + } V4L2m2mContext; + + typedef struct V4L2m2mPriv { +diff --git a/libavcodec/v4l2_m2m_dec.c b/libavcodec/v4l2_m2m_dec.c +index 45e9a8e9fe..eb6ecc8ed5 100644 +--- a/libavcodec/v4l2_m2m_dec.c ++++ b/libavcodec/v4l2_m2m_dec.c +@@ -23,6 +23,9 @@ + + #include + #include ++ ++#include "libavutil/hwcontext.h" ++#include "libavutil/hwcontext_drm.h" + #include "libavutil/pixfmt.h" + #include "libavutil/pixdesc.h" + #include "libavutil/opt.h" +@@ -30,6 +33,9 @@ + #include "libavcodec/decode.h" + #include "libavcodec/internal.h" + ++#include "libavcodec/hwaccels.h" ++#include "libavcodec/internal.h" ++ + #include "v4l2_context.h" + #include "v4l2_m2m.h" + #include "v4l2_fmt.h" +@@ -207,6 +213,15 @@ static av_cold int v4l2_decode_init(AVCodecContext *avctx) + capture->av_codec_id = AV_CODEC_ID_RAWVIDEO; + capture->av_pix_fmt = avctx->pix_fmt; + ++ /* the client requests the codec to generate DRM frames: ++ * - data[0] will therefore point to the returned AVDRMFrameDescriptor ++ * check the ff_v4l2_buffer_to_avframe conversion function. ++ * - the DRM frame format is passed in the DRM frame descriptor layer. ++ * check the v4l2_get_drm_frame function. ++ */ ++ if (ff_get_format(avctx, avctx->codec->pix_fmts) == AV_PIX_FMT_DRM_PRIME) ++ s->output_drm = 1; ++ + s->avctx = avctx; + ret = ff_v4l2_m2m_codec_init(priv); + if (ret) { +@@ -232,6 +247,11 @@ static const AVOption options[] = { + { NULL}, + }; + ++static const AVCodecHWConfigInternal *v4l2_m2m_hw_configs[] = { ++ HW_CONFIG_INTERNAL(DRM_PRIME), ++ NULL ++}; ++ + #define M2MDEC_CLASS(NAME) \ + static const AVClass v4l2_m2m_ ## NAME ## _dec_class = { \ + .class_name = #NAME "_v4l2m2m_decoder", \ +@@ -255,6 +275,9 @@ static const AVOption options[] = { + .bsfs = bsf_name, \ + .capabilities = AV_CODEC_CAP_HARDWARE | AV_CODEC_CAP_DELAY | AV_CODEC_CAP_AVOID_PROBING, \ + .caps_internal = FF_CODEC_CAP_SETS_PKT_DTS | FF_CODEC_CAP_INIT_CLEANUP, \ ++ .pix_fmts = (const enum AVPixelFormat[]) { AV_PIX_FMT_DRM_PRIME, \ ++ AV_PIX_FMT_NONE}, \ ++ .hw_configs = v4l2_m2m_hw_configs, \ + .wrapper_name = "v4l2m2m", \ + } + + +From 2b5cd753892dceba3b211053a6266c40aab38c55 Mon Sep 17 00:00:00 2001 +From: Lukas Rusak +Date: Thu, 16 Aug 2018 21:09:40 -0700 +Subject: [PATCH 04/11] libavcodec: v4l2m2m: depends on libdrm + +--- + configure | 1 + + libavcodec/v4l2_buffers.c | 2 +- + 2 files changed, 2 insertions(+), 1 deletion(-) + +diff --git a/configure b/configure +index 8569a60bf8..a049707dd6 100755 +--- a/configure ++++ b/configure +@@ -3401,6 +3401,7 @@ sndio_indev_deps="sndio" + sndio_outdev_deps="sndio" + v4l2_indev_deps_any="linux_videodev2_h sys_videoio_h" + v4l2_indev_suggest="libv4l2" ++v4l2_outdev_deps="libdrm" + v4l2_outdev_deps_any="linux_videodev2_h sys_videoio_h" + v4l2_outdev_suggest="libv4l2" + vfwcap_indev_deps="vfw32 vfwcap_defines" +diff --git a/libavcodec/v4l2_buffers.c b/libavcodec/v4l2_buffers.c +index 4bb2bf6f87..c36a73d1fa 100644 +--- a/libavcodec/v4l2_buffers.c ++++ b/libavcodec/v4l2_buffers.c +@@ -21,7 +21,7 @@ + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +-#include ++#include + #include + #include + #include + +From 09a0f1b99548a249991891ee4e02ae6613b545d7 Mon Sep 17 00:00:00 2001 +From: Lukas Rusak +Date: Thu, 16 Aug 2018 21:10:13 -0700 +Subject: [PATCH 05/11] libavcodec: v4l2m2m: set format_modifier to + DRM_FORMAT_MOD_LINEAR + +--- + libavcodec/v4l2_buffers.c | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/libavcodec/v4l2_buffers.c b/libavcodec/v4l2_buffers.c +index c36a73d1fa..072b77bbda 100644 +--- a/libavcodec/v4l2_buffers.c ++++ b/libavcodec/v4l2_buffers.c +@@ -328,10 +328,12 @@ static int v4l2_buffer_export_drm(V4L2Buffer* avbuf) + /* drm frame */ + avbuf->drm_frame.objects[i].size = avbuf->buf.m.planes[i].length; + avbuf->drm_frame.objects[i].fd = expbuf.fd; ++ avbuf->drm_frame.objects[i].format_modifier = DRM_FORMAT_MOD_LINEAR; + } else { + /* drm frame */ + avbuf->drm_frame.objects[0].size = avbuf->buf.length; + avbuf->drm_frame.objects[0].fd = expbuf.fd; ++ avbuf->drm_frame.objects[0].format_modifier = DRM_FORMAT_MOD_LINEAR; + } + } + + +From e8df5a982f705aaba1e03aef653169bc17a0d464 Mon Sep 17 00:00:00 2001 +From: Lukas Rusak +Date: Thu, 16 Aug 2018 21:10:53 -0700 +Subject: [PATCH 06/11] libavcodec: v4l2m2m: only mmap the buffer when it is + output type and drm prime is used + +--- + libavcodec/v4l2_buffers.c | 20 ++++++++++++++------ + 1 file changed, 14 insertions(+), 6 deletions(-) + +diff --git a/libavcodec/v4l2_buffers.c b/libavcodec/v4l2_buffers.c +index 072b77bbda..8162531973 100644 +--- a/libavcodec/v4l2_buffers.c ++++ b/libavcodec/v4l2_buffers.c +@@ -662,14 +662,22 @@ int ff_v4l2_buffer_initialize(V4L2Buffer* avbuf, int index) + + if (V4L2_TYPE_IS_MULTIPLANAR(ctx->type)) { + avbuf->plane_info[i].length = avbuf->buf.m.planes[i].length; +- avbuf->plane_info[i].mm_addr = mmap(NULL, avbuf->buf.m.planes[i].length, +- PROT_READ | PROT_WRITE, MAP_SHARED, +- buf_to_m2mctx(avbuf)->fd, avbuf->buf.m.planes[i].m.mem_offset); ++ ++ if ((V4L2_TYPE_IS_OUTPUT(ctx->type) && buf_to_m2mctx(avbuf)->output_drm) || ++ !buf_to_m2mctx(avbuf)->output_drm) { ++ avbuf->plane_info[i].mm_addr = mmap(NULL, avbuf->buf.m.planes[i].length, ++ PROT_READ | PROT_WRITE, MAP_SHARED, ++ buf_to_m2mctx(avbuf)->fd, avbuf->buf.m.planes[i].m.mem_offset); ++ } + } else { + avbuf->plane_info[i].length = avbuf->buf.length; +- avbuf->plane_info[i].mm_addr = mmap(NULL, avbuf->buf.length, +- PROT_READ | PROT_WRITE, MAP_SHARED, +- buf_to_m2mctx(avbuf)->fd, avbuf->buf.m.offset); ++ ++ if ((V4L2_TYPE_IS_OUTPUT(ctx->type) && buf_to_m2mctx(avbuf)->output_drm) || ++ !buf_to_m2mctx(avbuf)->output_drm) { ++ avbuf->plane_info[i].mm_addr = mmap(NULL, avbuf->buf.length, ++ PROT_READ | PROT_WRITE, MAP_SHARED, ++ buf_to_m2mctx(avbuf)->fd, avbuf->buf.m.offset); ++ } + } + + if (avbuf->plane_info[i].mm_addr == MAP_FAILED) + +From 4fb7664bb6be542b691323a03050cdd024585afc Mon Sep 17 00:00:00 2001 +From: Lukas Rusak +Date: Thu, 16 Aug 2018 21:11:38 -0700 +Subject: [PATCH 07/11] libavcodec: v4l2m2m: allow using software pixel formats + +--- + libavcodec/v4l2_m2m_dec.c | 11 ++++++++++- + 1 file changed, 10 insertions(+), 1 deletion(-) + +diff --git a/libavcodec/v4l2_m2m_dec.c b/libavcodec/v4l2_m2m_dec.c +index eb6ecc8ed5..3b2449ae6c 100644 +--- a/libavcodec/v4l2_m2m_dec.c ++++ b/libavcodec/v4l2_m2m_dec.c +@@ -219,8 +219,16 @@ static av_cold int v4l2_decode_init(AVCodecContext *avctx) + * - the DRM frame format is passed in the DRM frame descriptor layer. + * check the v4l2_get_drm_frame function. + */ +- if (ff_get_format(avctx, avctx->codec->pix_fmts) == AV_PIX_FMT_DRM_PRIME) ++ switch (ff_get_format(avctx, avctx->codec->pix_fmts)) { ++ case AV_PIX_FMT_DRM_PRIME: + s->output_drm = 1; ++ break; ++ case AV_PIX_FMT_NONE: ++ return 0; ++ break; ++ default: ++ break; ++ } + + s->avctx = avctx; + ret = ff_v4l2_m2m_codec_init(priv); +@@ -276,6 +284,7 @@ static const AVCodecHWConfigInternal *v4l2_m2m_hw_configs[] = { + .capabilities = AV_CODEC_CAP_HARDWARE | AV_CODEC_CAP_DELAY | AV_CODEC_CAP_AVOID_PROBING, \ + .caps_internal = FF_CODEC_CAP_SETS_PKT_DTS | FF_CODEC_CAP_INIT_CLEANUP, \ + .pix_fmts = (const enum AVPixelFormat[]) { AV_PIX_FMT_DRM_PRIME, \ ++ AV_PIX_FMT_NV12, \ + AV_PIX_FMT_NONE}, \ + .hw_configs = v4l2_m2m_hw_configs, \ + .wrapper_name = "v4l2m2m", \ + +From 27ae887df07992385b1afc9b532f978066e83774 Mon Sep 17 00:00:00 2001 +From: Lukas Rusak +Date: Mon, 24 Sep 2018 13:39:31 -0700 +Subject: [PATCH 08/11] libavcodec: v4l2m2m: implement hwcontext + +--- + libavcodec/v4l2_buffers.c | 22 ++++++++++++++++++++++ + libavcodec/v4l2_context.h | 2 ++ + libavcodec/v4l2_m2m.h | 2 ++ + libavcodec/v4l2_m2m_dec.c | 11 +++++++++++ + 4 files changed, 37 insertions(+) + +diff --git a/libavcodec/v4l2_buffers.c b/libavcodec/v4l2_buffers.c +index 8162531973..9c5d471c9b 100644 +--- a/libavcodec/v4l2_buffers.c ++++ b/libavcodec/v4l2_buffers.c +@@ -435,6 +435,7 @@ static int v4l2_buffer_buf_to_swframe(AVFrame *frame, V4L2Buffer *avbuf) + + frame->data[0] = (uint8_t *) v4l2_get_drm_frame(avbuf); + frame->format = AV_PIX_FMT_DRM_PRIME; ++ frame->hw_frames_ctx = av_buffer_ref(avbuf->context->frames_ref); + } else { + /* 1. get references to the actual data */ + for (i = 0; i < avbuf->num_planes; i++) { +@@ -635,6 +636,27 @@ int ff_v4l2_buffer_initialize(V4L2Buffer* avbuf, int index) + avbuf->buf.type = ctx->type; + avbuf->buf.index = index; + ++ if (buf_to_m2mctx(avbuf)->output_drm) { ++ AVHWFramesContext *hwframes; ++ ++ av_buffer_unref(&ctx->frames_ref); ++ ++ ctx->frames_ref = av_hwframe_ctx_alloc(buf_to_m2mctx(avbuf)->device_ref); ++ if (!ctx->frames_ref) { ++ ret = AVERROR(ENOMEM); ++ return ret; ++ } ++ ++ hwframes = (AVHWFramesContext*)ctx->frames_ref->data; ++ hwframes->format = AV_PIX_FMT_DRM_PRIME; ++ hwframes->sw_format = ctx->av_pix_fmt; ++ hwframes->width = ctx->width; ++ hwframes->height = ctx->height; ++ ret = av_hwframe_ctx_init(ctx->frames_ref); ++ if (ret < 0) ++ return ret; ++ } ++ + if (V4L2_TYPE_IS_MULTIPLANAR(ctx->type)) { + avbuf->buf.length = VIDEO_MAX_PLANES; + avbuf->buf.m.planes = avbuf->planes; +diff --git a/libavcodec/v4l2_context.h b/libavcodec/v4l2_context.h +index 22a9532444..e804e94131 100644 +--- a/libavcodec/v4l2_context.h ++++ b/libavcodec/v4l2_context.h +@@ -92,6 +92,8 @@ typedef struct V4L2Context { + */ + int done; + ++ AVBufferRef *frames_ref; ++ + } V4L2Context; + + /** +diff --git a/libavcodec/v4l2_m2m.h b/libavcodec/v4l2_m2m.h +index 4ee0be653b..61cb919771 100644 +--- a/libavcodec/v4l2_m2m.h ++++ b/libavcodec/v4l2_m2m.h +@@ -64,6 +64,8 @@ typedef struct V4L2m2mContext { + /* reference back to V4L2m2mPriv */ + void *priv; + ++ AVBufferRef *device_ref; ++ + /* generate DRM frames */ + int output_drm; + } V4L2m2mContext; +diff --git a/libavcodec/v4l2_m2m_dec.c b/libavcodec/v4l2_m2m_dec.c +index 3b2449ae6c..c6b865fde8 100644 +--- a/libavcodec/v4l2_m2m_dec.c ++++ b/libavcodec/v4l2_m2m_dec.c +@@ -35,6 +35,7 @@ + + #include "libavcodec/hwaccels.h" + #include "libavcodec/internal.h" ++#include "libavcodec/hwconfig.h" + + #include "v4l2_context.h" + #include "v4l2_m2m.h" +@@ -230,6 +231,16 @@ static av_cold int v4l2_decode_init(AVCodecContext *avctx) + break; + } + ++ s->device_ref = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_DRM); ++ if (!s->device_ref) { ++ ret = AVERROR(ENOMEM); ++ return ret; ++ } ++ ++ ret = av_hwdevice_ctx_init(s->device_ref); ++ if (ret < 0) ++ return ret; ++ + s->avctx = avctx; + ret = ff_v4l2_m2m_codec_init(priv); + if (ret) { + +From aee464209ec6ad060d352dfb638344a1f4db3ce4 Mon Sep 17 00:00:00 2001 +From: Lukas Rusak +Date: Mon, 4 May 2020 13:01:29 -0700 +Subject: [PATCH 09/11] libavcodec: v4l2m2m: allow lower minimum buffer values + +There is no reason to enforce a high minimum. In the context +of streaming only a few output buffers and capture buffers +are even needed for continuous playback. This also helps +alleviate memory pressure when decoding 4K media. +--- + libavcodec/v4l2_m2m.h | 2 +- + libavcodec/v4l2_m2m_dec.c | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/libavcodec/v4l2_m2m.h b/libavcodec/v4l2_m2m.h +index 61cb919771..feeb162812 100644 +--- a/libavcodec/v4l2_m2m.h ++++ b/libavcodec/v4l2_m2m.h +@@ -38,7 +38,7 @@ + + #define V4L_M2M_DEFAULT_OPTS \ + { "num_output_buffers", "Number of buffers in the output context",\ +- OFFSET(num_output_buffers), AV_OPT_TYPE_INT, { .i64 = 16 }, 6, INT_MAX, FLAGS } ++ OFFSET(num_output_buffers), AV_OPT_TYPE_INT, { .i64 = 16 }, 2, INT_MAX, FLAGS } + + typedef struct V4L2m2mContext { + char devname[PATH_MAX]; +diff --git a/libavcodec/v4l2_m2m_dec.c b/libavcodec/v4l2_m2m_dec.c +index c6b865fde8..b9725be377 100644 +--- a/libavcodec/v4l2_m2m_dec.c ++++ b/libavcodec/v4l2_m2m_dec.c +@@ -262,7 +262,7 @@ static av_cold int v4l2_decode_close(AVCodecContext *avctx) + static const AVOption options[] = { + V4L_M2M_DEFAULT_OPTS, + { "num_capture_buffers", "Number of buffers in the capture context", +- OFFSET(num_capture_buffers), AV_OPT_TYPE_INT, {.i64 = 20}, 20, INT_MAX, FLAGS }, ++ OFFSET(num_capture_buffers), AV_OPT_TYPE_INT, {.i64 = 20}, 2, INT_MAX, FLAGS }, + { NULL}, + }; + + +From ffc4419f456c00ab71cf93f792b0473c6de14e64 Mon Sep 17 00:00:00 2001 +From: Lukas Rusak +Date: Wed, 6 May 2020 11:12:58 -0700 +Subject: [PATCH 10/11] libavcodec: v4l2m2m: add option to specify pixel format + used by the decoder + +--- + libavcodec/v4l2_context.c | 9 +++++++++ + libavcodec/v4l2_m2m.h | 2 ++ + libavcodec/v4l2_m2m_dec.c | 1 + + 3 files changed, 12 insertions(+) + +diff --git a/libavcodec/v4l2_context.c b/libavcodec/v4l2_context.c +index 7a92df2c3e..fa2deae888 100644 +--- a/libavcodec/v4l2_context.c ++++ b/libavcodec/v4l2_context.c +@@ -531,6 +531,8 @@ static inline int v4l2_try_raw_format(V4L2Context* ctx, enum AVPixelFormat pixfm + + static int v4l2_get_raw_format(V4L2Context* ctx, enum AVPixelFormat *p) + { ++ V4L2m2mContext* s = ctx_to_m2mctx(ctx); ++ V4L2m2mPriv *priv = s->avctx->priv_data; + enum AVPixelFormat pixfmt = ctx->av_pix_fmt; + struct v4l2_fmtdesc fdesc; + int ret; +@@ -549,6 +551,13 @@ static int v4l2_get_raw_format(V4L2Context* ctx, enum AVPixelFormat *p) + if (ret) + return AVERROR(EINVAL); + ++ if (priv->pix_fmt != AV_PIX_FMT_NONE) { ++ if (fdesc.pixelformat != ff_v4l2_format_avfmt_to_v4l2(priv->pix_fmt)) { ++ fdesc.index++; ++ continue; ++ } ++ } ++ + pixfmt = ff_v4l2_format_v4l2_to_avfmt(fdesc.pixelformat, AV_CODEC_ID_RAWVIDEO); + ret = v4l2_try_raw_format(ctx, pixfmt); + if (ret){ +diff --git a/libavcodec/v4l2_m2m.h b/libavcodec/v4l2_m2m.h +index feeb162812..0e88bf9329 100644 +--- a/libavcodec/v4l2_m2m.h ++++ b/libavcodec/v4l2_m2m.h +@@ -30,6 +30,7 @@ + #include + + #include "libavcodec/avcodec.h" ++#include "libavutil/pixfmt.h" + #include "v4l2_context.h" + + #define container_of(ptr, type, member) ({ \ +@@ -78,6 +79,7 @@ typedef struct V4L2m2mPriv { + + int num_output_buffers; + int num_capture_buffers; ++ enum AVPixelFormat pix_fmt; + } V4L2m2mPriv; + + /** +diff --git a/libavcodec/v4l2_m2m_dec.c b/libavcodec/v4l2_m2m_dec.c +index b9725be377..6109deee8a 100644 +--- a/libavcodec/v4l2_m2m_dec.c ++++ b/libavcodec/v4l2_m2m_dec.c +@@ -263,6 +263,7 @@ static const AVOption options[] = { + V4L_M2M_DEFAULT_OPTS, + { "num_capture_buffers", "Number of buffers in the capture context", + OFFSET(num_capture_buffers), AV_OPT_TYPE_INT, {.i64 = 20}, 2, INT_MAX, FLAGS }, ++ { "pixel_format", "Pixel format to be used by the decoder", OFFSET(pix_fmt), AV_OPT_TYPE_PIXEL_FMT, {.i64 = AV_PIX_FMT_NONE}, AV_PIX_FMT_NONE, AV_PIX_FMT_NB, FLAGS }, + { NULL}, + }; + + +From 8595d06d4909bbec0aa14625fcfc869c6bcef696 Mon Sep 17 00:00:00 2001 +From: Lukas Rusak +Date: Mon, 24 Sep 2018 13:39:56 -0700 +Subject: [PATCH 11/11] libavcodec: v4l2m2m: implement flush + +--- + libavcodec/v4l2_m2m_dec.c | 36 ++++++++++++++++++++++++++++++++++++ + 1 file changed, 36 insertions(+) + +diff --git a/libavcodec/v4l2_m2m_dec.c b/libavcodec/v4l2_m2m_dec.c +index 6109deee8a..820cdf241f 100644 +--- a/libavcodec/v4l2_m2m_dec.c ++++ b/libavcodec/v4l2_m2m_dec.c +@@ -256,6 +256,41 @@ static av_cold int v4l2_decode_close(AVCodecContext *avctx) + return ff_v4l2_m2m_codec_end(avctx->priv_data); + } + ++static void v4l2_decode_flush(AVCodecContext *avctx) ++{ ++ V4L2m2mPriv *priv = avctx->priv_data; ++ V4L2m2mContext* s = priv->context; ++ V4L2Context* output = &s->output; ++ V4L2Context* capture = &s->capture; ++ int ret, i; ++ ++ ret = ff_v4l2_context_set_status(output, VIDIOC_STREAMOFF); ++ if (ret < 0) ++ av_log(avctx, AV_LOG_ERROR, "VIDIOC_STREAMOFF %s error: %d\n", output->name, ret); ++ ++ ret = ff_v4l2_context_set_status(output, VIDIOC_STREAMON); ++ if (ret < 0) ++ av_log(avctx, AV_LOG_ERROR, "VIDIOC_STREAMON %s error: %d\n", output->name, ret); ++ ++ for (i = 0; i < output->num_buffers; i++) { ++ if (output->buffers[i].status == V4L2BUF_IN_DRIVER) ++ output->buffers[i].status = V4L2BUF_AVAILABLE; ++ } ++ ++ struct v4l2_decoder_cmd cmd = { ++ .cmd = V4L2_DEC_CMD_START, ++ .flags = 0, ++ }; ++ ++ ret = ioctl(s->fd, VIDIOC_DECODER_CMD, &cmd); ++ if (ret < 0) ++ av_log(avctx, AV_LOG_ERROR, "VIDIOC_DECODER_CMD start error: %d\n", errno); ++ ++ s->draining = 0; ++ output->done = 0; ++ capture->done = 0; ++} ++ + #define OFFSET(x) offsetof(V4L2m2mPriv, x) + #define FLAGS AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_DECODING_PARAM + +@@ -292,6 +327,7 @@ static const AVCodecHWConfigInternal *v4l2_m2m_hw_configs[] = { + .init = v4l2_decode_init, \ + .receive_frame = v4l2_receive_frame, \ + .close = v4l2_decode_close, \ ++ .flush = v4l2_decode_flush, \ + .bsfs = bsf_name, \ + .capabilities = AV_CODEC_CAP_HARDWARE | AV_CODEC_CAP_DELAY | AV_CODEC_CAP_AVOID_PROBING, \ + .caps_internal = FF_CODEC_CAP_SETS_PKT_DTS | FF_CODEC_CAP_INIT_CLEANUP, \